From 0d58385db2ce20ceca50f3904af982e712ef00fa Mon Sep 17 00:00:00 2001 From: Jamie MacDonald Date: Wed, 12 Jul 2023 17:06:49 +0100 Subject: [PATCH 01/41] Remove Snyk jobs from CI scanning --- .github/workflows/security-scans.yml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/.github/workflows/security-scans.yml b/.github/workflows/security-scans.yml index 7348db48311f..a421295f4ab5 100644 --- a/.github/workflows/security-scans.yml +++ b/.github/workflows/security-scans.yml @@ -128,25 +128,4 @@ jobs: - name: Run Bandit 🔪 if: needs.changes.outputs.backend == 'true' - run: make lint-security - - snyk: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c - - name: Run Snyk Open Source to check for Python vulnerabilities - uses: snyk/actions/python-3.8@master - continue-on-error: true - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - command: monitor - args: --all-projects --org=rasa --skip-unresolved - - name: Run Snyk Open Source to check for JS vulnerabilities - uses: snyk/actions/node@master - continue-on-error: true - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - command: monitor - args: --org=rasa --yarn-workspaces --strict-out-of-sync=false --prune-repeated-subdependencies + run: make lint-security \ No newline at end of file From dc6f00de718dbcf244e800fe90912f0dfd6c45f8 Mon Sep 17 00:00:00 2001 From: Jamie MacDonald Date: Wed, 12 Jul 2023 17:10:28 +0100 Subject: [PATCH 02/41] Fix line formatting --- .github/workflows/security-scans.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/security-scans.yml b/.github/workflows/security-scans.yml index a421295f4ab5..b4cbf69cc92f 100644 --- a/.github/workflows/security-scans.yml +++ b/.github/workflows/security-scans.yml @@ -128,4 +128,4 @@ jobs: - name: Run Bandit 🔪 if: needs.changes.outputs.backend == 'true' - run: make lint-security \ No newline at end of file + run: make lint-security From 58e122fe459fe3a21441fc44f9a1082d490eb9f2 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Thu, 16 Nov 2023 10:06:17 +0100 Subject: [PATCH 03/41] Made use of drop-small-last-batch logic only possible in DIET and configurable (#12948) * Made use of drop-small-last-batch logic only possible in DIET and configurable --- changelog/12948.bugfix.md | 1 + rasa/nlu/classifiers/diet_classifier.py | 5 +++ rasa/utils/tensorflow/constants.py | 1 + rasa/utils/tensorflow/data_generator.py | 19 ++++++--- rasa/utils/train_utils.py | 5 +++ tests/nlu/classifiers/test_diet_classifier.py | 41 +++++++++++++------ 6 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 changelog/12948.bugfix.md diff --git a/changelog/12948.bugfix.md b/changelog/12948.bugfix.md new file mode 100644 index 000000000000..7479161831be --- /dev/null +++ b/changelog/12948.bugfix.md @@ -0,0 +1 @@ +Fixed UnexpecTEDIntentlessPolicy training errors that resulted from a change to batching behavior. Changed the batching behavior back to the original for all components. Made the changed batching behavior accessible in DietClassifier using `drop_small_last_batch: True`. diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 1cc65c89b3c9..bea4735da6fe 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -50,6 +50,7 @@ from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.shared.nlu.training_data.message import Message from rasa.utils.tensorflow.constants import ( + DROP_SMALL_LAST_BATCH, LABEL, IDS, HIDDEN_LAYERS_SIZES, @@ -288,6 +289,9 @@ def get_default_config() -> Dict[Text, Any]: # a few steps, as the compilation of the graph tends to take more time than # running it. It is recommended to not adjust the optimization parameter. RUN_EAGERLY: False, + # Determines whether the last batch should be dropped if it contains fewer + # than half a batch size of examples + DROP_SMALL_LAST_BATCH: False, } def __init__( @@ -931,6 +935,7 @@ def train(self, training_data: TrainingData) -> Resource: self.component_config[BATCH_STRATEGY], self.component_config[EVAL_NUM_EXAMPLES], self.component_config[RANDOM_SEED], + drop_small_last_batch=self.component_config[DROP_SMALL_LAST_BATCH], ) callbacks = train_utils.create_common_callbacks( self.component_config[EPOCHS], diff --git a/rasa/utils/tensorflow/constants.py b/rasa/utils/tensorflow/constants.py index 047db9878c67..39d5ea6d0560 100644 --- a/rasa/utils/tensorflow/constants.py +++ b/rasa/utils/tensorflow/constants.py @@ -113,3 +113,4 @@ USE_GPU = "use_gpu" RUN_EAGERLY = "run_eagerly" +DROP_SMALL_LAST_BATCH = "drop_small_last_batch" diff --git a/rasa/utils/tensorflow/data_generator.py b/rasa/utils/tensorflow/data_generator.py index a696f607c026..e54b95dad335 100644 --- a/rasa/utils/tensorflow/data_generator.py +++ b/rasa/utils/tensorflow/data_generator.py @@ -344,6 +344,7 @@ def __init__( epochs: int = 1, batch_strategy: Text = SEQUENCE, shuffle: bool = True, + drop_small_last_batch: bool = False, ): """Initializes the increasing batch size data generator. @@ -353,6 +354,8 @@ def __init__( epochs: The total number of epochs. batch_strategy: The batch strategy. shuffle: If 'True', data will be shuffled. + drop_small_last_batch: if 'True', the last batch in an epoch will be dropped + if it has less examples than half the batch size """ super().__init__(model_data, batch_size, batch_strategy, shuffle) @@ -370,6 +373,7 @@ def __init__( self._current_batch_size = 0 # create separate data variable that will store modified data for each batch self._data: Data = {} + self.drop_small_last_batch = drop_small_last_batch self.on_epoch_end() def __len__(self) -> int: @@ -381,11 +385,16 @@ def __len__(self) -> int: # data was rebalanced, so need to recalculate number of examples num_examples = self.model_data.number_of_examples(self._data) batch_size = self._current_batch_size - # keep last batch only if it has at least half a batch size of examples - last_batch_half_full = num_examples % batch_size >= math.ceil(batch_size / 2) - num_batches = num_examples // batch_size + int(last_batch_half_full) - # Return at least 1 if there is an example - return max(num_batches, int(num_examples > 0)) + if self.drop_small_last_batch: + # keep last batch only if it has at least half a batch size of examples + last_batch_half_full = num_examples % batch_size >= math.ceil( + batch_size / 2 + ) + num_batches = num_examples // batch_size + int(last_batch_half_full) + # Return at least 1 if there is an example + return max(num_batches, int(num_examples > 0)) + else: + return num_examples // batch_size + int(num_examples % batch_size > 0) def __getitem__(self, index: int) -> Tuple[Any, Any]: """Gets batch at position `index`. diff --git a/rasa/utils/train_utils.py b/rasa/utils/train_utils.py index 36de0370d210..764507d7e39d 100644 --- a/rasa/utils/train_utils.py +++ b/rasa/utils/train_utils.py @@ -302,6 +302,7 @@ def create_data_generators( eval_num_examples: int = 0, random_seed: Optional[int] = None, shuffle: bool = True, + drop_small_last_batch: bool = False, ) -> Tuple[RasaBatchDataGenerator, Optional[RasaBatchDataGenerator]]: """Create data generators for train and optional validation data. @@ -313,6 +314,8 @@ def create_data_generators( eval_num_examples: Number of examples to use for validation data. random_seed: The random seed. shuffle: Whether to shuffle data inside the data generator. + drop_small_last_batch: whether to drop the last batch if it has fewer than half + a batch size of examples Returns: The training data generator and optional validation data generator. @@ -328,6 +331,7 @@ def create_data_generators( epochs=epochs, batch_strategy=batch_strategy, shuffle=shuffle, + drop_small_last_batch=drop_small_last_batch, ) data_generator = RasaBatchDataGenerator( @@ -336,6 +340,7 @@ def create_data_generators( epochs=epochs, batch_strategy=batch_strategy, shuffle=shuffle, + drop_small_last_batch=drop_small_last_batch, ) return data_generator, validation_data_generator diff --git a/tests/nlu/classifiers/test_diet_classifier.py b/tests/nlu/classifiers/test_diet_classifier.py index 1f0c37a85faa..1fd84fdac47d 100644 --- a/tests/nlu/classifiers/test_diet_classifier.py +++ b/tests/nlu/classifiers/test_diet_classifier.py @@ -971,24 +971,35 @@ async def test_no_bilou_when_entity_recognition_off( @pytest.mark.timeout(120, func_only=True) @pytest.mark.parametrize( - "batch_size, expected_num_batches", + "batch_size, expected_num_batches, drop_small_last_batch", # the training dataset has 48 NLU examples [ - (1, 48), - (8, 6), - (15, 3), - (16, 3), - (18, 3), - (20, 2), - (32, 2), - (64, 1), - (128, 1), - (256, 1), + (1, 48, True), + (8, 6, True), + (15, 3, True), + (16, 3, True), + (18, 3, True), + (20, 2, True), + (32, 2, True), + (64, 1, True), + (128, 1, True), + (256, 1, True), + (1, 48, False), + (8, 6, False), + (15, 4, False), + (16, 3, False), + (18, 3, False), + (20, 3, False), + (32, 2, False), + (64, 1, False), + (128, 1, False), + (256, 1, False), ], ) async def test_dropping_of_last_partial_batch( batch_size: int, expected_num_batches: int, + drop_small_last_batch: bool, create_diet: Callable[..., DIETClassifier], train_and_preprocess: Callable[..., Tuple[TrainingData, List[GraphComponent]]], ): @@ -1012,7 +1023,9 @@ async def test_dropping_of_last_partial_batch( ) model_data = diet.preprocess_train_data(training_data) - data_generator, _ = train_utils.create_data_generators(model_data, batch_size, 1) + data_generator, _ = train_utils.create_data_generators( + model_data, batch_size, 1, drop_small_last_batch=drop_small_last_batch + ) assert len(data_generator) == expected_num_batches @@ -1041,6 +1054,8 @@ async def test_dropping_of_last_partial_batch_empty_data( ) model_data = diet.preprocess_train_data(training_data) - data_generator, _ = train_utils.create_data_generators(model_data, 64, 1) + data_generator, _ = train_utils.create_data_generators( + model_data, 64, 1, drop_small_last_batch=True + ) assert len(data_generator) == 0 From 048dc805b1781535a0b10705e4b104faec066abb Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Fri, 17 Nov 2023 15:11:59 +0100 Subject: [PATCH 04/41] adjusted release script to work with public remote (#12953) --- scripts/release.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/release.py b/scripts/release.py index d1ac98325f80..068bd14e5fd2 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -30,6 +30,10 @@ RELEASE_BRANCH_PATTERN = re.compile(r"^\d+\.\d+\.x$") +PUBLIC_REMOTE = "public" +DEFAULT_REMOTE = "origin" +FIRST_CALM_VERSION = "3.7.0" + def create_argument_parser() -> argparse.ArgumentParser: """Parse all the command line arguments for the release script.""" @@ -247,9 +251,9 @@ def create_commit(version: Version) -> None: check_call(["git", "commit", "-m", f"prepared release of version {version}"]) -def push_changes() -> None: - """Pushes the current branch to origin.""" - check_call(["git", "push", "origin", "HEAD"]) +def push_changes(remote: str = DEFAULT_REMOTE) -> None: + """Pushes the current branch to the specified remote.""" + check_call(["git", "push", remote, "HEAD"]) def ensure_clean_git() -> None: @@ -337,10 +341,11 @@ def main(args: argparse.Namespace) -> None: # never update changelog on a prerelease version generate_changelog(version) + remote = PUBLIC_REMOTE if str(version) < FIRST_CALM_VERSION else DEFAULT_REMOTE # alpha workflow on feature branch when a version bump is required if version.is_alpha and not git_current_branch_is_main_or_release(): create_commit(version) - push_changes() + push_changes(remote) print_done_message_same_branch(version) else: @@ -348,7 +353,7 @@ def main(args: argparse.Namespace) -> None: branch = create_release_branch(version) create_commit(version) - push_changes() + push_changes(remote) print_done_message(branch, base, version) From 241bf28342cb8b56e3d395d6136587286debbe9c Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Mon, 20 Nov 2023 10:10:11 +0100 Subject: [PATCH 05/41] Prepare release 3.6.14 (#12956) * prepared release of version 3.6.14 --- CHANGELOG.mdx | 7 +++++++ changelog/12948.bugfix.md | 1 - pyproject.toml | 2 +- rasa/version.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) delete mode 100644 changelog/12948.bugfix.md diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index 9a6177e3632c..b574a76b1b96 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,13 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.14] - 2023-11-17 + +Rasa 3.6.14 (2023-11-17) +### Bugfixes +- [#12948](https://github.com/rasahq/rasa/issues/12948): Fixed UnexpecTEDIntentlessPolicy training errors that resulted from a change to batching behavior. Changed the batching behavior back to the original for all components. Made the changed batching behavior accessible in DietClassifier using `drop_small_last_batch: True`. + + ## [3.6.13] - 2023-10-23 Rasa 3.6.13 (2023-10-23) diff --git a/changelog/12948.bugfix.md b/changelog/12948.bugfix.md deleted file mode 100644 index 7479161831be..000000000000 --- a/changelog/12948.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed UnexpecTEDIntentlessPolicy training errors that resulted from a change to batching behavior. Changed the batching behavior back to the original for all components. Made the changed batching behavior accessible in DietClassifier using `drop_small_last_batch: True`. diff --git a/pyproject.toml b/pyproject.toml index c3809092b5c2..4bf33f2a32d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.13" +version = "3.6.14" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index 7f4d3334548b..e4512ce41043 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.13" +__version__ = "3.6.14" From 609287ec3cdaa3151f867753878e4ffeb9344da9 Mon Sep 17 00:00:00 2001 From: Varun Shankar S Date: Thu, 30 Nov 2023 13:26:52 +0100 Subject: [PATCH 06/41] Port "connection timeout to action server" changes to 3.6.x - [ENG 689] (#12965) * Merge pull request #106 from RasaHQ/ENG-680-DEFAULT_KEEP_ALIVE_TIMEOUT Fix connection to action server - [ENG 680] --------- Co-authored-by: Tom Bocklisch --- rasa/core/agent.py | 96 +++++++++++++++++++---------------- rasa/core/constants.py | 2 + rasa/core/run.py | 82 ++++++++++++++---------------- rasa/utils/endpoints.py | 68 ++++++++++++++----------- tests/core/test_run.py | 3 -- tests/utils/test_endpoints.py | 42 +++------------ 6 files changed, 136 insertions(+), 157 deletions(-) diff --git a/rasa/core/agent.py b/rasa/core/agent.py index bf3d42236e70..47e3360f6a5b 100644 --- a/rasa/core/agent.py +++ b/rasa/core/agent.py @@ -112,53 +112,59 @@ async def _pull_model_and_fingerprint( logger.debug(f"Requesting model from server {model_server.url}...") - try: - params = model_server.combine_parameters() - async with model_server.session.request( - "GET", - model_server.url, - timeout=DEFAULT_REQUEST_TIMEOUT, - headers=headers, - params=params, - ) as resp: - if resp.status in [204, 304]: - logger.debug( - "Model server returned {} status code, " - "indicating that no new model is available. " - "Current fingerprint: {}" - "".format(resp.status, fingerprint) - ) - return None - elif resp.status == 404: - logger.debug( - "Model server could not find a model at the requested " - "endpoint '{}'. It's possible that no model has been " - "trained, or that the requested tag hasn't been " - "assigned.".format(model_server.url) - ) - return None - elif resp.status != 200: - logger.debug( - "Tried to fetch model from server, but server response " - "status code is {}. We'll retry later..." - "".format(resp.status) + async with model_server.session() as session: + try: + params = model_server.combine_parameters() + async with session.request( + "GET", + model_server.url, + timeout=DEFAULT_REQUEST_TIMEOUT, + headers=headers, + params=params, + ) as resp: + + if resp.status in [204, 304]: + logger.debug( + "Model server returned {} status code, " + "indicating that no new model is available. " + "Current fingerprint: {}" + "".format(resp.status, fingerprint) + ) + return None + elif resp.status == 404: + logger.debug( + "Model server could not find a model at the requested " + "endpoint '{}'. It's possible that no model has been " + "trained, or that the requested tag hasn't been " + "assigned.".format(model_server.url) + ) + return None + elif resp.status != 200: + logger.debug( + "Tried to fetch model from server, but server response " + "status code is {}. We'll retry later..." + "".format(resp.status) + ) + return None + + model_path = Path(model_directory) / resp.headers.get( + "filename", "model.tar.gz" ) - return None - model_path = Path(model_directory) / resp.headers.get( - "filename", "model.tar.gz" + with open(model_path, "wb") as file: + file.write(await resp.read()) + + logger.debug("Saved model to '{}'".format(os.path.abspath(model_path))) + + # return the new fingerprint + return resp.headers.get("ETag") + + except aiohttp.ClientError as e: + logger.debug( + "Tried to fetch model from server, but " + "couldn't reach server. We'll retry later... " + "Error: {}.".format(e) ) - with open(model_path, "wb") as file: - file.write(await resp.read()) - logger.debug("Saved model to '{}'".format(os.path.abspath(model_path))) - # return the new fingerprint - return resp.headers.get("ETag") - except aiohttp.ClientError as e: - logger.debug( - "Tried to fetch model from server, but " - "couldn't reach server. We'll retry later... " - "Error: {}.".format(e) - ) - return None + return None async def _run_model_pulling_worker(model_server: EndpointConfig, agent: Agent) -> None: diff --git a/rasa/core/constants.py b/rasa/core/constants.py index 973e4e7b3a99..40d65c3299bb 100644 --- a/rasa/core/constants.py +++ b/rasa/core/constants.py @@ -24,6 +24,8 @@ DEFAULT_LOCK_LIFETIME = 60 # in seconds +DEFAULT_KEEP_ALIVE_TIMEOUT = 120 # in seconds + BEARER_TOKEN_PREFIX = "Bearer " # The lowest priority is intended to be used by machine learning policies. diff --git a/rasa/core/run.py b/rasa/core/run.py index 5270162809dd..3a8133613c3f 100644 --- a/rasa/core/run.py +++ b/rasa/core/run.py @@ -1,9 +1,19 @@ import asyncio import logging import uuid +import platform import os from functools import partial -from typing import Any, List, Optional, TYPE_CHECKING, Text, Union, Dict +from typing import ( + Any, + Callable, + List, + Optional, + Text, + Tuple, + Union, + Dict, +) import rasa.core.utils from rasa.plugin import plugin_manager @@ -23,8 +33,6 @@ from sanic import Sanic from asyncio import AbstractEventLoop -if TYPE_CHECKING: - from aiohttp import ClientSession logger = logging.getLogger() # get the root logger @@ -80,6 +88,14 @@ def _create_app_without_api(cors: Optional[Union[Text, List[Text]]] = None) -> S return app +def _is_apple_silicon_system() -> bool: + # check if the system is MacOS + if platform.system().lower() != "darwin": + return False + # check for arm architecture, indicating apple silicon + return platform.machine().startswith("arm") or os.uname().machine.startswith("arm") + + def configure_app( input_channels: Optional[List["InputChannel"]] = None, cors: Optional[Union[Text, List[Text], None]] = None, @@ -99,6 +115,9 @@ def configure_app( syslog_port: Optional[int] = None, syslog_protocol: Optional[Text] = None, request_timeout: Optional[int] = None, + server_listeners: Optional[List[Tuple[Callable, Text]]] = None, + use_uvloop: Optional[bool] = True, + keep_alive_timeout: int = constants.DEFAULT_KEEP_ALIVE_TIMEOUT, ) -> Sanic: """Run the agent.""" rasa.core.utils.configure_file_logging( @@ -118,6 +137,14 @@ def configure_app( else: app = _create_app_without_api(cors) + app.config.KEEP_ALIVE_TIMEOUT = keep_alive_timeout + if _is_apple_silicon_system() or not use_uvloop: + app.config.USE_UVLOOP = False + # some library still sets the loop to uvloop, even if disabled for sanic + # using uvloop leads to breakingio errors, see + # https://rasahq.atlassian.net/browse/ENG-667 + asyncio.set_event_loop_policy(None) + if input_channels: channels.channel.register(input_channels, app, route=route) else: @@ -150,6 +177,10 @@ async def run_cmdline_io(running_app: Sanic) -> None: app.add_task(run_cmdline_io) + if server_listeners: + for (listener, event) in server_listeners: + app.register_listener(listener, event) + return app @@ -179,6 +210,7 @@ def serve_application( syslog_port: Optional[int] = None, syslog_protocol: Optional[Text] = None, request_timeout: Optional[int] = None, + server_listeners: Optional[List[Tuple[Callable, Text]]] = None, ) -> None: """Run the API entrypoint.""" if not channel and not credentials: @@ -204,6 +236,7 @@ def serve_application( syslog_port=syslog_port, syslog_protocol=syslog_protocol, request_timeout=request_timeout, + server_listeners=server_listeners, ) ssl_context = server.create_ssl_context( @@ -217,7 +250,7 @@ def serve_application( partial(load_agent_on_start, model_path, endpoints, remote_storage), "before_server_start", ) - app.register_listener(create_connection_pools, "after_server_start") + app.register_listener(close_resources, "after_server_stop") number_of_workers = rasa.core.utils.number_of_sanic_workers( @@ -279,44 +312,3 @@ async def close_resources(app: Sanic, _: AbstractEventLoop) -> None: event_broker = current_agent.tracker_store.event_broker if event_broker: await event_broker.close() - - action_endpoint = current_agent.action_endpoint - if action_endpoint: - await action_endpoint.session.close() - - model_server = current_agent.model_server - if model_server: - await model_server.session.close() - - -async def create_connection_pools(app: Sanic, _: AbstractEventLoop) -> None: - """Create connection pools for the agent's action server and model server.""" - current_agent = getattr(app.ctx, "agent", None) - if not current_agent: - logger.debug("No agent found after server start.") - return None - - create_action_endpoint_connection_pool(current_agent) - create_model_server_connection_pool(current_agent) - - return None - - -def create_action_endpoint_connection_pool(agent: Agent) -> Optional["ClientSession"]: - """Create a connection pool for the action endpoint.""" - action_endpoint = agent.action_endpoint - if not action_endpoint: - logger.debug("No action endpoint found after server start.") - return None - - return action_endpoint.session - - -def create_model_server_connection_pool(agent: Agent) -> Optional["ClientSession"]: - """Create a connection pool for the model server.""" - model_server = agent.model_server - if not model_server: - logger.debug("No model server endpoint found after server start.") - return None - - return model_server.session diff --git a/rasa/utils/endpoints.py b/rasa/utils/endpoints.py index 5e1032778e6b..31d1ea7228bc 100644 --- a/rasa/utils/endpoints.py +++ b/rasa/utils/endpoints.py @@ -1,8 +1,6 @@ import ssl -from functools import cached_property import aiohttp -import logging import os from aiohttp.client_exceptions import ContentTypeError from sanic.request import Request @@ -11,10 +9,11 @@ from rasa.shared.exceptions import FileNotFoundException import rasa.shared.utils.io import rasa.utils.io +import structlog from rasa.core.constants import DEFAULT_REQUEST_TIMEOUT -logger = logging.getLogger(__name__) +structlogger = structlog.get_logger() def read_endpoint_config( @@ -32,9 +31,13 @@ def read_endpoint_config( return EndpointConfig.from_dict(content[endpoint_type]) except FileNotFoundError: - logger.error( - "Failed to read endpoint configuration " - "from {}. No such file.".format(os.path.abspath(filename)) + structlogger.error( + "endpoint.read.failed_no_such_file", + filename=os.path.abspath(filename), + event_info=( + "Failed to read endpoint configuration file - " + "the file was not found." + ), ) return None @@ -56,9 +59,13 @@ def concat_url(base: Text, subpath: Optional[Text]) -> Text: """ if not subpath: if base.endswith("/"): - logger.debug( - f"The URL '{base}' has a trailing slash. Please make sure the " - f"target server supports trailing slashes for this endpoint." + structlogger.debug( + "endpoint.concat_url.trailing_slash", + url=base, + event_info=( + "The URL has a trailing slash. Please make sure the " + "target server supports trailing slashes for this endpoint." + ), ) return base @@ -95,7 +102,6 @@ def __init__( self.cafile = cafile self.kwargs = kwargs - @cached_property def session(self) -> aiohttp.ClientSession: """Creates and returns a configured aiohttp client session.""" # create authentication parameters @@ -164,23 +170,26 @@ async def request( f"'{os.path.abspath(self.cafile)}' does not exist." ) from e - async with self.session.request( - method, - url, - headers=headers, - params=self.combine_parameters(kwargs), - compress=compress, - ssl=sslcontext, - **kwargs, - ) as response: - if response.status >= 400: - raise ClientResponseError( - response.status, response.reason, await response.content.read() - ) - try: - return await response.json() - except ContentTypeError: - return None + async with self.session() as session: + async with session.request( + method, + url, + headers=headers, + params=self.combine_parameters(kwargs), + compress=compress, + ssl=sslcontext, + **kwargs, + ) as response: + if response.status >= 400: + raise ClientResponseError( + response.status, + response.reason, + await response.content.read(), + ) + try: + return await response.json() + except ContentTypeError: + return None @classmethod def from_dict(cls, data: Dict[Text, Any]) -> "EndpointConfig": @@ -263,7 +272,7 @@ def float_arg( try: return float(str(arg)) except (ValueError, TypeError): - logger.warning(f"Failed to convert '{arg}' to float.") + structlogger.warning("endpoint.float_arg.convert_failed", arg=arg, key=key) return default @@ -291,5 +300,6 @@ def int_arg( try: return int(str(arg)) except (ValueError, TypeError): - logger.warning(f"Failed to convert '{arg}' to int.") + + structlogger.warning("endpoint.int_arg.convert_failed", arg=arg, key=key) return default diff --git a/tests/core/test_run.py b/tests/core/test_run.py index 1ac276d43772..8eda15058c0d 100644 --- a/tests/core/test_run.py +++ b/tests/core/test_run.py @@ -1,7 +1,6 @@ import warnings from unittest.mock import Mock -import aiohttp import pytest from typing import Text @@ -84,8 +83,6 @@ async def test_close_resources(loop: AbstractEventLoop): broker = SQLEventBroker() app = Mock() app.ctx.agent.tracker_store.event_broker = broker - app.ctx.agent.action_endpoint.session = aiohttp.ClientSession() - app.ctx.agent.model_server.session = aiohttp.ClientSession() with warnings.catch_warnings() as record: await run.close_resources(app, loop) diff --git a/tests/utils/test_endpoints.py b/tests/utils/test_endpoints.py index 071e54ee9318..711f2fd25faa 100644 --- a/tests/utils/test_endpoints.py +++ b/tests/utils/test_endpoints.py @@ -1,4 +1,4 @@ -import logging +import structlog from pathlib import Path from typing import Text, Optional, Union from unittest.mock import Mock @@ -35,13 +35,14 @@ def test_concat_url(base, subpath, expected_result): assert endpoint_utils.concat_url(base, subpath) == expected_result -def test_warning_for_base_paths_with_trailing_slash(caplog): +def test_warning_for_base_paths_with_trailing_slash(): test_path = "base/" - - with caplog.at_level(logging.DEBUG, logger="rasa.utils.endpoints"): + with structlog.testing.capture_logs() as caplog: assert endpoint_utils.concat_url(test_path, None) == test_path - assert len(caplog.records) == 1 + assert len(caplog) == 1 + assert caplog[0]["event"] == "endpoint.concat_url.trailing_slash" + assert caplog[0]["log_level"] == "debug" async def test_endpoint_config(): @@ -88,7 +89,7 @@ async def test_endpoint_config(): # unfortunately, the mock library won't report any headers stored on # the session object, so we need to verify them separately - async with endpoint.session as s: + async with endpoint.session() as s: assert s._default_headers.get("X-Powered-By") == "Rasa" assert s._default_auth.login == "user" assert s._default_auth.password == "pass" @@ -231,32 +232,3 @@ def test_int_arg(value: Optional[Union[int, str]], default: int, expected_result if value is not None: request.args = {"key": value} assert endpoint_utils.int_arg(request, "key", default) == expected_result - - -async def test_endpoint_config_caches_session() -> None: - """Test that the EndpointConfig session is cached. - - Assert identity of the session object, which should not be recreated when calling - the property `session` multiple times. - """ - endpoint = endpoint_utils.EndpointConfig("https://example.com/") - session = endpoint.session - - assert endpoint.session is session - - # teardown - await endpoint.session.close() - - -async def test_endpoint_config_constructor_does_not_create_session_cached_property() -> None: # noqa: E501 - """Test that the instantiation of EndpointConfig does not create the session cached property.""" # noqa: E501 - endpoint = endpoint_utils.EndpointConfig("https://example.com/") - - assert endpoint.__dict__.get("url") == "https://example.com/" - assert endpoint.__dict__.get("session") is None - - # the property is created when it is accessed - async with endpoint.session as session: - assert session is not None - - assert endpoint.__dict__.get("session") is session From 7cfc7d07e3f5502ed7333ad1910d5d85891eb473 Mon Sep 17 00:00:00 2001 From: Varun Shankar S Date: Thu, 30 Nov 2023 19:51:17 +0100 Subject: [PATCH 07/41] prepared release of version 3.6.15 (#12967) * prepared release of version 3.6.15 --- CHANGELOG.mdx | 7 +++++++ Makefile | 2 +- pyproject.toml | 2 +- rasa/version.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index b574a76b1b96..fce125567d98 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,13 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.15] - 2023-11-30 + +Rasa 3.6.15 (2023-11-30) +### Bugfixes +- [#12965](https://github.com/rasahq/rasa/issues/12965): Fixed connection timeout to action server by setting KEEP_ALIVE_TIMEOUT to 120, and reverting changes introduced in #12886. + + ## [3.6.14] - 2023-11-17 Rasa 3.6.14 (2023-11-17) diff --git a/Makefile b/Makefile index f3ebb6135e25..23b5799e1d5f 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ prepare-tests-windows: # It will retry the installation 5 times if it fails # See: https://github.com/actions/virtual-environments/blob/main/images/win/scripts/ImageHelpers/ChocoHelpers.ps1 prepare-tests-windows-gha: - powershell -command "Choco-Install wget graphviz" + powershell -command "Install-ChocoPackage wget graphviz" test: clean # OMP_NUM_THREADS can improve overall performance using one thread by process (on tensorflow), avoiding overload diff --git a/pyproject.toml b/pyproject.toml index 4bf33f2a32d5..92d55359da8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.14" +version = "3.6.15" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index e4512ce41043..3d8e9f0ee007 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.14" +__version__ = "3.6.15" From 865902f3c576a4a66b9b9492f3e8234261c14e17 Mon Sep 17 00:00:00 2001 From: sancharigr Date: Mon, 11 Dec 2023 10:29:05 +0100 Subject: [PATCH 08/41] Additional load testing recommendations --- .../monitoring/load-testing-guidelines.mdx | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/docs/monitoring/load-testing-guidelines.mdx b/docs/docs/monitoring/load-testing-guidelines.mdx index ff40486853b5..23b104917177 100644 --- a/docs/docs/monitoring/load-testing-guidelines.mdx +++ b/docs/docs/monitoring/load-testing-guidelines.mdx @@ -17,6 +17,29 @@ In our tests we used the Rasa [HTTP-API](https://rasa.com/docs/rasa/pages/http-a | Up to 50,000 | 6vCPU | 16 GB | | Up to 80,000 | 6vCPU, with almost 90% CPU usage | 16 GB | +:::info This is the most optimal AWS setup tested on EKS with + +ec2: c5.2xlarge - 9.2rps/node throughput +ec2: c5.4xlarge - 19.5rps/node throughput +You can always choose a bigger compute efficient instance like c5.4xlarge with more CPU per node to maximize throughput per node + +::: + +| AWS | RasaPro | Rasa Action Server | +|--------------------------|----------------------------------------------|-------------------------------------------| +| EC2: C52xlarge | 3vCPU, 10Gb Memory, 3 Sanic Threads | 3vCPU, 2Gb Memory, 3 Sanic Threads | +| EC2: C54xlarge | 7vCPU, 16Gb Memory, 7 Sanic Threads | 7vCPU, 12Gb Memory, 7 Sanic Threads | + +### Some recommendations to improve latency +- Running action as a sidecar, saves about ~100ms on average trips from the action server on the concluded tests. Results may vary depending on the number of calls made to the action server. +- Sanic Workers must be mapped 1:1 to CPU for both Rasa Pro and Rasa Action Server +- Create `async` actions to avoid any blocking I/O +- Use KEDA for pre-emptive autoscaling of rasa pods in production based on http requests +- `enable_selective_domain: true` : Domain is only sent for actions that needs it. This massively trims the payload between the two pods. +- Consider using c5n.nxlarge machines which are more compute optimized and support better parallelization on http requests. + However, as they are low on memory, models need to be trained lightweight. + Not suitable if you want to run transformers + ### Debugging bot related issues while scaling up From ecdd8bee99f69fcf5a285abda6fa3ab2ecc11ed0 Mon Sep 17 00:00:00 2001 From: Tanja Date: Thu, 21 Dec 2023 10:20:13 +0100 Subject: [PATCH 09/41] [ENG-712] Fix dependency install failures on windows (#150) * prevent race condition in poetry build * update makefile --- .github/workflows/continous-integration.yml | 6 ++++++ Makefile | 2 +- changelog/712.misc.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog/712.misc.md diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index ea3d194566a3..b50753b1dffb 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -287,6 +287,12 @@ jobs: run: | poetry self add setuptools + - name: Prevent race condition in poetry build + # More context about race condition during poetry build can be found here: + # https://github.com/python-poetry/poetry/issues/7611#issuecomment-1747836233 + run: | + poetry config installer.max-workers 1 + - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.backend == 'true' diff --git a/Makefile b/Makefile index f3ebb6135e25..23b5799e1d5f 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ prepare-tests-windows: # It will retry the installation 5 times if it fails # See: https://github.com/actions/virtual-environments/blob/main/images/win/scripts/ImageHelpers/ChocoHelpers.ps1 prepare-tests-windows-gha: - powershell -command "Choco-Install wget graphviz" + powershell -command "Install-ChocoPackage wget graphviz" test: clean # OMP_NUM_THREADS can improve overall performance using one thread by process (on tensorflow), avoiding overload diff --git a/changelog/712.misc.md b/changelog/712.misc.md new file mode 100644 index 000000000000..020a19afe7f1 --- /dev/null +++ b/changelog/712.misc.md @@ -0,0 +1 @@ +Prevent race condition in poetry build to fix dependency install failures on windows. From 96eead79cce0aa7936dfd4827fc2dba5a56ed901 Mon Sep 17 00:00:00 2001 From: Tawakalt Date: Fri, 29 Dec 2023 22:49:27 +0100 Subject: [PATCH 10/41] update cryptography --- poetry.lock | 50 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/poetry.lock b/poetry.lock index b347db8c13bf..0f266f79a5ae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1090,35 +1090,35 @@ yaml = ["PyYAML (>=3.10)"] [[package]] name = "cryptography" -version = "41.0.3" +version = "41.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, - {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, - {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, - {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, ] [package.dependencies] @@ -7231,4 +7231,4 @@ transformers = ["sentencepiece", "transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "71bfb81a213fc85cbf7fe0ac10a9ac16363fd14c18647e1f0ee26b35e73e0747" +content-hash = "d38a12f16b69e12490c4ad2cebda4f8b8a9bdb45f6cdd883b44b4754dbf424c5" diff --git a/pyproject.toml b/pyproject.toml index 904736791eeb..efa2bce4abc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,7 +154,7 @@ structlog-sentry = "^2.0.2" dnspython = "2.3.0" wheel = ">=0.38.1" certifi = ">=2023.7.22" -cryptography = ">=41.0.2" +cryptography = ">=41.0.7" [[tool.poetry.dependencies.tensorflow-io-gcs-filesystem]] version = "==0.31" markers = "sys_platform == 'win32'" From 45cbefd0a901160376af4ceef83583ad8244ccc2 Mon Sep 17 00:00:00 2001 From: Tawakalt Date: Tue, 2 Jan 2024 11:34:22 +0100 Subject: [PATCH 11/41] add changelog --- changelog/12983.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/12983.bugfix.md diff --git a/changelog/12983.bugfix.md b/changelog/12983.bugfix.md new file mode 100644 index 000000000000..5de8934cfe59 --- /dev/null +++ b/changelog/12983.bugfix.md @@ -0,0 +1 @@ +Upgrade Cryptography to fix improper certificate validation. \ No newline at end of file From 652c1756ab695b55eb0c199ebe3a190482fdb29c Mon Sep 17 00:00:00 2001 From: sancharigr Date: Thu, 4 Jan 2024 20:25:20 +0530 Subject: [PATCH 12/41] Review changes made --- .../monitoring/load-testing-guidelines.mdx | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/docs/monitoring/load-testing-guidelines.mdx b/docs/docs/monitoring/load-testing-guidelines.mdx index 23b104917177..a794d73639da 100644 --- a/docs/docs/monitoring/load-testing-guidelines.mdx +++ b/docs/docs/monitoring/load-testing-guidelines.mdx @@ -12,33 +12,24 @@ In order to gather metrics on our system's ability to handle increased loads and In each test case we spawned the following number of concurrent users at peak concurrency using a [spawn rate](https://docs.locust.io/en/1.5.0/configuration.html#all-available-configuration-options) of 1000 users per second. In our tests we used the Rasa [HTTP-API](https://rasa.com/docs/rasa/pages/http-api) and the [Locust](https://locust.io/) open source load testing tool. + | Users | CPU | Memory | |--------------------------|----------------------------------------------|---------------| | Up to 50,000 | 6vCPU | 16 GB | | Up to 80,000 | 6vCPU, with almost 90% CPU usage | 16 GB | -:::info This is the most optimal AWS setup tested on EKS with - -ec2: c5.2xlarge - 9.2rps/node throughput -ec2: c5.4xlarge - 19.5rps/node throughput -You can always choose a bigger compute efficient instance like c5.4xlarge with more CPU per node to maximize throughput per node - -::: - -| AWS | RasaPro | Rasa Action Server | -|--------------------------|----------------------------------------------|-------------------------------------------| -| EC2: C52xlarge | 3vCPU, 10Gb Memory, 3 Sanic Threads | 3vCPU, 2Gb Memory, 3 Sanic Threads | -| EC2: C54xlarge | 7vCPU, 16Gb Memory, 7 Sanic Threads | 7vCPU, 12Gb Memory, 7 Sanic Threads | ### Some recommendations to improve latency -- Running action as a sidecar, saves about ~100ms on average trips from the action server on the concluded tests. Results may vary depending on the number of calls made to the action server. - Sanic Workers must be mapped 1:1 to CPU for both Rasa Pro and Rasa Action Server - Create `async` actions to avoid any blocking I/O -- Use KEDA for pre-emptive autoscaling of rasa pods in production based on http requests - `enable_selective_domain: true` : Domain is only sent for actions that needs it. This massively trims the payload between the two pods. -- Consider using c5n.nxlarge machines which are more compute optimized and support better parallelization on http requests. +- Consider using compute efficient machines on cloud which are optimized for high performance computing such as the C5 instances on AWS. However, as they are low on memory, models need to be trained lightweight. - Not suitable if you want to run transformers + + +| Machine | RasaPro | Rasa Action Server | +|--------------------------------|------------------------------------------------|--------------------------------------------------| +| AWS C5 or Azure F or Gcloud C2 | 3-7vCPU, 10-16Gb Memory, 3-7 Sanic Threads | 3-7vCPU, 2-12Gb Memory, 3-7 Sanic Threads | ### Debugging bot related issues while scaling up From 64546fcf70be2754e75550016c9e8916e898aa09 Mon Sep 17 00:00:00 2001 From: sancharigr Date: Fri, 5 Jan 2024 11:28:01 +0530 Subject: [PATCH 13/41] Add missing CI step condition --- .github/workflows/continous-integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index b50753b1dffb..587ad2ad26e4 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -290,6 +290,7 @@ jobs: - name: Prevent race condition in poetry build # More context about race condition during poetry build can be found here: # https://github.com/python-poetry/poetry/issues/7611#issuecomment-1747836233 + if: needs.changes.outputs.backend == 'true' run: | poetry config installer.max-workers 1 From 8f916e94faf43e4a9d5a240b29634afe701c6a30 Mon Sep 17 00:00:00 2001 From: sancharigr Date: Thu, 11 Jan 2024 18:32:19 +0530 Subject: [PATCH 14/41] Edit docs workflow to release docs for 3.6.x --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3e1a31913f6d..076f44d0c612 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -82,7 +82,7 @@ jobs: echo "build_docs=true" >> $GITHUB_OUTPUT else # Get latest tagged Rasa version - git fetch --depth=1 origin "+refs/tags/*:refs/tags/*" + git describe --tags --match="3.6.[0-9]*" --abbrev=0 HEAD # Fetch branch history TAG_NAME=${GITHUB_REF#refs/tags/} git fetch --prune --unshallow From e970b3a721ed87186edfb6e38069a49e971bea22 Mon Sep 17 00:00:00 2001 From: m-vdb Date: Tue, 16 Jan 2024 08:47:09 +0100 Subject: [PATCH 15/41] update changelog with entries from 3.5.x releases --- CHANGELOG.mdx | 113 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index fce125567d98..60752d9d683d 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -17,37 +17,37 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> ## [3.6.15] - 2023-11-30 - -Rasa 3.6.15 (2023-11-30) + +Rasa 3.6.15 (2023-11-30) ### Bugfixes - [#12965](https://github.com/rasahq/rasa/issues/12965): Fixed connection timeout to action server by setting KEEP_ALIVE_TIMEOUT to 120, and reverting changes introduced in #12886. ## [3.6.14] - 2023-11-17 - -Rasa 3.6.14 (2023-11-17) + +Rasa 3.6.14 (2023-11-17) ### Bugfixes - [#12948](https://github.com/rasahq/rasa/issues/12948): Fixed UnexpecTEDIntentlessPolicy training errors that resulted from a change to batching behavior. Changed the batching behavior back to the original for all components. Made the changed batching behavior accessible in DietClassifier using `drop_small_last_batch: True`. ## [3.6.13] - 2023-10-23 - -Rasa 3.6.13 (2023-10-23) + +Rasa 3.6.13 (2023-10-23) ### Bugfixes - [#12927](https://github.com/rasahq/rasa/issues/12927): Fix wrong conflicts that occur when rasa validate stories is run with slots that have active_loop set to null in mapping conditions. ## [3.6.12] - 2023-10-10 - -Rasa 3.6.12 (2023-10-10) + +Rasa 3.6.12 (2023-10-10) ### Bugfixes - [#12904](https://github.com/rasahq/rasa/issues/12904): Refresh headers used in requests (e.g. action server requests) made by `EndpointConfig` using its `headers` attribute. - [#12906](https://github.com/rasahq/rasa/issues/12906): Upgrade `pillow` to `10.0.1` to address security vulnerability CVE-2023-4863 found in `10.0.0` version. ## [3.6.11] - 2023-10-05 - -Rasa 3.6.11 (2023-10-05) + +Rasa 3.6.11 (2023-10-05) ### Bugfixes - [#12722](https://github.com/rasahq/rasa/issues/12722): Intent names will not be falsely abbreviated in interactive training (fixes OSS-413). @@ -59,8 +59,8 @@ Rasa 3.6.11 (2023-10-05) ## [3.6.10] - 2023-09-26 - -Rasa 3.6.10 (2023-09-26) + +Rasa 3.6.10 (2023-09-26) ### Improvements - [#12827](https://github.com/rasahq/rasa/issues/12827): Improved handling of last batch during DIET and TED training. The last batch is discarded if it contains less than half a batch size of data. - [#12852](https://github.com/rasahq/rasa/issues/12852): Added `username` to the connection parameters for `RedisLockStore` and `RedisTrackerStore` @@ -71,8 +71,8 @@ Rasa 3.6.10 (2023-09-26) ## [3.6.9] - 2023-09-15 - -Rasa 3.6.9 (2023-09-15) + +Rasa 3.6.9 (2023-09-15) ### Improvements - [#12778](https://github.com/rasahq/rasa/issues/12778): Added additional method `fingerprint_addon` to the `GraphComponent` interface to allow inclusion of external data into the fingerprint calculation of a component @@ -81,29 +81,29 @@ Rasa 3.6.9 (2023-09-15) ## [3.6.8] - 2023-08-30 - -Rasa 3.6.8 (2023-08-30) + +Rasa 3.6.8 (2023-08-30) No significant changes. ## [3.6.7] - 2023-08-29 - -Rasa 3.6.7 (2023-08-29) + +Rasa 3.6.7 (2023-08-29) ### Bugfixes - [#12768](https://github.com/rasahq/rasa/issues/12768): Updated certifi, cryptography, and scipy packages to address security vulnerabilities. ## [3.6.6] - 2023-08-23 - -Rasa 3.6.6 (2023-08-23) + +Rasa 3.6.6 (2023-08-23) ### Bugfixes - [#12755](https://github.com/rasahq/rasa/issues/12755): Updated setuptools and wheel to address security vulnerabilities. ## [3.6.5] - 2023-08-17 - -Rasa 3.6.5 (2023-08-17) + +Rasa 3.6.5 (2023-08-17) ### Improvements - [#12696](https://github.com/rasahq/rasa/issues/12696): Use the same session across requests in `RasaNLUHttpInterpreter` @@ -116,8 +116,8 @@ Rasa 3.6.5 (2023-08-17) ## [3.6.4] - 2023-07-21 - -Rasa 3.6.4 (2023-07-21) + +Rasa 3.6.4 (2023-07-21) ### Bugfixes - [#12575](https://github.com/rasahq/rasa/issues/12575): Extract conditional response variation and channel variation filtering logic into a separate component. Enable usage of this component in the NaturalLanguageGenerator subclasses (e.g. CallbackNaturalLanguageGenerator, TemplatedNaturalLanguageGenerator). @@ -128,8 +128,8 @@ Rasa 3.6.4 (2023-07-21) ## [3.6.3] - 2023-07-20 - -Rasa 3.6.3 (2023-07-20) + +Rasa 3.6.3 (2023-07-20) ### Improvements - [#12637](https://github.com/rasahq/rasa/issues/12637): Added a human readable component to structlog using the `event_info` key and made it the default rendered key if present. @@ -144,15 +144,15 @@ Rasa 3.6.3 (2023-07-20) ## [3.6.2] - 2023-07-06 - -Rasa 3.6.2 (2023-07-06) + +Rasa 3.6.2 (2023-07-06) ### Bugfixes - [#12602](https://github.com/rasahq/rasa/issues/12602): Resolves the issue of importing TensorFlow on Docker for ARM64 architecture. ## [3.6.1] - 2023-07-03 - -Rasa 3.6.1 (2023-07-03) + +Rasa 3.6.1 (2023-07-03) ### Improvements - [#12533](https://github.com/rasahq/rasa/issues/12533): Add building multi-platform Docker image (amd64/arm64) - [#12543](https://github.com/rasahq/rasa/issues/12543): Switch struct log to `FilteringBoundLogger` in order to retain log level set in the config. @@ -174,8 +174,8 @@ Rasa 3.6.1 (2023-07-03) ## [3.6.0] - 2023-06-14 - -Rasa 3.6.0 (2023-06-14) + +Rasa 3.6.0 (2023-06-14) ### Deprecations and Removals - [#12355](https://github.com/rasahq/rasa/issues/12355): Removed Python 3.7 support as [it reaches its end of life in June 2023](https://devguide.python.org/versions/) @@ -219,6 +219,55 @@ Rasa 3.6.0 (2023-06-14) ### Miscellaneous internal changes - [#12291](https://github.com/rasahq/rasa/issues/12291), [#12329](https://github.com/rasahq/rasa/issues/12329), [#12332](https://github.com/rasahq/rasa/issues/12332), [#12365](https://github.com/rasahq/rasa/issues/12365), [#12372](https://github.com/rasahq/rasa/issues/12372), [#12386](https://github.com/rasahq/rasa/issues/12386), [#12492](https://github.com/rasahq/rasa/issues/12492) +## [3.5.17] - 2023-12-05 + +Rasa 3.5.17 (2023-12-05) +### Improvements +- [#12851](https://github.com/rasahq/rasa/issues/12851): Added `username` to the connection parameters for `RedisLockStore` and `RedisTrackerStore` +- [#1493](https://github.com/rasahq/rasa/issues/1493): Telemetry data is only send for licensed users. + + +## [3.5.16] - 2023-08-30 + +Rasa 3.5.16 (2023-08-30) + +No significant changes. + + +## [3.5.15] - 2023-07-21 + +Rasa 3.5.15 (2023-07-21) + +No significant changes. + + +## [3.5.14] - 2023-07-12 + +Rasa 3.5.14 (2023-07-12) +### Bugfixes +- [#12639](https://github.com/rasahq/rasa/issues/12639): Fix the issue with the most recent model not being selected if the owner or permissions where modified on the model file. + +### Miscellaneous internal changes +- [#12649](https://github.com/rasahq/rasa/issues/12649) + + +## [3.5.13] - 2023-07-05 + +Rasa 3.5.13 (2023-07-05) +### Bugfixes +- [#12549](https://github.com/rasahq/rasa/issues/12549): Introduce a validation step in `rasa data validate` command to identify non-existent paths and empty domains. + + +## [3.5.12] - 2023-06-23 + +Rasa 3.5.12 (2023-06-23) +### Bugfixes +- [#12534](https://github.com/rasahq/rasa/issues/12534): Rich responses containing buttons with parentheses characters are now correctly parsed. + Previously any characters found between the first identified pair of `()` in response button took precedence. + +### Miscellaneous internal changes +- [#12512](https://github.com/rasahq/rasa/issues/12512) + ## [3.5.11] - 2023-06-08 From ce1f5d2a31e55723fde1f522a5338eb51b840385 Mon Sep 17 00:00:00 2001 From: Shailendra Paliwal Date: Fri, 19 Jan 2024 11:56:21 +0100 Subject: [PATCH 16/41] Get Full Retrieval Intent Name (#12998) * update full retrieval intent name * format * add test for test_update_full_retrieval_intent * fix linting * fix the structure of dict --- rasa/core/processor.py | 22 +++++++++++++ tests/core/test_processor.py | 64 +++++++++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/rasa/core/processor.py b/rasa/core/processor.py index ec6ef1260a3e..fc628c7a7247 100644 --- a/rasa/core/processor.py +++ b/rasa/core/processor.py @@ -66,7 +66,11 @@ ENTITIES, INTENT, INTENT_NAME_KEY, + INTENT_RESPONSE_KEY, PREDICTED_CONFIDENCE_KEY, + FULL_RETRIEVAL_INTENT_NAME_KEY, + RESPONSE_SELECTOR, + RESPONSE, TEXT, ) from rasa.utils.endpoints import EndpointConfig @@ -721,6 +725,7 @@ async def parse_message( message, tracker, only_output_properties ) + self._update_full_retrieval_intent(parse_data) structlogger.debug( "processor.message.parse", parse_data_text=copy.deepcopy(parse_data["text"]), @@ -732,6 +737,23 @@ async def parse_message( return parse_data + def _update_full_retrieval_intent(self, parse_data: Dict[Text, Any]) -> None: + """Update the parse data with the full retrieval intent. + + Args: + parse_data: Message parse data to update. + """ + intent_name = parse_data.get(INTENT, {}).get(INTENT_NAME_KEY) + response_selector = parse_data.get(RESPONSE_SELECTOR, {}) + all_retrieval_intents = response_selector.get("all_retrieval_intents", []) + if intent_name and intent_name in all_retrieval_intents: + retrieval_intent = ( + response_selector.get(intent_name, {}) + .get(RESPONSE, {}) + .get(INTENT_RESPONSE_KEY) + ) + parse_data[INTENT][FULL_RETRIEVAL_INTENT_NAME_KEY] = retrieval_intent + def _parse_message_with_graph( self, message: UserMessage, diff --git a/tests/core/test_processor.py b/tests/core/test_processor.py index 392d85c29745..d0581b1800ef 100644 --- a/tests/core/test_processor.py +++ b/tests/core/test_processor.py @@ -70,7 +70,12 @@ from rasa.core.http_interpreter import RasaNLUHttpInterpreter from rasa.core.processor import MessageProcessor from rasa.shared.core.trackers import DialogueStateTracker -from rasa.shared.nlu.constants import INTENT_NAME_KEY, METADATA_MODEL_ID +from rasa.shared.nlu.constants import ( + INTENT, + INTENT_NAME_KEY, + FULL_RETRIEVAL_INTENT_NAME_KEY, + METADATA_MODEL_ID, +) from rasa.shared.nlu.training_data.message import Message from rasa.utils.endpoints import EndpointConfig from rasa.shared.core.constants import ( @@ -1928,3 +1933,60 @@ async def test_run_anonymization_pipeline_mocked_pipeline( await processor.run_anonymization_pipeline(tracker) event_diff.assert_called_once() + + +async def test_update_full_retrieval_intent( + default_processor: MessageProcessor, +) -> None: + parse_data = { + "text": "I like sunny days in berlin", + "intent": {"name": "chitchat", "confidence": 0.9}, + "entities": [], + "response_selector": { + "all_retrieval_intents": ["faq", "chitchat"], + "faq": { + "response": { + "responses": [{"text": "Our return policy lasts 30 days."}], + "confidence": 1.0, + "intent_response_key": "faq/what_is_return_policy", + "utter_action": "utter_faq/what_is_return_policy", + }, + "ranking": [ + { + "confidence": 1.0, + "intent_response_key": "faq/what_is_return_policy", + }, + { + "confidence": 2.3378809862799945e-19, + "intent_response_key": "faq/how_can_i_track_my_order", + }, + ], + }, + "chitchat": { + "response": { + "responses": [ + { + "text": "The sun is out today! Isn't that great?", + }, + ], + "confidence": 1.0, + "intent_response_key": "chitchat/ask_weather", + "utter_action": "utter_chitchat/ask_weather", + }, + "ranking": [ + { + "confidence": 1.0, + "intent_response_key": "chitchat/ask_weather", + }, + {"confidence": 0.0, "intent_response_key": "chitchat/ask_name"}, + ], + }, + }, + } + + default_processor._update_full_retrieval_intent(parse_data) + + assert parse_data[INTENT][INTENT_NAME_KEY] == "chitchat" + # assert that parse_data["intent"] has a key called response + assert FULL_RETRIEVAL_INTENT_NAME_KEY in parse_data[INTENT] + assert parse_data[INTENT][FULL_RETRIEVAL_INTENT_NAME_KEY] == "chitchat/ask_weather" From 5339fae00c1cb527ba778916fd8b5e5d785b7d50 Mon Sep 17 00:00:00 2001 From: Shailendra Paliwal Date: Fri, 19 Jan 2024 12:23:27 +0100 Subject: [PATCH 17/41] add changelog (#13000) --- changelog/12998.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/12998.bugfix.md diff --git a/changelog/12998.bugfix.md b/changelog/12998.bugfix.md new file mode 100644 index 000000000000..63280d49c42d --- /dev/null +++ b/changelog/12998.bugfix.md @@ -0,0 +1 @@ +Fixes a bug that caused the `full_retrieval_intent_name` key to be missing in the published event. Rasa Analytics makes use of this key to get the Retrieval Intent Name \ No newline at end of file From 26210847931f4df166896f6ce4eb0334e093e23c Mon Sep 17 00:00:00 2001 From: Shailendra Paliwal Date: Fri, 19 Jan 2024 14:06:31 +0100 Subject: [PATCH 18/41] prepared release of version 3.6.16 (#13003) --- CHANGELOG.mdx | 11 +++++++++++ changelog/12983.bugfix.md | 1 - changelog/12998.bugfix.md | 1 - changelog/712.misc.md | 1 - pyproject.toml | 2 +- rasa/version.py | 2 +- 6 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 changelog/12983.bugfix.md delete mode 100644 changelog/12998.bugfix.md delete mode 100644 changelog/712.misc.md diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index 60752d9d683d..b3411889c069 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,17 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.16] - 2024-01-19 + +Rasa 3.6.16 (2024-01-19) +### Bugfixes +- [#12983](https://github.com/rasahq/rasa/issues/12983): Upgrade Cryptography to fix improper certificate validation. +- [#12998](https://github.com/rasahq/rasa/issues/12998): Fixes a bug that caused the `full_retrieval_intent_name` key to be missing in the published event. Rasa Analytics makes use of this key to get the Retrieval Intent Name + +### Miscellaneous internal changes +- [#712](https://github.com/rasahq/rasa/issues/712) + + ## [3.6.15] - 2023-11-30 Rasa 3.6.15 (2023-11-30) diff --git a/changelog/12983.bugfix.md b/changelog/12983.bugfix.md deleted file mode 100644 index 5de8934cfe59..000000000000 --- a/changelog/12983.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Upgrade Cryptography to fix improper certificate validation. \ No newline at end of file diff --git a/changelog/12998.bugfix.md b/changelog/12998.bugfix.md deleted file mode 100644 index 63280d49c42d..000000000000 --- a/changelog/12998.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixes a bug that caused the `full_retrieval_intent_name` key to be missing in the published event. Rasa Analytics makes use of this key to get the Retrieval Intent Name \ No newline at end of file diff --git a/changelog/712.misc.md b/changelog/712.misc.md deleted file mode 100644 index 020a19afe7f1..000000000000 --- a/changelog/712.misc.md +++ /dev/null @@ -1 +0,0 @@ -Prevent race condition in poetry build to fix dependency install failures on windows. diff --git a/pyproject.toml b/pyproject.toml index bd34f9e7ec4b..7b695117b412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.15" +version = "3.6.16" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index 3d8e9f0ee007..fc234e58601b 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.15" +__version__ = "3.6.16" From ece9224ac456575b67abfb12413c8640fb4de8be Mon Sep 17 00:00:00 2001 From: OgnjenFrancuski Date: Mon, 12 Feb 2024 16:00:02 +0100 Subject: [PATCH 19/41] Update aiohttp and aioresponse packages, due to reported vulnerability on aiohttp --- poetry.lock | 241 +++++++++++++++++++++++++------------------------ pyproject.toml | 4 +- 2 files changed, 125 insertions(+), 120 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0f266f79a5ae..e3eb767e760a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -45,134 +45,121 @@ files = [ [[package]] name = "aiogram" -version = "2.25.1" +version = "2.15" description = "Is a pretty simple and fully asynchronous framework for Telegram Bot API" category = "main" optional = false -python-versions = ">=3.7" +python-versions = "*" files = [ - {file = "aiogram-2.25.1-py3-none-any.whl", hash = "sha256:7bb770cd0459f1dbaea00578bf13fb2e6a1812f22adf94a988c11a7c0d5f33e1"}, - {file = "aiogram-2.25.1.tar.gz", hash = "sha256:59ad78fc0ebbef1fd471c15778a4594b60117e0d7373bc2ce7bcd192074d527d"}, + {file = "aiogram-2.15-py3-none-any.whl", hash = "sha256:5d4dae610625893fe53e07c01c9e95671fd863718caab692baac948e3746ab87"}, + {file = "aiogram-2.15.tar.gz", hash = "sha256:13c740c52ee1301af8a9905e0a412754bcff03deb83dfdf1c578d9249ab35026"}, ] [package.dependencies] -aiohttp = ">=3.8.0,<3.9.0" -Babel = ">=2.9.1,<2.10.0" -certifi = ">=2021.10.8" -magic-filter = ">=1.0.9" +aiohttp = ">=3.7.2,<4.0.0" +Babel = ">=2.8.0" +certifi = ">=2020.6.20" [package.extras] -fast = ["ujson (>=1.35)", "uvloop (>=0.16.0,<0.17.0)"] +fast = ["ujson (>=1.35)", "uvloop (>=0.14.0,<0.15.0)"] proxy = ["aiohttp-socks (>=0.5.3,<0.6.0)"] [[package]] name = "aiohttp" -version = "3.8.4" +version = "3.9.3" description = "Async http client/server framework (asyncio)" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, - {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, - {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, - {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, - {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, - {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, - {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, - {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, - {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, - {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, - {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, - {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, ] [package.dependencies] aiosignal = ">=1.1.2" -async-timeout = ">=4.0.0a3,<5.0" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<4.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["Brotli", "aiodns", "brotlicffi"] [[package]] name = "aiohttp-retry" @@ -191,18 +178,18 @@ aiohttp = "*" [[package]] name = "aioresponses" -version = "0.7.4" +version = "0.7.6" description = "Mock out requests made by ClientSession from aiohttp package" category = "dev" optional = false python-versions = "*" files = [ - {file = "aioresponses-0.7.4-py2.py3-none-any.whl", hash = "sha256:1160486b5ea96fcae6170cf2bdef029b9d3a283b7dbeabb3d7f1182769bfb6b7"}, - {file = "aioresponses-0.7.4.tar.gz", hash = "sha256:9b8c108b36354c04633bad0ea752b55d956a7602fe3e3234b939fc44af96f1d8"}, + {file = "aioresponses-0.7.6-py2.py3-none-any.whl", hash = "sha256:d2c26defbb9b440ea2685ec132e90700907fd10bcca3e85ec2f157219f0d26f7"}, + {file = "aioresponses-0.7.6.tar.gz", hash = "sha256:f795d9dbda2d61774840e7e32f5366f45752d1adc1b74c9362afd017296c7ee1"}, ] [package.dependencies] -aiohttp = ">=2.0.0,<4.0.0" +aiohttp = ">=3.3.0,<4.0.0" [[package]] name = "aiormq" @@ -2716,18 +2703,6 @@ files = [ {file = "locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632"}, ] -[[package]] -name = "magic-filter" -version = "1.0.9" -description = "This package provides magic filter based on dynamic attribute getter" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "magic-filter-1.0.9.tar.gz", hash = "sha256:d0f1ffa5ff1fbe5105fd5f293c79b5d3795f336ea0f6129c636959a687bf422a"}, - {file = "magic_filter-1.0.9-py3-none-any.whl", hash = "sha256:51002312a8972fa514b998b7ff89340c98be3fc499967c1f5f2af98d13baf8d5"}, -] - [[package]] name = "markdown" version = "3.4.3" @@ -4347,6 +4322,15 @@ files = [ {file = "python_crfsuite-0.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ab333b7cda21b1b5ab76f8e16e5f00654360df057e2092b273681d64850f714"}, {file = "python_crfsuite-0.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee6153ed8a26adaea645445b997c55d67505476976dfc40f4bbd46200b66de15"}, {file = "python_crfsuite-0.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c32292f722e293cee87aac0d793abd8b53dc05126e4b6f0202c41e0e7d027005"}, + {file = "python_crfsuite-0.9.9-cp311-cp311-win32.whl", hash = "sha256:3e8bbacff1d86cbc18e1d52f617c85521029127a09c86ed428cd8238384a9db3"}, + {file = "python_crfsuite-0.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:83f87b652108110263aa83c220baccc36c911f04cf422cf6632a5f42121bce6a"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:92f72eb554dac53218805958747b4cd417bd76039f083f66cc9881987d88e167"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5cdd1e823fa9dfe611a573a0c1371941e887ad8b8ffbc25e2d87e4cd6d4f22af"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54f94d6fabb14ad8106dc65b5d38665bb0abb16527d4a6aa2ba233670c3480db"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:65ebfab892c49c49b5e1030318d144c559e449bdf86b12983fa7ba0e88f7abdf"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5c1bb3d2802bb777affbe2855fc723aa32345ead2fe26ad18c7e1417bc104c53"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-win32.whl", hash = "sha256:abef974a2b520c0204cb15b0b799fdbc1c1a0af4be2b9ad7548800de95975345"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:ccc4c4f1cd47c74553d03d915c7dd7c06fc23b41310f30f35f2e5c09cdeb9297"}, {file = "python_crfsuite-0.9.9-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:01d279f1c8225aaf66290563312708c1905dce84f70ee5e374ecfb2dec1c2343"}, {file = "python_crfsuite-0.9.9-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f29e94fcb9e2f8f52c7323668752b34dcffef91751b0d1e0789ecbbc0069842"}, {file = "python_crfsuite-0.9.9-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497a8b9c0df152ada10732c07853942a8725cb66fe5b6fe1a64f768ecf583291"}, @@ -4491,6 +4475,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -5562,6 +5547,7 @@ files = [ {file = "SQLAlchemy-1.4.49-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:03db81b89fe7ef3857b4a00b63dedd632d6183d4ea5a31c5d8a92e000a41fc71"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:95b9df9afd680b7a3b13b38adf6e3a38995da5e162cc7524ef08e3be4e5ed3e1"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a63e43bf3f668c11bb0444ce6e809c1227b8f067ca1068898f3008a273f52b09"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca46de16650d143a928d10842939dab208e8d8c3a9a8757600cae9b7c579c5cd"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f835c050ebaa4e48b18403bed2c0fda986525896efd76c245bdd4db995e51a4c"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c21b172dfb22e0db303ff6419451f0cac891d2e911bb9fbf8003d717f1bcf91"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-win32.whl", hash = "sha256:5fb1ebdfc8373b5a291485757bd6431de8d7ed42c27439f543c81f6c8febd729"}, @@ -5571,26 +5557,35 @@ files = [ {file = "SQLAlchemy-1.4.49-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5debe7d49b8acf1f3035317e63d9ec8d5e4d904c6e75a2a9246a119f5f2fdf3d"}, {file = "SQLAlchemy-1.4.49-cp311-cp311-win32.whl", hash = "sha256:82b08e82da3756765c2e75f327b9bf6b0f043c9c3925fb95fb51e1567fa4ee87"}, {file = "SQLAlchemy-1.4.49-cp311-cp311-win_amd64.whl", hash = "sha256:171e04eeb5d1c0d96a544caf982621a1711d078dbc5c96f11d6469169bd003f1"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f23755c384c2969ca2f7667a83f7c5648fcf8b62a3f2bbd883d805454964a800"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8396e896e08e37032e87e7fbf4a15f431aa878c286dc7f79e616c2feacdb366c"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66da9627cfcc43bbdebd47bfe0145bb662041472393c03b7802253993b6b7c90"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-win32.whl", hash = "sha256:9a06e046ffeb8a484279e54bda0a5abfd9675f594a2e38ef3133d7e4d75b6214"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-win_amd64.whl", hash = "sha256:7cf8b90ad84ad3a45098b1c9f56f2b161601e4670827d6b892ea0e884569bd1d"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:36e58f8c4fe43984384e3fbe6341ac99b6b4e083de2fe838f0fdb91cebe9e9cb"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b31e67ff419013f99ad6f8fc73ee19ea31585e1e9fe773744c0f3ce58c039c30"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc22807a7e161c0d8f3da34018ab7c97ef6223578fcdd99b1d3e7ed1100a5db"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c14b29d9e1529f99efd550cd04dbb6db6ba5d690abb96d52de2bff4ed518bc95"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c40f3470e084d31247aea228aa1c39bbc0904c2b9ccbf5d3cfa2ea2dac06f26d"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-win32.whl", hash = "sha256:706bfa02157b97c136547c406f263e4c6274a7b061b3eb9742915dd774bbc264"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-win_amd64.whl", hash = "sha256:a7f7b5c07ae5c0cfd24c2db86071fb2a3d947da7bd487e359cc91e67ac1c6d2e"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:4afbbf5ef41ac18e02c8dc1f86c04b22b7a2125f2a030e25bbb4aff31abb224b"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24e300c0c2147484a002b175f4e1361f102e82c345bf263242f0449672a4bccf"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:393cd06c3b00b57f5421e2133e088df9cabcececcea180327e43b937b5a7caa5"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:201de072b818f8ad55c80d18d1a788729cccf9be6d9dc3b9d8613b053cd4836d"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7653ed6817c710d0c95558232aba799307d14ae084cc9b1f4c389157ec50df5c"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-win32.whl", hash = "sha256:647e0b309cb4512b1f1b78471fdaf72921b6fa6e750b9f891e09c6e2f0e5326f"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-win_amd64.whl", hash = "sha256:ab73ed1a05ff539afc4a7f8cf371764cdf79768ecb7d2ec691e3ff89abbc541e"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:37ce517c011560d68f1ffb28af65d7e06f873f191eb3a73af5671e9c3fada08a"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1878ce508edea4a879015ab5215546c444233881301e97ca16fe251e89f1c55"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95ab792ca493891d7a45a077e35b418f68435efb3e1706cb8155e20e86a9013c"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e8e608983e6f85d0852ca61f97e521b62e67969e6e640fe6c6b575d4db68557"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccf956da45290df6e809ea12c54c02ace7f8ff4d765d6d3dfb3655ee876ce58d"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-win32.whl", hash = "sha256:f167c8175ab908ce48bd6550679cc6ea20ae169379e73c7720a28f89e53aa532"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-win_amd64.whl", hash = "sha256:45806315aae81a0c202752558f0df52b42d11dd7ba0097bf71e253b4215f34f4"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b6d0c4b15d65087738a6e22e0ff461b407533ff65a73b818089efc8eb2b3e1de"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a843e34abfd4c797018fd8d00ffffa99fd5184c421f190b6ca99def4087689bd"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:738d7321212941ab19ba2acf02a68b8ee64987b248ffa2101630e8fccb549e0d"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1c890421651b45a681181301b3497e4d57c0d01dc001e10438a40e9a9c25ee77"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d26f280b8f0a8f497bc10573849ad6dc62e671d2468826e5c748d04ed9e670d5"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-win32.whl", hash = "sha256:ec2268de67f73b43320383947e74700e95c6770d0c68c4e615e9897e46296294"}, @@ -7040,6 +7035,16 @@ files = [ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecee4132c6cd2ce5308e21672015ddfed1ff975ad0ac8d27168ea82e71413f55"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2020f391008ef874c6d9e208b24f28e31bcb85ccff4f335f15a3251d222b92d9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2feecf86e1f7a86517cab34ae6c2f081fd2d0dac860cb0c0ded96d799d20b335"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:240b1686f38ae665d1b15475966fe0472f78e71b1b4903c143a842659c8e4cb9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9008dad07d71f68487c91e96579c8567c98ca4c3881b9b113bc7b33e9fd78b8"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6447e9f3ba72f8e2b985a1da758767698efa72723d5b59accefd716e9e8272bf"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:acae32e13a4153809db37405f5eba5bac5fbe2e2ba61ab227926a22901051c0a"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49ef582b7a1152ae2766557f0550a9fcbf7bbd76f43fbdc94dd3bf07cc7168be"}, + {file = "wrapt-1.14.1-cp311-cp311-win32.whl", hash = "sha256:358fe87cc899c6bb0ddc185bf3dbfa4ba646f05b1b0b9b5a27c2cb92c2cea204"}, + {file = "wrapt-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:26046cd03936ae745a502abf44dac702a5e6880b2b01c29aea8ddf3353b68224"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, @@ -7231,4 +7236,4 @@ transformers = ["sentencepiece", "transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "d38a12f16b69e12490c4ad2cebda4f8b8a9bdb45f6cdd883b44b4754dbf424c5" +content-hash = "c554d0f11eb367109ccd80c3694de7f07d35b52a09b1279d2b9743e6c155f43c" diff --git a/pyproject.toml b/pyproject.toml index 7b695117b412..01b534b61908 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,7 @@ sanic-jwt = "^1.6.0" sanic-routing = "^0.7.2" websockets = ">=10.0,<11.0" cloudpickle = ">=1.2,<2.3" -aiohttp = ">=3.6,!=3.7.4.post0,<3.9" +aiohttp = ">=3.9.0,<3.10" questionary = ">=1.5.1,<1.11.0" prompt-toolkit = "^3.0,<3.0.29" python-socketio = ">=4.4,<6" @@ -313,7 +313,7 @@ pytest-xdist = "^3.2.1" pytest = "^7.1.3" freezegun = "^1.0.0" responses = "^0.22.0" -aioresponses = "^0.7.2" +aioresponses = "^0.7.6" moto = "~=4.1.2" fakeredis = "^2.11.2" mongomock = "^4.1.2" From 9386a7345490fb8e8e27175b99a0a71da9485e90 Mon Sep 17 00:00:00 2001 From: OgnjenFrancuski Date: Mon, 12 Feb 2024 16:05:07 +0100 Subject: [PATCH 20/41] Add changelog --- changelog/13007.misc.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/13007.misc.md diff --git a/changelog/13007.misc.md b/changelog/13007.misc.md new file mode 100644 index 000000000000..4329f20aed22 --- /dev/null +++ b/changelog/13007.misc.md @@ -0,0 +1,2 @@ +aiohttp - update from 3.6 to 3.9 +aioresponses - update from 0.7.4 to 0.7.6 \ No newline at end of file From d7a26bc23e42deedd773479fd03de453bc6c83e1 Mon Sep 17 00:00:00 2001 From: OgnjenFrancuski Date: Tue, 13 Feb 2024 13:36:16 +0100 Subject: [PATCH 21/41] prepared release of version 3.6.17 --- CHANGELOG.mdx | 7 +++++++ changelog/13007.misc.md | 2 -- pyproject.toml | 2 +- rasa/version.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 changelog/13007.misc.md diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index b3411889c069..8efe6d303a21 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,13 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.17] - 2024-02-13 + +Rasa 3.6.17 (2024-02-13) +### Miscellaneous internal changes +- [#13007](https://github.com/rasahq/rasa/issues/13007) + + ## [3.6.16] - 2024-01-19 Rasa 3.6.16 (2024-01-19) diff --git a/changelog/13007.misc.md b/changelog/13007.misc.md deleted file mode 100644 index 4329f20aed22..000000000000 --- a/changelog/13007.misc.md +++ /dev/null @@ -1,2 +0,0 @@ -aiohttp - update from 3.6 to 3.9 -aioresponses - update from 0.7.4 to 0.7.6 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 01b534b61908..913610027cf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.16" +version = "3.6.17" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index fc234e58601b..6a7745b5540c 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.16" +__version__ = "3.6.17" From 56c0c483fc1176812d8d05c11e6b0d3379ab920d Mon Sep 17 00:00:00 2001 From: sancharigr Date: Mon, 19 Feb 2024 16:50:21 +0100 Subject: [PATCH 22/41] Delete auto main to branch merge workflow --- .../automatic-release-to-main-merger.yml | 113 ------------------ 1 file changed, 113 deletions(-) delete mode 100644 .github/workflows/automatic-release-to-main-merger.yml diff --git a/.github/workflows/automatic-release-to-main-merger.yml b/.github/workflows/automatic-release-to-main-merger.yml deleted file mode 100644 index 6ed4cca20af8..000000000000 --- a/.github/workflows/automatic-release-to-main-merger.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Automatic main branch merger -on: - # whenever a pull request is merged into a release branch, - # open a pull request to merge changes down to the main branch - pull_request: - branches: - - '[0-9]+.[0-9]+.x' - # Don't merge 2.8.x into main - - '!2.8.x' - # Don't merge 3.0, 3.1 and 3.2 into main - - '!3.0.x' - - '!3.1.x' - - '!3.2.x' - - types: - # means that the PR is closed, we still have to check if it was merged - - closed - -env: - # keep this in sync with the automatic-pr-approver workflow - LABEL_TYPE: type:release-branch-port - LABEL_STATUS: status:ready-to-merge - -jobs: - update_merge_pr: - runs-on: ubuntu-22.04 - - # only run this workflow if a pull request has been merged - # don't run this workflow on pull request from forks, permissions will be missing anyway - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflows-in-forked-repositories - if: github.event.pull_request.merged == true && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' - - steps: - - name: Checkout git repository 🕝 - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c - - - name: Fetch git tags 🎨 - # see https://github.com/actions/checkout/issues/206#issuecomment-617937725 - run: git fetch --prune --unshallow --tags - - - name: Get branch name ✍️ - id: get-branch-name - run: | - GITHUB_BRANCH=${GITHUB_REF/refs\/heads\//} - echo "release_branch=${GITHUB_BRANCH}" >> $GITHUB_OUTPUT - echo "new_branch=merge-${GITHUB_BRANCH}-main-${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT - - - name: Get GitHub labels 🏷 - id: get-github-labels - run: | - LATEST_RASA_MINOR=$(git tag --list | grep -P '^\d+\.\d+\.\d+$' | tail -n1 | sed -e 's/.\([0-9]\)*$/.0/g') - echo "Latest minor: ${LATEST_RASA_MINOR}" - # bash doesn't support nested variable access - CURRENT_RASA_MINOR=${GITHUB_REF/refs\/heads\//} - CURRENT_RASA_MINOR=${CURRENT_RASA_MINOR/\.x/\.0} - - if [[ ${LATEST_RASA_MINOR} == ${CURRENT_RASA_MINOR} ]] - then - echo "labels=${LABEL_TYPE},${LABEL_STATUS}" >> $GITHUB_OUTPUT - else - echo "labels=${LABEL_TYPE}" >> $GITHUB_OUTPUT - fi - - - name: Create new branch 🐣 - id: create-new-branch - if: always() - uses: peterjgrainger/action-create-branch@64aa569aea81305305c6e92bd236d8c427debff8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: ${{ steps.get-branch-name.outputs.new_branch }} - - - name: Open pull request ☄️ - if: ${{ steps.create-new-branch.conclusion == 'success' }} - uses: repo-sync/pull-request@7e79a9f5dc3ad0ce53138f01df2fad14a04831c5 - with: - # using this token to make sure it triggers other actions - github_token: ${{ secrets.RASABOT_GITHUB_TOKEN }} - source_branch: ${{ steps.get-branch-name.outputs.new_branch }} - destination_branch: main - pr_title: Merge ${{ steps.get-branch-name.outputs.release_branch }} into main - pr_template: .github/PULL_REQUEST_AUTOMATIC_TEMPLATE.md - pr_label: ${{ steps.get-github-labels.outputs.labels }} - pr_reviewer: ${{ github.event.pull_request.user.login }} - - - name: Close outdated release-merge PRs 🧹 - id: close-outdated-release-merge-prs - run: | - # fetch all open merge-PRs that have been opened from the current release branch - gh pr list -S "is:open label:${LABEL_TYPE} head:merge-${{ steps.get-branch-name.outputs.release_branch }}-main" > prs.txt - less prs.txt - - # delete newly opened PR from the list - awk '!/${{ steps.get-branch-name.outputs.new_branch }}/' prs.txt > temp && mv temp prs.txt - - # extract the PR ids - awk '{print $1}' prs.txt > pr_ids.txt - - # close all outdated PRs - while read id; do - gh pr close $id -d - done Date: Fri, 23 Feb 2024 13:29:56 +0000 Subject: [PATCH 23/41] [ATO-2122] Backport `rasa export` Kafka bugfix to `3.6.x` (#13017) * backport bugfix, update CI workflow * add changelog entry, update dead link in Docker build docs --- .github/workflows/continous-integration.yml | 110 +++++++++++++--- Makefile | 2 +- changelog/13017.bugfix.md | 1 + docs/docs/docker/building-in-docker.mdx | 2 +- rasa/core/brokers/kafka.py | 4 +- .../core/brokers/test_kafka.py | 14 ++- .../core/brokers/test_pika.py | 3 + tests/integration_tests/core/test_exporter.py | 117 ++++++++++++++++++ 8 files changed, 228 insertions(+), 25 deletions(-) create mode 100644 changelog/13017.bugfix.md create mode 100644 tests/integration_tests/core/test_exporter.py diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 587ad2ad26e4..7a69bbb4e807 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -618,10 +618,6 @@ jobs: POSTGRES_PORT: 5432 POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - RABBITMQ_HOST: localhost - RABBITMQ_PORT: 5672 - RABBITMQ_USER: guest - RABBITMQ_PASSWORD: guest services: redis: @@ -653,12 +649,6 @@ jobs: # mapping container ports to the host - 5432:5432 - rabbitmq: - # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck - image: healthcheck/rabbitmq - ports: - - 5672:5672 - mongodb: image: mongodb/mongodb-community-server:6.0.4-ubuntu2204 options: >- @@ -728,6 +718,94 @@ jobs: if grep 'The lock file is not up to date' .output; then exit 1; fi make prepare-tests-ubuntu + - name: Test Code with Services 🩺 + if: needs.changes.outputs.backend == 'true' + env: + JOBS: 2 + INTEGRATION_TEST_PYTEST_MARKERS: '"(not sequential) and (not broker)"' + PYTHONIOENCODING: "utf-8" + run: | + make test-integration + + broker_integration_test: + name: Run Broker Integration Tests + if: github.ref_type != 'tag' + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: [changes] + env: + RABBITMQ_HOST: localhost + RABBITMQ_PORT: 5672 + RABBITMQ_USER: guest + RABBITMQ_PASSWORD: guest + + services: + rabbitmq: + # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck + image: healthcheck/rabbitmq + ports: + - 5672:5672 + + steps: + - name: Checkout git repository 🕝 + if: needs.changes.outputs.backend == 'true' + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + + - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍 + if: needs.changes.outputs.backend == 'true' + uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + + - name: Read Poetry Version 🔢 + if: needs.changes.outputs.backend == 'true' + run: | + echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV + shell: bash + + - name: Install poetry 🦄 + if: needs.changes.outputs.backend == 'true' + uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Load Poetry Cached Libraries ⬇ + id: cache-poetry + if: needs.changes.outputs.backend == 'true' + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + with: + path: .venv + key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} + + - name: Clear Poetry cache + if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.backend == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests') + run: rm -r .venv + + # Poetry >= 1.1.0b uses virtualenv to create a virtual environment. + # The virtualenv simply doesn't work on Windows with our setup, + # that's why we use venv to create virtual environment + - name: Create virtual environment + if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')) && needs.changes.outputs.backend == 'true' + run: python -m venv create .venv + + - name: Set up virtual environment + if: needs.changes.outputs.backend == 'true' + # Poetry on Windows cannot pick up the virtual environments directory properly, + # and it creates a new one every time the pipeline runs. + # This step solves this problem — it tells poetry to always use `.venv` directory inside + # the project itself, which also makes it easier for us to determine the correct directory + # that needs to be cached. + run: poetry config virtualenvs.in-project true + + - name: Install Dependencies (Linux) 📦 + if: needs.changes.outputs.backend == 'true' + run: | + sudo apt-get -y install libpq-dev + make install-full | tee .output + if grep 'The lock file is not up to date' .output; then exit 1; fi + make prepare-tests-ubuntu + make prepare-spacy + - name: Run kafka and zookeeper containers for integration testing if: needs.changes.outputs.backend == 'true' run: | @@ -737,11 +815,16 @@ jobs: if: needs.changes.outputs.backend == 'true' env: JOBS: 2 - INTEGRATION_TEST_PYTEST_MARKERS: '"not sequential"' + INTEGRATION_TEST_PYTEST_MARKERS: "broker" PYTHONIOENCODING: "utf-8" run: | make test-integration + - name: Stop kafka and zookeeper containers for integration testing + if: needs.changes.outputs.backend == 'true' + run: | + docker-compose -f tests_deployment/docker-compose.kafka.yml down + sequential_integration_test: name: Run Sequential Integration Tests if: github.ref_type != 'tag' @@ -841,11 +924,6 @@ jobs: run: | make test-integration - - name: Stop kafka and zookeeper containers for integration testing - if: needs.changes.outputs.backend == 'true' - run: | - docker-compose -f tests_deployment/docker-compose.kafka.yml down - build_docker_base_images_and_set_env: name: Build Docker base images and setup environment runs-on: ubuntu-22.04 diff --git a/Makefile b/Makefile index 23b5799e1d5f..62367bc17c4f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ JOBS ?= 1 INTEGRATION_TEST_FOLDER = tests/integration_tests/ -INTEGRATION_TEST_PYTEST_MARKERS ?= "sequential or not sequential" +INTEGRATION_TEST_PYTEST_MARKERS ?= "sequential or broker or ((not sequential) and (not broker))" PLATFORM ?= "linux/amd64" help: diff --git a/changelog/13017.bugfix.md b/changelog/13017.bugfix.md new file mode 100644 index 000000000000..654e62aafa2c --- /dev/null +++ b/changelog/13017.bugfix.md @@ -0,0 +1 @@ +Flush messages when Kafka producer is closed. This is to ensure that all messages in the producer's internal queue are sent to the broker. diff --git a/docs/docs/docker/building-in-docker.mdx b/docs/docs/docker/building-in-docker.mdx index a6027c51f7e2..9866ad95e063 100644 --- a/docs/docs/docker/building-in-docker.mdx +++ b/docs/docs/docker/building-in-docker.mdx @@ -67,7 +67,7 @@ The initial project files should all be there, as well as a `models` directory t :::note If you run into permission errors, it may be because the `rasa/rasa` images run as user `1001` as a best practice, to avoid giving the container `root` permissions. -Hence, all files created by these containers will be owned by user `1001`. See the [Docker documentation](https://docs.docker.com/edge/engine/reference/commandline/run/) +Hence, all files created by these containers will be owned by user `1001`. See the [Docker documentation](https://docs.docker.com/reference/cli/docker/container/run/) if you want to run the containers as a different user. ::: diff --git a/rasa/core/brokers/kafka.py b/rasa/core/brokers/kafka.py index 7183be12746a..66e77c2ca385 100644 --- a/rasa/core/brokers/kafka.py +++ b/rasa/core/brokers/kafka.py @@ -260,9 +260,11 @@ def _publish(self, event: Dict[Text, Any]) -> None: on_delivery=delivery_report, ) - def _close(self) -> None: + async def close(self) -> None: self._cancelled = True self._poll_thread.join() + if self.producer: + self.producer.flush() @rasa.shared.utils.common.lazy_property def rasa_environment(self) -> Optional[Text]: diff --git a/tests/integration_tests/core/brokers/test_kafka.py b/tests/integration_tests/core/brokers/test_kafka.py index 89fcdbb2d7bd..6be6eaa89d48 100644 --- a/tests/integration_tests/core/brokers/test_kafka.py +++ b/tests/integration_tests/core/brokers/test_kafka.py @@ -1,9 +1,12 @@ +import pytest + from rasa.core.brokers.kafka import KafkaEventBroker from pytest import LogCaptureFixture import logging.config -def test_kafka_event_broker_valid(): +@pytest.mark.broker +async def test_kafka_event_broker_valid(): broker = KafkaEventBroker( url="localhost", topic="rasa", @@ -19,11 +22,11 @@ def test_kafka_event_broker_valid(): ) assert broker.producer.poll() == 1 finally: - broker.producer.flush() - broker._close() + await broker.close() -def test_kafka_event_broker_buffer_error_is_handled(caplog: LogCaptureFixture): +@pytest.mark.broker +async def test_kafka_event_broker_buffer_error_is_handled(caplog: LogCaptureFixture): broker = KafkaEventBroker( url="localhost", topic="rasa", @@ -48,5 +51,4 @@ def test_kafka_event_broker_buffer_error_is_handled(caplog: LogCaptureFixture): assert "Queue full" in caplog.text assert broker.producer.poll() == 1 finally: - broker.producer.flush() - broker._close() + await broker.close() diff --git a/tests/integration_tests/core/brokers/test_pika.py b/tests/integration_tests/core/brokers/test_pika.py index eb27f9ba9f09..b514b1f91c09 100644 --- a/tests/integration_tests/core/brokers/test_pika.py +++ b/tests/integration_tests/core/brokers/test_pika.py @@ -16,6 +16,7 @@ ) +@pytest.mark.broker async def test_pika_event_broker_connect(): broker = PikaEventBroker( host=RABBITMQ_HOST, @@ -31,6 +32,7 @@ async def test_pika_event_broker_connect(): await broker.close() +@pytest.mark.broker @pytest.mark.xdist_group("rabbitmq") async def test_pika_event_broker_publish_after_restart( docker_client: docker.DockerClient, @@ -102,6 +104,7 @@ async def test_pika_event_broker_publish_after_restart( rabbitmq_container.remove() +@pytest.mark.broker @pytest.mark.xdist_group("rabbitmq") @pytest.mark.parametrize("host_component", ["localhost", "myuser:mypassword@localhost"]) async def test_pika_event_broker_connect_with_path_and_query_params_in_url( diff --git a/tests/integration_tests/core/test_exporter.py b/tests/integration_tests/core/test_exporter.py new file mode 100644 index 000000000000..2b3e8b83edb4 --- /dev/null +++ b/tests/integration_tests/core/test_exporter.py @@ -0,0 +1,117 @@ +import textwrap +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from pytest import MonkeyPatch + +from rasa.core.brokers.kafka import KafkaEventBroker +from rasa.core.exporter import Exporter +from rasa.core.tracker_store import InMemoryTrackerStore +from rasa.shared.core.domain import Domain +from rasa.shared.core.events import ActionExecuted +from rasa.shared.core.trackers import DialogueStateTracker + + +@pytest.mark.broker +async def test_exporter_publishes_to_kafka_broker_success( + tmp_path: Path, +) -> None: + tracker_store = InMemoryTrackerStore(domain=Domain.empty()) + tracker = DialogueStateTracker.from_events( + "test_export", + [ + ActionExecuted("action_listen"), + ], + ) + + await tracker_store.save(tracker) + + kafka_broker = KafkaEventBroker( + url="localhost", + topic="rasa", + sasl_username="admin", + sasl_password="password", + partition_by_sender=True, + ) + + endpoints_file = tmp_path / "endpoints.yml" + endpoints_file.write_text( + textwrap.dedent( + """ + event_broker: + type: kafka + topic: rasa + url: localhost:9092 + client_id: kafka-python-rasa + partition_by_sender: true + security_protocol: SASL_PLAINTEXT + sasl_username: admin + sasl_password: password + sasl_mechanism: PLAIN + """ + ) + ) + + exporter = Exporter(tracker_store, kafka_broker, str(endpoints_file)) + + published_events = await exporter.publish_events() + assert published_events == 1 + + +@pytest.mark.broker +async def test_exporter_publishes_to_kafka_broker_fail( + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + tracker_store = InMemoryTrackerStore(domain=Domain.empty()) + tracker = DialogueStateTracker.from_events( + "test_export", + [ + ActionExecuted("action_listen"), + ], + ) + + await tracker_store.save(tracker) + + kafka_broker = KafkaEventBroker( + url="localhost", + topic="rasa", + sasl_username="admin", + sasl_password="password", + partition_by_sender=True, + ) + + endpoints_file = tmp_path / "endpoints.yml" + endpoints_file.write_text( + textwrap.dedent( + """ + event_broker: + type: kafka + topic: rasa + url: localhost:9092 + client_id: kafka-python-rasa + partition_by_sender: true + security_protocol: SASL_PLAINTEXT + sasl_username: admin + sasl_password: password + sasl_mechanism: PLAIN + """ + ) + ) + + exporter = Exporter(tracker_store, kafka_broker, str(endpoints_file)) + + # patch the exporter to raise an exception when publishing events + monkeypatch.setattr(exporter, "publish_events", Mock(side_effect=Exception)) + + with pytest.raises(Exception) as error: + await exporter.publish_events() + assert "Producer terminating with 1 messages" in str(error.value) + assert ( + "still in queue or transit: use flush() to wait for " + "outstanding message delivery" in str(error.value) + ) + # necessary for producer teardown + await kafka_broker.close() From 50667b5469a20798cde508d16893ddbaa267ad96 Mon Sep 17 00:00:00 2001 From: Anca Lita <27920906+ancalita@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:35:36 +0000 Subject: [PATCH 24/41] prepared release of version 3.6.18 --- CHANGELOG.mdx | 7 +++++++ changelog/13017.bugfix.md | 1 - pyproject.toml | 2 +- rasa/version.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) delete mode 100644 changelog/13017.bugfix.md diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index 8efe6d303a21..a0d7ebd27de6 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,13 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.18] - 2024-02-23 + +Rasa 3.6.18 (2024-02-23) +### Bugfixes +- [#13017](https://github.com/rasahq/rasa/issues/13017): Flush messages when Kafka producer is closed. This is to ensure that all messages in the producer's internal queue are sent to the broker. + + ## [3.6.17] - 2024-02-13 Rasa 3.6.17 (2024-02-13) diff --git a/changelog/13017.bugfix.md b/changelog/13017.bugfix.md deleted file mode 100644 index 654e62aafa2c..000000000000 --- a/changelog/13017.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Flush messages when Kafka producer is closed. This is to ensure that all messages in the producer's internal queue are sent to the broker. diff --git a/pyproject.toml b/pyproject.toml index 913610027cf5..f52ef4094521 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.17" +version = "3.6.18" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index 6a7745b5540c..bbd3bcf024cb 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.17" +__version__ = "3.6.18" From 0df5212ad0a925216357f1226142484ca5883735 Mon Sep 17 00:00:00 2001 From: Shailendra Paliwal Date: Mon, 4 Mar 2024 17:26:53 +0100 Subject: [PATCH 25/41] SQL event.timestamp is not always unique (#13019) * return ID instead of timestamp * order by both timestamp and ID * order events by ID * add timestamp ordering to events * fix linting error * change event timestamps in test * add context in docstring and comment * add test to tracker store * undo changes to assertion * add changelog --- changelog/13019.bugfix.md | 1 + rasa/core/exporter.py | 8 +++++++- rasa/core/tracker_store.py | 5 ++++- tests/core/test_exporter.py | 6 +++--- tests/core/test_tracker_stores.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 changelog/13019.bugfix.md diff --git a/changelog/13019.bugfix.md b/changelog/13019.bugfix.md new file mode 100644 index 000000000000..9ce820d61dbe --- /dev/null +++ b/changelog/13019.bugfix.md @@ -0,0 +1 @@ +Changed the ordering of returned events to order by ID (previously timestamp) in SQL Tracker Store diff --git a/rasa/core/exporter.py b/rasa/core/exporter.py index 58c567dbdbe2..2961b6b6fc94 100644 --- a/rasa/core/exporter.py +++ b/rasa/core/exporter.py @@ -160,7 +160,7 @@ def _validate_all_requested_ids_exist( self, conversation_ids_in_tracker_store: Set[Text] ) -> None: """Warn user if `self.requested_conversation_ids` contains IDs not found in - `conversation_ids_in_tracker_store` + `conversation_ids_in_tracker_store`. Args: conversation_ids_in_tracker_store: Set of conversation IDs contained in @@ -241,6 +241,12 @@ async def _fetch_events_within_time_range(self) -> AsyncIterator[Dict[Text, Any] continue events = self._get_events_for_conversation_id(_events, conversation_id) + + # the order of events was changed after ATO-2192 + # more context: https://github.com/RasaHQ/rasa/pull/13019 + # we should sort the events by timestamp to keep the order + events.sort(key=lambda x: x["timestamp"]) + # the conversation IDs are needed in the event publishing for event in events: if ( diff --git a/rasa/core/tracker_store.py b/rasa/core/tracker_store.py index 93632cd3d779..a91f9f5c6b87 100644 --- a/rasa/core/tracker_store.py +++ b/rasa/core/tracker_store.py @@ -1291,6 +1291,9 @@ def _event_query( self, session: "Session", sender_id: Text, fetch_events_from_all_sessions: bool ) -> "Query": """Provide the query to retrieve the conversation events for a specific sender. + The events are ordered by ID to ensure correct sequence of events. + As `timestamp` is not guaranteed to be unique and low-precision (float), it + cannot be used to order the events. Args: session: Current database session. @@ -1325,7 +1328,7 @@ def _event_query( ) ) - return event_query.order_by(self.SQLEvent.timestamp) + return event_query.order_by(self.SQLEvent.id) async def save(self, tracker: DialogueStateTracker) -> None: """Update database with events from the current conversation.""" diff --git a/tests/core/test_exporter.py b/tests/core/test_exporter.py index b74849c6bb33..0ea78f4b397f 100644 --- a/tests/core/test_exporter.py +++ b/tests/core/test_exporter.py @@ -73,9 +73,9 @@ async def test_fetch_events_within_time_range(): conversation_ids = ["some-id", "another-id"] # prepare events from different senders and different timestamps - event_1 = random_user_uttered_event(3) - event_2 = random_user_uttered_event(2) - event_3 = random_user_uttered_event(1) + event_1 = random_user_uttered_event(1) + event_2 = random_user_uttered_event(3) + event_3 = random_user_uttered_event(2) events = {conversation_ids[0]: [event_1, event_2], conversation_ids[1]: [event_3]} def _get_tracker(conversation_id: Text) -> DialogueStateTracker: diff --git a/tests/core/test_tracker_stores.py b/tests/core/test_tracker_stores.py index fb4a891b097e..4a05f370e5ff 100644 --- a/tests/core/test_tracker_stores.py +++ b/tests/core/test_tracker_stores.py @@ -614,6 +614,36 @@ async def test_sql_additional_events_with_session_start(domain: Domain): assert isinstance(additional_events[0], UserUttered) +async def test_tracker_store_retrieve_ordered_by_id( + domain: Domain, +): + tracker_store_kwargs = {"host": "sqlite:///"} + tracker_store = SQLTrackerStore(domain, **tracker_store_kwargs) + events = [ + SessionStarted(timestamp=1), + UserUttered("Hola", {"name": "greet"}, timestamp=2), + BotUttered("Hi", timestamp=2), + UserUttered("How are you?", {"name": "greet"}, timestamp=2), + BotUttered("I am good, whats up", timestamp=2), + UserUttered("Ciao", {"name": "greet"}, timestamp=2), + BotUttered("Bye", timestamp=2), + ] + sender_id = "test_sql_tracker_store_events_order" + tracker = DialogueStateTracker.from_events(sender_id, events) + await tracker_store.save(tracker) + + # Save other tracker to ensure that we don't run into problems with other senders + other_tracker = DialogueStateTracker.from_events("other-sender", [SessionStarted()]) + await tracker_store.save(other_tracker) + + # Retrieve tracker with events since latest SessionStarted + tracker = await tracker_store.retrieve(sender_id) + + assert len(tracker.events) == 7 + # assert the order of events is same as the order in which they were added + assert all((event == tracker.events[i] for i, event in enumerate(events))) + + @pytest.mark.parametrize( "tracker_store_type,tracker_store_kwargs", [(MockedMongoTrackerStore, {}), (SQLTrackerStore, {"host": "sqlite:///"})], From c90c2211264cde9c3e319b93b2a7de2e0baf2a85 Mon Sep 17 00:00:00 2001 From: Shailendra Paliwal Date: Mon, 4 Mar 2024 17:49:17 +0100 Subject: [PATCH 26/41] prepared release of version 3.6.19 (#13020) --- CHANGELOG.mdx | 7 +++++++ changelog/13019.bugfix.md | 1 - pyproject.toml | 2 +- rasa/version.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) delete mode 100644 changelog/13019.bugfix.md diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index a0d7ebd27de6..a9a2a21fb56b 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,13 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.19] - 2024-03-04 + +Rasa 3.6.19 (2024-03-04) +### Bugfixes +- [#13019](https://github.com/rasahq/rasa/issues/13019): Changed the ordering of returned events to order by ID (previously timestamp) in SQL Tracker Store + + ## [3.6.18] - 2024-02-23 Rasa 3.6.18 (2024-02-23) diff --git a/changelog/13019.bugfix.md b/changelog/13019.bugfix.md deleted file mode 100644 index 9ce820d61dbe..000000000000 --- a/changelog/13019.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Changed the ordering of returned events to order by ID (previously timestamp) in SQL Tracker Store diff --git a/pyproject.toml b/pyproject.toml index f52ef4094521..b9c98cf2486f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.18" +version = "3.6.19" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index bbd3bcf024cb..12d8ae480621 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.18" +__version__ = "3.6.19" From 9dd97dc6f1abbd8419dc04861a9f96959c6c554e Mon Sep 17 00:00:00 2001 From: sancharigr Date: Wed, 13 Mar 2024 17:49:07 +0100 Subject: [PATCH 27/41] Update slack release notifications channels --- .github/workflows/continous-integration.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 7a69bbb4e807..8b6c0ab7dbe2 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -1264,10 +1264,9 @@ jobs: sentry-cli releases set-commits --auto "rasa-$GITHUB_TAG" sentry-cli releases finalize "rasa-$GITHUB_TAG" - - name: Notify Slack & Publish Release Notes 🗞 + - name: Publish Release Notes 🗞 env: GH_RELEASE_NOTES_TOKEN: ${{ secrets.GH_RELEASE_NOTES_TOKEN }} - SLACK_WEBHOOK_TOKEN: ${{ secrets.SLACK_WEBHOOK_TOKEN }} GITHUB_TAG: ${{ github.ref }} GITHUB_REPO_SLUG: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1275,7 +1274,16 @@ jobs: GITHUB_TAG=${GITHUB_TAG/refs\/tags\//} pip install -U github3.py pep440-version-utils python3 scripts/publish_gh_release_notes.py - ./scripts/ping_slack_about_package_release.sh + + - name: Notify Slack of successful release + # notification will be sent to the #product channel on slack, webhook url is added as repository secret + if: success() + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }} + uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a + with: + args: "💥 New *Rasa Open Source * version `${{ github.ref_name }}` has been released!" + send_slack_notification_for_release_on_failure: name: Notify Slack & Publish Release Notes @@ -1290,7 +1298,7 @@ jobs: # send notification if 'deploy' is skipped (previous needed job failed) or failed if: needs.deploy.result != 'success' env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_TOKEN }} + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }} uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a with: args: "⛔️ *Rasa Open Source* version `${{ github.ref_name }}` could not be released 😱! Please check out GitHub Actions: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" From 3a6ddaaef686540e8965918c80d108bf95a2f351 Mon Sep 17 00:00:00 2001 From: sancharigr Date: Fri, 12 Apr 2024 14:38:44 +0200 Subject: [PATCH 28/41] Update poetry lock for 1.8.2 --- poetry.lock | 310 +++---------------------------------------------- pyproject.toml | 1 + 2 files changed, 16 insertions(+), 295 deletions(-) diff --git a/poetry.lock b/poetry.lock index e3eb767e760a..a99ed6046eab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "absl-py" version = "1.4.0" description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -16,7 +15,6 @@ files = [ name = "aio-pika" version = "8.2.3" description = "Wrapper for the aiormq for asyncio and humans." -category = "main" optional = false python-versions = ">3.6, <4" files = [ @@ -35,7 +33,6 @@ develop = ["aiomisc (>=16.0,<17.0)", "coverage (!=4.3)", "coveralls", "nox", "py name = "aiofiles" version = "23.1.0" description = "File support for asyncio." -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -47,7 +44,6 @@ files = [ name = "aiogram" version = "2.15" description = "Is a pretty simple and fully asynchronous framework for Telegram Bot API" -category = "main" optional = false python-versions = "*" files = [ @@ -68,7 +64,6 @@ proxy = ["aiohttp-socks (>=0.5.3,<0.6.0)"] name = "aiohttp" version = "3.9.3" description = "Async http client/server framework (asyncio)" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -165,7 +160,6 @@ speedups = ["Brotli", "aiodns", "brotlicffi"] name = "aiohttp-retry" version = "2.8.3" description = "Simple retry client for aiohttp" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -180,7 +174,6 @@ aiohttp = "*" name = "aioresponses" version = "0.7.6" description = "Mock out requests made by ClientSession from aiohttp package" -category = "dev" optional = false python-versions = "*" files = [ @@ -195,7 +188,6 @@ aiohttp = ">=3.3.0,<4.0.0" name = "aiormq" version = "6.4.2" description = "Pure python AMQP asynchronous client library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -214,7 +206,6 @@ develop = ["aiomisc (>=16.0,<17.0)", "coverage (!=4.3)", "coveralls", "pylava", name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -229,7 +220,6 @@ frozenlist = ">=1.1.0" name = "analytics-python" version = "1.4.post1" description = "The hassle-free way to integrate analytics into any python application." -category = "dev" optional = false python-versions = "*" files = [ @@ -251,7 +241,6 @@ test = ["flake8 (==3.7.9)", "mock (==2.0.0)", "pylint (==1.9.3)"] name = "anyio" version = "3.7.1" description = "High level compatibility layer for multiple asynchronous event loop implementations" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -273,7 +262,6 @@ trio = ["trio (<0.22)"] name = "apscheduler" version = "3.9.1.post1" description = "In-process task scheduler with Cron-like capabilities" -category = "main" optional = false python-versions = "!=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" files = [ @@ -285,7 +273,7 @@ files = [ pytz = "*" setuptools = ">=0.7" six = ">=1.4.0" -tzlocal = ">=2.0,<3.0.0 || >=4.0.0" +tzlocal = ">=2.0,<3.dev0 || >=4.dev0" [package.extras] asyncio = ["trollius"] @@ -304,7 +292,6 @@ zookeeper = ["kazoo"] name = "astunparse" version = "1.6.3" description = "An AST unparser for Python" -category = "main" optional = false python-versions = "*" files = [ @@ -320,7 +307,6 @@ wheel = ">=0.23.0,<1.0" name = "async-generator" version = "1.10" description = "Async generators and context managers for Python 3.5+" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -332,7 +318,6 @@ files = [ name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -344,7 +329,6 @@ files = [ name = "attrs" version = "22.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -362,7 +346,6 @@ tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy name = "azure-core" version = "1.27.1" description = "Microsoft Azure Core Library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -382,7 +365,6 @@ aio = ["aiohttp (>=3.0)"] name = "azure-storage-blob" version = "12.15.0" description = "Microsoft Azure Blob Storage Client Library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -403,7 +385,6 @@ aio = ["azure-core[aio] (>=1.26.0,<2.0.0)"] name = "babel" version = "2.9.1" description = "Internationalization utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -418,7 +399,6 @@ pytz = ">=2015.7" name = "backoff" version = "1.10.0" description = "Function decoration for backoff and retry" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -430,7 +410,6 @@ files = [ name = "backports-zoneinfo" version = "0.2.1" description = "Backport of the standard library zoneinfo module" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -459,7 +438,6 @@ tzdata = ["tzdata"] name = "bandit" version = "1.7.5" description = "Security oriented static analyser for python code." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -483,7 +461,6 @@ yaml = ["PyYAML"] name = "bidict" version = "0.22.1" description = "The bidirectional mapping library for Python." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -500,7 +477,6 @@ test = ["hypothesis", "pytest", "pytest-benchmark[histogram]", "pytest-cov", "py name = "black" version = "22.12.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -536,7 +512,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "blis" version = "0.7.9" description = "The Blis BLAS-like linear algebra library, as a self-contained C-extension." -category = "main" optional = true python-versions = "*" files = [ @@ -577,7 +552,6 @@ numpy = ">=1.15.0" name = "boto3" version = "1.27.1" description = "The AWS SDK for Python" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -597,7 +571,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.30.1" description = "Low-level, data-driven core of boto 3." -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -617,7 +590,6 @@ crt = ["awscrt (==0.16.9)"] name = "cachecontrol" version = "0.12.14" description = "httplib2 caching for requests" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -637,7 +609,6 @@ redis = ["redis (>=2.10.5)"] name = "cachetools" version = "5.3.1" description = "Extensible memoizing collections and decorators" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -649,7 +620,6 @@ files = [ name = "catalogue" version = "2.0.8" description = "Super lightweight function registries for your library" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -661,7 +631,6 @@ files = [ name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -673,7 +642,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -750,7 +718,6 @@ pycparser = "*" name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -835,7 +802,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -850,7 +816,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "click-default-group" version = "1.2.2" description = "Extends click.Group to invoke a command without explicit subcommand name" -category = "dev" optional = false python-versions = "*" files = [ @@ -864,7 +829,6 @@ click = "*" name = "cloudpickle" version = "2.2.1" description = "Extended pickling support for Python objects" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -876,7 +840,6 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -888,7 +851,6 @@ files = [ name = "colorclass" version = "2.2.2" description = "Colorful worry-free console applications for Linux, Mac OS X, and Windows." -category = "main" optional = false python-versions = ">=2.6" files = [ @@ -900,7 +862,6 @@ files = [ name = "coloredlogs" version = "15.0.1" description = "Colored terminal output for Python's logging module" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -918,7 +879,6 @@ cron = ["capturer (>=2.4)"] name = "colorhash" version = "1.2.1" description = "Generate color based on any object" -category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -930,7 +890,6 @@ files = [ name = "confection" version = "0.1.0" description = "The sweetest config system for Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -946,7 +905,6 @@ srsly = ">=2.4.0,<3.0.0" name = "confluent-kafka" version = "2.1.1" description = "Confluent's Python client for Apache Kafka" -category = "main" optional = false python-versions = "*" files = [ @@ -993,7 +951,6 @@ schema-registry = ["requests"] name = "coverage" version = "6.5.0" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1059,7 +1016,6 @@ toml = ["tomli"] name = "coveralls" version = "3.3.1" description = "Show coverage stats online via coveralls.io" -category = "dev" optional = false python-versions = ">= 3.5" files = [ @@ -1068,7 +1024,7 @@ files = [ ] [package.dependencies] -coverage = ">=4.1,<6.0.0 || >6.1,<6.1.1 || >6.1.1,<7.0" +coverage = ">=4.1,<6.0.dev0 || >6.1,<6.1.1 || >6.1.1,<7.0" docopt = ">=0.6.1" requests = ">=1.0.0" @@ -1079,7 +1035,6 @@ yaml = ["PyYAML (>=3.10)"] name = "cryptography" version = "41.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1125,7 +1080,6 @@ test-randomorder = ["pytest-randomly"] name = "cycler" version = "0.11.0" description = "Composable style cycles" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1137,7 +1091,6 @@ files = [ name = "cymem" version = "2.0.7" description = "Manage calls to calloc/free through Cython" -category = "main" optional = true python-versions = "*" files = [ @@ -1175,7 +1128,6 @@ files = [ name = "dask" version = "2022.10.2" description = "Parallel PyData with Task Scheduling" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1204,7 +1156,6 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest- name = "databind" version = "1.5.3" description = "Databind is a library inspired by jackson-databind to de-/serialize Python dataclasses. The `databind` package will install the full suite of databind packages. Compatible with Python 3.7 and newer." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1220,7 +1171,6 @@ files = [ name = "databind-core" version = "1.5.3" description = "Databind is a library inspired by jackson-databind to de-/serialize Python dataclasses. Compatible with Python 3.7 and newer." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1237,7 +1187,6 @@ typing-extensions = ">=3.10.0" name = "databind-json" version = "1.5.3" description = "De-/serialize Python dataclasses to or from JSON payloads. Compatible with Python 3.7 and newer." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1254,7 +1203,6 @@ typing-extensions = ">=3.10.0" name = "datadog" version = "0.45.0" description = "The Datadog Python library" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -1269,7 +1217,6 @@ requests = ">=2.6.0" name = "datadog-api-client" version = "2.14.0" description = "Collection of all Datadog Public endpoints" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1293,7 +1240,6 @@ zstandard = ["zstandard"] name = "deprecated" version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1311,7 +1257,6 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] name = "dnspython" version = "2.3.0" description = "DNS toolkit" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1332,7 +1277,6 @@ wmi = ["wmi (>=1.5.1,<2.0.0)"] name = "docker" version = "6.1.3" description = "A Python library for the Docker Engine API." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1354,7 +1298,6 @@ ssh = ["paramiko (>=2.4.3)"] name = "docopt" version = "0.6.2" description = "Pythonic argument parser, that will make you smile" -category = "main" optional = false python-versions = "*" files = [ @@ -1365,7 +1308,6 @@ files = [ name = "docspec" version = "2.1.2" description = "Docspec is a JSON object specification for representing API documentation of programming languages." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1381,7 +1323,6 @@ Deprecated = ">=1.2.12,<2.0.0" name = "docspec-python" version = "2.0.2" description = "A parser based on lib2to3 producing docspec data from Python source code." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1397,7 +1338,6 @@ docspec = ">=2.0.2,<3.0.0" name = "docstring-parser" version = "0.11" description = "\"Parse Python docstrings in reST, Google and Numpydoc format\"" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1411,7 +1351,6 @@ test = ["black", "pytest"] name = "exceptiongroup" version = "1.1.2" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1426,7 +1365,6 @@ test = ["pytest (>=6)"] name = "execnet" version = "1.9.0" description = "execnet: rapid multi-Python deployment" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1441,7 +1379,6 @@ testing = ["pre-commit"] name = "fakeredis" version = "2.16.0" description = "Python implementation of redis API, can be used for testing purposes." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1461,7 +1398,6 @@ lua = ["lupa (>=1.14,<2.0)"] name = "fbmessenger" version = "6.0.0" description = "A python library to communicate with the Facebook Messenger API's" -category = "main" optional = false python-versions = "*" files = [ @@ -1476,7 +1412,6 @@ requests = ">=2.0" name = "filelock" version = "3.12.2" description = "A platform independent file lock." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1492,7 +1427,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "fire" version = "0.5.0" description = "A library for automatically generating command line interfaces." -category = "main" optional = false python-versions = "*" files = [ @@ -1507,7 +1441,6 @@ termcolor = "*" name = "flatbuffers" version = "23.5.26" description = "The FlatBuffers serialization format for Python" -category = "main" optional = false python-versions = "*" files = [ @@ -1519,7 +1452,6 @@ files = [ name = "fonttools" version = "4.40.0" description = "Tools to manipulate font files" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1577,7 +1509,6 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "freezegun" version = "1.2.2" description = "Let your Python tests travel through time" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1592,7 +1523,6 @@ python-dateutil = ">=2.7" name = "frozenlist" version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1676,7 +1606,6 @@ files = [ name = "fsspec" version = "2023.6.0" description = "File-system specification" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1712,7 +1641,6 @@ tqdm = ["tqdm"] name = "future" version = "0.18.3" description = "Clean single-source support for Python 3 and 2" -category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1723,7 +1651,6 @@ files = [ name = "gast" version = "0.4.0" description = "Python AST that abstracts the underlying Python version" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1735,7 +1662,6 @@ files = [ name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1750,7 +1676,6 @@ smmap = ">=3.0.1,<6" name = "github3-py" version = "3.2.0" description = "Python wrapper for the GitHub API(http://developer.github.com/v3)" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1768,7 +1693,6 @@ uritemplate = ">=3.0.0" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1783,7 +1707,6 @@ gitdb = ">=4.0.1,<5" name = "google-api-core" version = "2.11.1" description = "Google API client core library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1806,7 +1729,6 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] name = "google-auth" version = "2.21.0" description = "Google Authentication Library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1832,7 +1754,6 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] name = "google-auth-oauthlib" version = "1.0.0" description = "Google Authentication Library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1851,7 +1772,6 @@ tool = ["click (>=6.0.0)"] name = "google-cloud-core" version = "2.3.3" description = "Google Cloud API client core library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1860,7 +1780,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.6,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] @@ -1870,7 +1790,6 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)"] name = "google-cloud-storage" version = "2.10.0" description = "Google Cloud Storage API client library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1879,7 +1798,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" google-resumable-media = ">=2.3.2" @@ -1892,7 +1811,6 @@ protobuf = ["protobuf (<5.0.0dev)"] name = "google-crc32c" version = "1.5.0" description = "A python wrapper of the C library 'Google CRC32C'" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1973,7 +1891,6 @@ testing = ["pytest"] name = "google-pasta" version = "0.2.0" description = "pasta is an AST-based Python refactoring library" -category = "main" optional = false python-versions = "*" files = [ @@ -1989,7 +1906,6 @@ six = "*" name = "google-resumable-media" version = "2.5.0" description = "Utilities for Google Media Downloads and Resumable Uploads" -category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -2008,7 +1924,6 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] name = "googleapis-common-protos" version = "1.59.1" description = "Common protobufs used in Google APIs" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2026,7 +1941,6 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -2104,7 +2018,6 @@ test = ["objgraph", "psutil"] name = "grpcio" version = "1.56.0" description = "HTTP/2-based RPC framework" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2162,7 +2075,6 @@ protobuf = ["grpcio-tools (>=1.56.0)"] name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2174,7 +2086,6 @@ files = [ name = "h5py" version = "3.9.0" description = "Read and write HDF5 files from Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2208,7 +2119,6 @@ numpy = ">=1.17.3" name = "httpcore" version = "0.16.3" description = "A minimal low-level HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2220,17 +2130,16 @@ files = [ anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = ">=1.0.0,<2.0.0" +sniffio = "==1.*" [package.extras] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "httptools" version = "0.5.0" description = "A collection of framework independent HTTP protocol utils." -category = "main" optional = false python-versions = ">=3.5.0" files = [ @@ -2284,7 +2193,6 @@ test = ["Cython (>=0.29.24,<0.30.0)"] name = "httpx" version = "0.23.3" description = "The next generation HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2300,15 +2208,14 @@ sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<13)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "huggingface-hub" version = "0.16.2" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -2341,7 +2248,6 @@ typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "typ name = "humanfriendly" version = "10.0" description = "Human friendly output for text interfaces using Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2356,7 +2262,6 @@ pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_ve name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2368,7 +2273,6 @@ files = [ name = "importlib-metadata" version = "6.7.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2388,7 +2292,6 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2407,7 +2310,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "incremental" version = "22.10.0" description = "\"A small library that versions your Python projects.\"" -category = "dev" optional = false python-versions = "*" files = [ @@ -2423,7 +2325,6 @@ scripts = ["click (>=6.0)", "twisted (>=16.4.0)"] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2435,7 +2336,6 @@ files = [ name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" -category = "dev" optional = false python-versions = "*" files = [ @@ -2450,7 +2350,6 @@ six = "*" name = "jax" version = "0.4.13" description = "Differentiate, compile, and transform Numpy code." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2481,7 +2380,6 @@ tpu = ["jaxlib (==0.4.13)", "libtpu-nightly (==0.1.dev20230622)"] name = "jieba" version = "0.42.1" description = "Chinese Words Segmentation Utilities" -category = "main" optional = true python-versions = "*" files = [ @@ -2492,7 +2390,6 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2510,7 +2407,6 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2522,7 +2418,6 @@ files = [ name = "joblib" version = "1.2.0" description = "Lightweight pipelining with Python functions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2534,7 +2429,6 @@ files = [ name = "jsonpickle" version = "3.0.1" description = "Python library for serializing any arbitrary object graph into JSON" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2551,7 +2445,6 @@ testing-libs = ["simplejson", "ujson"] name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2573,7 +2466,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "keras" version = "2.12.0" description = "Deep learning for humans." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2584,7 +2476,6 @@ files = [ name = "kiwisolver" version = "1.4.4" description = "A fast implementation of the Cassowary constraint solver" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2662,7 +2553,6 @@ files = [ name = "langcodes" version = "3.3.0" description = "Tools for labeling human languages with IETF language tags" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2677,7 +2567,6 @@ data = ["language-data (>=1.1,<2.0)"] name = "libclang" version = "16.0.0" description = "Clang Python Bindings, mirrored from the official LLVM repo: https://github.com/llvm/llvm-project/tree/main/clang/bindings/python, to make the installation process easier." -category = "main" optional = false python-versions = "*" files = [ @@ -2695,7 +2584,6 @@ files = [ name = "locket" version = "1.0.0" description = "File-based locks for Python on Linux and Windows" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2707,7 +2595,6 @@ files = [ name = "markdown" version = "3.4.3" description = "Python implementation of John Gruber's Markdown." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2725,7 +2612,6 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2750,7 +2636,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2820,7 +2705,6 @@ files = [ name = "matplotlib" version = "3.5.3" description = "Python plotting package" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2875,7 +2759,6 @@ python-dateutil = ">=2.7" name = "mattermostwrapper" version = "2.2" description = "A mattermost api v4 wrapper to interact with api" -category = "main" optional = false python-versions = "*" files = [ @@ -2889,7 +2772,6 @@ requests = "*" name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2901,7 +2783,6 @@ files = [ name = "memory-profiler" version = "0.61.0" description = "A module for monitoring memory usage of a python program" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -2916,7 +2797,6 @@ psutil = "*" name = "ml-dtypes" version = "0.2.0" description = "" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2941,8 +2821,8 @@ files = [ [package.dependencies] numpy = [ - {version = ">1.20", markers = "python_version <= \"3.9\""}, {version = ">=1.21.2", markers = "python_version > \"3.9\""}, + {version = ">1.20", markers = "python_version <= \"3.9\""}, ] [package.extras] @@ -2952,7 +2832,6 @@ dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"] name = "mongomock" version = "4.1.2" description = "Fake pymongo stub for testing simple MongoDB-dependent code" -category = "dev" optional = false python-versions = "*" files = [ @@ -2968,7 +2847,6 @@ sentinels = "*" name = "monotonic" version = "1.6" description = "An implementation of time.monotonic() for Python 2 & < 3.3" -category = "dev" optional = false python-versions = "*" files = [ @@ -2980,7 +2858,6 @@ files = [ name = "moto" version = "4.1.12" description = "" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3027,7 +2904,6 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] name = "msgpack" version = "1.0.5" description = "MessagePack serializer" -category = "main" optional = false python-versions = "*" files = [ @@ -3100,7 +2976,6 @@ files = [ name = "multidict" version = "5.2.0" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3182,7 +3057,6 @@ files = [ name = "murmurhash" version = "1.0.9" description = "Cython bindings for MurmurHash" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3220,7 +3094,6 @@ files = [ name = "mypy" version = "1.0.1" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3267,7 +3140,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "0.4.4" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -3278,7 +3150,6 @@ files = [ name = "networkx" version = "2.6.3" description = "Python package for creating and manipulating graphs and networks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3297,7 +3168,6 @@ test = ["codecov (>=2.1)", "pytest (>=6.2)", "pytest-cov (>=2.12)"] name = "nr-util" version = "0.8.12" description = "General purpose Python utility library." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -3313,7 +3183,6 @@ typing-extensions = ">=3.0.0" name = "numpy" version = "1.22.3" description = "NumPy is the fundamental package for array computing with Python." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3343,7 +3212,6 @@ files = [ name = "numpy" version = "1.23.5" description = "NumPy is the fundamental package for array computing with Python." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3381,7 +3249,6 @@ files = [ name = "oauthlib" version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3398,7 +3265,6 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] name = "opt-einsum" version = "3.3.0" description = "Optimizing numpys einsum function" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -3417,7 +3283,6 @@ tests = ["pytest", "pytest-cov", "pytest-pep8"] name = "packaging" version = "20.9" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3432,7 +3297,6 @@ pyparsing = ">=2.0.2" name = "pamqp" version = "3.2.1" description = "RabbitMQ Focused AMQP low-level library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3448,7 +3312,6 @@ testing = ["coverage", "flake8", "flake8-comprehensions", "flake8-deprecated", " name = "partd" version = "1.4.0" description = "Appendable key-value storage" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3467,7 +3330,6 @@ complete = ["blosc", "numpy (>=1.9.0)", "pandas (>=0.19.0)", "pyzmq"] name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3479,7 +3341,6 @@ files = [ name = "pathy" version = "0.10.2" description = "pathlib.Path subclasses for local and cloud bucket storage" -category = "main" optional = true python-versions = ">= 3.6" files = [ @@ -3502,7 +3363,6 @@ test = ["mock", "pytest", "pytest-coverage", "typer-cli"] name = "pbr" version = "5.11.1" description = "Python Build Reasonableness" -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -3514,7 +3374,6 @@ files = [ name = "pep440-version-utils" version = "0.3.0" description = "Utilities to deal with pep440 versioning" -category = "dev" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -3529,7 +3388,6 @@ packaging = ">=20.3,<21.0" name = "pillow" version = "10.0.1" description = "Python Imaging Library (Fork)" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3597,7 +3455,6 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3609,7 +3466,6 @@ files = [ name = "platformdirs" version = "3.8.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3625,7 +3481,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3641,7 +3496,6 @@ testing = ["pytest", "pytest-benchmark"] name = "portalocker" version = "2.7.0" description = "Wraps the portalocker recipe for easy usage" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -3661,7 +3515,6 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p name = "preshed" version = "3.0.8" description = "Cython hash table that trusts the keys are pre-hashed" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3703,7 +3556,6 @@ murmurhash = ">=0.28.0,<1.1.0" name = "prompt-toolkit" version = "3.0.28" description = "Library for building powerful interactive command lines in Python" -category = "main" optional = false python-versions = ">=3.6.2" files = [ @@ -3718,7 +3570,6 @@ wcwidth = "*" name = "protobuf" version = "4.23.3" description = "" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3741,7 +3592,6 @@ files = [ name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3768,7 +3618,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "psycopg2-binary" version = "2.9.6" description = "psycopg2 - Python-PostgreSQL Database Adapter" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3840,7 +3689,6 @@ files = [ name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -3852,7 +3700,6 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -3867,7 +3714,6 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3879,7 +3725,6 @@ files = [ name = "pydantic" version = "1.10.9" description = "Data validation and settings management using python type hints" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3932,7 +3777,6 @@ email = ["email-validator (>=1.0.3)"] name = "pydoc-markdown" version = "4.7.0" description = "Create Python API documentation in Markdown format." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -3959,7 +3803,6 @@ yapf = ">=0.30.0" name = "pydot" version = "1.4.2" description = "Python interface to Graphviz's Dot" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3974,7 +3817,6 @@ pyparsing = ">=2.1.4" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3989,7 +3831,6 @@ plugins = ["importlib-metadata"] name = "pyjwt" version = "2.7.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4010,7 +3851,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pykwalify" version = "1.8.0" description = "Python lib/cli for JSON/YAML schema validation" -category = "main" optional = false python-versions = "*" files = [ @@ -4027,7 +3867,6 @@ python-dateutil = ">=2.8.0" name = "pymongo" version = "4.3.3" description = "Python driver for MongoDB " -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4122,7 +3961,6 @@ zstd = ["zstandard"] name = "pyparsing" version = "3.1.0" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -4137,7 +3975,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyreadline3" version = "3.4.1" description = "A python implementation of GNU readline." -category = "main" optional = false python-versions = "*" files = [ @@ -4149,7 +3986,6 @@ files = [ name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4186,7 +4022,6 @@ files = [ name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4209,7 +4044,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.20.3" description = "Pytest support for asyncio" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4228,7 +4062,6 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4247,7 +4080,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-sanic" version = "1.9.1" description = "a pytest plugin for Sanic" -category = "dev" optional = false python-versions = ">=3.7" files = [] @@ -4269,7 +4101,6 @@ resolved_reference = "4092e8005fbbdc29b892cd8b09399894d86b1ac7" name = "pytest-timeout" version = "2.1.0" description = "pytest plugin to abort hanging tests" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -4284,7 +4115,6 @@ pytest = ">=5.0.0" name = "pytest-xdist" version = "3.3.1" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4305,7 +4135,6 @@ testing = ["filelock"] name = "python-crfsuite" version = "0.9.9" description = "Python binding for CRFsuite" -category = "main" optional = false python-versions = "*" files = [ @@ -4365,7 +4194,6 @@ files = [ name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -4380,7 +4208,6 @@ six = ">=1.5" name = "python-engineio" version = "4.5.1" description = "Engine.IO server and client for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4397,7 +4224,6 @@ docs = ["sphinx"] name = "python-socketio" version = "5.8.0" description = "Socket.IO server and client for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4417,7 +4243,6 @@ client = ["requests (>=2.21.0)", "websocket-client (>=0.54.0)"] name = "pytz" version = "2022.7.1" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -4429,7 +4254,6 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" -category = "main" optional = false python-versions = "*" files = [ @@ -4453,7 +4277,6 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4514,7 +4337,6 @@ files = [ name = "questionary" version = "1.10.0" description = "Python library to build pretty command line user prompts ⭐️" -category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -4532,7 +4354,6 @@ docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphin name = "randomname" version = "0.1.5" description = "Generate random adj-noun names like docker and github." -category = "main" optional = false python-versions = "*" files = [ @@ -4546,7 +4367,6 @@ fire = "*" name = "rasa-sdk" version = "3.6.2" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" -category = "main" optional = false python-versions = ">=3.8,<3.11" files = [ @@ -4570,7 +4390,6 @@ wheel = ">=0.38.1" name = "redis" version = "4.6.0" description = "Python client for Redis database and key-value store" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4589,7 +4408,6 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" name = "regex" version = "2022.10.31" description = "Alternative regular expression module, to replace re." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4687,7 +4505,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4709,7 +4526,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-oauthlib" version = "1.3.1" description = "OAuthlib authentication support for Requests." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -4728,7 +4544,6 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] name = "requests-toolbelt" version = "1.0.0" description = "A utility belt for advanced users of python-requests" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -4743,7 +4558,6 @@ requests = ">=2.0.1,<3.0.0" name = "responses" version = "0.22.0" description = "A utility library for mocking out the `requests` Python library." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4764,7 +4578,6 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rfc3986" version = "1.5.0" description = "Validating URI References per RFC 3986" -category = "dev" optional = false python-versions = "*" files = [ @@ -4782,7 +4595,6 @@ idna2008 = ["idna"] name = "rich" version = "13.4.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -4802,7 +4614,6 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "rocketchat-api" version = "1.30.0" description = "Python API wrapper for Rocket.Chat" -category = "main" optional = false python-versions = "*" files = [ @@ -4818,7 +4629,6 @@ requests = "*" name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -4833,7 +4643,6 @@ pyasn1 = ">=0.1.3" name = "ruamel-yaml" version = "0.17.21" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "main" optional = false python-versions = ">=3" files = [ @@ -4852,7 +4661,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] name = "ruamel-yaml-clib" version = "0.2.7" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -4899,7 +4707,6 @@ files = [ name = "ruff" version = "0.0.255" description = "An extremely fast Python linter, written in Rust." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4926,7 +4733,6 @@ files = [ name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -4944,7 +4750,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "sanic" version = "21.12.2" description = "A web server and web framework that's written to go fast. Build fast. Run fast." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4962,17 +4767,16 @@ uvloop = {version = ">=0.5.3", markers = "sys_platform != \"win32\" and implemen websockets = ">=10.0" [package.extras] -all = ["bandit", "beautifulsoup4", "black", "chardet (>=3.0.0,<4.0.0)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "m2r2", "mistune (<2.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "sphinx (>=2.1.2)", "sphinx-rtd-theme (>=0.4.3)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] -dev = ["bandit", "beautifulsoup4", "black", "chardet (>=3.0.0,<4.0.0)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] +all = ["bandit", "beautifulsoup4", "black", "chardet (==3.*)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "m2r2", "mistune (<2.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "sphinx (>=2.1.2)", "sphinx-rtd-theme (>=0.4.3)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] +dev = ["bandit", "beautifulsoup4", "black", "chardet (==3.*)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] docs = ["docutils", "m2r2", "mistune (<2.0.0)", "pygments", "sphinx (>=2.1.2)", "sphinx-rtd-theme (>=0.4.3)"] ext = ["sanic-ext"] -test = ["bandit", "beautifulsoup4", "black", "chardet (>=3.0.0,<4.0.0)", "coverage (==5.3)", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "types-ujson", "uvicorn (<0.15.0)"] +test = ["bandit", "beautifulsoup4", "black", "chardet (==3.*)", "coverage (==5.3)", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "types-ujson", "uvicorn (<0.15.0)"] [[package]] name = "sanic-cors" version = "2.0.1" description = "A Sanic extension adding a decorator for CORS support. Based on flask-cors by Cory Dolphin." -category = "main" optional = false python-versions = "*" files = [ @@ -4987,7 +4791,6 @@ sanic = ">=21.9.3" name = "sanic-jwt" version = "1.8.0" description = "JWT oauth flow for Sanic" -category = "main" optional = false python-versions = "*" files = [ @@ -5006,7 +4809,6 @@ docs = ["Jinja2 (<3.1)", "Sphinx"] name = "sanic-routing" version = "0.7.2" description = "Core routing component for Sanic" -category = "main" optional = false python-versions = "*" files = [ @@ -5018,7 +4820,6 @@ files = [ name = "sanic-testing" version = "22.6.0" description = "Core testing clients for Sanic" -category = "dev" optional = false python-versions = "*" files = [ @@ -5033,7 +4834,6 @@ httpx = ">=0.18,<0.24" name = "scikit-learn" version = "1.1.3" description = "A set of python modules for machine learning and data mining" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5076,7 +4876,6 @@ tests = ["black (>=22.3.0)", "flake8 (>=3.8.2)", "matplotlib (>=3.1.2)", "mypy ( name = "scipy" version = "1.10.1" description = "Fundamental algorithms for scientific computing in Python" -category = "main" optional = false python-versions = "<3.12,>=3.8" files = [ @@ -5115,7 +4914,6 @@ test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeo name = "sentencepiece" version = "0.1.99" description = "SentencePiece python wrapper" -category = "main" optional = true python-versions = "*" files = [ @@ -5170,7 +4968,6 @@ files = [ name = "sentinels" version = "1.0.0" description = "Various objects to denote special meanings in python" -category = "dev" optional = false python-versions = "*" files = [ @@ -5181,7 +4978,6 @@ files = [ name = "sentry-sdk" version = "1.14.0" description = "Python client for Sentry (https://sentry.io)" -category = "main" optional = false python-versions = "*" files = [ @@ -5220,7 +5016,6 @@ tornado = ["tornado (>=5)"] name = "setuptools" version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5237,7 +5032,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -5249,7 +5043,6 @@ files = [ name = "sklearn-crfsuite" version = "0.3.6" description = "CRFsuite (python-crfsuite) wrapper which provides interface simlar to scikit-learn" -category = "main" optional = false python-versions = "*" files = [ @@ -5267,7 +5060,6 @@ tqdm = ">=2.0" name = "slack-sdk" version = "3.21.3" description = "The Slack API Platform SDK for Python" -category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -5283,7 +5075,6 @@ testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "We name = "smart-open" version = "6.3.0" description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" -category = "main" optional = true python-versions = ">=3.6,<4.0" files = [ @@ -5305,7 +5096,6 @@ webhdfs = ["requests"] name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -5317,7 +5107,6 @@ files = [ name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5329,7 +5118,6 @@ files = [ name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -category = "dev" optional = false python-versions = "*" files = [ @@ -5341,7 +5129,6 @@ files = [ name = "spacy" version = "3.4.4" description = "Industrial-strength Natural Language Processing (NLP) in Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5428,7 +5215,6 @@ transformers = ["spacy-transformers (>=1.1.2,<1.2.0)"] name = "spacy" version = "3.5.4" description = "Industrial-strength Natural Language Processing (NLP) in Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5515,7 +5301,6 @@ transformers = ["spacy-transformers (>=1.1.2,<1.3.0)"] name = "spacy-legacy" version = "3.0.12" description = "Legacy registered functions for spaCy backwards compatibility" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5527,7 +5312,6 @@ files = [ name = "spacy-loggers" version = "1.0.4" description = "Logging utilities for SpaCy" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5539,7 +5323,6 @@ files = [ name = "sqlalchemy" version = "1.4.49" description = "Database Abstraction Library" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -5594,7 +5377,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -5621,7 +5404,6 @@ sqlcipher = ["sqlcipher3-binary"] name = "srsly" version = "2.4.6" description = "Modern high-performance serialization utilities for Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5662,7 +5444,6 @@ catalogue = ">=2.0.3,<2.1.0" name = "stevedore" version = "5.1.0" description = "Manage dynamic plugins for Python applications" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -5677,7 +5458,6 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" name = "structlog" version = "23.1.0" description = "Structured Logging for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5695,7 +5475,6 @@ typing = ["mypy", "rich", "twisted"] name = "structlog-sentry" version = "2.0.3" description = "Sentry integration for structlog" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -5711,7 +5490,6 @@ structlog = "*" name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5726,7 +5504,6 @@ widechars = ["wcwidth"] name = "tarsafe" version = "0.0.4" description = "A safe subclass of the TarFile class for interacting with tar files. Can be used as a direct drop-in replacement for safe usage of extractall()" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -5738,7 +5515,6 @@ files = [ name = "tensorboard" version = "2.12.3" description = "TensorBoard lets you watch Tensors Flow" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5763,7 +5539,6 @@ wheel = ">=0.26" name = "tensorboard-data-server" version = "0.7.1" description = "Fast data loading for TensorBoard" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5776,7 +5551,6 @@ files = [ name = "tensorflow" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5826,7 +5600,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-cpu-aws" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5864,7 +5637,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-estimator" version = "2.12.0" description = "TensorFlow Estimator." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5875,7 +5647,6 @@ files = [ name = "tensorflow-hub" version = "0.13.0" description = "TensorFlow Hub is a library to foster the publication, discovery, and consumption of reusable parts of machine learning models." -category = "main" optional = false python-versions = "*" files = [ @@ -5894,7 +5665,6 @@ make-nearest-neighbour-index = ["annoy", "apache-beam"] name = "tensorflow-intel" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5932,7 +5702,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-io-gcs-filesystem" version = "0.31.0" description = "TensorFlow IO" -category = "main" optional = false python-versions = ">=3.7, <3.12" files = [ @@ -5968,7 +5737,6 @@ tensorflow-rocm = ["tensorflow-rocm (>=2.11.0,<2.12.0)"] name = "tensorflow-io-gcs-filesystem" version = "0.32.0" description = "TensorFlow IO" -category = "main" optional = false python-versions = ">=3.7, <3.12" files = [ @@ -5999,7 +5767,6 @@ tensorflow-rocm = ["tensorflow-rocm (>=2.12.0,<2.13.0)"] name = "tensorflow-macos" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -6041,7 +5808,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-metal" version = "0.8.0" description = "TensorFlow acceleration for Mac GPUs." -category = "main" optional = true python-versions = "*" files = [ @@ -6063,7 +5829,6 @@ wheel = ">=0.35,<1.0" name = "tensorflow-text" version = "2.12.0" description = "TF.Text is a TensorFlow library of text related ops, modules, and subgraphs." -category = "main" optional = false python-versions = "*" files = [ @@ -6089,7 +5854,6 @@ tests = ["absl-py", "pytest", "tensorflow-datasets (>=3.2.0)"] name = "termcolor" version = "2.3.0" description = "ANSI color formatting for output in terminal" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6104,7 +5868,6 @@ tests = ["pytest", "pytest-cov"] name = "terminaltables" version = "3.1.10" description = "Generate simple tables in terminals from a nested list of strings." -category = "main" optional = false python-versions = ">=2.6" files = [ @@ -6116,7 +5879,6 @@ files = [ name = "thinc" version = "8.1.10" description = "A refreshing functional take on deep learning, compatible with your favorite libraries" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6192,7 +5954,6 @@ torch = ["torch (>=1.6.0)"] name = "threadpoolctl" version = "3.1.0" description = "threadpoolctl" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -6204,7 +5965,6 @@ files = [ name = "tokenizers" version = "0.13.3" description = "Fast and Customizable Tokenizers" -category = "main" optional = true python-versions = "*" files = [ @@ -6259,7 +6019,6 @@ testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -6271,7 +6030,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6283,7 +6041,6 @@ files = [ name = "tomli-w" version = "1.0.0" description = "A lil' TOML writer" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6295,7 +6052,6 @@ files = [ name = "toolz" version = "0.12.0" description = "List processing tools and functional utilities" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -6307,7 +6063,6 @@ files = [ name = "towncrier" version = "22.12.0" description = "Building newsfiles for your project." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6330,7 +6085,6 @@ dev = ["furo", "packaging", "sphinx (>=5)", "twisted"] name = "tqdm" version = "4.65.0" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6351,7 +6105,6 @@ telegram = ["requests"] name = "transformers" version = "4.26.0" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" -category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -6418,7 +6171,6 @@ vision = ["Pillow"] name = "twilio" version = "8.2.2" description = "Twilio API client and TwiML generator" -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -6437,7 +6189,6 @@ requests = ">=2.0.0" name = "typer" version = "0.7.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6458,7 +6209,6 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. name = "typer" version = "0.9.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6480,7 +6230,6 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. name = "types-pyopenssl" version = "23.2.0.1" description = "Typing stubs for pyOpenSSL" -category = "dev" optional = false python-versions = "*" files = [ @@ -6495,7 +6244,6 @@ cryptography = ">=35.0.0" name = "types-python-dateutil" version = "2.8.19.13" description = "Typing stubs for python-dateutil" -category = "dev" optional = false python-versions = "*" files = [ @@ -6507,7 +6255,6 @@ files = [ name = "types-pytz" version = "2022.7.1.2" description = "Typing stubs for pytz" -category = "dev" optional = false python-versions = "*" files = [ @@ -6519,7 +6266,6 @@ files = [ name = "types-redis" version = "4.6.0.2" description = "Typing stubs for redis" -category = "dev" optional = false python-versions = "*" files = [ @@ -6535,7 +6281,6 @@ types-pyOpenSSL = "*" name = "types-requests" version = "2.31.0.1" description = "Typing stubs for requests" -category = "dev" optional = false python-versions = "*" files = [ @@ -6550,7 +6295,6 @@ types-urllib3 = "*" name = "types-setuptools" version = "67.8.0.0" description = "Typing stubs for setuptools" -category = "dev" optional = false python-versions = "*" files = [ @@ -6562,7 +6306,6 @@ files = [ name = "types-toml" version = "0.10.8.6" description = "Typing stubs for toml" -category = "dev" optional = false python-versions = "*" files = [ @@ -6574,7 +6317,6 @@ files = [ name = "types-urllib3" version = "1.26.25.13" description = "Typing stubs for urllib3" -category = "dev" optional = false python-versions = "*" files = [ @@ -6586,7 +6328,6 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6598,7 +6339,6 @@ files = [ name = "typing-utils" version = "0.1.0" description = "utils to inspect Python type annotations" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -6613,7 +6353,6 @@ test = ["pytest"] name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" -category = "main" optional = false python-versions = ">=2" files = [ @@ -6625,7 +6364,6 @@ files = [ name = "tzlocal" version = "5.0.1" description = "tzinfo object for the local timezone" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6644,7 +6382,6 @@ devenv = ["black", "check-manifest", "flake8", "pyroma", "pytest (>=4.3)", "pyte name = "ujson" version = "5.8.0" description = "Ultra fast JSON encoder and decoder for Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -6715,7 +6452,6 @@ files = [ name = "uritemplate" version = "4.1.1" description = "Implementation of RFC 6570 URI Templates" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6727,7 +6463,6 @@ files = [ name = "urllib3" version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -6744,7 +6479,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "uvloop" version = "0.17.0" description = "Fast implementation of asyncio event loop on top of libuv" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6789,7 +6523,6 @@ test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "my name = "wasabi" version = "0.10.1" description = "A lightweight console printing and formatting toolkit" -category = "main" optional = true python-versions = "*" files = [ @@ -6801,7 +6534,6 @@ files = [ name = "wasabi" version = "1.1.2" description = "A lightweight console printing and formatting toolkit" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6813,7 +6545,6 @@ files = [ name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6853,7 +6584,6 @@ watchmedo = ["PyYAML (>=3.10)"] name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "main" optional = false python-versions = "*" files = [ @@ -6865,7 +6595,6 @@ files = [ name = "webexteamssdk" version = "1.6.1" description = "Community-developed Python SDK for the Webex Teams APIs" -category = "main" optional = false python-versions = "*" files = [ @@ -6883,7 +6612,6 @@ requests-toolbelt = "*" name = "websocket-client" version = "1.6.1" description = "WebSocket client for Python with low level API options" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6900,7 +6628,6 @@ test = ["websockets"] name = "websockets" version = "10.4" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6979,7 +6706,6 @@ files = [ name = "werkzeug" version = "2.3.6" description = "The comprehensive WSGI web application library." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -6997,7 +6723,6 @@ watchdog = ["watchdog (>=2.3)"] name = "wheel" version = "0.40.0" description = "A built-package format for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7012,7 +6737,6 @@ test = ["pytest (>=6.0.0)"] name = "wrapt" version = "1.14.1" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -7096,7 +6820,6 @@ files = [ name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" -category = "dev" optional = false python-versions = ">=3.4" files = [ @@ -7108,7 +6831,6 @@ files = [ name = "yapf" version = "0.40.1" description = "A formatter for Python code." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -7125,7 +6847,6 @@ tomli = ">=2.0.1" name = "yarl" version = "1.9.2" description = "Yet another URL library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7213,7 +6934,6 @@ multidict = ">=4.0" name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7236,4 +6956,4 @@ transformers = ["sentencepiece", "transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "c554d0f11eb367109ccd80c3694de7f07d35b52a09b1279d2b9743e6c155f43c" +content-hash = "8e1bef6b78365110ec598219ee747edb505f82a76fd03bdcffcec7a299b39513" diff --git a/pyproject.toml b/pyproject.toml index b9c98cf2486f..53c05df1c74e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ license = "Apache-2.0" [[tool.poetry.source]] name = "internal repository mirroring psycopg binary for macos" url = "https://europe-west3-python.pkg.dev/rasa-releases/psycopg-binary/simple/" +priority = "supplemental" [tool.towncrier] package = "rasa" From 635afda18016583f5da4f0f9f2369bc0d2000495 Mon Sep 17 00:00:00 2001 From: sancharigr Date: Fri, 12 Apr 2024 14:44:10 +0200 Subject: [PATCH 29/41] Update CI and dockerfiles --- .github/poetry_version.txt | 2 +- docker/docker-bake.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/poetry_version.txt b/.github/poetry_version.txt index 8856dd07eb95..271f9b9b1d5e 100644 --- a/.github/poetry_version.txt +++ b/.github/poetry_version.txt @@ -1,2 +1,2 @@ # The poetry version is stored in a separate file due to the https://github.com/python-poetry/poetry/issues/3316 -poetry-version=1.4.2 +poetry-version=1.8.2 diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl index 00e1471d0804..89934a81accf 100644 --- a/docker/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -21,7 +21,7 @@ variable "BASE_BUILDER_IMAGE_HASH" { # keep this in sync with the version in .github/poetry_version.txt # the variable is set automatically for builds in CI variable "POETRY_VERSION" { - default = "1.4.2" + default = "1.8.2" } group "base-images" { From d87389690f0c0fc34cde927e6f967483b857c301 Mon Sep 17 00:00:00 2001 From: sancharigr Date: Fri, 12 Apr 2024 14:49:09 +0200 Subject: [PATCH 30/41] Add a changelog entry --- changelog/13030.misc.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/13030.misc.md diff --git a/changelog/13030.misc.md b/changelog/13030.misc.md new file mode 100644 index 000000000000..6dd2c81ee85b --- /dev/null +++ b/changelog/13030.misc.md @@ -0,0 +1 @@ +Update poetry version to `1.8.2` From 772c387c29452ec522240b213c230001248903a5 Mon Sep 17 00:00:00 2001 From: Anca Lita <27920906+ancalita@users.noreply.github.com> Date: Thu, 18 Apr 2024 15:25:58 +0100 Subject: [PATCH 31/41] Add upper limit version constraint for scipy on 3.6.x (#13031) * update pyproject and poetry.lock on 3.6.x * add changelog entry --- changelog/13031.misc.md | 1 + poetry.lock | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog/13031.misc.md diff --git a/changelog/13031.misc.md b/changelog/13031.misc.md new file mode 100644 index 000000000000..9a6c1b307a41 --- /dev/null +++ b/changelog/13031.misc.md @@ -0,0 +1 @@ +Add upper limit constraint for scipy dependency version in `pyproject.toml`. diff --git a/poetry.lock b/poetry.lock index a99ed6046eab..58af70bada4b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6956,4 +6956,4 @@ transformers = ["sentencepiece", "transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "8e1bef6b78365110ec598219ee747edb505f82a76fd03bdcffcec7a299b39513" +content-hash = "4c84d994449f859816e48dd00d77f31f6f9d964e29a9f6060300c51d923786e0" diff --git a/pyproject.toml b/pyproject.toml index 53c05df1c74e..7b983f4be729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,7 +194,7 @@ version = ">=1.4.1,<1.7.3" python = "~=3.7.0" [[tool.poetry.dependencies.scipy]] -version = ">=1.10.0" +version = ">=1.10.0,<1.11.0" python = ">=3.8,<3.11" [[tool.poetry.dependencies.scikit-learn]] From c195a25166a4d63ff7fdd51b1e7cba19f0137f4c Mon Sep 17 00:00:00 2001 From: Anca Lita <27920906+ancalita@users.noreply.github.com> Date: Thu, 18 Apr 2024 15:27:41 +0100 Subject: [PATCH 32/41] prepared release of version 3.6.20 --- CHANGELOG.mdx | 7 +++++++ changelog/13030.misc.md | 1 - changelog/13031.misc.md | 1 - pyproject.toml | 2 +- rasa/version.py | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 changelog/13030.misc.md delete mode 100644 changelog/13031.misc.md diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index a9a2a21fb56b..a97640d0772b 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,13 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.20] - 2024-04-18 + +Rasa 3.6.20 (2024-04-18) +### Miscellaneous internal changes +- [#13030](https://github.com/rasahq/rasa/issues/13030), [#13031](https://github.com/rasahq/rasa/issues/13031) + + ## [3.6.19] - 2024-03-04 Rasa 3.6.19 (2024-03-04) diff --git a/changelog/13030.misc.md b/changelog/13030.misc.md deleted file mode 100644 index 6dd2c81ee85b..000000000000 --- a/changelog/13030.misc.md +++ /dev/null @@ -1 +0,0 @@ -Update poetry version to `1.8.2` diff --git a/changelog/13031.misc.md b/changelog/13031.misc.md deleted file mode 100644 index 9a6c1b307a41..000000000000 --- a/changelog/13031.misc.md +++ /dev/null @@ -1 +0,0 @@ -Add upper limit constraint for scipy dependency version in `pyproject.toml`. diff --git a/pyproject.toml b/pyproject.toml index 7b983f4be729..0944c09460d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.19" +version = "3.6.20" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index 12d8ae480621..2e1490a6a8c3 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.19" +__version__ = "3.6.20" From a51504b8cc73036c5c20e75e6c846061f80343a0 Mon Sep 17 00:00:00 2001 From: Vlada Anicic <115999837+rasa-aadlv@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:04:01 +0100 Subject: [PATCH 33/41] [INFRA-1124] Update deprecated dependencies --- .github/workflows/continous-integration.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 8b6c0ab7dbe2..00f2f8aab5f8 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -396,7 +396,7 @@ jobs: - name: Store coverage reports if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 with: name: ${{ matrix.test }}-coverage path: | @@ -547,7 +547,7 @@ jobs: - name: Store coverage reports if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 with: name: ${{ matrix.test }}-coverage path: | @@ -574,7 +574,7 @@ jobs: - name: Get backend coverage reports if: needs.changes.outputs.backend == 'true' - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8 with: path: ${{ github.workspace }}/tests_coverage From 66296b28b34f3400e55e61d5360c6c722427d004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radovan=20=C5=BDivkovi=C4=87?= <115744253+radovanZRasa@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:29:55 +0100 Subject: [PATCH 34/41] Fix 3.6.x build (#13068) * Update poetry install GitHub action * Store coverage results separately for each os and python version * Enable CI run on Windows * Update Gr1N/setup-poetry action * Use different GitHub actions to install poetry on ubuntu and windows * Bump GitHub Actions runner OS version to ubuntu-24.04 * Check for Ubuntu 24.04 when installing poetry * Update ubuntu source index before installing packages necessary for tests * Update zookeeper and kafka container setup using docker compose * Disable docs check * Fix tests code coverage processing --------- Co-authored-by: Vlada Anicic --- .github/workflows/automatic-pr-update.yml | 2 +- .github/workflows/ci-docs-tests.yml | 10 +- .github/workflows/ci-github-actions.yml | 2 +- .../ci-model-regression-on-schedule.yml | 10 +- .github/workflows/ci-model-regression.yml | 14 +- .github/workflows/continous-integration.yml | 179 ++++++++++-------- .../workflows/dependabot-batch-updater.yml | 2 +- .github/workflows/documentation.yml | 10 +- .github/workflows/nightly_release.yml | 6 +- .github/workflows/rasa-install-cron-check.yml | 2 +- .github/workflows/security-scans.yml | 6 +- .github/workflows/semgrep-check.yml | 2 +- .github/workflows/spellcheck.yml | 2 +- Makefile | 2 +- 14 files changed, 136 insertions(+), 113 deletions(-) diff --git a/.github/workflows/automatic-pr-update.yml b/.github/workflows/automatic-pr-update.yml index 4842a120451b..6621d5985d04 100644 --- a/.github/workflows/automatic-pr-update.yml +++ b/.github/workflows/automatic-pr-update.yml @@ -7,7 +7,7 @@ jobs: # thats's all. single step is needed - if PR is mergeable according to # branch protection rules it will be merged automatically mergepal: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.repository == 'RasaHQ/rasa' steps: diff --git a/.github/workflows/ci-docs-tests.yml b/.github/workflows/ci-docs-tests.yml index 7a5da8f35554..366fe1cdb0f8 100644 --- a/.github/workflows/ci-docs-tests.yml +++ b/.github/workflows/ci-docs-tests.yml @@ -16,7 +16,7 @@ env: jobs: changes: name: Check for file changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: docs: ${{ steps.filter.outputs.docs }} steps: @@ -29,9 +29,9 @@ jobs: test_documentation: name: Test Documentation - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] - if: needs.changes.outputs.docs == 'true' + if: needs.changes.outputs.docs == 'true' && false # disabled as docs are moved out in new versions steps: - name: Checkout git repository 🕝 @@ -100,9 +100,9 @@ jobs: documentation_lint: name: Documentation Linting Checks - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] - if: needs.changes.outputs.docs == 'true' + if: needs.changes.outputs.docs == 'true' && false # disabled as docs are moved out in new versions steps: - name: Checkout git repository 🕝 diff --git a/.github/workflows/ci-github-actions.yml b/.github/workflows/ci-github-actions.yml index 4f354bc1aa43..161c0b027b0f 100644 --- a/.github/workflows/ci-github-actions.yml +++ b/.github/workflows/ci-github-actions.yml @@ -14,7 +14,7 @@ env: jobs: test: name: Run Tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 #missing matrix strategy: fail-fast: false diff --git a/.github/workflows/ci-model-regression-on-schedule.yml b/.github/workflows/ci-model-regression-on-schedule.yml index b4463fd67a62..5ac4710cf288 100644 --- a/.github/workflows/ci-model-regression-on-schedule.yml +++ b/.github/workflows/ci-model-regression-on-schedule.yml @@ -17,7 +17,7 @@ env: jobs: read_test_configuration: name: Reads tests configuration - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} matrix_length: ${{ steps.set-matrix.outputs.matrix_length }} @@ -53,7 +53,7 @@ jobs: deploy_runner_gpu: name: Deploy Github Runner - GPU needs: read_test_configuration - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout @@ -503,7 +503,7 @@ jobs: combine_reports: name: Combine reports - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - model_regression_test_gpu if: always() && needs.model_regression_test_gpu.result == 'success' @@ -542,7 +542,7 @@ jobs: analyse_performance: name: Analyse Performance - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: always() && github.event_name == 'schedule' needs: - model_regression_test_gpu @@ -691,7 +691,7 @@ jobs: needs: - deploy_runner_gpu - model_regression_test_gpu - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: diff --git a/.github/workflows/ci-model-regression.yml b/.github/workflows/ci-model-regression.yml index 84f6d34a5392..3e65fcec2ebd 100644 --- a/.github/workflows/ci-model-regression.yml +++ b/.github/workflows/ci-model-regression.yml @@ -27,7 +27,7 @@ jobs: read_test_configuration: name: Reads tests configuration if: ${{ github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} matrix_length: ${{ steps.set-matrix.outputs.matrix_length }} @@ -158,7 +158,7 @@ jobs: deploy_runner_gpu: name: Deploy Github Runner - GPU needs: read_test_configuration - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: ${{ contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }} steps: @@ -492,7 +492,7 @@ jobs: - read_test_configuration env: ACCELERATOR_TYPE: "CPU" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: max-parallel: 3 matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}} @@ -733,7 +733,7 @@ jobs: combine_reports: name: Combine reports - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - model_regression_test_cpu - model_regression_test_gpu @@ -790,7 +790,7 @@ jobs: set_job_success_status: name: Set job success status - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - combine_reports if: ${{ always() && needs.combine_reports.result == 'success' }} @@ -807,7 +807,7 @@ jobs: add_comment_results: name: Add a comment with the results - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - combine_reports if: ${{ always() && needs.combine_reports.result == 'success' }} @@ -901,7 +901,7 @@ jobs: needs: - deploy_runner_gpu - model_regression_test_gpu - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: ${{ always() && needs.deploy_runner_gpu.result != 'skipped' && contains(github.event.pull_request.labels.*.name, 'runner:gpu') && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }} steps: diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 00f2f8aab5f8..67594243b1f1 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -44,7 +44,7 @@ env: jobs: changes: name: Check for file changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: # Both of the outputs below are strings but only one exists at any given time backend: ${{ steps.changed-files.outputs.backend || steps.run-all.outputs.backend }} @@ -87,7 +87,7 @@ jobs: # Runs on pushes to main exclusively name: Wait for docs tests if: github.ref_type != 'tag' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] steps: @@ -110,7 +110,7 @@ jobs: quality: name: Code Quality if: github.ref_type != 'tag' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] steps: @@ -132,14 +132,12 @@ jobs: - name: Install poetry 🦄 if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} - - - name: Inject setuptools into poetry's runtime environment - if: needs.changes.outputs.backend == 'true' - run: | - poetry self add setuptools + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Load Poetry Cached Libraries ⬇ id: cache-poetry @@ -210,7 +208,7 @@ jobs: changelog: name: Check for changelog - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout git repository 🕝 @@ -249,7 +247,7 @@ jobs: - test-full-model-training - test-other-unit-tests - test-performance - os: [ubuntu-22.04, windows-2019] + os: [ubuntu-24.04, windows-2019] python-version: [3.8, 3.9, "3.10"] steps: @@ -276,16 +274,20 @@ jobs: echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV shell: bash - - name: Install poetry 🦄 - if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + - name: Install poetry (Ubuntu) 🦄 + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - - name: Inject setuptools into poetry's runtime environment - if: needs.changes.outputs.backend == 'true' - run: | - poetry self add setuptools + - name: Install poetry (Windows) 🦄 + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec #v9 + with: + poetry-version: ${{ env.POETRY_VERSION }} - name: Prevent race condition in poetry build # More context about race condition during poetry build can be found here: @@ -323,7 +325,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Install Dependencies (Linux) 📦 - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' # Poetry intermittently fails to install dependency if it is not PEP 517 compliant # This is a workaround for that issue run: | @@ -351,7 +353,7 @@ jobs: make prepare-tests-windows-gha - name: Add github workflow problem matchers - if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-24.04' # only annotate based on test runs on ubuntu: otherwise # all errors will be duplicated for each python / os combination # therefore, we only enable for the one where most tests are run @@ -370,7 +372,7 @@ jobs: Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 0 - name: Install ddtrace on Linux - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' run: poetry run pip install -U 'ddtrace<2.0.0' - name: Install ddtrace on Windows @@ -389,18 +391,21 @@ jobs: DD_ARGS: --ddtrace --ddtrace-patch-all run: | make ${{ matrix.test }} - if [[ "${{ matrix.os }}" != "windows-2019" ]]; then - mv .coverage ${{ github.workspace }}/${{ matrix.test }}-coverage - fi shell: bash # bash shell is a way to make code run for both Linux and Windows + - name: Move coverage report + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + run: | + mv .coverage ${{ github.workspace }}/.coverage.${{ matrix.test }} + - name: Store coverage reports - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 with: - name: ${{ matrix.test }}-coverage + name: ${{ matrix.test }}-${{ matrix.python-version }}-${{ matrix.os }}-coverage + include-hidden-files: true path: | - ${{ github.workspace }}/${{ matrix.test }}-coverage + ${{ github.workspace }}/.coverage.${{ matrix.test }} test-flaky: name: Run Flaky Tests @@ -411,7 +416,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, windows-2019] + os: [ubuntu-24.04, windows-2019] python-version: [3.8, 3.9, "3.10"] steps: @@ -438,16 +443,20 @@ jobs: echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV shell: bash - - name: Install poetry 🦄 - if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + - name: Install poetry (Ubuntu) 🦄 + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - - name: Inject setuptools into poetry's runtime environment - if: needs.changes.outputs.backend == 'true' - run: | - poetry self add setuptools + - name: Install poetry (Windows) 🦄 + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec #v9 + with: + poetry-version: ${{ env.POETRY_VERSION }} - name: Load Poetry Cached Libraries ⬇ id: cache-poetry @@ -478,7 +487,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Install Dependencies (Linux) 📦 - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' run: | sudo apt-get -y install libpq-dev make install-full | tee .output @@ -502,7 +511,7 @@ jobs: make prepare-tests-windows-gha - name: Add github workflow problem matchers - if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-24.04' # only annotate based on test runs on ubuntu: otherwise # all errors will be duplicated for each python / os combination # therefore, we only enable for the one where most tests are run @@ -521,7 +530,7 @@ jobs: Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 0 - name: Install ddtrace on Linux - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' run: poetry run pip install -U 'ddtrace<2.0.0' - name: Install ddtrace on Windows @@ -540,26 +549,30 @@ jobs: DD_ARGS: --ddtrace --ddtrace-patch-all run: | make test-flaky - if [[ "${{ matrix.os }}" != "windows-2019" ]]; then - mv .coverage ${{ github.workspace }}/test-flaky-coverage - fi shell: bash # bash shell is a way to make code run for both Linux and Windows + - name: Move coverage report + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + run: | + mv .coverage ${{ github.workspace }}/.coverage.test_flaky_coverage + - name: Store coverage reports - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + uses: actions/upload-artifact@v4 with: - name: ${{ matrix.test }}-coverage + name: ${{ matrix.test }}-${{ matrix.python-version }}-${{ matrix.os }}-coverage + include-hidden-files: true path: | - ${{ github.workspace }}/${{ matrix.test }}-coverage + ${{ github.workspace }}/.coverage.test_flaky_coverage upload_coverage_reports: name: Upload coverage reports to codeclimate if: github.ref_type != 'tag' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # Always upload results even if tests failed needs: - test + - test-flaky - changes steps: @@ -574,25 +587,23 @@ jobs: - name: Get backend coverage reports if: needs.changes.outputs.backend == 'true' - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8 + uses: actions/download-artifact@v4 #v4.1.8 with: path: ${{ github.workspace }}/tests_coverage + pattern: "*-coverage" + merge-multiple: true - - name: Merge all reports + - name: List all coverages if: needs.changes.outputs.backend == 'true' run: | - subs=`ls ${{ github.workspace }}/tests_coverage` - download_dir="${{ github.workspace }}/tests_coverage" - final_dir="${{ github.workspace }}/tests_coverage/final" - - # Downloaded artifacts go into folders, gotta extract them all into one folder for upload - mkdir "${final_dir}/" - for i in $subs; do - mv "${download_dir}/$i"/* "${final_dir}/" - done + ls -l ${{ github.workspace }} + ls -l ${{ github.workspace }}/tests_coverage + - name: Merge all reports + if: needs.changes.outputs.backend == 'true' + run: | pip install coverage - coverage combine "${final_dir}/"* + coverage combine ${{ github.workspace }}/tests_coverage/ coverage xml - name: Upload reports to codeclimate @@ -608,7 +619,7 @@ jobs: integration_test: name: Run Non-Sequential Integration Tests if: github.ref_type != 'tag' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: [changes] env: @@ -678,9 +689,12 @@ jobs: - name: Install poetry 🦄 if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Load Poetry Cached Libraries ⬇ id: cache-poetry @@ -730,7 +744,7 @@ jobs: broker_integration_test: name: Run Broker Integration Tests if: github.ref_type != 'tag' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: [changes] env: @@ -765,9 +779,12 @@ jobs: - name: Install poetry 🦄 if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Load Poetry Cached Libraries ⬇ id: cache-poetry @@ -809,7 +826,7 @@ jobs: - name: Run kafka and zookeeper containers for integration testing if: needs.changes.outputs.backend == 'true' run: | - docker-compose -f tests_deployment/docker-compose.kafka.yml up -d + docker compose -f tests_deployment/docker-compose.kafka.yml up -d - name: Test Code with Services 🩺 if: needs.changes.outputs.backend == 'true' @@ -823,12 +840,12 @@ jobs: - name: Stop kafka and zookeeper containers for integration testing if: needs.changes.outputs.backend == 'true' run: | - docker-compose -f tests_deployment/docker-compose.kafka.yml down + docker compose -f tests_deployment/docker-compose.kafka.yml down sequential_integration_test: name: Run Sequential Integration Tests if: github.ref_type != 'tag' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: [changes] env: @@ -873,9 +890,12 @@ jobs: - name: Install poetry 🦄 if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Load Poetry Cached Libraries ⬇ id: cache-poetry @@ -926,7 +946,7 @@ jobs: build_docker_base_images_and_set_env: name: Build Docker base images and setup environment - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: base_image_hash: ${{ steps.check_image.outputs.base_image_hash }} base_mitie_image_hash: ${{ steps.check_image.outputs.base_mitie_image_hash }} @@ -1109,7 +1129,7 @@ jobs: docker: name: Build Docker - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes, build_docker_base_images_and_set_env] env: IMAGE_TAG: ${{ needs.build_docker_base_images_and_set_env.outputs.image_tag }} @@ -1211,7 +1231,7 @@ jobs: deploy: name: Deploy to PyPI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # deploy will only be run when there is a tag available if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && github.repository == 'RasaHQ/rasa' @@ -1232,9 +1252,12 @@ jobs: shell: bash - name: Install poetry 🦄 - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Copy Segment write key to the package env: @@ -1287,7 +1310,7 @@ jobs: send_slack_notification_for_release_on_failure: name: Notify Slack & Publish Release Notes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # run this job when the workflow is triggered by a tag push if: always() && github.repository == 'RasaHQ/rasa' && github.ref_type == 'tag' needs: diff --git a/.github/workflows/dependabot-batch-updater.yml b/.github/workflows/dependabot-batch-updater.yml index 2065260d3bfc..dd08a4cf6c7c 100644 --- a/.github/workflows/dependabot-batch-updater.yml +++ b/.github/workflows/dependabot-batch-updater.yml @@ -6,7 +6,7 @@ on: jobs: update_dependencies: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Update dependencies steps: - uses: RasaHQ/dependabot-batch-updater@f049cbb0bbd3754bcb5ab154a79f00cd780fc633 # v1.0 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 076f44d0c612..224bc01da6f4 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -30,7 +30,7 @@ env: jobs: changes: name: Check for file changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # don't run this for pull requests of forks if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' outputs: @@ -59,7 +59,7 @@ jobs: evaluate_release_tag: name: Evaluate release tag - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # don't run this for main branches of forks and on documentation branch if: github.repository == 'RasaHQ/rasa' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request' outputs: @@ -100,7 +100,7 @@ jobs: prebuild_docs: name: Prebuild Docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [evaluate_release_tag] # don't run this for main branches of forks, would fail anyways if: github.repository == 'RasaHQ/rasa' && needs.evaluate_release_tag.outputs.build_docs == 'true' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request' @@ -189,7 +189,7 @@ jobs: preview_docs: name: Preview Docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] # don't run this for pull requests from forks if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' @@ -285,7 +285,7 @@ jobs: publish_docs: name: Publish Docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # don't run this for main branches of forks; only run on documentation branch if: github.repository == 'RasaHQ/rasa' && github.ref == 'refs/heads/documentation' diff --git a/.github/workflows/nightly_release.yml b/.github/workflows/nightly_release.yml index c7639d0cbf16..772f06a8a48a 100644 --- a/.github/workflows/nightly_release.yml +++ b/.github/workflows/nightly_release.yml @@ -9,7 +9,7 @@ on: jobs: run_script_and_tag_nightly_release: name: Run release script and tag a new nightly release - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: tag_name: ${{ steps.set_tagname.outputs.tag_name }} @@ -52,7 +52,7 @@ jobs: deploy: name: Deploy to PyPI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # deploy will only be run when there is a tag available needs: run_script_and_tag_nightly_release # only run after all other stages succeeded @@ -112,7 +112,7 @@ jobs: docker: name: Build Docker - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: run_script_and_tag_nightly_release env: GCLOUD_VERSION: "297.0.1" diff --git a/.github/workflows/rasa-install-cron-check.yml b/.github/workflows/rasa-install-cron-check.yml index 91fa6eb6ee1d..2cc143c35681 100644 --- a/.github/workflows/rasa-install-cron-check.yml +++ b/.github/workflows/rasa-install-cron-check.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: - [ubuntu-22.04, ubuntu-18.04, macos-latest, windows-2019, windows-2022] + [ubuntu-24.04, ubuntu-18.04, macos-latest, windows-2019, windows-2022] python-version: [3.8, 3.9, '3.10'] fail-fast: false diff --git a/.github/workflows/security-scans.yml b/.github/workflows/security-scans.yml index b4cbf69cc92f..5ae74653b87f 100644 --- a/.github/workflows/security-scans.yml +++ b/.github/workflows/security-scans.yml @@ -11,7 +11,7 @@ concurrency: jobs: changes: name: Check for file changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: backend: ${{ steps.filter.outputs.backend }} docker: ${{ steps.filter.outputs.docker }} @@ -26,7 +26,7 @@ jobs: trivy: name: Detecting hardcoded secrets - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c with: @@ -75,7 +75,7 @@ jobs: bandit: name: Detect python security issues - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] steps: diff --git a/.github/workflows/semgrep-check.yml b/.github/workflows/semgrep-check.yml index 846f92494b26..0e6665738406 100644 --- a/.github/workflows/semgrep-check.yml +++ b/.github/workflows/semgrep-check.yml @@ -14,7 +14,7 @@ jobs: # User-definable name of this GitHub Actions job: name: Semgrep Workflow Security Scan # If you are self-hosting, change the following `runs-on` value: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: # A Docker image with Semgrep installed. Do not change this. diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 4f90d2724ca9..3359884634e9 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -7,7 +7,7 @@ on: jobs: spellcheck: name: Typo CI (GitHub Action) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 4 if: "!contains(github.event.head_commit.message, '[ci skip]')" steps: diff --git a/Makefile b/Makefile index 62367bc17c4f..a58eda4a3939 100644 --- a/Makefile +++ b/Makefile @@ -127,7 +127,7 @@ prepare-tests-macos: brew install wget graphviz || true prepare-tests-ubuntu: - sudo apt-get -y install graphviz graphviz-dev python-tk + sudo apt-get update && sudo apt-get -y install graphviz graphviz-dev python3-tk prepare-tests-windows: choco install wget graphviz From 2bb1d779d4f4acaf70b6dfa35dd1899dccbb1ae6 Mon Sep 17 00:00:00 2001 From: Tanja Date: Fri, 10 Jan 2025 13:52:45 +0100 Subject: [PATCH 35/41] Replace pickle with safer alternatives (#13067) * Update slack release notification step * [ENG-1424] Use `pickle` alternatives (#1453) * use json.dump and json.load in count_vectors_featurizer and lexical_syntactic_featurizer instead of pickle * update load and persist in sklearn intent classifier * update persist and load in dietclassifier * update load and persist in sklearn intent classifier * use json.dump and json.load in tracker featurizers * update persist and load of TEDPolicy * updated unexpected intent policy persist and load of model utilities. * save and load fake features * rename patterns.pkl to patterns.json * update poetry.lock * ruff formatting * move skops import * add comments * clean up save_features and load_features * WIP: update model data saving and loading * add tests for save and load features * update tests for test_tracker_featurizer * update tests for test_tracker_featurizer * WIP: serialization of feature arrays. * update serialization and deserialization for feature array * remove not needed tests/utils/tensorflow/test_model_data_storage.py * start writing tests for feature array * update feature array tests * update tests * fix linting * add changelog * add new dependencies to .github/dependabot.yml * fix some tests * fix loading and saving of unexpected intent ted policy * fix linting issue * fix converting of features in cvf and lsf * fix lint issues * convert vocab in cvf * fix linting * update crf entity extractor * fix to_dict of crf_token * addressed type issues * ruff formatting * fix typing and lint issues * remove cloudpickle dependency * update logistic_regression_classifier and remove joblib as dependency * update formatting of pyproject.toml * next try: update formatting of pyproject.toml * update logging * update poetry.lock * refactor loading of lexical_syntactic_featurizer * rename FeatureMetadata.type -> FeatureMetadata.data_type * clean up tests test_features.py and test_crf_entity_extractor.py * update test_feature_array.py * check for type when loading tracker featurizer. * update changelog * fix line too long * move import of skops * Prepared release of version 3.10.9.dev1 (#1496) * prepared release of version 3.10.9.dev1 * update minimum model version * Check for 'step_id' and 'active_flow' keys in the metadata when adding 'ActionExecuted' event to flows paths stack. * fix parsing of commands * improve logging * formatting * add changelog * fix parse commands for multi step * [ATO-2985] - Windows model loading test (#1537) * Add test for model loading on windows * Improve the error message logged when handling the user message * Add a changelog * Fix Code Quality - line too long * Rasa-sdk-update (#1546) * all rasa-sdk micro updates * update poetry lock * update rasa-sdk in lock file * Remove trailing white sapce * Prepared release of version 3.10.11 (#1570) * prepared release of version 3.10.11 * add comments again in pyproject.toml * update poetry.lock * revert changes in github workflows * undo changes in pyproject.toml * update changelog * revert changes in github workflows * update poetry.lock * update poetry.lock * update pyproject.toml * update poetry.lock * update setuptools = '>=65.5.1,<75.6.0' * update setuptools = '~75.3.0' * reformat code * undo deleting of ping_slack_about_package_release.sh * fix formatting and type issues * downgrade setuptools to 70.3.0 * fixing logging issues (?) --------- Co-authored-by: sancharigr --- .github/workflows/continous-integration.yml | 1 - changelog/1424.bugfix.md | 19 + poetry.lock | 454 +++++++++++++----- pyproject.toml | 9 +- .../featurizers/single_state_featurizer.py | 23 +- rasa/core/featurizers/tracker_featurizers.py | 133 ++++- rasa/core/policies/ted_policy.py | 91 ++-- .../core/policies/unexpected_intent_policy.py | 22 +- rasa/nlu/classifiers/diet_classifier.py | 63 ++- .../logistic_regression_classifier.py | 31 +- .../classifiers/sklearn_intent_classifier.py | 53 +- rasa/nlu/extractors/crf_entity_extractor.py | 155 +++--- .../count_vectors_featurizer.py | 59 ++- .../lexical_syntactic_featurizer.py | 69 ++- .../sparse_featurizer/regex_featurizer.py | 8 +- rasa/shared/nlu/training_data/features.py | 122 ++++- rasa/shared/utils/io.py | 1 + rasa/utils/common.py | 7 +- rasa/utils/io.py | 76 +-- rasa/utils/tensorflow/feature_array.py | 370 ++++++++++++++ rasa/utils/tensorflow/model_data.py | 198 +------- scripts/ping_slack_about_package_release.sh | 1 - .../featurizers/test_tracker_featurizer.py | 38 +- .../extractors/test_crf_entity_extractor.py | 121 ++++- .../shared/nlu/training_data/test_features.py | 219 ++++++++- tests/utils/tensorflow/test_feature_array.py | 197 ++++++++ tests/utils/test_io.py | 18 - 27 files changed, 1942 insertions(+), 616 deletions(-) create mode 100644 changelog/1424.bugfix.md create mode 100644 rasa/utils/tensorflow/feature_array.py mode change 100755 => 100644 scripts/ping_slack_about_package_release.sh create mode 100644 tests/utils/tensorflow/test_feature_array.py diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 67594243b1f1..b6f2bd8d38f3 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -1307,7 +1307,6 @@ jobs: with: args: "💥 New *Rasa Open Source * version `${{ github.ref_name }}` has been released!" - send_slack_notification_for_release_on_failure: name: Notify Slack & Publish Release Notes runs-on: ubuntu-24.04 diff --git a/changelog/1424.bugfix.md b/changelog/1424.bugfix.md new file mode 100644 index 000000000000..d71648f6da3c --- /dev/null +++ b/changelog/1424.bugfix.md @@ -0,0 +1,19 @@ +Replace `pickle` and `joblib` with safer alternatives, e.g. `json`, `safetensors`, and `skops`, for +serializing components. + +**Note**: This is a model breaking change. Please retrain your model. + +If you have a custom component that inherits from one of the components listed below and modified the `persist` or +`load` method, make sure to update your code. Please contact us in case you encounter any problems. + +Affected components: + +- `CountVectorFeaturizer` +- `LexicalSyntacticFeaturizer` +- `LogisticRegressionClassifier` +- `SklearnIntentClassifier` +- `DIETClassifier` +- `CRFEntityExtractor` +- `TrackerFeaturizer` +- `TEDPolicy` +- `UnexpectedIntentTEDPolicy` \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 58af70bada4b..ba2092a01ff4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1412,7 +1412,7 @@ requests = ">=2.0" name = "filelock" version = "3.12.2" description = "A platform independent file lock." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, @@ -2214,18 +2214,18 @@ socks = ["socksio (==1.*)"] [[package]] name = "huggingface-hub" -version = "0.16.2" +version = "0.27.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -optional = true -python-versions = ">=3.7.0" +optional = false +python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.16.2-py3-none-any.whl", hash = "sha256:92facff575c11a8cf4b35d184ae67867a577a1b30865edcd8a9c5a48d2202133"}, - {file = "huggingface_hub-0.16.2.tar.gz", hash = "sha256:205abbf02a3408129a309f09e6d1a88d0c82de296b498682a813d9baa91c272f"}, + {file = "huggingface_hub-0.27.0-py3-none-any.whl", hash = "sha256:8f2e834517f1f1ddf1ecc716f91b120d7333011b7485f665a9a412eacb1a2a81"}, + {file = "huggingface_hub-0.27.0.tar.gz", hash = "sha256:902cce1a1be5739f5589e560198a65a8edcfd3b830b1666f36e4b961f0454fac"}, ] [package.dependencies] filelock = "*" -fsspec = "*" +fsspec = ">=2023.5.0" packaging = ">=20.9" pyyaml = ">=5.1" requests = "*" @@ -2233,16 +2233,18 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -inference = ["aiohttp", "pydantic"] -quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] +hf-transfer = ["hf-transfer (>=0.1.4)"] +inference = ["aiohttp"] +quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.5.0)"] tensorflow = ["graphviz", "pydot", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["torch"] -typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +tensorflow-testing = ["keras (<3.0)", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["safetensors[torch]", "torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] [[package]] name = "humanfriendly" @@ -2416,30 +2418,30 @@ files = [ [[package]] name = "joblib" -version = "1.2.0" +version = "1.4.2" description = "Lightweight pipelining with Python functions" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, - {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, ] [[package]] name = "jsonpickle" -version = "3.0.1" -description = "Python library for serializing any arbitrary object graph into JSON" +version = "3.0.4" +description = "Serialize any Python object to JSON" optional = false python-versions = ">=3.7" files = [ - {file = "jsonpickle-3.0.1-py2.py3-none-any.whl", hash = "sha256:130d8b293ea0add3845de311aaba55e6d706d0bb17bc123bd2c8baf8a39ac77c"}, - {file = "jsonpickle-3.0.1.tar.gz", hash = "sha256:032538804795e73b94ead410800ac387fdb6de98f8882ac957fcd247e3a85200"}, + {file = "jsonpickle-3.0.4-py3-none-any.whl", hash = "sha256:04ae7567a14269579e3af66b76bda284587458d7e8a204951ca8f71a3309952e"}, + {file = "jsonpickle-3.0.4.tar.gz", hash = "sha256:a1b14c8d6221cd8f394f2a97e735ea1d7edc927fbd135b26f2f8700657c8c62b"}, ] [package.extras] -docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] -testing = ["ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] -testing-libs = ["simplejson", "ujson"] +docs = ["furo", "rst.linker (>=1.9)", "sphinx"] +packaging = ["build", "twine"] +testing = ["bson", "ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-benchmark", "pytest-benchmark[histogram]", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-ruff (>=0.2.1)", "scikit-learn", "scipy", "scipy (>=1.9.3)", "simplejson", "sqlalchemy", "ujson"] [[package]] name = "jsonschema" @@ -4746,6 +4748,138 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "safetensors" +version = "0.4.5" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "safetensors-0.4.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a63eaccd22243c67e4f2b1c3e258b257effc4acd78f3b9d397edc8cf8f1298a7"}, + {file = "safetensors-0.4.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:23fc9b4ec7b602915cbb4ec1a7c1ad96d2743c322f20ab709e2c35d1b66dad27"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6885016f34bef80ea1085b7e99b3c1f92cb1be78a49839203060f67b40aee761"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:133620f443450429322f238fda74d512c4008621227fccf2f8cf4a76206fea7c"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4fb3e0609ec12d2a77e882f07cced530b8262027f64b75d399f1504ffec0ba56"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0f1dd769f064adc33831f5e97ad07babbd728427f98e3e1db6902e369122737"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6d156bdb26732feada84f9388a9f135528c1ef5b05fae153da365ad4319c4c5"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e347d77e2c77eb7624400ccd09bed69d35c0332f417ce8c048d404a096c593b"}, + {file = "safetensors-0.4.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9f556eea3aec1d3d955403159fe2123ddd68e880f83954ee9b4a3f2e15e716b6"}, + {file = "safetensors-0.4.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9483f42be3b6bc8ff77dd67302de8ae411c4db39f7224dec66b0eb95822e4163"}, + {file = "safetensors-0.4.5-cp310-none-win32.whl", hash = "sha256:7389129c03fadd1ccc37fd1ebbc773f2b031483b04700923c3511d2a939252cc"}, + {file = "safetensors-0.4.5-cp310-none-win_amd64.whl", hash = "sha256:e98ef5524f8b6620c8cdef97220c0b6a5c1cef69852fcd2f174bb96c2bb316b1"}, + {file = "safetensors-0.4.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:21f848d7aebd5954f92538552d6d75f7c1b4500f51664078b5b49720d180e47c"}, + {file = "safetensors-0.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bb07000b19d41e35eecef9a454f31a8b4718a185293f0d0b1c4b61d6e4487971"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09dedf7c2fda934ee68143202acff6e9e8eb0ddeeb4cfc24182bef999efa9f42"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:59b77e4b7a708988d84f26de3ebead61ef1659c73dcbc9946c18f3b1786d2688"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d3bc83e14d67adc2e9387e511097f254bd1b43c3020440e708858c684cbac68"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39371fc551c1072976073ab258c3119395294cf49cdc1f8476794627de3130df"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6c19feda32b931cae0acd42748a670bdf56bee6476a046af20181ad3fee4090"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a659467495de201e2f282063808a41170448c78bada1e62707b07a27b05e6943"}, + {file = "safetensors-0.4.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bad5e4b2476949bcd638a89f71b6916fa9a5cae5c1ae7eede337aca2100435c0"}, + {file = "safetensors-0.4.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a3a315a6d0054bc6889a17f5668a73f94f7fe55121ff59e0a199e3519c08565f"}, + {file = "safetensors-0.4.5-cp311-none-win32.whl", hash = "sha256:a01e232e6d3d5cf8b1667bc3b657a77bdab73f0743c26c1d3c5dd7ce86bd3a92"}, + {file = "safetensors-0.4.5-cp311-none-win_amd64.whl", hash = "sha256:cbd39cae1ad3e3ef6f63a6f07296b080c951f24cec60188378e43d3713000c04"}, + {file = "safetensors-0.4.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:473300314e026bd1043cef391bb16a8689453363381561b8a3e443870937cc1e"}, + {file = "safetensors-0.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:801183a0f76dc647f51a2d9141ad341f9665602a7899a693207a82fb102cc53e"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1524b54246e422ad6fb6aea1ac71edeeb77666efa67230e1faf6999df9b2e27f"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3139098e3e8b2ad7afbca96d30ad29157b50c90861084e69fcb80dec7430461"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65573dc35be9059770808e276b017256fa30058802c29e1038eb1c00028502ea"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd33da8e9407559f8779c82a0448e2133737f922d71f884da27184549416bfed"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3685ce7ed036f916316b567152482b7e959dc754fcc4a8342333d222e05f407c"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dde2bf390d25f67908278d6f5d59e46211ef98e44108727084d4637ee70ab4f1"}, + {file = "safetensors-0.4.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7469d70d3de970b1698d47c11ebbf296a308702cbaae7fcb993944751cf985f4"}, + {file = "safetensors-0.4.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a6ba28118636a130ccbb968bc33d4684c48678695dba2590169d5ab03a45646"}, + {file = "safetensors-0.4.5-cp312-none-win32.whl", hash = "sha256:c859c7ed90b0047f58ee27751c8e56951452ed36a67afee1b0a87847d065eec6"}, + {file = "safetensors-0.4.5-cp312-none-win_amd64.whl", hash = "sha256:b5a8810ad6a6f933fff6c276eae92c1da217b39b4d8b1bc1c0b8af2d270dc532"}, + {file = "safetensors-0.4.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:25e5f8e2e92a74f05b4ca55686234c32aac19927903792b30ee6d7bd5653d54e"}, + {file = "safetensors-0.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:81efb124b58af39fcd684254c645e35692fea81c51627259cdf6d67ff4458916"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:585f1703a518b437f5103aa9cf70e9bd437cb78eea9c51024329e4fb8a3e3679"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4b99fbf72e3faf0b2f5f16e5e3458b93b7d0a83984fe8d5364c60aa169f2da89"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b17b299ca9966ca983ecda1c0791a3f07f9ca6ab5ded8ef3d283fff45f6bcd5f"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:76ded72f69209c9780fdb23ea89e56d35c54ae6abcdec67ccb22af8e696e449a"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2783956926303dcfeb1de91a4d1204cd4089ab441e622e7caee0642281109db3"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d94581aab8c6b204def4d7320f07534d6ee34cd4855688004a4354e63b639a35"}, + {file = "safetensors-0.4.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:67e1e7cb8678bb1b37ac48ec0df04faf689e2f4e9e81e566b5c63d9f23748523"}, + {file = "safetensors-0.4.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:dbd280b07e6054ea68b0cb4b16ad9703e7d63cd6890f577cb98acc5354780142"}, + {file = "safetensors-0.4.5-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:77d9b228da8374c7262046a36c1f656ba32a93df6cc51cd4453af932011e77f1"}, + {file = "safetensors-0.4.5-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:500cac01d50b301ab7bb192353317035011c5ceeef0fca652f9f43c000bb7f8d"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75331c0c746f03158ded32465b7d0b0e24c5a22121743662a2393439c43a45cf"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:670e95fe34e0d591d0529e5e59fd9d3d72bc77b1444fcaa14dccda4f36b5a38b"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:098923e2574ff237c517d6e840acada8e5b311cb1fa226019105ed82e9c3b62f"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13ca0902d2648775089fa6a0c8fc9e6390c5f8ee576517d33f9261656f851e3f"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f0032bedc869c56f8d26259fe39cd21c5199cd57f2228d817a0e23e8370af25"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f4b15f51b4f8f2a512341d9ce3475cacc19c5fdfc5db1f0e19449e75f95c7dc8"}, + {file = "safetensors-0.4.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f6594d130d0ad933d885c6a7b75c5183cb0e8450f799b80a39eae2b8508955eb"}, + {file = "safetensors-0.4.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:60c828a27e852ded2c85fc0f87bf1ec20e464c5cd4d56ff0e0711855cc2e17f8"}, + {file = "safetensors-0.4.5-cp37-none-win32.whl", hash = "sha256:6d3de65718b86c3eeaa8b73a9c3d123f9307a96bbd7be9698e21e76a56443af5"}, + {file = "safetensors-0.4.5-cp37-none-win_amd64.whl", hash = "sha256:5a2d68a523a4cefd791156a4174189a4114cf0bf9c50ceb89f261600f3b2b81a"}, + {file = "safetensors-0.4.5-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:e7a97058f96340850da0601a3309f3d29d6191b0702b2da201e54c6e3e44ccf0"}, + {file = "safetensors-0.4.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:63bfd425e25f5c733f572e2246e08a1c38bd6f2e027d3f7c87e2e43f228d1345"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3664ac565d0e809b0b929dae7ccd74e4d3273cd0c6d1220c6430035befb678e"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:313514b0b9b73ff4ddfb4edd71860696dbe3c1c9dc4d5cc13dbd74da283d2cbf"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31fa33ee326f750a2f2134a6174773c281d9a266ccd000bd4686d8021f1f3dac"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09566792588d77b68abe53754c9f1308fadd35c9f87be939e22c623eaacbed6b"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309aaec9b66cbf07ad3a2e5cb8a03205663324fea024ba391594423d0f00d9fe"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53946c5813b8f9e26103c5efff4a931cc45d874f45229edd68557ffb35ffb9f8"}, + {file = "safetensors-0.4.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:868f9df9e99ad1e7f38c52194063a982bc88fedc7d05096f4f8160403aaf4bd6"}, + {file = "safetensors-0.4.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9cc9449bd0b0bc538bd5e268221f0c5590bc5c14c1934a6ae359d44410dc68c4"}, + {file = "safetensors-0.4.5-cp38-none-win32.whl", hash = "sha256:83c4f13a9e687335c3928f615cd63a37e3f8ef072a3f2a0599fa09f863fb06a2"}, + {file = "safetensors-0.4.5-cp38-none-win_amd64.whl", hash = "sha256:b98d40a2ffa560653f6274e15b27b3544e8e3713a44627ce268f419f35c49478"}, + {file = "safetensors-0.4.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:cf727bb1281d66699bef5683b04d98c894a2803442c490a8d45cd365abfbdeb2"}, + {file = "safetensors-0.4.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:96f1d038c827cdc552d97e71f522e1049fef0542be575421f7684756a748e457"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:139fbee92570ecea774e6344fee908907db79646d00b12c535f66bc78bd5ea2c"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c36302c1c69eebb383775a89645a32b9d266878fab619819ce660309d6176c9b"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d641f5b8149ea98deb5ffcf604d764aad1de38a8285f86771ce1abf8e74c4891"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4db6a61d968de73722b858038c616a1bebd4a86abe2688e46ca0cc2d17558f2"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b75a616e02f21b6f1d5785b20cecbab5e2bd3f6358a90e8925b813d557666ec1"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:788ee7d04cc0e0e7f944c52ff05f52a4415b312f5efd2ee66389fb7685ee030c"}, + {file = "safetensors-0.4.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:87bc42bd04fd9ca31396d3ca0433db0be1411b6b53ac5a32b7845a85d01ffc2e"}, + {file = "safetensors-0.4.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4037676c86365a721a8c9510323a51861d703b399b78a6b4486a54a65a975fca"}, + {file = "safetensors-0.4.5-cp39-none-win32.whl", hash = "sha256:1500418454529d0ed5c1564bda376c4ddff43f30fce9517d9bee7bcce5a8ef50"}, + {file = "safetensors-0.4.5-cp39-none-win_amd64.whl", hash = "sha256:9d1a94b9d793ed8fe35ab6d5cea28d540a46559bafc6aae98f30ee0867000cab"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fdadf66b5a22ceb645d5435a0be7a0292ce59648ca1d46b352f13cff3ea80410"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d42ffd4c2259f31832cb17ff866c111684c87bd930892a1ba53fed28370c918c"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd8a1f6d2063a92cd04145c7fd9e31a1c7d85fbec20113a14b487563fdbc0597"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:951d2fcf1817f4fb0ef0b48f6696688a4e852a95922a042b3f96aaa67eedc920"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ac85d9a8c1af0e3132371d9f2d134695a06a96993c2e2f0bbe25debb9e3f67a"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e3cec4a29eb7fe8da0b1c7988bc3828183080439dd559f720414450de076fcab"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:21742b391b859e67b26c0b2ac37f52c9c0944a879a25ad2f9f9f3cd61e7fda8f"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c7db3006a4915151ce1913652e907cdede299b974641a83fbc092102ac41b644"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f68bf99ea970960a237f416ea394e266e0361895753df06e3e06e6ea7907d98b"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8158938cf3324172df024da511839d373c40fbfaa83e9abf467174b2910d7b4c"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:540ce6c4bf6b58cb0fd93fa5f143bc0ee341c93bb4f9287ccd92cf898cc1b0dd"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bfeaa1a699c6b9ed514bd15e6a91e74738b71125a9292159e3d6b7f0a53d2cde"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:01c8f00da537af711979e1b42a69a8ec9e1d7112f208e0e9b8a35d2c381085ef"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a0dd565f83b30f2ca79b5d35748d0d99dd4b3454f80e03dfb41f0038e3bdf180"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:023b6e5facda76989f4cba95a861b7e656b87e225f61811065d5c501f78cdb3f"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9633b663393d5796f0b60249549371e392b75a0b955c07e9c6f8708a87fc841f"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78dd8adfb48716233c45f676d6e48534d34b4bceb50162c13d1f0bdf6f78590a"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e8deb16c4321d61ae72533b8451ec4a9af8656d1c61ff81aa49f966406e4b68"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:52452fa5999dc50c4decaf0c53aa28371f7f1e0fe5c2dd9129059fbe1e1599c7"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d5f23198821e227cfc52d50fa989813513db381255c6d100927b012f0cfec63d"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f4beb84b6073b1247a773141a6331117e35d07134b3bb0383003f39971d414bb"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:68814d599d25ed2fdd045ed54d370d1d03cf35e02dce56de44c651f828fb9b7b"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b6453c54c57c1781292c46593f8a37254b8b99004c68d6c3ce229688931a22"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adaa9c6dead67e2dd90d634f89131e43162012479d86e25618e821a03d1eb1dc"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73e7d408e9012cd17511b382b43547850969c7979efc2bc353f317abaf23c84c"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:775409ce0fcc58b10773fdb4221ed1eb007de10fe7adbdf8f5e8a56096b6f0bc"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:834001bed193e4440c4a3950a31059523ee5090605c907c66808664c932b549c"}, + {file = "safetensors-0.4.5.tar.gz", hash = "sha256:d73de19682deabb02524b3d5d1f8b3aaba94c72f1bbfc7911b9b9d5d391c0310"}, +] + +[package.extras] +all = ["safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[pinned-tf]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] +dev = ["safetensors[all]"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] +mlx = ["mlx (>=0.0.9)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] +pinned-tf = ["safetensors[numpy]", "tensorflow (==2.11.0)"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] +torch = ["safetensors[numpy]", "torch (>=1.10)"] + [[package]] name = "sanic" version = "21.12.2" @@ -5014,19 +5148,18 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "68.0.0" +version = "70.3.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, - {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, + {file = "setuptools-70.3.0-py3-none-any.whl", hash = "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"}, + {file = "setuptools-70.3.0.tar.gz", hash = "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -5056,6 +5189,28 @@ six = "*" tabulate = "*" tqdm = ">=2.0" +[[package]] +name = "skops" +version = "0.9.0" +description = "A set of tools to push scikit-learn based models to and pull from Hugging Face Hub" +optional = false +python-versions = ">=3.8" +files = [ + {file = "skops-0.9.0-py3-none-any.whl", hash = "sha256:05645199bf6976e1f6dbba4a0704799cd5d2fcef18a98b069b4c84744e1a80a1"}, + {file = "skops-0.9.0.tar.gz", hash = "sha256:3e39333d65f26d5863ad44db5001b4cfe6a29642274ac37af54fb834813aee3f"}, +] + +[package.dependencies] +huggingface-hub = ">=0.17.0" +packaging = ">=17.0" +scikit-learn = ">=0.24" +tabulate = ">=0.8.8" + +[package.extras] +docs = ["fairlearn (>=0.7.0)", "matplotlib (>=3.3)", "numpydoc (>=1.0.0)", "pandas (>=1)", "scikit-learn-intelex (>=2021.7.1)", "sphinx (>=3.2.0)", "sphinx-gallery (>=0.7.0)", "sphinx-issues (>=1.2.0)", "sphinx-prompt (>=1.3.0)", "sphinx-rtd-theme (>=1)"] +rich = ["rich (>=12)"] +tests = ["catboost (>=1.0)", "fairlearn (>=0.7.0)", "flake8 (>=3.8.2)", "flaky (>=3.7.0)", "lightgbm (>=3)", "matplotlib (>=3.3)", "pandas (>=1)", "pytest (>=5.0.1)", "pytest-cov (>=2.9.0)", "quantile-forest (>=1.0.0)", "rich (>=12)", "types-requests (>=2.28.5)", "xgboost (>=1.6)"] + [[package]] name = "slack-sdk" version = "3.21.3" @@ -5963,56 +6118,129 @@ files = [ [[package]] name = "tokenizers" -version = "0.13.3" -description = "Fast and Customizable Tokenizers" +version = "0.15.2" +description = "" optional = true -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, - {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee0b1b311d65beab83d7a41c56a1e46ab732a9eed4460648e8eb0bd69fc2d059"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ef4215284df1277dadbcc5e17d4882bda19f770d02348e73523f7e7d8b8d396"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4d53976079cff8a033f778fb9adca2d9d69d009c02fa2d71a878b5f3963ed30"}, - {file = "tokenizers-0.13.3-cp310-cp310-win32.whl", hash = "sha256:1f0e3b4c2ea2cd13238ce43548959c118069db7579e5d40ec270ad77da5833ce"}, - {file = "tokenizers-0.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:89649c00d0d7211e8186f7a75dfa1db6996f65edce4b84821817eadcc2d3c79e"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:56b726e0d2bbc9243872b0144515ba684af5b8d8cd112fb83ee1365e26ec74c8"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc5c022ce692e1f499d745af293ab9ee6f5d92538ed2faf73f9708c89ee59ce6"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f55c981ac44ba87c93e847c333e58c12abcbb377a0c2f2ef96e1a266e4184ff2"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f247eae99800ef821a91f47c5280e9e9afaeed9980fc444208d5aa6ba69ff148"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b3e3215d048e94f40f1c95802e45dcc37c5b05eb46280fc2ccc8cd351bff839"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ba2b0bf01777c9b9bc94b53764d6684554ce98551fec496f71bc5be3a03e98b"}, - {file = "tokenizers-0.13.3-cp311-cp311-win32.whl", hash = "sha256:cc78d77f597d1c458bf0ea7c2a64b6aa06941c7a99cb135b5969b0278824d808"}, - {file = "tokenizers-0.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:ecf182bf59bd541a8876deccf0360f5ae60496fd50b58510048020751cf1724c"}, - {file = "tokenizers-0.13.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:0527dc5436a1f6bf2c0327da3145687d3bcfbeab91fed8458920093de3901b44"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07cbb2c307627dc99b44b22ef05ff4473aa7c7cc1fec8f0a8b37d8a64b1a16d2"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4560dbdeaae5b7ee0d4e493027e3de6d53c991b5002d7ff95083c99e11dd5ac0"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64064bd0322405c9374305ab9b4c07152a1474370327499911937fd4a76d004b"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8c6e2ab0f2e3d939ca66aa1d596602105fe33b505cd2854a4c1717f704c51de"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win32.whl", hash = "sha256:6cc29d410768f960db8677221e497226e545eaaea01aa3613fa0fdf2cc96cff4"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fc2a7fdf864554a0dacf09d32e17c0caa9afe72baf9dd7ddedc61973bae352d8"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:8791dedba834c1fc55e5f1521be325ea3dafb381964be20684b92fdac95d79b7"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:d607a6a13718aeb20507bdf2b96162ead5145bbbfa26788d6b833f98b31b26e1"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3791338f809cd1bf8e4fee6b540b36822434d0c6c6bc47162448deee3f77d425"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2f35f30e39e6aab8716f07790f646bdc6e4a853816cc49a95ef2a9016bf9ce6"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310204dfed5aa797128b65d63538a9837cbdd15da2a29a77d67eefa489edda26"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0f9b92ea052305166559f38498b3b0cae159caea712646648aaa272f7160963"}, - {file = "tokenizers-0.13.3-cp38-cp38-win32.whl", hash = "sha256:9a3fa134896c3c1f0da6e762d15141fbff30d094067c8f1157b9fdca593b5806"}, - {file = "tokenizers-0.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:8e7b0cdeace87fa9e760e6a605e0ae8fc14b7d72e9fc19c578116f7287bb873d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:00cee1e0859d55507e693a48fa4aef07060c4bb6bd93d80120e18fea9371c66d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a23ff602d0797cea1d0506ce69b27523b07e70f6dda982ab8cf82402de839088"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70ce07445050b537d2696022dafb115307abdffd2a5c106f029490f84501ef97"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:280ffe95f50eaaf655b3a1dc7ff1d9cf4777029dbbc3e63a74e65a056594abc3"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97acfcec592f7e9de8cadcdcda50a7134423ac8455c0166b28c9ff04d227b371"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd7730c98a3010cd4f523465867ff95cd9d6430db46676ce79358f65ae39797b"}, - {file = "tokenizers-0.13.3-cp39-cp39-win32.whl", hash = "sha256:48625a108029cb1ddf42e17a81b5a3230ba6888a70c9dc14e81bc319e812652d"}, - {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, - {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, -] - -[package.extras] -dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] + {file = "tokenizers-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:52f6130c9cbf70544287575a985bf44ae1bda2da7e8c24e97716080593638012"}, + {file = "tokenizers-0.15.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:054c1cc9c6d68f7ffa4e810b3d5131e0ba511b6e4be34157aa08ee54c2f8d9ee"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a9b9b070fdad06e347563b88c278995735292ded1132f8657084989a4c84a6d5"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea621a7eef4b70e1f7a4e84dd989ae3f0eeb50fc8690254eacc08acb623e82f1"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf7fd9a5141634fa3aa8d6b7be362e6ae1b4cda60da81388fa533e0b552c98fd"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44f2a832cd0825295f7179eaf173381dc45230f9227ec4b44378322d900447c9"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b9ec69247a23747669ec4b0ca10f8e3dfb3545d550258129bd62291aabe8605"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40b6a4c78da863ff26dbd5ad9a8ecc33d8a8d97b535172601cf00aee9d7ce9ce"}, + {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5ab2a4d21dcf76af60e05af8063138849eb1d6553a0d059f6534357bce8ba364"}, + {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a47acfac7e511f6bbfcf2d3fb8c26979c780a91e06fb5b9a43831b2c0153d024"}, + {file = "tokenizers-0.15.2-cp310-none-win32.whl", hash = "sha256:064ff87bb6acdbd693666de9a4b692add41308a2c0ec0770d6385737117215f2"}, + {file = "tokenizers-0.15.2-cp310-none-win_amd64.whl", hash = "sha256:3b919afe4df7eb6ac7cafd2bd14fb507d3f408db7a68c43117f579c984a73843"}, + {file = "tokenizers-0.15.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:89cd1cb93e4b12ff39bb2d626ad77e35209de9309a71e4d3d4672667b4b256e7"}, + {file = "tokenizers-0.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfed5c64e5be23d7ee0f0e98081a25c2a46b0b77ce99a4f0605b1ec43dd481fa"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a907d76dcfda37023ba203ab4ceeb21bc5683436ebefbd895a0841fd52f6f6f2"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ea60479de6fc7b8ae756b4b097572372d7e4032e2521c1bbf3d90c90a99ff0"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:48e2b9335be2bc0171df9281385c2ed06a15f5cf121c44094338306ab7b33f2c"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:112a1dd436d2cc06e6ffdc0b06d55ac019a35a63afd26475205cb4b1bf0bfbff"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4620cca5c2817177ee8706f860364cc3a8845bc1e291aaf661fb899e5d1c45b0"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccd73a82751c523b3fc31ff8194702e4af4db21dc20e55b30ecc2079c5d43cb7"}, + {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:107089f135b4ae7817affe6264f8c7a5c5b4fd9a90f9439ed495f54fcea56fb4"}, + {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0ff110ecc57b7aa4a594396525a3451ad70988e517237fe91c540997c4e50e29"}, + {file = "tokenizers-0.15.2-cp311-none-win32.whl", hash = "sha256:6d76f00f5c32da36c61f41c58346a4fa7f0a61be02f4301fd30ad59834977cc3"}, + {file = "tokenizers-0.15.2-cp311-none-win_amd64.whl", hash = "sha256:cc90102ed17271cf0a1262babe5939e0134b3890345d11a19c3145184b706055"}, + {file = "tokenizers-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f86593c18d2e6248e72fb91c77d413a815153b8ea4e31f7cd443bdf28e467670"}, + {file = "tokenizers-0.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0774bccc6608eca23eb9d620196687c8b2360624619623cf4ba9dc9bd53e8b51"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d0222c5b7c9b26c0b4822a82f6a7011de0a9d3060e1da176f66274b70f846b98"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3835738be1de66624fff2f4f6f6684775da4e9c00bde053be7564cbf3545cc66"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0143e7d9dcd811855c1ce1ab9bf5d96d29bf5e528fd6c7824d0465741e8c10fd"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db35825f6d54215f6b6009a7ff3eedee0848c99a6271c870d2826fbbedf31a38"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f5e64b0389a2be47091d8cc53c87859783b837ea1a06edd9d8e04004df55a5c"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e0480c452217edd35eca56fafe2029fb4d368b7c0475f8dfa3c5c9c400a7456"}, + {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a33ab881c8fe70474980577e033d0bc9a27b7ab8272896e500708b212995d834"}, + {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a308a607ca9de2c64c1b9ba79ec9a403969715a1b8ba5f998a676826f1a7039d"}, + {file = "tokenizers-0.15.2-cp312-none-win32.whl", hash = "sha256:b8fcfa81bcb9447df582c5bc96a031e6df4da2a774b8080d4f02c0c16b42be0b"}, + {file = "tokenizers-0.15.2-cp312-none-win_amd64.whl", hash = "sha256:38d7ab43c6825abfc0b661d95f39c7f8af2449364f01d331f3b51c94dcff7221"}, + {file = "tokenizers-0.15.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:38bfb0204ff3246ca4d5e726e8cc8403bfc931090151e6eede54d0e0cf162ef0"}, + {file = "tokenizers-0.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c861d35e8286a53e06e9e28d030b5a05bcbf5ac9d7229e561e53c352a85b1fc"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:936bf3842db5b2048eaa53dade907b1160f318e7c90c74bfab86f1e47720bdd6"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:620beacc3373277700d0e27718aa8b25f7b383eb8001fba94ee00aeea1459d89"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2735ecbbf37e52db4ea970e539fd2d450d213517b77745114f92867f3fc246eb"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:473c83c5e2359bb81b0b6fde870b41b2764fcdd36d997485e07e72cc3a62264a"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968fa1fb3c27398b28a4eca1cbd1e19355c4d3a6007f7398d48826bbe3a0f728"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:865c60ae6eaebdde7da66191ee9b7db52e542ed8ee9d2c653b6d190a9351b980"}, + {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7c0d8b52664ab2d4a8d6686eb5effc68b78608a9008f086a122a7b2996befbab"}, + {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f33dfbdec3784093a9aebb3680d1f91336c56d86cc70ddf88708251da1fe9064"}, + {file = "tokenizers-0.15.2-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d44ba80988ff9424e33e0a49445072ac7029d8c0e1601ad25a0ca5f41ed0c1d6"}, + {file = "tokenizers-0.15.2-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:dce74266919b892f82b1b86025a613956ea0ea62a4843d4c4237be2c5498ed3a"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0ef06b9707baeb98b316577acb04f4852239d856b93e9ec3a299622f6084e4be"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c73e2e74bbb07910da0d37c326869f34113137b23eadad3fc00856e6b3d9930c"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4eeb12daf02a59e29f578a865f55d87cd103ce62bd8a3a5874f8fdeaa82e336b"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ba9f6895af58487ca4f54e8a664a322f16c26bbb442effd01087eba391a719e"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccec77aa7150e38eec6878a493bf8c263ff1fa8a62404e16c6203c64c1f16a26"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3f40604f5042ff210ba82743dda2b6aa3e55aa12df4e9f2378ee01a17e2855e"}, + {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5645938a42d78c4885086767c70923abad047163d809c16da75d6b290cb30bbe"}, + {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:05a77cbfebe28a61ab5c3891f9939cc24798b63fa236d84e5f29f3a85a200c00"}, + {file = "tokenizers-0.15.2-cp37-none-win32.whl", hash = "sha256:361abdc068e8afe9c5b818769a48624687fb6aaed49636ee39bec4e95e1a215b"}, + {file = "tokenizers-0.15.2-cp37-none-win_amd64.whl", hash = "sha256:7ef789f83eb0f9baeb4d09a86cd639c0a5518528f9992f38b28e819df397eb06"}, + {file = "tokenizers-0.15.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4fe1f74a902bee74a3b25aff180fbfbf4f8b444ab37c4d496af7afd13a784ed2"}, + {file = "tokenizers-0.15.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c4b89038a684f40a6b15d6b09f49650ac64d951ad0f2a3ea9169687bbf2a8ba"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d05a1b06f986d41aed5f2de464c003004b2df8aaf66f2b7628254bcbfb72a438"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:508711a108684111ec8af89d3a9e9e08755247eda27d0ba5e3c50e9da1600f6d"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:daa348f02d15160cb35439098ac96e3a53bacf35885072611cd9e5be7d333daa"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:494fdbe5932d3416de2a85fc2470b797e6f3226c12845cadf054dd906afd0442"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2d60f5246f4da9373f75ff18d64c69cbf60c3bca597290cea01059c336d2470"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93268e788825f52de4c7bdcb6ebc1fcd4a5442c02e730faa9b6b08f23ead0e24"}, + {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6fc7083ab404019fc9acafe78662c192673c1e696bd598d16dc005bd663a5cf9"}, + {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:41e39b41e5531d6b2122a77532dbea60e171ef87a3820b5a3888daa847df4153"}, + {file = "tokenizers-0.15.2-cp38-none-win32.whl", hash = "sha256:06cd0487b1cbfabefb2cc52fbd6b1f8d4c37799bd6c6e1641281adaa6b2504a7"}, + {file = "tokenizers-0.15.2-cp38-none-win_amd64.whl", hash = "sha256:5179c271aa5de9c71712e31cb5a79e436ecd0d7532a408fa42a8dbfa4bc23fd9"}, + {file = "tokenizers-0.15.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82f8652a74cc107052328b87ea8b34291c0f55b96d8fb261b3880216a9f9e48e"}, + {file = "tokenizers-0.15.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:02458bee6f5f3139f1ebbb6d042b283af712c0981f5bc50edf771d6b762d5e4f"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c9a09cd26cca2e1c349f91aa665309ddb48d71636370749414fbf67bc83c5343"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:158be8ea8554e5ed69acc1ce3fbb23a06060bd4bbb09029431ad6b9a466a7121"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ddba9a2b0c8c81633eca0bb2e1aa5b3a15362b1277f1ae64176d0f6eba78ab1"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ef5dd1d39797044642dbe53eb2bc56435308432e9c7907728da74c69ee2adca"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:454c203164e07a860dbeb3b1f4a733be52b0edbb4dd2e5bd75023ffa8b49403a"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cf6b7f1d4dc59af960e6ffdc4faffe6460bbfa8dce27a58bf75755ffdb2526d"}, + {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2ef09bbc16519f6c25d0c7fc0c6a33a6f62923e263c9d7cca4e58b8c61572afb"}, + {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c9a2ebdd2ad4ec7a68e7615086e633857c85e2f18025bd05d2a4399e6c5f7169"}, + {file = "tokenizers-0.15.2-cp39-none-win32.whl", hash = "sha256:918fbb0eab96fe08e72a8c2b5461e9cce95585d82a58688e7f01c2bd546c79d0"}, + {file = "tokenizers-0.15.2-cp39-none-win_amd64.whl", hash = "sha256:524e60da0135e106b254bd71f0659be9f89d83f006ea9093ce4d1fab498c6d0d"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6a9b648a58281c4672212fab04e60648fde574877d0139cd4b4f93fe28ca8944"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7c7d18b733be6bbca8a55084027f7be428c947ddf871c500ee603e375013ffba"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:13ca3611de8d9ddfbc4dc39ef54ab1d2d4aaa114ac8727dfdc6a6ec4be017378"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:237d1bf3361cf2e6463e6c140628e6406766e8b27274f5fcc62c747ae3c6f094"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de19c4dc503c612847edf833c82e9f73cd79926a384af9d801dcf93f110cea4e"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea09acd2fe3324174063d61ad620dec3bcf042b495515f27f638270a7d466e8b"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cf27fd43472e07b57cf420eee1e814549203d56de00b5af8659cb99885472f1f"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7ca22bd897537a0080521445d91a58886c8c04084a6a19e6c78c586e0cfa92a5"}, + {file = "tokenizers-0.15.2.tar.gz", hash = "sha256:e6e9c6e019dd5484be5beafc775ae6c925f4c69a3487040ed09b45e13df2cb91"}, +] + +[package.dependencies] +huggingface_hub = ">=0.16.4,<1.0" + +[package.extras] +dev = ["tokenizers[testing]"] +docs = ["setuptools_rust", "sphinx", "sphinx_rtd_theme"] testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] [[package]] @@ -6103,69 +6331,71 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.26.0" +version = "4.36.2" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = true -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" files = [ - {file = "transformers-4.26.0-py3-none-any.whl", hash = "sha256:6a902eee6098d9a737faadf185b8df5a169acc695ebbde5a81b90528f43e665f"}, - {file = "transformers-4.26.0.tar.gz", hash = "sha256:d7859bd83829a3682ca632197ee5c72556e1063d199ab84eec35c4f23b3d73a3"}, + {file = "transformers-4.36.2-py3-none-any.whl", hash = "sha256:462066c4f74ee52516f12890dcc9ec71d1a5e97998db621668455117a54330f6"}, + {file = "transformers-4.36.2.tar.gz", hash = "sha256:d8068e897e47793281501e547d2bbdfc5b8556409c2cb6c3d9e2ca77d4c0b4ec"}, ] [package.dependencies] filelock = "*" -huggingface-hub = ">=0.11.0,<1.0" +huggingface-hub = ">=0.19.3,<1.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" requests = "*" -tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" +safetensors = ">=0.3.1" +tokenizers = ">=0.14,<0.19" tqdm = ">=4.27" [package.extras] -accelerate = ["accelerate (>=0.10.0)"] -all = ["Pillow", "accelerate (>=0.10.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio"] +accelerate = ["accelerate (>=0.21.0)"] +agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.10,!=1.12.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.10.0)", "deepspeed (>=0.6.5)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.10.0)", "beautifulsoup4", "black (==22.3)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.6.5)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.10.0)", "beautifulsoup4", "black (==22.3)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flake8 (>=3.8.3)", "flax (>=0.4.1)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (==22.3)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flake8 (>=3.8.3)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (==22.3)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flake8 (>=3.8.3)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -docs = ["Pillow", "accelerate (>=0.10.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio"] +deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.14,<0.19)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] -fairscale = ["fairscale (>0.3)"] -flax = ["flax (>=0.4.1)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] -integrations = ["optuna", "ray[tune]", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.4)"] +natten = ["natten (>=0.14.6)"] onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "black (==22.3)", "datasets (!=2.5.0)", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)"] -ray = ["ray[tune]"] +quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<2.0.0)"] +ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf (<=3.20.2)", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic (<2)", "starlette", "uvicorn"] sigopt = ["sigopt"] sklearn = ["scikit-learn"] speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (==22.3)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.12)", "tensorflow-text", "tf2onnx"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "tensorboard", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] -tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["torch (>=1.7,!=1.12.0)"] +tokenizers = ["tokenizers (>=0.14,<0.19)"] +torch = ["accelerate (>=0.21.0)", "torch (>=1.10,!=1.12.0)"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torchhub = ["filelock", "huggingface-hub (>=0.11.0,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.2)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "tqdm (>=4.27)"] -video = ["decord (==0.6.0)"] -vision = ["Pillow"] +torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "tqdm (>=4.27)"] +video = ["av (==9.2.0)", "decord (==0.6.0)"] +vision = ["Pillow (>=10.0.1,<=15.0)"] [[package]] name = "twilio" @@ -6956,4 +7186,4 @@ transformers = ["sentencepiece", "transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "4c84d994449f859816e48dd00d77f31f6f9d964e29a9f6060300c51d923786e0" +content-hash = "c1c51259ab3b886039dcf7eb746a45815d4b8afcaa4bdbe179c891810aee553f" diff --git a/pyproject.toml b/pyproject.toml index 0944c09460d6..d2a48c527aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,6 @@ sanic-cors = "~2.0.0" sanic-jwt = "^1.6.0" sanic-routing = "^0.7.2" websockets = ">=10.0,<11.0" -cloudpickle = ">=1.2,<2.3" aiohttp = ">=3.9.0,<3.10" questionary = ">=1.5.1,<1.11.0" prompt-toolkit = "^3.0,<3.0.29" @@ -133,10 +132,9 @@ psycopg2-binary = ">=2.8.2,<2.10.0" python-dateutil = "~2.8" protobuf = ">=4.23.3,< 4.23.4" tensorflow_hub = "^0.13.0" -setuptools = ">=65.5.1" +setuptools = "~70.3.0" ujson = ">=1.35,<6.0" regex = ">=2020.6,<2022.11" -joblib = ">=0.15.1,<1.3.0" sentry-sdk = ">=0.17.0,<1.15.0" aio-pika = ">=6.7.1,<8.2.4" aiogram = "<2.26" @@ -156,6 +154,9 @@ dnspython = "2.3.0" wheel = ">=0.38.1" certifi = ">=2023.7.22" cryptography = ">=41.0.7" +skops = "0.9.0" +safetensors = "~0.4.5" + [[tool.poetry.dependencies.tensorflow-io-gcs-filesystem]] version = "==0.31" markers = "sys_platform == 'win32'" @@ -285,7 +286,7 @@ version = "~3.2.0" optional = true [tool.poetry.dependencies.transformers] -version = ">=4.13.0, <=4.26.0" +version = "~4.36.2" optional = true [tool.poetry.dependencies.sentencepiece] diff --git a/rasa/core/featurizers/single_state_featurizer.py b/rasa/core/featurizers/single_state_featurizer.py index 7d8c504084c1..0a6c921491a4 100644 --- a/rasa/core/featurizers/single_state_featurizer.py +++ b/rasa/core/featurizers/single_state_featurizer.py @@ -1,7 +1,8 @@ import logging +from typing import List, Optional, Dict, Text, Set, Any + import numpy as np import scipy.sparse -from typing import List, Optional, Dict, Text, Set, Any from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization from rasa.nlu.extractors.extractor import EntityTagSpec @@ -362,6 +363,26 @@ def encode_all_labels( for action in domain.action_names_or_texts ] + def to_dict(self) -> Dict[str, Any]: + return { + "action_texts": self.action_texts, + "entity_tag_specs": self.entity_tag_specs, + "feature_states": self._default_feature_states, + } + + @classmethod + def create_from_dict( + cls, data: Dict[str, Any] + ) -> Optional["SingleStateFeaturizer"]: + if not data: + return None + + featurizer = SingleStateFeaturizer() + featurizer.action_texts = data["action_texts"] + featurizer._default_feature_states = data["feature_states"] + featurizer.entity_tag_specs = data["entity_tag_specs"] + return featurizer + class IntentTokenizerSingleStateFeaturizer(SingleStateFeaturizer): """A SingleStateFeaturizer for use with policies that predict intent labels.""" diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index 42df6e4e1187..9c6dbca92d47 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -1,11 +1,9 @@ from __future__ import annotations -from pathlib import Path -from collections import defaultdict -from abc import abstractmethod -import jsonpickle -import logging -from tqdm import tqdm +import logging +from abc import abstractmethod +from collections import defaultdict +from pathlib import Path from typing import ( Tuple, List, @@ -18,25 +16,30 @@ Set, DefaultDict, cast, + Type, + Callable, + ClassVar, ) + import numpy as np +from tqdm import tqdm -from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer -from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization -from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError import rasa.shared.core.trackers import rasa.shared.utils.io -from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME -from rasa.shared.nlu.training_data.features import Features -from rasa.shared.core.trackers import DialogueStateTracker -from rasa.shared.core.domain import State, Domain -from rasa.shared.core.events import Event, ActionExecuted, UserUttered +from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError +from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization +from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer from rasa.shared.core.constants import ( USER, ACTION_UNLIKELY_INTENT_NAME, PREVIOUS_ACTION, ) +from rasa.shared.core.domain import State, Domain +from rasa.shared.core.events import Event, ActionExecuted, UserUttered +from rasa.shared.core.trackers import DialogueStateTracker from rasa.shared.exceptions import RasaException +from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME +from rasa.shared.nlu.training_data.features import Features from rasa.utils.tensorflow.constants import LABEL_PAD_ID from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray @@ -64,6 +67,10 @@ def __str__(self) -> Text: class TrackerFeaturizer: """Base class for actual tracker featurizers.""" + # Class registry to store all subclasses + _registry: ClassVar[Dict[str, Type["TrackerFeaturizer"]]] = {} + _featurizer_type: str = "TrackerFeaturizer" + def __init__( self, state_featurizer: Optional[SingleStateFeaturizer] = None ) -> None: @@ -74,6 +81,36 @@ def __init__( """ self.state_featurizer = state_featurizer + @classmethod + def register(cls, featurizer_type: str) -> Callable: + """Decorator to register featurizer subclasses.""" + + def wrapper(subclass: Type["TrackerFeaturizer"]) -> Type["TrackerFeaturizer"]: + cls._registry[featurizer_type] = subclass + # Store the type identifier in the class for serialization + subclass._featurizer_type = featurizer_type + return subclass + + return wrapper + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer": + """Create featurizer instance from dictionary.""" + featurizer_type = data.pop("type") + + if featurizer_type not in cls._registry: + raise ValueError(f"Unknown featurizer type: {featurizer_type}") + + # Get the correct subclass and instantiate it + subclass = cls._registry[featurizer_type] + return subclass.create_from_dict(data) + + @classmethod + @abstractmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer": + """Each subclass must implement its own creation from dict method.""" + pass + @staticmethod def _create_states( tracker: DialogueStateTracker, @@ -465,9 +502,7 @@ def persist(self, path: Union[Text, Path]) -> None: self.state_featurizer.entity_tag_specs = [] # noinspection PyTypeChecker - rasa.shared.utils.io.write_text_file( - str(jsonpickle.encode(self)), featurizer_file - ) + rasa.shared.utils.io.dump_obj_as_json_to_file(featurizer_file, self.to_dict()) @staticmethod def load(path: Union[Text, Path]) -> Optional[TrackerFeaturizer]: @@ -481,7 +516,17 @@ def load(path: Union[Text, Path]) -> Optional[TrackerFeaturizer]: """ featurizer_file = Path(path) / FEATURIZER_FILE if featurizer_file.is_file(): - return jsonpickle.decode(rasa.shared.utils.io.read_file(featurizer_file)) + data = rasa.shared.utils.io.read_json_file(featurizer_file) + + if "type" not in data: + logger.error( + f"Couldn't load featurizer for policy. " + f"File '{featurizer_file}' does not contain all " + f"necessary information. 'type' is missing." + ) + return None + + return TrackerFeaturizer.from_dict(data) logger.error( f"Couldn't load featurizer for policy. " @@ -508,7 +553,16 @@ def _remove_action_unlikely_intent_from_events(events: List[Event]) -> List[Even ) ] + def to_dict(self) -> Dict[str, Any]: + return { + "type": self.__class__._featurizer_type, + "state_featurizer": ( + self.state_featurizer.to_dict() if self.state_featurizer else None + ), + } + +@TrackerFeaturizer.register("FullDialogueTrackerFeaturizer") class FullDialogueTrackerFeaturizer(TrackerFeaturizer): """Creates full dialogue training data for time distributed architectures. @@ -646,7 +700,20 @@ def prediction_states( return trackers_as_states + def to_dict(self) -> Dict[str, Any]: + return super().to_dict() + @classmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "FullDialogueTrackerFeaturizer": + state_featurizer = SingleStateFeaturizer.create_from_dict( + data["state_featurizer"] + ) + return cls( + state_featurizer, + ) + + +@TrackerFeaturizer.register("MaxHistoryTrackerFeaturizer") class MaxHistoryTrackerFeaturizer(TrackerFeaturizer): """Truncates the tracker history into `max_history` long sequences. @@ -887,7 +954,25 @@ def prediction_states( return trackers_as_states + def to_dict(self) -> Dict[str, Any]: + data = super().to_dict() + data.update( + { + "remove_duplicates": self.remove_duplicates, + "max_history": self.max_history, + } + ) + return data + + @classmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "MaxHistoryTrackerFeaturizer": + state_featurizer = SingleStateFeaturizer.create_from_dict( + data["state_featurizer"] + ) + return cls(state_featurizer, data["max_history"], data["remove_duplicates"]) + +@TrackerFeaturizer.register("IntentMaxHistoryTrackerFeaturizer") class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer): """Truncates the tracker history into `max_history` long sequences. @@ -1166,6 +1251,18 @@ def prediction_states( return trackers_as_states + def to_dict(self) -> Dict[str, Any]: + return super().to_dict() + + @classmethod + def create_from_dict( + cls, data: Dict[str, Any] + ) -> "IntentMaxHistoryTrackerFeaturizer": + state_featurizer = SingleStateFeaturizer.create_from_dict( + data["state_featurizer"] + ) + return cls(state_featurizer, data["max_history"], data["remove_duplicates"]) + def _is_prev_action_unlikely_intent_in_state(state: State) -> bool: prev_action_name = state.get(PREVIOUS_ACTION, {}).get(ACTION_NAME) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index c5f895e3ce64..af96af627de6 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -1,15 +1,15 @@ from __future__ import annotations -import logging -from rasa.engine.recipes.default_recipe import DefaultV1Recipe +import logging from pathlib import Path from collections import defaultdict import contextlib +from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type import numpy as np import tensorflow as tf -from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type +from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.graph import ExecutionContext from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage @@ -49,18 +49,22 @@ from rasa.shared.core.events import EntitiesAdded, Event from rasa.shared.core.domain import Domain from rasa.shared.nlu.training_data.message import Message -from rasa.shared.nlu.training_data.features import Features +from rasa.shared.nlu.training_data.features import ( + Features, + save_features, + load_features, +) import rasa.shared.utils.io import rasa.utils.io from rasa.utils import train_utils -from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel -from rasa.utils.tensorflow import rasa_layers -from rasa.utils.tensorflow.model_data import ( - RasaModelData, - FeatureSignature, +from rasa.utils.tensorflow.feature_array import ( FeatureArray, - Data, + serialize_nested_feature_arrays, + deserialize_nested_feature_arrays, ) +from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel +from rasa.utils.tensorflow import rasa_layers +from rasa.utils.tensorflow.model_data import RasaModelData, FeatureSignature, Data from rasa.utils.tensorflow.model_data_utils import convert_to_data_format from rasa.utils.tensorflow.constants import ( LABEL, @@ -961,22 +965,32 @@ def persist_model_utilities(self, model_path: Path) -> None: model_path: Path where model is to be persisted """ model_filename = self._metadata_filename() - rasa.utils.io.json_pickle( - model_path / f"{model_filename}.priority.pkl", self.priority - ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.meta.pkl", self.config + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{model_filename}.priority.json", self.priority ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.data_example.pkl", self.data_example + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{model_filename}.meta.json", self.config ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.fake_features.pkl", self.fake_features + # save data example + serialize_nested_feature_arrays( + self.data_example, + str(model_path / f"{model_filename}.data_example.st"), + str(model_path / f"{model_filename}.data_example_metadata.json"), ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.label_data.pkl", + # save label data + serialize_nested_feature_arrays( dict(self._label_data.data) if self._label_data is not None else {}, + str(model_path / f"{model_filename}.label_data.st"), + str(model_path / f"{model_filename}.label_data_metadata.json"), + ) + # save fake features + metadata = save_features( + self.fake_features, str(model_path / f"{model_filename}.fake_features.st") + ) + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{model_filename}.fake_features_metadata.json", metadata ) + entity_tag_specs = ( [tag_spec._asdict() for tag_spec in self._entity_tag_specs] if self._entity_tag_specs @@ -994,18 +1008,29 @@ def _load_model_utilities(cls, model_path: Path) -> Dict[Text, Any]: model_path: Path where model is to be persisted. """ tf_model_file = model_path / f"{cls._metadata_filename()}.tf_model" - loaded_data = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.data_example.pkl" + + # load data example + loaded_data = deserialize_nested_feature_arrays( + str(model_path / f"{cls._metadata_filename()}.data_example.st"), + str(model_path / f"{cls._metadata_filename()}.data_example_metadata.json"), ) - label_data = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.label_data.pkl" + # load label data + loaded_label_data = deserialize_nested_feature_arrays( + str(model_path / f"{cls._metadata_filename()}.label_data.st"), + str(model_path / f"{cls._metadata_filename()}.label_data_metadata.json"), ) - fake_features = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.fake_features.pkl" + label_data = RasaModelData(data=loaded_label_data) + + # load fake features + metadata = rasa.shared.utils.io.read_json_file( + model_path / f"{cls._metadata_filename()}.fake_features_metadata.json" ) - label_data = RasaModelData(data=label_data) - priority = rasa.utils.io.json_unpickle( - model_path / f"{cls._metadata_filename()}.priority.pkl" + fake_features = load_features( + str(model_path / f"{cls._metadata_filename()}.fake_features.st"), metadata + ) + + priority = rasa.shared.utils.io.read_json_file( + model_path / f"{cls._metadata_filename()}.priority.json" ) entity_tag_specs = rasa.shared.utils.io.read_json_file( model_path / f"{cls._metadata_filename()}.entity_tag_specs.json" @@ -1023,8 +1048,8 @@ def _load_model_utilities(cls, model_path: Path) -> Dict[Text, Any]: ) for tag_spec in entity_tag_specs ] - model_config = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.meta.pkl" + model_config = rasa.shared.utils.io.read_json_file( + model_path / f"{cls._metadata_filename()}.meta.json" ) return { @@ -1070,7 +1095,7 @@ def _load( ) -> TEDPolicy: featurizer = TrackerFeaturizer.load(model_path) - if not (model_path / f"{cls._metadata_filename()}.data_example.pkl").is_file(): + if not (model_path / f"{cls._metadata_filename()}.data_example.st").is_file(): return cls( config, model_storage, diff --git a/rasa/core/policies/unexpected_intent_policy.py b/rasa/core/policies/unexpected_intent_policy.py index d5b39a561b82..ca788662f133 100644 --- a/rasa/core/policies/unexpected_intent_policy.py +++ b/rasa/core/policies/unexpected_intent_policy.py @@ -5,6 +5,7 @@ import numpy as np import tensorflow as tf + import rasa.utils.common from rasa.engine.graph import ExecutionContext from rasa.engine.recipes.default_recipe import DefaultV1Recipe @@ -16,6 +17,7 @@ from rasa.shared.core.trackers import DialogueStateTracker from rasa.shared.core.constants import SLOTS, ACTIVE_LOOP, ACTION_UNLIKELY_INTENT_NAME from rasa.shared.core.events import UserUttered, ActionExecuted +import rasa.shared.utils.io from rasa.shared.nlu.constants import ( INTENT, TEXT, @@ -103,8 +105,6 @@ ) from rasa.utils.tensorflow import layers from rasa.utils.tensorflow.model_data import RasaModelData, FeatureArray, Data - -import rasa.utils.io as io_utils from rasa.core.exceptions import RasaCoreException from rasa.shared.utils import common @@ -881,9 +881,12 @@ def persist_model_utilities(self, model_path: Path) -> None: model_path: Path where model is to be persisted """ super().persist_model_utilities(model_path) - io_utils.pickle_dump( - model_path / f"{self._metadata_filename()}.label_quantiles.pkl", - self.label_quantiles, + + from safetensors.numpy import save_file + + save_file( + {str(k): np.array(v) for k, v in self.label_quantiles.items()}, + model_path / f"{self._metadata_filename()}.label_quantiles.st", ) @classmethod @@ -894,9 +897,14 @@ def _load_model_utilities(cls, model_path: Path) -> Dict[Text, Any]: model_path: Path where model is to be persisted. """ model_utilties = super()._load_model_utilities(model_path) - label_quantiles = io_utils.pickle_load( - model_path / f"{cls._metadata_filename()}.label_quantiles.pkl" + + from safetensors.numpy import load_file + + loaded_label_quantiles = load_file( + model_path / f"{cls._metadata_filename()}.label_quantiles.st" ) + label_quantiles = {int(k): list(v) for k, v in loaded_label_quantiles.items()} + model_utilties.update({"label_quantiles": label_quantiles}) return model_utilties diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index bea4735da6fe..b53eb5db8d76 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1,18 +1,17 @@ from __future__ import annotations + import copy import logging from collections import defaultdict from pathlib import Path - -from rasa.exceptions import ModelNotFound -from rasa.nlu.featurizers.featurizer import Featurizer +from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type import numpy as np import scipy.sparse import tensorflow as tf -from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type - +from rasa.exceptions import ModelNotFound +from rasa.nlu.featurizers.featurizer import Featurizer from rasa.engine.graph import ExecutionContext, GraphComponent from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource @@ -20,18 +19,21 @@ from rasa.nlu.extractors.extractor import EntityExtractorMixin from rasa.nlu.classifiers.classifier import IntentClassifier import rasa.shared.utils.io -import rasa.utils.io as io_utils import rasa.nlu.utils.bilou_utils as bilou_utils from rasa.shared.constants import DIAGNOSTIC_DATA from rasa.nlu.extractors.extractor import EntityTagSpec from rasa.nlu.classifiers import LABEL_RANKING_LENGTH from rasa.utils import train_utils from rasa.utils.tensorflow import rasa_layers +from rasa.utils.tensorflow.feature_array import ( + FeatureArray, + serialize_nested_feature_arrays, + deserialize_nested_feature_arrays, +) from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel from rasa.utils.tensorflow.model_data import ( RasaModelData, FeatureSignature, - FeatureArray, ) from rasa.nlu.constants import TOKENS_NAMES, DEFAULT_TRANSFORMER_SIZE from rasa.shared.nlu.constants import ( @@ -118,7 +120,6 @@ POSSIBLE_TAGS = [ENTITY_ATTRIBUTE_TYPE, ENTITY_ATTRIBUTE_ROLE, ENTITY_ATTRIBUTE_GROUP] - DIETClassifierT = TypeVar("DIETClassifierT", bound="DIETClassifier") @@ -1085,18 +1086,24 @@ def persist(self) -> None: self.model.save(str(tf_model_file)) - io_utils.pickle_dump( - model_path / f"{file_name}.data_example.pkl", self._data_example - ) - io_utils.pickle_dump( - model_path / f"{file_name}.sparse_feature_sizes.pkl", - self._sparse_feature_sizes, + # save data example + serialize_nested_feature_arrays( + self._data_example, + model_path / f"{file_name}.data_example.st", + model_path / f"{file_name}.data_example_metadata.json", ) - io_utils.pickle_dump( - model_path / f"{file_name}.label_data.pkl", + # save label data + serialize_nested_feature_arrays( dict(self._label_data.data) if self._label_data is not None else {}, + model_path / f"{file_name}.label_data.st", + model_path / f"{file_name}.label_data_metadata.json", ) - io_utils.json_pickle( + + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{file_name}.sparse_feature_sizes.json", + self._sparse_feature_sizes, + ) + rasa.shared.utils.io.dump_obj_as_json_to_file( model_path / f"{file_name}.index_label_id_mapping.json", self.index_label_id_mapping, ) @@ -1185,15 +1192,22 @@ def _load_from_files( ]: file_name = cls.__name__ - data_example = io_utils.pickle_load( - model_path / f"{file_name}.data_example.pkl" + # load data example + data_example = deserialize_nested_feature_arrays( + str(model_path / f"{file_name}.data_example.st"), + str(model_path / f"{file_name}.data_example_metadata.json"), ) - label_data = io_utils.pickle_load(model_path / f"{file_name}.label_data.pkl") - label_data = RasaModelData(data=label_data) - sparse_feature_sizes = io_utils.pickle_load( - model_path / f"{file_name}.sparse_feature_sizes.pkl" + # load label data + loaded_label_data = deserialize_nested_feature_arrays( + str(model_path / f"{file_name}.label_data.st"), + str(model_path / f"{file_name}.label_data_metadata.json"), + ) + label_data = RasaModelData(data=loaded_label_data) + + sparse_feature_sizes = rasa.shared.utils.io.read_json_file( + model_path / f"{file_name}.sparse_feature_sizes.json" ) - index_label_id_mapping = io_utils.json_unpickle( + index_label_id_mapping = rasa.shared.utils.io.read_json_file( model_path / f"{file_name}.index_label_id_mapping.json" ) entity_tag_specs = rasa.shared.utils.io.read_json_file( @@ -1213,7 +1227,6 @@ def _load_from_files( for tag_spec in entity_tag_specs ] - # jsonpickle converts dictionary keys to strings index_label_id_mapping = { int(key): value for key, value in index_label_id_mapping.items() } diff --git a/rasa/nlu/classifiers/logistic_regression_classifier.py b/rasa/nlu/classifiers/logistic_regression_classifier.py index c652d20af9c0..46303a11697e 100644 --- a/rasa/nlu/classifiers/logistic_regression_classifier.py +++ b/rasa/nlu/classifiers/logistic_regression_classifier.py @@ -1,20 +1,19 @@ import logging from typing import Any, Text, Dict, List, Type, Tuple -import joblib from scipy.sparse import hstack, vstack, csr_matrix from sklearn.linear_model import LogisticRegression +from rasa.engine.graph import ExecutionContext, GraphComponent +from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage -from rasa.engine.recipes.default_recipe import DefaultV1Recipe -from rasa.engine.graph import ExecutionContext, GraphComponent from rasa.nlu.classifiers import LABEL_RANKING_LENGTH -from rasa.nlu.featurizers.featurizer import Featurizer from rasa.nlu.classifiers.classifier import IntentClassifier -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message +from rasa.nlu.featurizers.featurizer import Featurizer from rasa.shared.nlu.constants import TEXT, INTENT +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.utils.tensorflow.constants import RANKING_LENGTH logger = logging.getLogger(__name__) @@ -158,9 +157,11 @@ def process(self, messages: List[Message]) -> List[Message]: def persist(self) -> None: """Persist this model into the passed directory.""" + import skops.io as sio + with self._model_storage.write_to(self._resource) as model_dir: - path = model_dir / f"{self._resource.name}.joblib" - joblib.dump(self.clf, path) + path = model_dir / f"{self._resource.name}.skops" + sio.dump(self.clf, path) logger.debug(f"Saved intent classifier to '{path}'.") @classmethod @@ -173,9 +174,21 @@ def load( **kwargs: Any, ) -> "LogisticRegressionClassifier": """Loads trained component (see parent class for full docstring).""" + import skops.io as sio + try: with model_storage.read_from(resource) as model_dir: - classifier = joblib.load(model_dir / f"{resource.name}.joblib") + classifier_file = model_dir / f"{resource.name}.skops" + unknown_types = sio.get_untrusted_types(file=classifier_file) + + if unknown_types: + logger.debug( + f"Untrusted types ({unknown_types}) found when " + f"loading {classifier_file}!", + ) + raise ValueError() + + classifier = sio.load(classifier_file, trusted=unknown_types) component = cls( config, execution_context.node_name, model_storage, resource ) diff --git a/rasa/nlu/classifiers/sklearn_intent_classifier.py b/rasa/nlu/classifiers/sklearn_intent_classifier.py index 5c941d3d8806..3aa656f0f3ba 100644 --- a/rasa/nlu/classifiers/sklearn_intent_classifier.py +++ b/rasa/nlu/classifiers/sklearn_intent_classifier.py @@ -1,6 +1,6 @@ from __future__ import annotations + import logging -from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer import typing import warnings from typing import Any, Dict, List, Optional, Text, Tuple, Type @@ -8,18 +8,18 @@ import numpy as np import rasa.shared.utils.io -import rasa.utils.io as io_utils from rasa.engine.graph import GraphComponent, ExecutionContext from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage -from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU from rasa.nlu.classifiers import LABEL_RANKING_LENGTH +from rasa.nlu.classifiers.classifier import IntentClassifier +from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer +from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU from rasa.shared.exceptions import RasaException from rasa.shared.nlu.constants import TEXT -from rasa.nlu.classifiers.classifier import IntentClassifier -from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.utils.tensorflow.constants import FEATURIZERS logger = logging.getLogger(__name__) @@ -266,14 +266,20 @@ def predict(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: def persist(self) -> None: """Persist this model into the passed directory.""" + import skops.io as sio + with self._model_storage.write_to(self._resource) as model_dir: file_name = self.__class__.__name__ - classifier_file_name = model_dir / f"{file_name}_classifier.pkl" - encoder_file_name = model_dir / f"{file_name}_encoder.pkl" + classifier_file_name = model_dir / f"{file_name}_classifier.skops" + encoder_file_name = model_dir / f"{file_name}_encoder.json" if self.clf and self.le: - io_utils.json_pickle(encoder_file_name, self.le.classes_) - io_utils.json_pickle(classifier_file_name, self.clf.best_estimator_) + # convert self.le.classes_ (numpy array of strings) to a list in order + # to use json dump + rasa.shared.utils.io.dump_obj_as_json_to_file( + encoder_file_name, list(self.le.classes_) + ) + sio.dump(self.clf.best_estimator_, classifier_file_name) @classmethod def load( @@ -286,21 +292,36 @@ def load( ) -> SklearnIntentClassifier: """Loads trained component (see parent class for full docstring).""" from sklearn.preprocessing import LabelEncoder + import skops.io as sio try: with model_storage.read_from(resource) as model_dir: file_name = cls.__name__ - classifier_file = model_dir / f"{file_name}_classifier.pkl" + classifier_file = model_dir / f"{file_name}_classifier.skops" if classifier_file.exists(): - classifier = io_utils.json_unpickle(classifier_file) + unknown_types = sio.get_untrusted_types(file=classifier_file) - encoder_file = model_dir / f"{file_name}_encoder.pkl" - classes = io_utils.json_unpickle(encoder_file) - encoder = LabelEncoder() - encoder.classes_ = classes + if unknown_types: + logger.error( + f"Untrusted types ({unknown_types}) found when " + f"loading {classifier_file}!" + ) + raise ValueError() + else: + classifier = sio.load(classifier_file, trusted=unknown_types) + + encoder_file = model_dir / f"{file_name}_encoder.json" + classes = rasa.shared.utils.io.read_json_file(encoder_file) - return cls(config, model_storage, resource, classifier, encoder) + encoder = LabelEncoder() + intent_classifier = cls( + config, model_storage, resource, classifier, encoder + ) + # convert list of strings (class labels) back to numpy array of + # strings + intent_classifier.transform_labels_str2num(classes) + return intent_classifier except ValueError: logger.debug( f"Failed to load '{cls.__name__}' from model storage. Resource " diff --git a/rasa/nlu/extractors/crf_entity_extractor.py b/rasa/nlu/extractors/crf_entity_extractor.py index 1332c250d55a..357d6044fec8 100644 --- a/rasa/nlu/extractors/crf_entity_extractor.py +++ b/rasa/nlu/extractors/crf_entity_extractor.py @@ -1,12 +1,12 @@ from __future__ import annotations -from collections import OrderedDict -from enum import Enum import logging import typing +from collections import OrderedDict +from enum import Enum +from typing import Any, Dict, List, Optional, Text, Tuple, Callable, Type import numpy as np -from typing import Any, Dict, List, Optional, Text, Tuple, Callable, Type import rasa.nlu.utils.bilou_utils as bilou_utils import rasa.shared.utils.io @@ -15,13 +15,12 @@ from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage +from rasa.nlu.constants import TOKENS_NAMES +from rasa.nlu.extractors.extractor import EntityExtractorMixin from rasa.nlu.test import determine_token_labels from rasa.nlu.tokenizers.spacy_tokenizer import POS_TAG_KEY -from rasa.nlu.extractors.extractor import EntityExtractorMixin from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message -from rasa.nlu.constants import TOKENS_NAMES +from rasa.shared.constants import DOCS_URL_COMPONENTS from rasa.shared.nlu.constants import ( TEXT, ENTITIES, @@ -32,7 +31,8 @@ SPLIT_ENTITIES_BY_COMMA, SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE, ) -from rasa.shared.constants import DOCS_URL_COMPONENTS +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.utils.tensorflow.constants import BILOU_FLAG, FEATURIZERS logger = logging.getLogger(__name__) @@ -41,6 +41,9 @@ from sklearn_crfsuite import CRF +CONFIG_FEATURES = "features" + + class CRFToken: def __init__( self, @@ -60,6 +63,29 @@ def __init__( self.entity_role_tag = entity_role_tag self.entity_group_tag = entity_group_tag + def to_dict(self) -> Dict[str, Any]: + return { + "text": self.text, + "pos_tag": self.pos_tag, + "pattern": self.pattern, + "dense_features": [str(x) for x in list(self.dense_features)], + "entity_tag": self.entity_tag, + "entity_role_tag": self.entity_role_tag, + "entity_group_tag": self.entity_group_tag, + } + + @classmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "CRFToken": + return cls( + data["text"], + data["pos_tag"], + data["pattern"], + np.array([float(x) for x in data["dense_features"]]), + data["entity_tag"], + data["entity_role_tag"], + data["entity_group_tag"], + ) + class CRFEntityExtractorOptions(str, Enum): """Features that can be used for the 'CRFEntityExtractor'.""" @@ -137,7 +163,7 @@ def get_default_config() -> Dict[Text, Any]: # "is the preceding token in title case?" # POS features require SpacyTokenizer # pattern feature require RegexFeaturizer - CRFEntityExtractor.CONFIG_FEATURES: [ + CONFIG_FEATURES: [ [ CRFEntityExtractorOptions.LOW, CRFEntityExtractorOptions.TITLE, @@ -200,7 +226,7 @@ def __init__( ) def _validate_configuration(self) -> None: - if len(self.component_config.get(self.CONFIG_FEATURES, [])) % 2 != 1: + if len(self.component_config.get(CONFIG_FEATURES, [])) % 2 != 1: raise ValueError( "Need an odd number of crf feature lists to have a center word." ) @@ -251,9 +277,11 @@ def train(self, training_data: TrainingData) -> Resource: ] dataset = [self._convert_to_crf_tokens(example) for example in entity_examples] - self._train_model(dataset) + self.entity_taggers = self.train_model( + dataset, self.component_config, self.crf_order + ) - self.persist() + self.persist(dataset) return self._resource @@ -299,7 +327,9 @@ def extract_entities(self, message: Message) -> List[Dict[Text, Any]]: if include_tag_features: self._add_tag_to_crf_token(crf_tokens, predictions) - features = self._crf_tokens_to_features(crf_tokens, include_tag_features) + features = self._crf_tokens_to_features( + crf_tokens, self.component_config, include_tag_features + ) predictions[tag_name] = entity_tagger.predict_marginals_single(features) # convert predictions into a list of tags and a list of confidences @@ -389,27 +419,25 @@ def load( **kwargs: Any, ) -> CRFEntityExtractor: """Loads trained component (see parent class for full docstring).""" - import joblib - try: - entity_taggers = OrderedDict() with model_storage.read_from(resource) as model_dir: - # We have to load in the same order as we persisted things as otherwise - # the predictions might be off - file_names = sorted(model_dir.glob("**/*.pkl")) - if not file_names: - logger.debug( - "Failed to load model for 'CRFEntityExtractor'. " - "Maybe you did not provide enough training data and " - "no model was trained." - ) - return cls(config, model_storage, resource) + dataset = rasa.shared.utils.io.read_json_file( + model_dir / "crf_dataset.json" + ) + crf_order = rasa.shared.utils.io.read_json_file( + model_dir / "crf_order.json" + ) + + dataset = [ + [CRFToken.create_from_dict(token_data) for token_data in sub_list] + for sub_list in dataset + ] - for file_name in file_names: - name = file_name.stem[1:] - entity_taggers[name] = joblib.load(file_name) + entity_taggers = cls.train_model(dataset, config, crf_order) - return cls(config, model_storage, resource, entity_taggers) + entity_extractor = cls(config, model_storage, resource, entity_taggers) + entity_extractor.crf_order = crf_order + return entity_extractor except ValueError: logger.warning( f"Failed to load {cls.__name__} from model storage. Resource " @@ -417,23 +445,29 @@ def load( ) return cls(config, model_storage, resource) - def persist(self) -> None: + def persist(self, dataset: List[List[CRFToken]]) -> None: """Persist this model into the passed directory.""" - import joblib - with self._model_storage.write_to(self._resource) as model_dir: - if self.entity_taggers: - for idx, (name, entity_tagger) in enumerate( - self.entity_taggers.items() - ): - model_file_name = model_dir / f"{idx}{name}.pkl" - joblib.dump(entity_tagger, model_file_name) + data_to_store = [ + [token.to_dict() for token in sub_list] for sub_list in dataset + ] + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_dir / "crf_dataset.json", data_to_store + ) + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_dir / "crf_order.json", self.crf_order + ) + + @classmethod def _crf_tokens_to_features( - self, crf_tokens: List[CRFToken], include_tag_features: bool = False + cls, + crf_tokens: List[CRFToken], + config: Dict[str, Any], + include_tag_features: bool = False, ) -> List[Dict[Text, Any]]: """Convert the list of tokens into discrete features.""" - configured_features = self.component_config[self.CONFIG_FEATURES] + configured_features = config[CONFIG_FEATURES] sentence_features = [] for token_idx in range(len(crf_tokens)): @@ -444,28 +478,31 @@ def _crf_tokens_to_features( half_window_size = window_size // 2 window_range = range(-half_window_size, half_window_size + 1) - token_features = self._create_features_for_token( + token_features = cls._create_features_for_token( crf_tokens, token_idx, half_window_size, window_range, include_tag_features, + config, ) sentence_features.append(token_features) return sentence_features + @classmethod def _create_features_for_token( - self, + cls, crf_tokens: List[CRFToken], token_idx: int, half_window_size: int, window_range: range, include_tag_features: bool, + config: Dict[str, Any], ) -> Dict[Text, Any]: """Convert a token into discrete features including words before and after.""" - configured_features = self.component_config[self.CONFIG_FEATURES] + configured_features = config[CONFIG_FEATURES] prefixes = [str(i) for i in window_range] token_features = {} @@ -505,13 +542,13 @@ def _create_features_for_token( # set in the training data, 'matched' is either 'True' or # 'False' depending on whether the token actually matches the # pattern or not - regex_patterns = self.function_dict[feature](token) + regex_patterns = cls.function_dict[feature](token) for pattern_name, matched in regex_patterns.items(): token_features[ f"{prefix}:{feature}:{pattern_name}" ] = matched else: - value = self.function_dict[feature](token) + value = cls.function_dict[feature](token) token_features[f"{prefix}:{feature}"] = value return token_features @@ -635,38 +672,46 @@ def _get_tags(self, message: Message) -> Dict[Text, List[Text]]: return tags - def _train_model(self, df_train: List[List[CRFToken]]) -> None: + @classmethod + def train_model( + cls, + df_train: List[List[CRFToken]], + config: Dict[str, Any], + crf_order: List[str], + ) -> OrderedDict[str, CRF]: """Train the crf tagger based on the training data.""" import sklearn_crfsuite - self.entity_taggers = OrderedDict() + entity_taggers = OrderedDict() - for tag_name in self.crf_order: + for tag_name in crf_order: logger.debug(f"Training CRF for '{tag_name}'.") # add entity tag features for second level CRFs include_tag_features = tag_name != ENTITY_ATTRIBUTE_TYPE X_train = ( - self._crf_tokens_to_features(sentence, include_tag_features) + cls._crf_tokens_to_features(sentence, config, include_tag_features) for sentence in df_train ) y_train = ( - self._crf_tokens_to_tags(sentence, tag_name) for sentence in df_train + cls._crf_tokens_to_tags(sentence, tag_name) for sentence in df_train ) entity_tagger = sklearn_crfsuite.CRF( algorithm="lbfgs", # coefficient for L1 penalty - c1=self.component_config["L1_c"], + c1=config["L1_c"], # coefficient for L2 penalty - c2=self.component_config["L2_c"], + c2=config["L2_c"], # stop earlier - max_iterations=self.component_config["max_iterations"], + max_iterations=config["max_iterations"], # include transitions that are possible, but not observed all_possible_transitions=True, ) entity_tagger.fit(X_train, y_train) - self.entity_taggers[tag_name] = entity_tagger + entity_taggers[tag_name] = entity_tagger logger.debug("Training finished.") + + return entity_taggers diff --git a/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py b/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py index 0c76b71fa6ea..98cecba9ca3e 100644 --- a/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py +++ b/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py @@ -1,29 +1,31 @@ from __future__ import annotations + import logging import re +from typing import Any, Dict, List, Optional, Text, Tuple, Set, Type, Union + +import numpy as np import scipy.sparse -from typing import Any, Dict, List, Optional, Text, Tuple, Set, Type -from rasa.nlu.tokenizers.tokenizer import Tokenizer +from sklearn.feature_extraction.text import CountVectorizer import rasa.shared.utils.io from rasa.engine.graph import GraphComponent, ExecutionContext from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage -from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer -from rasa.nlu.utils.spacy_utils import SpacyModel -from rasa.shared.constants import DOCS_URL_COMPONENTS -import rasa.utils.io as io_utils -from sklearn.feature_extraction.text import CountVectorizer -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message -from rasa.shared.exceptions import RasaException, FileIOException from rasa.nlu.constants import ( TOKENS_NAMES, MESSAGE_ATTRIBUTES, DENSE_FEATURIZABLE_ATTRIBUTES, ) +from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer +from rasa.nlu.tokenizers.tokenizer import Tokenizer +from rasa.nlu.utils.spacy_utils import SpacyModel +from rasa.shared.constants import DOCS_URL_COMPONENTS +from rasa.shared.exceptions import RasaException, FileIOException from rasa.shared.nlu.constants import TEXT, INTENT, INTENT_RESPONSE_KEY, ACTION_NAME +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData BUFFER_SLOTS_PREFIX = "buf_" @@ -686,6 +688,31 @@ def _is_any_model_trained( """Check if any model got trained.""" return any(value is not None for value in attribute_vocabularies.values()) + @staticmethod + def convert_vocab( + vocab: Dict[str, Union[int, Optional[Dict[str, int]]]], to_int: bool + ) -> Dict[str, Union[None, int, np.int64, Dict[str, Union[int, np.int64]]]]: + """Converts numpy integers in the vocabulary to Python integers.""" + + def convert_value(value: int) -> Union[int, np.int64]: + """Helper function to convert a single value based on to_int flag.""" + return int(value) if to_int else np.int64(value) + + result_dict: Dict[ + str, Union[None, int, np.int64, Dict[str, Union[int, np.int64]]] + ] = {} + for key, sub_dict in vocab.items(): + if isinstance(sub_dict, int): + result_dict[key] = convert_value(sub_dict) + elif not sub_dict: + result_dict[key] = None + else: + result_dict[key] = { + sub_key: convert_value(value) for sub_key, value in sub_dict.items() + } + + return result_dict + def persist(self) -> None: """Persist this model into the passed directory. @@ -699,17 +726,18 @@ def persist(self) -> None: attribute_vocabularies = self._collect_vectorizer_vocabularies() if self._is_any_model_trained(attribute_vocabularies): # Definitely need to persist some vocabularies - featurizer_file = model_dir / "vocabularies.pkl" + featurizer_file = model_dir / "vocabularies.json" # Only persist vocabulary from one attribute if `use_shared_vocab`. # Can be loaded and distributed to all attributes. - vocab = ( + loaded_vocab = ( attribute_vocabularies[TEXT] if self.use_shared_vocab else attribute_vocabularies ) + vocab = self.convert_vocab(loaded_vocab, to_int=True) - io_utils.json_pickle(featurizer_file, vocab) + rasa.shared.utils.io.dump_obj_as_json_to_file(featurizer_file, vocab) # Dump OOV words separately as they might have been modified during # training @@ -784,8 +812,9 @@ def load( """Loads trained component (see parent class for full docstring).""" try: with model_storage.read_from(resource) as model_dir: - featurizer_file = model_dir / "vocabularies.pkl" - vocabulary = io_utils.json_unpickle(featurizer_file) + featurizer_file = model_dir / "vocabularies.json" + vocabulary = rasa.shared.utils.io.read_json_file(featurizer_file) + vocabulary = cls.convert_vocab(vocabulary, to_int=False) share_vocabulary = config["use_shared_vocab"] diff --git a/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py b/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py index 92312197755a..2c4ee3928348 100644 --- a/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py +++ b/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py @@ -1,9 +1,7 @@ from __future__ import annotations + import logging from collections import OrderedDict - -import scipy.sparse -import numpy as np from typing import ( Any, Dict, @@ -17,30 +15,34 @@ Union, ) +import numpy as np +import scipy.sparse + +import rasa.shared.utils.io +import rasa.utils.io from rasa.engine.graph import ExecutionContext, GraphComponent from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage +from rasa.nlu.constants import TOKENS_NAMES +from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer from rasa.nlu.tokenizers.spacy_tokenizer import POS_TAG_KEY, SpacyTokenizer from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer -from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer -from rasa.nlu.constants import TOKENS_NAMES from rasa.shared.constants import DOCS_URL_COMPONENTS -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message -from rasa.shared.nlu.constants import TEXT from rasa.shared.exceptions import InvalidConfigException -import rasa.shared.utils.io -import rasa.utils.io +from rasa.shared.nlu.constants import TEXT +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData logger = logging.getLogger(__name__) - END_OF_SENTENCE = "EOS" BEGIN_OF_SENTENCE = "BOS" FEATURES = "features" +SEPERATOR = "###" + @DefaultV1Recipe.register( DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=True @@ -72,7 +74,7 @@ class LexicalSyntacticFeaturizer(SparseFeaturizer, GraphComponent): of the token at position `t+1`. """ - FILENAME_FEATURE_TO_IDX_DICT = "feature_to_idx_dict.pkl" + FILENAME_FEATURE_TO_IDX_DICT = "feature_to_idx_dict.json" # NOTE: "suffix5" of the token "is" will be "is". Hence, when combining multiple # prefixes, short words will be represented/encoded repeatedly. @@ -489,6 +491,32 @@ def create( """Creates a new untrained component (see parent class for full docstring).""" return cls(config, model_storage, resource, execution_context) + @staticmethod + def _restructure_feature_to_idx_dict( + loaded_data: Dict[str, Dict[str, int]], + ) -> Dict[Tuple[int, str], Dict[str, int]]: + """Reconstructs the feature to idx dict. + + When storing the feature_to_idx_dict to disk, we need to convert the tuple (key) + into a string to be able to store it via json. When loading the data + we need to reconstruct the tuple from the stored string. + + Args: + loaded_data: The loaded feature to idx dict from file. + + Returns: + The reconstructed feature_to_idx_dict + """ + feature_to_idx_dict = {} + for tuple_string, feature_value in loaded_data.items(): + # Example of tuple_string: "1###low" + index, feature_name = tuple_string.split(SEPERATOR) + + feature_key = (int(index), feature_name) + feature_to_idx_dict[feature_key] = feature_value + + return feature_to_idx_dict + @classmethod def load( cls, @@ -501,10 +529,13 @@ def load( """Loads trained component (see parent class for full docstring).""" try: with model_storage.read_from(resource) as model_path: - feature_to_idx_dict = rasa.utils.io.json_unpickle( + loaded_data = rasa.shared.utils.io.read_json_file( model_path / cls.FILENAME_FEATURE_TO_IDX_DICT, - encode_non_string_keys=True, ) + + # convert the key back into tuple + feature_to_idx_dict = cls._restructure_feature_to_idx_dict(loaded_data) + return cls( config=config, model_storage=model_storage, @@ -529,9 +560,13 @@ def persist(self) -> None: if not self._feature_to_idx_dict: return None + # as we cannot dump tuples, convert the tuple into a string + restructured_feature_dict = { + f"{k[0]}{SEPERATOR}{k[1]}": v for k, v in self._feature_to_idx_dict.items() + } + with self._model_storage.write_to(self._resource) as model_path: - rasa.utils.io.json_pickle( + rasa.shared.utils.io.dump_obj_as_json_to_file( model_path / self.FILENAME_FEATURE_TO_IDX_DICT, - self._feature_to_idx_dict, - encode_non_string_keys=True, + restructured_feature_dict, ) diff --git a/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py b/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py index fee53fd5b4f6..baed7f2c4852 100644 --- a/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py +++ b/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py @@ -1,11 +1,13 @@ from __future__ import annotations + import logging import re from typing import Any, Dict, List, Optional, Text, Tuple, Type + import numpy as np import scipy.sparse -from rasa.nlu.tokenizers.tokenizer import Tokenizer +from rasa.nlu.tokenizers.tokenizer import Tokenizer import rasa.shared.utils.io import rasa.utils.io import rasa.nlu.utils.pattern_utils as pattern_utils @@ -240,7 +242,7 @@ def load( try: with model_storage.read_from(resource) as model_dir: - patterns_file_name = model_dir / "patterns.pkl" + patterns_file_name = model_dir / "patterns.json" known_patterns = rasa.shared.utils.io.read_json_file(patterns_file_name) except (ValueError, FileNotFoundError): logger.warning( @@ -258,7 +260,7 @@ def load( def _persist(self) -> None: with self._model_storage.write_to(self._resource) as model_dir: - regex_file = model_dir / "patterns.pkl" + regex_file = model_dir / "patterns.json" rasa.shared.utils.io.dump_obj_as_json_to_file( regex_file, self.known_patterns ) diff --git a/rasa/shared/nlu/training_data/features.py b/rasa/shared/nlu/training_data/features.py index d981c1563bb0..0c5553df20c8 100644 --- a/rasa/shared/nlu/training_data/features.py +++ b/rasa/shared/nlu/training_data/features.py @@ -1,15 +1,133 @@ from __future__ import annotations -from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set + import itertools +from dataclasses import dataclass +from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set import numpy as np import scipy.sparse +from safetensors.numpy import save_file, load_file -import rasa.shared.utils.io import rasa.shared.nlu.training_data.util +import rasa.shared.utils.io from rasa.shared.nlu.constants import FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE +@dataclass +class FeatureMetadata: + data_type: str + attribute: str + origin: Union[str, List[str]] + is_sparse: bool + shape: tuple + safetensors_key: str + + +def save_features( + features_dict: Dict[Text, List[Features]], file_name: str +) -> Dict[str, Any]: + """Save a dictionary of Features lists to disk using safetensors. + + Args: + features_dict: Dictionary mapping strings to lists of Features objects + file_name: File to save the features to + + Returns: + The metadata to reconstruct the features. + """ + # All tensors are stored in a single safetensors file + tensors_to_save = {} + # Metadata will be stored separately + metadata = {} + + for key, features_list in features_dict.items(): + feature_metadata_list = [] + + for idx, feature in enumerate(features_list): + # Create a unique key for this tensor in the safetensors file + safetensors_key = f"{key}_{idx}" + + # Convert sparse matrices to dense if needed + if feature.is_sparse(): + # For sparse matrices, use the COO format + coo = feature.features.tocoo() # type:ignore[union-attr] + # Save data, row indices and col indices separately + tensors_to_save[f"{safetensors_key}_data"] = coo.data + tensors_to_save[f"{safetensors_key}_row"] = coo.row + tensors_to_save[f"{safetensors_key}_col"] = coo.col + else: + tensors_to_save[safetensors_key] = feature.features + + # Store metadata + metadata_item = FeatureMetadata( + data_type=feature.type, + attribute=feature.attribute, + origin=feature.origin, + is_sparse=feature.is_sparse(), + shape=feature.features.shape, + safetensors_key=safetensors_key, + ) + feature_metadata_list.append(vars(metadata_item)) + + metadata[key] = feature_metadata_list + + # Save tensors + save_file(tensors_to_save, file_name) + + return metadata + + +def load_features( + filename: str, metadata: Dict[str, Any] +) -> Dict[Text, List[Features]]: + """Load Features dictionary from disk. + + Args: + filename: File name of the safetensors file. + metadata: Metadata to reconstruct the features. + + Returns: + Dictionary mapping strings to lists of Features objects + """ + # Load tensors + tensors = load_file(filename) + + # Reconstruct the features dictionary + features_dict: Dict[Text, List[Features]] = {} + + for key, feature_metadata_list in metadata.items(): + features_list = [] + + for meta in feature_metadata_list: + safetensors_key = meta["safetensors_key"] + + if meta["is_sparse"]: + # Reconstruct sparse matrix from COO format + data = tensors[f"{safetensors_key}_data"] + row = tensors[f"{safetensors_key}_row"] + col = tensors[f"{safetensors_key}_col"] + + features_matrix = scipy.sparse.coo_matrix( + (data, (row, col)), shape=tuple(meta["shape"]) + ).tocsr() # Convert back to CSR format + else: + features_matrix = tensors[safetensors_key] + + # Reconstruct Features object + features = Features( + features=features_matrix, + feature_type=meta["data_type"], + attribute=meta["attribute"], + origin=meta["origin"], + ) + + features_list.append(features) + + features_dict[key] = features_list + + return features_dict + + class Features: """Stores the features produced by any featurizer.""" diff --git a/rasa/shared/utils/io.py b/rasa/shared/utils/io.py index de2b1bc28f6c..3b13b5c18063 100644 --- a/rasa/shared/utils/io.py +++ b/rasa/shared/utils/io.py @@ -12,6 +12,7 @@ import warnings import random import string + import portalocker from ruamel import yaml as yaml diff --git a/rasa/utils/common.py b/rasa/utils/common.py index 27b754664317..f99f54042a1f 100644 --- a/rasa/utils/common.py +++ b/rasa/utils/common.py @@ -8,6 +8,7 @@ import tempfile import warnings from pathlib import Path +from socket import SOCK_DGRAM, SOCK_STREAM from types import TracebackType from typing import ( Any, @@ -24,8 +25,9 @@ Tuple, ) -from socket import SOCK_DGRAM, SOCK_STREAM import numpy as np + +import rasa.shared.utils.io import rasa.utils.io from rasa.constants import ( DEFAULT_LOG_LEVEL_LIBRARIES, @@ -36,7 +38,6 @@ ) from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL, TCP_PROTOCOL from rasa.shared.exceptions import RasaException -import rasa.shared.utils.io logger = logging.getLogger(__name__) @@ -153,7 +154,7 @@ def configure_logging_from_file(logging_config_file: Text) -> None: try: logging.config.dictConfig(logging_config_dict) except (ValueError, TypeError, AttributeError, ImportError) as e: - logging.debug( + logger.debug( f"The logging config file {logging_config_file} could not " f"be applied because it failed validation against " f"the built-in Python logging schema. " diff --git a/rasa/utils/io.py b/rasa/utils/io.py index 3388ef98b049..da0800c61f0a 100644 --- a/rasa/utils/io.py +++ b/rasa/utils/io.py @@ -2,13 +2,13 @@ import filecmp import logging import os -import pickle +import re import tempfile import warnings -import re from asyncio import AbstractEventLoop from pathlib import Path -from typing import Text, Any, Union, List, Type, Callable, TYPE_CHECKING, Pattern +from typing import Text, Any, List, Type, Callable, TYPE_CHECKING, Pattern + from typing_extensions import Protocol import rasa.shared.constants @@ -81,29 +81,6 @@ def enable_async_loop_debugging( return event_loop -def pickle_dump(filename: Union[Text, Path], obj: Any) -> None: - """Saves object to file. - - Args: - filename: the filename to save the object to - obj: the object to store - """ - with open(filename, "wb") as f: - pickle.dump(obj, f) - - -def pickle_load(filename: Union[Text, Path]) -> Any: - """Loads an object from a file. - - Args: - filename: the filename to load the object from - - Returns: the loaded object - """ - with open(filename, "rb") as f: - return pickle.load(f) - - def create_temporary_file(data: Any, suffix: Text = "", mode: Text = "w+") -> Text: """Creates a tempfile.NamedTemporaryFile object for data.""" encoding = None if "b" in mode else rasa.shared.utils.io.DEFAULT_ENCODING @@ -124,7 +101,6 @@ def create_temporary_directory() -> Text: def create_path(file_path: Text) -> None: """Makes sure all directories in the 'file_path' exists.""" - parent_dir = os.path.dirname(os.path.abspath(file_path)) if not os.path.exists(parent_dir): os.makedirs(parent_dir) @@ -160,8 +136,8 @@ def create_validator( function: Callable[[Text], bool], error_message: Text ) -> Type["Validator"]: """Helper method to create `Validator` classes from callable functions. Should be - removed when questionary supports `Validator` objects.""" - + removed when questionary supports `Validator` objects. + """ from prompt_toolkit.validation import Validator, ValidationError from prompt_toolkit.document import Document @@ -175,48 +151,6 @@ def validate(document: Document) -> None: return FunctionValidator -def json_unpickle( - file_name: Union[Text, Path], encode_non_string_keys: bool = False -) -> Any: - """Unpickle an object from file using json. - - Args: - file_name: the file to load the object from - encode_non_string_keys: If set to `True` then jsonpickle will encode non-string - dictionary keys instead of coercing them into strings via `repr()`. - - Returns: the object - """ - import jsonpickle.ext.numpy as jsonpickle_numpy - import jsonpickle - - jsonpickle_numpy.register_handlers() - - file_content = rasa.shared.utils.io.read_file(file_name) - return jsonpickle.loads(file_content, keys=encode_non_string_keys) - - -def json_pickle( - file_name: Union[Text, Path], obj: Any, encode_non_string_keys: bool = False -) -> None: - """Pickle an object to a file using json. - - Args: - file_name: the file to store the object to - obj: the object to store - encode_non_string_keys: If set to `True` then jsonpickle will encode non-string - dictionary keys instead of coercing them into strings via `repr()`. - """ - import jsonpickle.ext.numpy as jsonpickle_numpy - import jsonpickle - - jsonpickle_numpy.register_handlers() - - rasa.shared.utils.io.write_text_file( - jsonpickle.dumps(obj, keys=encode_non_string_keys), file_name - ) - - def get_emoji_regex() -> Pattern: """Returns regex to identify emojis.""" return re.compile( diff --git a/rasa/utils/tensorflow/feature_array.py b/rasa/utils/tensorflow/feature_array.py new file mode 100644 index 000000000000..9af50e3ceb66 --- /dev/null +++ b/rasa/utils/tensorflow/feature_array.py @@ -0,0 +1,370 @@ +from typing import Dict, Any, List, Tuple, Optional, Union + +import numpy as np +import scipy.sparse +from safetensors.numpy import load_file +from safetensors.numpy import save_file + +import rasa.shared.utils.io + + +def _recursive_serialize( + array: Any, prefix: str, data_dict: Dict[str, Any], metadata: List[Dict[str, Any]] +) -> None: + """Recursively serialize arrays and matrices for high dimensional data.""" + if isinstance(array, np.ndarray) and array.ndim <= 2: + data_key = f"{prefix}_array" + data_dict[data_key] = array + metadata.append({"type": "dense", "key": data_key, "shape": array.shape}) + + elif isinstance(array, list) and all([isinstance(v, float) for v in array]): + data_key = f"{prefix}_list" + data_dict[data_key] = np.array(array, dtype=np.float32) + metadata.append({"type": "list", "key": data_key}) + + elif isinstance(array, list) and all([isinstance(v, int) for v in array]): + data_key = f"{prefix}_list" + data_dict[data_key] = np.array(array, dtype=np.int64) + metadata.append({"type": "list", "key": data_key}) + + elif isinstance(array, scipy.sparse.spmatrix): + data_key_data = f"{prefix}_data" + data_key_row = f"{prefix}_row" + data_key_col = f"{prefix}_col" + array = array.tocoo() + data_dict.update( + { + data_key_data: array.data, + data_key_row: array.row, + data_key_col: array.col, + } + ) + metadata.append({"type": "sparse", "key": prefix, "shape": array.shape}) + + elif isinstance(array, list) or isinstance(array, np.ndarray): + group_metadata = {"type": "group", "subcomponents": []} + for idx, item in enumerate(array): + new_prefix = f"{prefix}_{idx}" + _recursive_serialize( + item, new_prefix, data_dict, group_metadata["subcomponents"] + ) + metadata.append(group_metadata) + + +def _serialize_nested_data( + nested_data: Dict[str, Dict[str, List["FeatureArray"]]], + prefix: str, + data_dict: Dict[str, np.ndarray], + metadata: List[Dict[str, Union[str, List]]], +) -> None: + """Handle serialization across dictionary and list levels.""" + for outer_key, inner_dict in nested_data.items(): + inner_metadata = {"key": outer_key, "components": []} + + for inner_key, feature_arrays in inner_dict.items(): + array_metadata = { + "key": inner_key, + "number_of_dimensions": feature_arrays[0].number_of_dimensions, + "features": [], + } + + for idx, feature_array in enumerate(feature_arrays): + feature_prefix = f"{prefix}_{outer_key}_{inner_key}_{idx}" + _recursive_serialize( + feature_array.tolist(), + feature_prefix, + data_dict, + array_metadata["features"], + ) + + inner_metadata["components"].append( # type:ignore[attr-defined] + array_metadata + ) + + metadata.append(inner_metadata) + + +def serialize_nested_feature_arrays( + nested_feature_array: Dict[str, Dict[str, List["FeatureArray"]]], + data_filename: str, + metadata_filename: str, +) -> None: + data_dict: Dict[str, np.ndarray] = {} + metadata: List[Dict[str, Union[str, List]]] = [] + + _serialize_nested_data(nested_feature_array, "component", data_dict, metadata) + + # Save serialized data and metadata + save_file(data_dict, data_filename) + rasa.shared.utils.io.dump_obj_as_json_to_file(metadata_filename, metadata) + + +def _recursive_deserialize( + metadata: List[Dict[str, Any]], data: Dict[str, Any] +) -> List[Any]: + """Recursively deserialize arrays and matrices for high dimensional data.""" + result = [] + + for item in metadata: + if item["type"] == "dense": + key = item["key"] + array = np.asarray(data[key]).reshape(item["shape"]) + result.append(array) + + elif item["type"] == "list": + key = item["key"] + result.append(list(data[key])) + + elif item["type"] == "sparse": + data_vals = data[f"{item['key']}_data"] + row_vals = data[f"{item['key']}_row"] + col_vals = data[f"{item['key']}_col"] + sparse_matrix = scipy.sparse.coo_matrix( + (data_vals, (row_vals, col_vals)), shape=item["shape"] + ) + result.append(sparse_matrix) + elif item["type"] == "group": + sublist = _recursive_deserialize(item["subcomponents"], data) + result.append(sublist) + + return result + + +def _deserialize_nested_data( + metadata: List[Dict[str, Any]], data_dict: Dict[str, Any] +) -> Dict[str, Dict[str, List["FeatureArray"]]]: + """Handle deserialization across all dictionary and list levels.""" + result: Dict[str, Dict[str, List["FeatureArray"]]] = {} + + for outer_item in metadata: + outer_key = outer_item["key"] + result[outer_key] = {} + + for inner_item in outer_item["components"]: + inner_key = inner_item["key"] + feature_arrays = [] + + # Reconstruct the list of FeatureArrays + for feature_item in inner_item["features"]: + # Reconstruct the list of FeatureArrays + feature_array_data = _recursive_deserialize([feature_item], data_dict) + # Prepare the input for the FeatureArray; + # ensure it is np.ndarray compatible + input_array = np.array(feature_array_data[0], dtype=object) + feature_array = FeatureArray( + input_array, inner_item["number_of_dimensions"] + ) + feature_arrays.append(feature_array) + + result[outer_key][inner_key] = feature_arrays + + return result + + +def deserialize_nested_feature_arrays( + data_filename: str, metadata_filename: str +) -> Dict[str, Dict[str, List["FeatureArray"]]]: + metadata = rasa.shared.utils.io.read_json_file(metadata_filename) + data_dict = load_file(data_filename) + + return _deserialize_nested_data(metadata, data_dict) + + +class FeatureArray(np.ndarray): + """Stores any kind of features ready to be used by a RasaModel. + + Next to the input numpy array of features, it also received the number of + dimensions of the features. + As our features can have 1 to 4 dimensions we might have different number of numpy + arrays stacked. The number of dimensions helps us to figure out how to handle this + particular feature array. Also, it is automatically determined whether the feature + array is sparse or not and the number of units is determined as well. + + Subclassing np.array: https://numpy.org/doc/stable/user/basics.subclassing.html + """ + + def __new__( + cls, input_array: np.ndarray, number_of_dimensions: int + ) -> "FeatureArray": + """Create and return a new object. See help(type) for accurate signature.""" + FeatureArray._validate_number_of_dimensions(number_of_dimensions, input_array) + + feature_array = np.asarray(input_array).view(cls) + + if number_of_dimensions <= 2: + feature_array.units = input_array.shape[-1] + feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) + elif number_of_dimensions == 3: + feature_array.units = input_array[0].shape[-1] + feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) + elif number_of_dimensions == 4: + feature_array.units = input_array[0][0].shape[-1] + feature_array.is_sparse = isinstance( + input_array[0][0], scipy.sparse.spmatrix + ) + else: + raise ValueError( + f"Number of dimensions '{number_of_dimensions}' currently not " + f"supported." + ) + + feature_array.number_of_dimensions = number_of_dimensions + + return feature_array + + def __init__( + self, input_array: Any, number_of_dimensions: int, **kwargs: Any + ) -> None: + """Initialize. FeatureArray. + + Needed in order to avoid 'Invalid keyword argument number_of_dimensions + to function FeatureArray.__init__ ' + Args: + input_array: the array that contains features + number_of_dimensions: number of dimensions in input_array + """ + super().__init__(**kwargs) + self.number_of_dimensions = number_of_dimensions + + def __array_finalize__(self, obj: Optional[np.ndarray]) -> None: + """This method is called when the system allocates a new array from obj. + + Args: + obj: A subclass (subtype) of ndarray. + """ + if obj is None: + return + + self.units = getattr(obj, "units", None) + self.number_of_dimensions = getattr( + obj, "number_of_dimensions", None + ) # type: ignore[assignment] + self.is_sparse = getattr(obj, "is_sparse", None) + + default_attributes = { + "units": self.units, + "number_of_dimensions": self.number_of_dimensions, + "is_spare": self.is_sparse, + } + self.__dict__.update(default_attributes) + + # pytype: disable=attribute-error + def __array_ufunc__( + self, ufunc: Any, method: str, *inputs: Any, **kwargs: Any + ) -> Any: + """Overwrite this method as we are subclassing numpy array. + + Args: + ufunc: The ufunc object that was called. + method: A string indicating which Ufunc method was called + (one of "__call__", "reduce", "reduceat", "accumulate", "outer", + "inner"). + *inputs: A tuple of the input arguments to the ufunc. + **kwargs: Any additional arguments + + Returns: + The result of the operation. + """ + f = { + "reduce": ufunc.reduce, + "accumulate": ufunc.accumulate, + "reduceat": ufunc.reduceat, + "outer": ufunc.outer, + "at": ufunc.at, + "__call__": ufunc, + } + # convert the inputs to np.ndarray to prevent recursion, call the function, + # then cast it back as FeatureArray + output = FeatureArray( + f[method](*(i.view(np.ndarray) for i in inputs), **kwargs), + number_of_dimensions=kwargs["number_of_dimensions"], + ) + output.__dict__ = self.__dict__ # carry forward attributes + return output + + def __reduce__(self) -> Tuple[Any, Any, Any]: + """Needed in order to pickle this object. + + Returns: + A tuple. + """ + pickled_state = super(FeatureArray, self).__reduce__() + if isinstance(pickled_state, str): + raise TypeError("np array __reduce__ returned string instead of tuple.") + new_state = pickled_state[2] + ( + self.number_of_dimensions, + self.is_sparse, + self.units, + ) + return pickled_state[0], pickled_state[1], new_state + + def __setstate__(self, state: Any, **kwargs: Any) -> None: + """Sets the state. + + Args: + state: The state argument must be a sequence that contains the following + elements version, shape, dtype, isFortan, rawdata. + **kwargs: Any additional parameter + """ + # Needed in order to load the object + self.number_of_dimensions = state[-3] + self.is_sparse = state[-2] + self.units = state[-1] + super(FeatureArray, self).__setstate__(state[0:-3], **kwargs) + + # pytype: enable=attribute-error + + @staticmethod + def _validate_number_of_dimensions( + number_of_dimensions: int, input_array: np.ndarray + ) -> None: + """Validates if the input array has given number of dimensions. + + Args: + number_of_dimensions: number of dimensions + input_array: input array + + Raises: ValueError in case the dimensions do not match + """ + # when loading the feature arrays from disk, the shape represents + # the correct number of dimensions + if len(input_array.shape) == number_of_dimensions: + return + + _sub_array = input_array + dim = 0 + # Go number_of_dimensions into the given input_array + for i in range(1, number_of_dimensions + 1): + _sub_array = _sub_array[0] + if isinstance(_sub_array, scipy.sparse.spmatrix): + dim = i + break + if isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: + # sequence dimension is 0, we are dealing with "fake" features + dim = i + break + + # If the resulting sub_array is sparse, the remaining number of dimensions + # should be at least 2 + if isinstance(_sub_array, scipy.sparse.spmatrix): + if dim > 2: + raise ValueError( + f"Given number of dimensions '{number_of_dimensions}' does not " + f"match dimensions of given input array: {input_array}." + ) + elif isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: + # sequence dimension is 0, we are dealing with "fake" features, + # but they should be of dim 2 + if dim > 2: + raise ValueError( + f"Given number of dimensions '{number_of_dimensions}' does not " + f"match dimensions of given input array: {input_array}." + ) + # If the resulting sub_array is dense, the sub_array should be a single number + elif not np.issubdtype(type(_sub_array), np.integer) and not isinstance( + _sub_array, (np.float32, np.float64) + ): + raise ValueError( + f"Given number of dimensions '{number_of_dimensions}' does not match " + f"dimensions of given input array: {input_array}." + ) diff --git a/rasa/utils/tensorflow/model_data.py b/rasa/utils/tensorflow/model_data.py index 128ff6cbd575..393756972305 100644 --- a/rasa/utils/tensorflow/model_data.py +++ b/rasa/utils/tensorflow/model_data.py @@ -20,6 +20,8 @@ import scipy.sparse from sklearn.model_selection import train_test_split +from rasa.utils.tensorflow.feature_array import FeatureArray + logger = logging.getLogger(__name__) @@ -37,199 +39,6 @@ def ragged_array_to_ndarray(ragged_array: Iterable[np.ndarray]) -> np.ndarray: return np.array(ragged_array, dtype=object) -class FeatureArray(np.ndarray): - """Stores any kind of features ready to be used by a RasaModel. - - Next to the input numpy array of features, it also received the number of - dimensions of the features. - As our features can have 1 to 4 dimensions we might have different number of numpy - arrays stacked. The number of dimensions helps us to figure out how to handle this - particular feature array. Also, it is automatically determined whether the feature - array is sparse or not and the number of units is determined as well. - - Subclassing np.array: https://numpy.org/doc/stable/user/basics.subclassing.html - """ - - def __new__( - cls, input_array: np.ndarray, number_of_dimensions: int - ) -> "FeatureArray": - """Create and return a new object. See help(type) for accurate signature.""" - FeatureArray._validate_number_of_dimensions(number_of_dimensions, input_array) - - feature_array = np.asarray(input_array).view(cls) - - if number_of_dimensions <= 2: - feature_array.units = input_array.shape[-1] - feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) - elif number_of_dimensions == 3: - feature_array.units = input_array[0].shape[-1] - feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) - elif number_of_dimensions == 4: - feature_array.units = input_array[0][0].shape[-1] - feature_array.is_sparse = isinstance( - input_array[0][0], scipy.sparse.spmatrix - ) - else: - raise ValueError( - f"Number of dimensions '{number_of_dimensions}' currently not " - f"supported." - ) - - feature_array.number_of_dimensions = number_of_dimensions - - return feature_array - - def __init__( - self, input_array: Any, number_of_dimensions: int, **kwargs: Any - ) -> None: - """Initialize. FeatureArray. - - Needed in order to avoid 'Invalid keyword argument number_of_dimensions - to function FeatureArray.__init__ ' - Args: - input_array: the array that contains features - number_of_dimensions: number of dimensions in input_array - """ - super().__init__(**kwargs) - self.number_of_dimensions = number_of_dimensions - - def __array_finalize__(self, obj: Optional[np.ndarray]) -> None: - """This method is called when the system allocates a new array from obj. - - Args: - obj: A subclass (subtype) of ndarray. - """ - if obj is None: - return - - self.units = getattr(obj, "units", None) - self.number_of_dimensions = getattr(obj, "number_of_dimensions", None) # type: ignore[assignment] # noqa:E501 - self.is_sparse = getattr(obj, "is_sparse", None) - - default_attributes = { - "units": self.units, - "number_of_dimensions": self.number_of_dimensions, - "is_spare": self.is_sparse, - } - self.__dict__.update(default_attributes) - - # pytype: disable=attribute-error - def __array_ufunc__( - self, ufunc: Any, method: Text, *inputs: Any, **kwargs: Any - ) -> Any: - """Overwrite this method as we are subclassing numpy array. - - Args: - ufunc: The ufunc object that was called. - method: A string indicating which Ufunc method was called - (one of "__call__", "reduce", "reduceat", "accumulate", "outer", - "inner"). - *inputs: A tuple of the input arguments to the ufunc. - **kwargs: Any additional arguments - - Returns: - The result of the operation. - """ - f = { - "reduce": ufunc.reduce, - "accumulate": ufunc.accumulate, - "reduceat": ufunc.reduceat, - "outer": ufunc.outer, - "at": ufunc.at, - "__call__": ufunc, - } - # convert the inputs to np.ndarray to prevent recursion, call the function, - # then cast it back as FeatureArray - output = FeatureArray( - f[method](*(i.view(np.ndarray) for i in inputs), **kwargs), - number_of_dimensions=kwargs["number_of_dimensions"], - ) - output.__dict__ = self.__dict__ # carry forward attributes - return output - - def __reduce__(self) -> Tuple[Any, Any, Any]: - """Needed in order to pickle this object. - - Returns: - A tuple. - """ - pickled_state = super(FeatureArray, self).__reduce__() - if isinstance(pickled_state, str): - raise TypeError("np array __reduce__ returned string instead of tuple.") - new_state = pickled_state[2] + ( - self.number_of_dimensions, - self.is_sparse, - self.units, - ) - return pickled_state[0], pickled_state[1], new_state - - def __setstate__(self, state: Any, **kwargs: Any) -> None: - """Sets the state. - - Args: - state: The state argument must be a sequence that contains the following - elements version, shape, dtype, isFortan, rawdata. - **kwargs: Any additional parameter - """ - # Needed in order to load the object - self.number_of_dimensions = state[-3] - self.is_sparse = state[-2] - self.units = state[-1] - super(FeatureArray, self).__setstate__(state[0:-3], **kwargs) - - # pytype: enable=attribute-error - - @staticmethod - def _validate_number_of_dimensions( - number_of_dimensions: int, input_array: np.ndarray - ) -> None: - """Validates if the the input array has given number of dimensions. - - Args: - number_of_dimensions: number of dimensions - input_array: input array - - Raises: ValueError in case the dimensions do not match - """ - _sub_array = input_array - dim = 0 - # Go number_of_dimensions into the given input_array - for i in range(1, number_of_dimensions + 1): - _sub_array = _sub_array[0] - if isinstance(_sub_array, scipy.sparse.spmatrix): - dim = i - break - if isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: - # sequence dimension is 0, we are dealing with "fake" features - dim = i - break - - # If the resulting sub_array is sparse, the remaining number of dimensions - # should be at least 2 - if isinstance(_sub_array, scipy.sparse.spmatrix): - if dim > 2: - raise ValueError( - f"Given number of dimensions '{number_of_dimensions}' does not " - f"match dimensions of given input array: {input_array}." - ) - elif isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: - # sequence dimension is 0, we are dealing with "fake" features, - # but they should be of dim 2 - if dim > 2: - raise ValueError( - f"Given number of dimensions '{number_of_dimensions}' does not " - f"match dimensions of given input array: {input_array}." - ) - # If the resulting sub_array is dense, the sub_array should be a single number - elif not np.issubdtype(type(_sub_array), np.integer) and not isinstance( - _sub_array, (np.float32, np.float64) - ): - raise ValueError( - f"Given number of dimensions '{number_of_dimensions}' does not match " - f"dimensions of given input array: {input_array}." - ) - - class FeatureSignature(NamedTuple): """Signature of feature arrays. @@ -270,8 +79,7 @@ def __init__( label_sub_key: Optional[Text] = None, data: Optional[Data] = None, ) -> None: - """ - Initializes the RasaModelData object. + """Initializes the RasaModelData object. Args: label_key: the key of a label used for balancing, etc. diff --git a/scripts/ping_slack_about_package_release.sh b/scripts/ping_slack_about_package_release.sh old mode 100755 new mode 100644 index ef97ead7a178..b8ee68b4b6cd --- a/scripts/ping_slack_about_package_release.sh +++ b/scripts/ping_slack_about_package_release.sh @@ -7,4 +7,3 @@ if [[ ${GITHUB_TAG} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then --data "{\"text\":\"💥 New *Rasa Open Source* version ${GITHUB_TAG} has been released! https://github.com/RasaHQ/rasa/releases/tag/${GITHUB_TAG}\"}" \ "https://hooks.slack.com/services/T0GHWFTS8/BMTQQL47K/${SLACK_WEBHOOK_TOKEN}" fi - diff --git a/tests/core/featurizers/test_tracker_featurizer.py b/tests/core/featurizers/test_tracker_featurizer.py index 99ffea6e9641..20a0a08f558d 100644 --- a/tests/core/featurizers/test_tracker_featurizer.py +++ b/tests/core/featurizers/test_tracker_featurizer.py @@ -34,7 +34,25 @@ def test_fail_to_load_non_existent_featurizer(): assert TrackerFeaturizer.load("non_existent_class") is None -def test_persist_and_load_tracker_featurizer(tmp_path: Text, moodbot_domain: Domain): +def test_persist_and_load_full_dialogue_tracker_featurizer( + tmp_path: Text, moodbot_domain: Domain +): + state_featurizer = SingleStateFeaturizer() + state_featurizer.prepare_for_training(moodbot_domain) + tracker_featurizer = FullDialogueTrackerFeaturizer(state_featurizer) + + tracker_featurizer.persist(tmp_path) + + loaded_tracker_featurizer = TrackerFeaturizer.load(tmp_path) + + assert loaded_tracker_featurizer is not None + assert loaded_tracker_featurizer.state_featurizer is not None + assert loaded_tracker_featurizer.to_dict() == tracker_featurizer.to_dict() + + +def test_persist_and_load_max_history_tracker_featurizer( + tmp_path: Text, moodbot_domain: Domain +): state_featurizer = SingleStateFeaturizer() state_featurizer.prepare_for_training(moodbot_domain) tracker_featurizer = MaxHistoryTrackerFeaturizer(state_featurizer) @@ -45,6 +63,23 @@ def test_persist_and_load_tracker_featurizer(tmp_path: Text, moodbot_domain: Dom assert loaded_tracker_featurizer is not None assert loaded_tracker_featurizer.state_featurizer is not None + assert loaded_tracker_featurizer.to_dict() == tracker_featurizer.to_dict() + + +def test_persist_and_load_intent_max_history_tracker_featurizer( + tmp_path: Text, moodbot_domain: Domain +): + state_featurizer = SingleStateFeaturizer() + state_featurizer.prepare_for_training(moodbot_domain) + tracker_featurizer = IntentMaxHistoryTrackerFeaturizer(state_featurizer) + + tracker_featurizer.persist(tmp_path) + + loaded_tracker_featurizer = TrackerFeaturizer.load(tmp_path) + + assert loaded_tracker_featurizer is not None + assert loaded_tracker_featurizer.state_featurizer is not None + assert loaded_tracker_featurizer.to_dict() == tracker_featurizer.to_dict() def test_convert_action_labels_to_ids(domain: Domain): @@ -127,7 +162,6 @@ def compare_featurized_states( """Compares two lists of featurized states and returns True if they are identical and False otherwise. """ - if len(states1) != len(states2): return False diff --git a/tests/nlu/extractors/test_crf_entity_extractor.py b/tests/nlu/extractors/test_crf_entity_extractor.py index b2342ab30fb7..bf095385017b 100644 --- a/tests/nlu/extractors/test_crf_entity_extractor.py +++ b/tests/nlu/extractors/test_crf_entity_extractor.py @@ -1,23 +1,25 @@ import copy from typing import Dict, Text, List, Any, Callable +import numpy as np import pytest from rasa.engine.graph import ExecutionContext from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage +from rasa.nlu.constants import SPACY_DOCS +from rasa.nlu.extractors.crf_entity_extractor import ( + CRFEntityExtractor, + CRFEntityExtractorOptions, + CRFToken, +) from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer from rasa.nlu.tokenizers.spacy_tokenizer import SpacyTokenizer -from rasa.nlu.constants import SPACY_DOCS from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer from rasa.nlu.utils.spacy_utils import SpacyModel, SpacyNLP from rasa.shared.importers.rasa import RasaFileImporter from rasa.shared.nlu.constants import TEXT, ENTITIES from rasa.shared.nlu.training_data.message import Message -from rasa.nlu.extractors.crf_entity_extractor import ( - CRFEntityExtractor, - CRFEntityExtractorOptions, -) @pytest.fixture() @@ -204,7 +206,7 @@ def test_crf_use_dense_features( spacy_featurizer.process([message]) text_data = crf_extractor._convert_to_crf_tokens(message) - features = crf_extractor._crf_tokens_to_features(text_data) + features = crf_extractor._crf_tokens_to_features(text_data, component_config) assert "0:text_dense_features" in features[0] dense_features, _ = message.get_dense_features(TEXT, []) @@ -249,3 +251,110 @@ def test_process_unfeaturized_input( assert processed_message.get(TEXT) == message_text assert processed_message.get(ENTITIES) == [] + + +@pytest.fixture +def sample_data(): + return { + "text": "apple", + "pos_tag": "NOUN", + "pattern": {"length": 5, "is_capitalized": False}, + "dense_features": np.array([0.1, 0.2, 0.3]), + "entity_tag": "B-FOOD", + "entity_role_tag": "INGREDIENT", + "entity_group_tag": "ITEM", + } + + +@pytest.fixture +def sample_token(sample_data): + return CRFToken( + sample_data["text"], + sample_data["pos_tag"], + sample_data["pattern"], + sample_data["dense_features"], + sample_data["entity_tag"], + sample_data["entity_role_tag"], + sample_data["entity_group_tag"], + ) + + +def test_crf_token_to_dict(sample_data, sample_token): + token_dict = sample_token.to_dict() + + assert token_dict["text"] == sample_data["text"] + assert token_dict["pos_tag"] == sample_data["pos_tag"] + assert token_dict["pattern"] == sample_data["pattern"] + assert token_dict["dense_features"] == [ + str(x) for x in sample_data["dense_features"] + ] + assert token_dict["entity_tag"] == sample_data["entity_tag"] + assert token_dict["entity_role_tag"] == sample_data["entity_role_tag"] + assert token_dict["entity_group_tag"] == sample_data["entity_group_tag"] + + +def test_crf_token_create_from_dict(sample_data): + dict_data = { + "text": sample_data["text"], + "pos_tag": sample_data["pos_tag"], + "pattern": sample_data["pattern"], + "dense_features": [str(x) for x in sample_data["dense_features"]], + "entity_tag": sample_data["entity_tag"], + "entity_role_tag": sample_data["entity_role_tag"], + "entity_group_tag": sample_data["entity_group_tag"], + } + + token = CRFToken.create_from_dict(dict_data) + + assert token.text == sample_data["text"] + assert token.pos_tag == sample_data["pos_tag"] + assert token.pattern == sample_data["pattern"] + np.testing.assert_array_equal(token.dense_features, sample_data["dense_features"]) + assert token.entity_tag == sample_data["entity_tag"] + assert token.entity_role_tag == sample_data["entity_role_tag"] + assert token.entity_group_tag == sample_data["entity_group_tag"] + + +def test_crf_token_roundtrip_conversion(sample_token): + token_dict = sample_token.to_dict() + new_token = CRFToken.create_from_dict(token_dict) + + assert new_token.text == sample_token.text + assert new_token.pos_tag == sample_token.pos_tag + assert new_token.pattern == sample_token.pattern + np.testing.assert_array_equal(new_token.dense_features, sample_token.dense_features) + assert new_token.entity_tag == sample_token.entity_tag + assert new_token.entity_role_tag == sample_token.entity_role_tag + assert new_token.entity_group_tag == sample_token.entity_group_tag + + +def test_crf_token_empty_dense_features(sample_data): + sample_data["dense_features"] = np.array([]) + token = CRFToken( + sample_data["text"], + sample_data["pos_tag"], + sample_data["pattern"], + sample_data["dense_features"], + sample_data["entity_tag"], + sample_data["entity_role_tag"], + sample_data["entity_group_tag"], + ) + token_dict = token.to_dict() + new_token = CRFToken.create_from_dict(token_dict) + np.testing.assert_array_equal(new_token.dense_features, np.array([])) + + +def test_crf_token_empty_pattern(sample_data): + sample_data["pattern"] = {} + token = CRFToken( + sample_data["text"], + sample_data["pos_tag"], + sample_data["pattern"], + sample_data["dense_features"], + sample_data["entity_tag"], + sample_data["entity_role_tag"], + sample_data["entity_group_tag"], + ) + token_dict = token.to_dict() + new_token = CRFToken.create_from_dict(token_dict) + assert new_token.pattern == {} diff --git a/tests/shared/nlu/training_data/test_features.py b/tests/shared/nlu/training_data/test_features.py index bd0f29fa046b..457e9648f28f 100644 --- a/tests/shared/nlu/training_data/test_features.py +++ b/tests/shared/nlu/training_data/test_features.py @@ -1,17 +1,56 @@ import itertools +import os +import tempfile +from pathlib import Path from typing import Optional, Text, List, Dict, Tuple, Any import numpy as np import pytest import scipy.sparse -from rasa.shared.nlu.training_data.features import Features from rasa.shared.nlu.constants import ( FEATURE_TYPE_SENTENCE, FEATURE_TYPE_SEQUENCE, TEXT, INTENT, ) +from rasa.shared.nlu.training_data.features import ( + Features, + FeatureMetadata, + save_features, + load_features, +) + + +@pytest.fixture +def safe_tensors_tmp_file() -> str: + with tempfile.NamedTemporaryFile(delete=False, suffix=".safetensors") as f: + yield f.name + os.unlink(f.name) + + +@pytest.fixture +def dense_features() -> Features: + features_matrix = np.array([[1, 2, 3], [4, 5, 6]]) + return Features( + features=features_matrix, + feature_type="dense", + attribute="test", + origin="test_origin", + ) + + +@pytest.fixture +def sparse_features() -> Features: + features_matrix = scipy.sparse.csr_matrix( + ([1, 2, 3], ([0, 1, 1], [0, 1, 2])), shape=(2, 3) + ) + return Features( + features=features_matrix, + feature_type="sparse", + attribute="test", + origin="test_origin", + ) @pytest.mark.parametrize( @@ -181,6 +220,7 @@ def _generate_feature_list_and_modifications( instantiate `Features` that differ from the aforementioned list of features in exactly one property (i.e. type, sequence length (if the given `type` is sequence type only), attribute, origin) + Args: is_sparse: whether all features should be sparse type: the type to be used for all features @@ -190,7 +230,6 @@ def _generate_feature_list_and_modifications( a list of kwargs dictionaries that can be used to instantiate `Features` that differ from the aforementioned list of features in exactly one property """ - seq_len = 3 first_dim = 1 if type == FEATURE_TYPE_SENTENCE else 3 @@ -467,3 +506,179 @@ def test_reduce_raises_if_combining_different_origins_or_attributes(differ: Text expected_origin = ["origin-1"] with pytest.raises(ValueError, match=message): Features.reduce(features_list, expected_origins=expected_origin) + + +def test_feature_metadata(): + metadata = FeatureMetadata( + data_type="dense", + attribute="text", + origin="test", + is_sparse=False, + shape=(10, 5), + safetensors_key="key_0", + ) + + assert metadata.data_type == "dense" + assert metadata.attribute == "text" + assert metadata.origin == "test" + assert not metadata.is_sparse + assert metadata.shape == (10, 5) + assert metadata.safetensors_key == "key_0" + + +def test_save_dense_features(safe_tensors_tmp_file: str, dense_features: Features): + features_dict = {"test_key": [dense_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "test_key" in metadata + assert len(metadata["test_key"]) == 1 + assert metadata["test_key"][0]["data_type"] == "dense" + assert metadata["test_key"][0]["shape"] == (2, 3) + assert not metadata["test_key"][0]["is_sparse"] + assert Path(safe_tensors_tmp_file).exists() + + +def test_save_sparse_features(safe_tensors_tmp_file: str, sparse_features: Features): + features_dict = {"test_key": [sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "test_key" in metadata + assert len(metadata["test_key"]) == 1 + assert metadata["test_key"][0]["data_type"] == "sparse" + assert metadata["test_key"][0]["shape"] == (2, 3) + assert metadata["test_key"][0]["is_sparse"] + assert Path(safe_tensors_tmp_file).exists() + + +def test_save_mixed_features( + safe_tensors_tmp_file: str, dense_features: Features, sparse_features: Features +): + features_dict = {"test_key": [dense_features, sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "test_key" in metadata + assert len(metadata["test_key"]) == 2 + assert metadata["test_key"][0]["data_type"] == "dense" + assert metadata["test_key"][1]["data_type"] == "sparse" + assert Path(safe_tensors_tmp_file).exists() + + +def test_save_multiple_keys( + safe_tensors_tmp_file: str, dense_features: Features, sparse_features: Features +): + features_dict = {"dense_key": [dense_features], "sparse_key": [sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "dense_key" in metadata + assert "sparse_key" in metadata + assert metadata["dense_key"][0]["data_type"] == "dense" + assert metadata["sparse_key"][0]["data_type"] == "sparse" + assert Path(safe_tensors_tmp_file).exists() + + +@pytest.fixture +def setup_save_load( + safe_tensors_tmp_file: str, dense_features: Features, sparse_features: Features +) -> Tuple[str, Dict[str, Any], Dict[str, List[Features]]]: + features_dict = {"dense_key": [dense_features], "sparse_key": [sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + return safe_tensors_tmp_file, metadata, features_dict + + +def test_load_dense_features( + setup_save_load: Tuple[str, Dict[str, Any], Dict[str, List[Features]]], +): + temp_file, metadata, original_dict = setup_save_load + loaded_dict = load_features(temp_file, metadata) + + assert "dense_key" in loaded_dict + assert len(loaded_dict["dense_key"]) == 1 + assert not loaded_dict["dense_key"][0].is_sparse() + np.testing.assert_array_equal( + loaded_dict["dense_key"][0].features, original_dict["dense_key"][0].features + ) + + +def test_load_sparse_features( + setup_save_load: Tuple[str, Dict[str, Any], Dict[str, List[Features]]], +): + temp_file, metadata, original_dict = setup_save_load + loaded_dict = load_features(temp_file, metadata) + + assert "sparse_key" in loaded_dict + assert len(loaded_dict["sparse_key"]) == 1 + assert loaded_dict["sparse_key"][0].is_sparse() + assert ( + loaded_dict["sparse_key"][0].features != original_dict["sparse_key"][0].features + ).nnz == 0 + + +def test_load_preserves_metadata( + setup_save_load: Tuple[str, Dict[str, Any], Dict[str, List[Features]]], +): + temp_file, metadata, original_dict = setup_save_load + loaded_dict = load_features(temp_file, metadata) + + for key in original_dict: + for orig_feat, loaded_feat in zip(original_dict[key], loaded_dict[key]): + assert orig_feat.type == loaded_feat.type + assert orig_feat.attribute == loaded_feat.attribute + assert orig_feat.origin == loaded_feat.origin + + +def test_load_nonexistent_file(): + with pytest.raises(Exception): + load_features("nonexistent.safetensors", {}) + + +def test_load_invalid_metadata(safe_tensors_tmp_file: str, dense_features: Features): + features_dict = {"test_key": [dense_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + # Corrupt the metadata + metadata["test_key"][0]["safetensors_key"] = "invalid_key" + + with pytest.raises(Exception): + load_features(safe_tensors_tmp_file, metadata) + + +def test_end_to_end(safe_tensors_tmp_file: str): + # Create test data + dense_matrix = np.array([[1, 2], [3, 4]]) + sparse_matrix = scipy.sparse.csr_matrix(([1, 2], ([0, 1], [0, 1])), shape=(2, 2)) + + features_dict = { + "group1": [ + Features(dense_matrix, "dense", "test1", "origin1"), + Features(sparse_matrix, "sparse", "test2", "origin2"), + ], + "group2": [ + Features(dense_matrix * 2, "dense", "test3", ["origin3", "origin4"]) + ], + } + + # Save features + metadata = save_features(features_dict, safe_tensors_tmp_file) + + # Load features + loaded_dict = load_features(safe_tensors_tmp_file, metadata) + + # Verify structure + assert set(loaded_dict.keys()) == set(features_dict.keys()) + assert len(loaded_dict["group1"]) == 2 + assert len(loaded_dict["group2"]) == 1 + + # Verify dense features + np.testing.assert_array_equal( + loaded_dict["group1"][0].features, features_dict["group1"][0].features + ) + + # Verify sparse features + assert ( + loaded_dict["group1"][1].features != features_dict["group1"][1].features + ).nnz == 0 + + # Verify metadata + assert loaded_dict["group1"][0].type == "dense" + assert loaded_dict["group1"][1].type == "sparse" + assert loaded_dict["group2"][0].origin == ["origin3", "origin4"] diff --git a/tests/utils/tensorflow/test_feature_array.py b/tests/utils/tensorflow/test_feature_array.py new file mode 100644 index 000000000000..95be7ba993a6 --- /dev/null +++ b/tests/utils/tensorflow/test_feature_array.py @@ -0,0 +1,197 @@ +import numpy as np +import scipy.sparse + +from rasa.utils.tensorflow.feature_array import ( + _recursive_serialize, + _serialize_nested_data, + _deserialize_nested_data, +) +from rasa.utils.tensorflow.model_data import RasaModelData + + +def test_recursive_serialize_numpy_array(): + data_dict = {} + metadata = [] + + _recursive_serialize(np.array([1, 2, 3]), "test_array", data_dict, metadata) + assert "test_array_array" in data_dict + assert metadata[0] == {"type": "dense", "key": "test_array_array", "shape": (3,)} + + +def test_recursive_serialize_floats(): + data_dict = {} + metadata = [] + + _recursive_serialize([1.0, 2.0, 3.0], "test_list", data_dict, metadata) + assert "test_list_list" in data_dict + assert metadata[0] == {"type": "list", "key": "test_list_list"} + + +def test_recursive_serialize_sparse_matrix(): + data_dict = {} + metadata = [] + + sparse_matrix = scipy.sparse.random(5, 10, density=0.1, format="coo") + _recursive_serialize(sparse_matrix, "test_sparse", data_dict, metadata) + assert "test_sparse_data" in data_dict + assert "test_sparse_row" in data_dict + assert "test_sparse_col" in data_dict + assert metadata[0] == { + "type": "sparse", + "key": "test_sparse", + "shape": sparse_matrix.shape, + } + + +def test_serialize_model_data(model_data: RasaModelData): + nested_data = model_data.data + + data_dict = {} + metadata = [] + _serialize_nested_data(nested_data, "component", data_dict, metadata) + + assert len(metadata) == 5 + + assert metadata[0]["key"] == "text" + assert len(metadata[0]["components"]) == 1 + assert metadata[0]["components"][0]["key"] == "sentence" + assert metadata[0]["components"][0]["number_of_dimensions"] == 3 + assert len(metadata[0]["components"][0]["features"]) == 2 + assert metadata[0]["components"][0]["features"][0]["type"] == "group" + assert len(metadata[0]["components"][0]["features"][0]["subcomponents"]) == 5 + assert ( + metadata[0]["components"][0]["features"][0]["subcomponents"][0]["type"] + == "dense" + ) + assert metadata[0]["components"][0]["features"][0]["subcomponents"][0]["shape"] == ( + 5, + 14, + ) + assert metadata[0]["components"][0]["features"][1]["type"] == "group" + assert len(metadata[0]["components"][0]["features"][1]["subcomponents"]) == 5 + assert ( + metadata[0]["components"][0]["features"][1]["subcomponents"][0]["type"] + == "sparse" + ) + assert metadata[0]["components"][0]["features"][1]["subcomponents"][0]["shape"] == ( + 5, + 10, + ) + + assert metadata[3]["key"] == "label" + assert len(metadata[3]["components"]) == 1 + assert metadata[3]["components"][0]["key"] == "ids" + assert metadata[3]["components"][0]["number_of_dimensions"] == 1 + assert metadata[3]["components"][0]["features"][0]["type"] == "list" + assert ( + metadata[3]["components"][0]["features"][0]["key"] + == "component_label_ids_0_list" + ) + + assert len(data_dict) == 87 + assert ( + data_dict["component_label_ids_0_list"] + == model_data.data["label"]["ids"][0].view(np.ndarray) + ).all() + + +def test_serialize_and_deserialize_model_data(model_data: RasaModelData): + actual_data = model_data.data + + data_dict = {} + metadata = [] + _serialize_nested_data(actual_data, "component", data_dict, metadata) + + loaded_data = _deserialize_nested_data(metadata, data_dict) + + assert len(actual_data) == len(loaded_data) + + assert len(actual_data["text"]["sentence"]) == len(loaded_data["text"]["sentence"]) + + # text.sentence has a dimension of 3 + assert len(actual_data["text"]["sentence"][0]) == len( + loaded_data["text"]["sentence"][0] + ) + # assert that the numpy arrays of the actual and loaded data in + # text.sentence are the same + for i in range(0, 5): + assert ( + actual_data["text"]["sentence"][0][i] + == loaded_data["text"]["sentence"][0][i] + ).all() + assert len(actual_data["text"]["sentence"][1]) == len( + loaded_data["text"]["sentence"][1] + ) + # assert that the sparse matrices of the actual and loaded data in + # text.sentence are the same + for i in range(0, 5): + assert ( + actual_data["text"]["sentence"][1][i] + == loaded_data["text"]["sentence"][1][i] + ).data.all() + + # action_text.sequence has a dimension of 4 + assert len(actual_data["action_text"]["sequence"]) == len( + loaded_data["action_text"]["sequence"] + ) + assert len(actual_data["action_text"]["sequence"][0]) == len( + loaded_data["action_text"]["sequence"][0] + ) + # assert that the sparse matrices of the actual and loaded data in + # action_text.sequence are the same + for i in range(0, 5): + for j in range(0, len(actual_data["action_text"]["sequence"][0][i])): + assert ( + actual_data["action_text"]["sequence"][0][i][j] + == loaded_data["action_text"]["sequence"][0][i][j] + ).data.all() + assert len(actual_data["action_text"]["sequence"][1]) == len( + loaded_data["action_text"]["sequence"][1] + ) + # assert that the numpy array of the actual and loaded data in + # action_text.sequence are the same + for i in range(0, 5): + for j in range(0, len(actual_data["action_text"]["sequence"][1][i])): + assert ( + actual_data["action_text"]["sequence"][1][i][j] + == loaded_data["action_text"]["sequence"][1][i][j] + ).all() + + # dialogue.sentence has a dimension of 3 + assert len(actual_data["dialogue"]["sentence"]) == len( + loaded_data["dialogue"]["sentence"] + ) + assert len(actual_data["dialogue"]["sentence"][0]) == len( + loaded_data["dialogue"]["sentence"][0] + ) + # assert that the numpy array of the actual and loaded data in + # dialogue.sentence are the same + for i in range(0, 5): + assert ( + actual_data["dialogue"]["sentence"][0][i] + == loaded_data["dialogue"]["sentence"][0][i] + ).all() + + # label.ids has a dimension of 4 + assert len(actual_data["label"]["ids"]) == len(loaded_data["label"]["ids"]) + # assert that the numpy array of the actual and loaded data in + # label.ids are the same + assert ( + actual_data["label"]["ids"][0].view(np.ndarray) + == loaded_data["label"]["ids"][0].view(np.ndarray) + ).all() + + # entities.tag_ids has a dimension of 3 + assert len(actual_data["entities"]["tag_ids"]) == len( + loaded_data["entities"]["tag_ids"] + ) + assert len(actual_data["entities"]["tag_ids"][0]) == len( + loaded_data["entities"]["tag_ids"][0] + ) + # assert that the numpy array of the actual and loaded data in + # entities.tag_ids are the same + for i in range(0, 5): + assert ( + actual_data["entities"]["tag_ids"][0][i] + == loaded_data["entities"]["tag_ids"][0][i] + ).all() diff --git a/tests/utils/test_io.py b/tests/utils/test_io.py index ac788373a422..3042b113aaf3 100644 --- a/tests/utils/test_io.py +++ b/tests/utils/test_io.py @@ -1,5 +1,3 @@ -from pathlib import Path -from typing import Dict, Text import pytest from _pytest.tmpdir import TempPathFactory from prompt_toolkit.document import Document @@ -71,22 +69,6 @@ def is_valid(user_input) -> None: assert e.value.message == error_message -@pytest.mark.parametrize( - "input,kwargs,expected", - [ - ({(1, 2): 3}, {}, {repr((1, 2)): 3}), - ({(1, 2): 3}, {"encode_non_string_keys": True}, {(1, 2): 3}), - ], -) -def test_write_and_load_dict_via_jsonpickle( - tmp_path: Path, input: Dict, kwargs: Dict[Text, bool], expected: Dict -): - file_name = tmp_path / "bla.pkl" - rasa.utils.io.json_pickle(file_name=file_name, obj=input, **kwargs) - loaded = rasa.utils.io.json_unpickle(file_name=file_name, **kwargs) - assert loaded == expected - - def test_empty_directories_are_equal(tmp_path_factory: TempPathFactory): dir1 = tmp_path_factory.mktemp("dir1") dir2 = tmp_path_factory.mktemp("dir2") From 2d9c7aa5543bf68825487aee1f48d2d7ffa68a51 Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Mon, 13 Jan 2025 13:48:40 +0100 Subject: [PATCH 36/41] prepared release of version 3.6.21 --- CHANGELOG.mdx | 25 +++++++++++++++++++++++++ changelog/1424.bugfix.md | 19 ------------------- pyproject.toml | 3 +-- rasa/version.py | 2 +- 4 files changed, 27 insertions(+), 22 deletions(-) delete mode 100644 changelog/1424.bugfix.md diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index a97640d0772b..d1290c72e19e 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,6 +16,31 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> +## [3.6.21] - 2025-01-13 + +Rasa 3.6.21 (2025-01-13) +### Bugfixes +- [#1424](https://github.com/rasahq/rasa/issues/1424): Replace `pickle` and `joblib` with safer alternatives, e.g. `json`, `safetensors`, and `skops`, for + serializing components. + + **Note**: This is a model breaking change. Please retrain your model. + + If you have a custom component that inherits from one of the components listed below and modified the `persist` or + `load` method, make sure to update your code. Please contact us in case you encounter any problems. + + Affected components: + + - `CountVectorFeaturizer` + - `LexicalSyntacticFeaturizer` + - `LogisticRegressionClassifier` + - `SklearnIntentClassifier` + - `DIETClassifier` + - `CRFEntityExtractor` + - `TrackerFeaturizer` + - `TEDPolicy` + - `UnexpectedIntentTEDPolicy` + + ## [3.6.20] - 2024-04-18 Rasa 3.6.20 (2024-04-18) diff --git a/changelog/1424.bugfix.md b/changelog/1424.bugfix.md deleted file mode 100644 index d71648f6da3c..000000000000 --- a/changelog/1424.bugfix.md +++ /dev/null @@ -1,19 +0,0 @@ -Replace `pickle` and `joblib` with safer alternatives, e.g. `json`, `safetensors`, and `skops`, for -serializing components. - -**Note**: This is a model breaking change. Please retrain your model. - -If you have a custom component that inherits from one of the components listed below and modified the `persist` or -`load` method, make sure to update your code. Please contact us in case you encounter any problems. - -Affected components: - -- `CountVectorFeaturizer` -- `LexicalSyntacticFeaturizer` -- `LogisticRegressionClassifier` -- `SklearnIntentClassifier` -- `DIETClassifier` -- `CRFEntityExtractor` -- `TrackerFeaturizer` -- `TEDPolicy` -- `UnexpectedIntentTEDPolicy` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d2a48c527aab..c37977e46891 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.20" +version = "3.6.21" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] @@ -156,7 +156,6 @@ certifi = ">=2023.7.22" cryptography = ">=41.0.7" skops = "0.9.0" safetensors = "~0.4.5" - [[tool.poetry.dependencies.tensorflow-io-gcs-filesystem]] version = "==0.31" markers = "sys_platform == 'win32'" diff --git a/rasa/version.py b/rasa/version.py index 2e1490a6a8c3..3b9621033a5d 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.20" +__version__ = "3.6.21" From 1d6f66c23c59d01c259ddb21ae285b682273e5db Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Mon, 13 Jan 2025 15:32:08 +0100 Subject: [PATCH 37/41] Update minimum compatible version to 3.6.21 --- rasa/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/constants.py b/rasa/constants.py index ef973613bc96..ba883c66b0e0 100644 --- a/rasa/constants.py +++ b/rasa/constants.py @@ -18,7 +18,7 @@ CONFIG_TELEMETRY_ENABLED = "enabled" CONFIG_TELEMETRY_DATE = "date" -MINIMUM_COMPATIBLE_VERSION = "3.5.0" +MINIMUM_COMPATIBLE_VERSION = "3.6.21" GLOBAL_USER_CONFIG_PATH = os.path.expanduser("~/.config/rasa/global.yml") From 140f901f53d68260db0a1e437ae26e75d4514282 Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Tue, 14 Jan 2025 11:06:56 +0100 Subject: [PATCH 38/41] Update version to 3.6.21 in test_read_long_resource_names_windows --- tests/engine/storage/test_local_model_storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/engine/storage/test_local_model_storage.py b/tests/engine/storage/test_local_model_storage.py index 9324c399c948..789aa0638568 100644 --- a/tests/engine/storage/test_local_model_storage.py +++ b/tests/engine/storage/test_local_model_storage.py @@ -2,21 +2,21 @@ import uuid from datetime import datetime from pathlib import Path -from tarsafe import TarSafe import freezegun import pytest from _pytest.monkeypatch import MonkeyPatch from _pytest.tmpdir import TempPathFactory +from tarsafe import TarSafe import rasa.shared.utils.io -from rasa.engine.graph import SchemaNode, GraphSchema, GraphModelConfiguration +from rasa.engine.graph import GraphModelConfiguration, GraphSchema, SchemaNode from rasa.engine.storage.local_model_storage import ( - LocalModelStorage, MODEL_ARCHIVE_METADATA_FILE, + LocalModelStorage, ) -from rasa.engine.storage.storage import ModelStorage, ModelMetadata from rasa.engine.storage.resource import Resource +from rasa.engine.storage.storage import ModelMetadata, ModelStorage from rasa.exceptions import UnsupportedModelVersionError from rasa.shared.core.domain import Domain from rasa.shared.data import TrainingType @@ -96,7 +96,7 @@ def test_read_long_resource_names_windows( domain: Domain, ): model_dir = tmp_path_factory.mktemp("model_dir") - version = "3.5.0" + version = "3.6.21" # full path length > 260 chars # but each component of the path needs to be below 255 chars From f28c69e4672cd978745f5a73396c79c26017d612 Mon Sep 17 00:00:00 2001 From: Diana Muse Date: Tue, 26 Aug 2025 17:46:38 +0100 Subject: [PATCH 39/41] feat: update references of bitnami image to binamilegacy (#13137) * feat: update references of bitnami image to binamilegacy * update `actions/cache` to version 4.2.3, as existing version is deprecated and causing all jobs using it to error out --------- Co-authored-by: Github Actions --- .devcontainer/docker-compose.yml | 4 +-- .github/workflows/ci-docs-tests.yml | 8 ++--- .github/workflows/ci-github-actions.yml | 2 +- .../ci-model-regression-on-schedule.yml | 2 +- .github/workflows/ci-model-regression.yml | 4 +-- .github/workflows/continous-integration.yml | 34 +++++++++---------- .github/workflows/documentation.yml | 10 +++--- .github/workflows/security-scans.yml | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index d870d0d16a2e..6224abc328cd 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -29,7 +29,7 @@ services: # user: vscode db: - image: "bitnami/postgresql:11.15.0" + image: "bitnamilegacy/postgresql:11.15.0" restart: unless-stopped volumes: - postgres-data:/bitnami/postgresql @@ -47,7 +47,7 @@ services: redis: restart: unless-stopped - image: "bitnami/redis:6.2.7" + image: "bitnamilegacy/redis:6.2.7" environment: REDIS_PASSWORD: "redis" expose: diff --git a/.github/workflows/ci-docs-tests.yml b/.github/workflows/ci-docs-tests.yml index 366fe1cdb0f8..ba6ee10b0b6d 100644 --- a/.github/workflows/ci-docs-tests.yml +++ b/.github/workflows/ci-docs-tests.yml @@ -59,7 +59,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -78,7 +78,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Load Yarn Cached Packages ⬇ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} @@ -130,7 +130,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -149,7 +149,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Load Yarn Cached Packages ⬇ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} diff --git a/.github/workflows/ci-github-actions.yml b/.github/workflows/ci-github-actions.yml index 161c0b027b0f..2883f6f64e31 100644 --- a/.github/workflows/ci-github-actions.yml +++ b/.github/workflows/ci-github-actions.yml @@ -45,7 +45,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} diff --git a/.github/workflows/ci-model-regression-on-schedule.yml b/.github/workflows/ci-model-regression-on-schedule.yml index 5ac4710cf288..37247cb7fc99 100644 --- a/.github/workflows/ci-model-regression-on-schedule.yml +++ b/.github/workflows/ci-model-regression-on-schedule.yml @@ -297,7 +297,7 @@ jobs: poetry-version: ${{ env.POETRY_VERSION }} - name: Load Poetry Cached Libraries ⬇ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' with: path: ~/.cache/pypoetry/virtualenvs diff --git a/.github/workflows/ci-model-regression.yml b/.github/workflows/ci-model-regression.yml index 3e65fcec2ebd..04f6674b9235 100644 --- a/.github/workflows/ci-model-regression.yml +++ b/.github/workflows/ci-model-regression.yml @@ -385,7 +385,7 @@ jobs: poetry-version: ${{ env.POETRY_VERSION }} - name: Load Poetry Cached Libraries ⬇ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' with: path: ~/.cache/pypoetry/virtualenvs @@ -630,7 +630,7 @@ jobs: poetry-version: ${{ env.POETRY_VERSION }} - name: Load Poetry Cached Libraries ⬇ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' with: path: ~/.cache/pypoetry/virtualenvs diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index b6f2bd8d38f3..c5cf83fcd0ba 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -142,7 +142,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -247,12 +247,12 @@ jobs: - test-full-model-training - test-other-unit-tests - test-performance - os: [ubuntu-24.04, windows-2019] + os: [ubuntu-24.04, windows-2022] python-version: [3.8, 3.9, "3.10"] steps: - name: Run DataDog Agent - if: needs.changes.outputs.backend == 'true' && (matrix.os != 'windows-2019' || contains(github.event.pull_request.labels.*.name, 'tools:datadog-windows')) + if: needs.changes.outputs.backend == 'true' && (matrix.os != 'windows-2022' || contains(github.event.pull_request.labels.*.name, 'tools:datadog-windows')) run: | docker run --name dd_agent -p 8126:8126 -d -e "DD_API_KEY=${{ secrets.DD_API_KEY }}" -e "DD_INSIDE_CI=true" -e "DD_HOSTNAME=none" -e "DD_SITE=datadoghq.eu" -e GITHUB_ACTIONS=true -e CI=true datadog/agent:latest docker ps --all --filter name=dd_agent --filter status=running --no-trunc --format "{{.ID}} {{.Status}}" @@ -284,7 +284,7 @@ jobs: installer-parallel: true - name: Install poetry (Windows) 🦄 - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec #v9 with: poetry-version: ${{ env.POETRY_VERSION }} @@ -299,7 +299,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} @@ -335,7 +335,7 @@ jobs: make prepare-tests-ubuntu - name: Install Dependencies (Windows) 📦 - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' # Restoring cache doesn't work properly on Windows due to symlinks. # We create symlinks for spacy models, that's why we need to clean them up # before caching the dependencies directory. @@ -361,7 +361,7 @@ jobs: run: pip install pytest-github-actions-annotate-failures - name: Disable "LongPathsEnabled" option on Windows - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-2022' # On Windows laptops, a default preset prevents path names from being longer than # 260 characters. Some of our users can't enable this setting due to company policies. # We implemented a fix for model storage. The Windows container in GitHub @@ -376,7 +376,7 @@ jobs: run: poetry run pip install -U 'ddtrace<2.0.0' - name: Install ddtrace on Windows - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' run: | .\.venv\Scripts\activate py -m pip install -U 'ddtrace<2.0.0' @@ -416,12 +416,12 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, windows-2019] + os: [ubuntu-24.04, windows-2022] python-version: [3.8, 3.9, "3.10"] steps: - name: Run DataDog Agent - if: needs.changes.outputs.backend == 'true' && (matrix.os != 'windows-2019' || contains(github.event.pull_request.labels.*.name, 'tools:datadog-windows')) + if: needs.changes.outputs.backend == 'true' && (matrix.os != 'windows-2022' || contains(github.event.pull_request.labels.*.name, 'tools:datadog-windows')) run: | docker run --name dd_agent -p 8126:8126 -d -e "DD_API_KEY=${{ secrets.DD_API_KEY }}" -e "DD_INSIDE_CI=true" -e "DD_HOSTNAME=none" -e "DD_SITE=datadoghq.eu" -e GITHUB_ACTIONS=true -e CI=true datadog/agent:latest docker ps --all --filter name=dd_agent --filter status=running --no-trunc --format "{{.ID}} {{.Status}}" @@ -453,7 +453,7 @@ jobs: installer-parallel: true - name: Install poetry (Windows) 🦄 - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec #v9 with: poetry-version: ${{ env.POETRY_VERSION }} @@ -461,7 +461,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} @@ -495,7 +495,7 @@ jobs: make prepare-tests-ubuntu - name: Install Dependencies (Windows) 📦 - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' # Restoring cache doesn't work properly on Windows due to symlinks. # We create symlinks for spacy models, that's why we need to clean them up # before caching the dependencies' directory. @@ -519,7 +519,7 @@ jobs: run: pip install pytest-github-actions-annotate-failures - name: Disable "LongPathsEnabled" option on Windows - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-2022' # On Windows laptops, a default preset prevents path names from being longer than # 260 characters. Some of our users can't enable this setting due to company policies. # We implemented a fix for model storage. The Windows container in GitHub @@ -534,7 +534,7 @@ jobs: run: poetry run pip install -U 'ddtrace<2.0.0' - name: Install ddtrace on Windows - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' run: | .\.venv\Scripts\activate py -m pip install -U 'ddtrace<2.0.0' @@ -699,7 +699,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} @@ -789,7 +789,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 224bc01da6f4..04c536960223 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -131,7 +131,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -149,7 +149,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Load Yarn Cached Packages ⬇ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} @@ -226,7 +226,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.docs == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -246,7 +246,7 @@ jobs: - name: Load Yarn Cached Packages ⬇ if: needs.changes.outputs.docs == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} @@ -299,7 +299,7 @@ jobs: node-version: "12.x" - name: Load Yarn Cached Packages ⬇ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} diff --git a/.github/workflows/security-scans.yml b/.github/workflows/security-scans.yml index 5ae74653b87f..612518e1db1a 100644 --- a/.github/workflows/security-scans.yml +++ b/.github/workflows/security-scans.yml @@ -104,7 +104,7 @@ jobs: - name: Load Poetry Cached Libraries ⬇ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} From 32ba52791569a4dd8cf8ae7a5f6edd0c38a2713d Mon Sep 17 00:00:00 2001 From: Maxime Vdb Date: Thu, 18 Dec 2025 09:04:16 +0100 Subject: [PATCH 40/41] Revise README for community engagement and future direction Updated README to reflect changes in community and project focus. --- README.md | 479 +++++++++--------------------------------------------- 1 file changed, 81 insertions(+), 398 deletions(-) diff --git a/README.md b/README.md index a9dae57dce95..c535027d142b 100644 --- a/README.md +++ b/README.md @@ -2,450 +2,133 @@
-[![Join the chat on Rasa Community Forum](https://img.shields.io/badge/forum-join%20discussions-brightgreen.svg)](https://forum.rasa.com/?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Join the Agent Engineering Community](https://img.shields.io/badge/Community-Join%20the%20Discussion-blueviolet)](https://info.rasa.com/community?utm_source=github&utm_medium=website&utm_campaign=) +[![Try Hello Rasa](https://img.shields.io/badge/Playground-Try%20Hello%20Rasa-ff69b4)](https://hello.rasa.ai/?utm_source=github&utm_medium=website&utm_campaign=) [![PyPI version](https://badge.fury.io/py/rasa.svg)](https://badge.fury.io/py/rasa) [![Supported Python Versions](https://img.shields.io/pypi/pyversions/rasa.svg)](https://pypi.python.org/pypi/rasa) [![Build Status](https://github.com/RasaHQ/rasa/workflows/Continuous%20Integration/badge.svg)](https://github.com/RasaHQ/rasa/actions) -[![Coverage Status](https://api.codeclimate.com/v1/badges/756dc6fea1d5d3e127f7/test_coverage)](https://codeclimate.com/github/RasaHQ/rasa/) [![Documentation Status](https://img.shields.io/badge/docs-stable-brightgreen.svg)](https://rasa.com/docs) -![Documentation Build](https://img.shields.io/netlify/d2e447e4-5a5e-4dc7-be5d-7c04ae7ff706?label=Documentation%20Build) -[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B8141%2Fgit%40github.com%3ARasaHQ%2Frasa.git.svg?type=shield)](https://app.fossa.com/projects/custom%2B8141%2Fgit%40github.com%3ARasaHQ%2Frasa.git?ref=badge_shield) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://github.com/orgs/RasaHQ/projects/23)
-
- -💡 **We're migrating issues to Jira** 💡 +
-Starting January 2023, issues for Rasa Open Source are located in -[this Jira board](https://rasa-open-source.atlassian.net/browse/OSS). You can browse issues without being logged in; -if you want to create issues, you'll need to create a Jira account. +
+

🚧 Note: Maintenance Mode 🚧

+

+ Rasa Open Source is currently in maintenance mode. +
+ The future of building AI agents with Rasa is Hello Rasa and CALM. +

+
-An image of Sara, the Rasa mascot bird, holding a flag that reads Open Source with one wing, and a wrench in the other - -Rasa is an open source machine learning framework to automate text and voice-based conversations. With Rasa, you can build contextual assistants on: -- Facebook Messenger -- Slack -- Google Hangouts -- Webex Teams -- Microsoft Bot Framework -- Rocket.Chat -- Mattermost -- Telegram -- Twilio -- Your own custom conversational channels - -or voice assistants as: -- Alexa Skills -- Google Home Actions +## 🚀 The Future of Rasa: Hello Rasa -Rasa helps you build contextual assistants capable of having layered conversations with -lots of back-and-forth. In order for a human to have a meaningful exchange with a contextual -assistant, the assistant needs to be able to use context to build on things that were previously -discussed – Rasa enables you to build assistants that can do this in a scalable way. +**Building reliable AI agents just got easier.** -There's a lot more background information in this -[blog post](https://medium.com/rasa-blog/a-new-approach-to-conversational-software-2e64a5d05f2a). +[**Hello Rasa**](https://hello.rasa.ai/?utm_source=github&utm_medium=website&utm_campaign=) is our new interactive playground for prototyping AI agents. It combines LLM fluency with the reliability of business logic using our **CALM** (Conversational AI with Language Models) engine. ---- -- 🤔 [Learn more about Rasa](https://rasa.community/) +### Why switch to Hello Rasa? -- 🤓 [Read The Docs](https://rasa.com/docs/rasa/) +* **No setup required:** Open the playground, pick a template (Banking, Telecom, Support), and start building in your browser. +* **No NLU training:** We have moved beyond intents. The LLM handles dialogue understanding while you define the business flows. +* **Built-in copilot:** A specialized AI assistant helps you generate code, debug flows, and expand your agent instantly. +* **Production ready:** Hello Rasa is not just a toy. Export your agent to the Rasa Platform when you are ready to scale. -- 😁 [Install Rasa](https://rasa.com/docs/rasa/installation/environment-set-up) - -- 🚀 [Dive deeper in the learning center](https://learning.rasa.com/) - -- 🤗 [Contribute](#how-to-contribute) - -- ❓ [Get enterprise-grade support](https://rasa.com/support/) - -- 🏢 [Explore the features of our commercial platform](https://rasa.com/product/rasa-platform/) - -- 📚 [Learn more about research papers that leverage Rasa](https://scholar.google.com/scholar?oi=bibs&hl=en&authuser=1&cites=16243802403383697687,353275993797024115,14567308604105196228,9067977709825839723,9855847065463746011&as_sdt=5) +### Core concepts +* **CALM:** Combines LLM flexibility with strict business logic. The LLM understands the user; the code enforces the rules. +* **Flows:** Describe logical steps (e.g., collect money, transfer funds) rather than rigid dialogue trees. +* **Inspector:** See real-time decision-making. No black boxes. +👉 **[Start building for free at Hello Rasa](https://hello.rasa.ai/?utm_source=github&utm_medium=website&utm_campaign=)** --- -## Where to get help - -There is extensive documentation in the [Rasa Docs](https://rasa.com/docs/rasa). -Make sure to select the correct version so you are looking at -the docs for the version you installed. - -Please use [Rasa Community Forum](https://forum.rasa.com) for quick answers to -questions. - -### README Contents: -- [How to contribute](#how-to-contribute) -- [Development Internals](#development-internals) -- [Releases](#releases) -- [License](#license) - -### How to contribute -We are very happy to receive and merge your contributions into this repository! - -To contribute via pull request, follow these steps: - -1. Create an issue describing the feature you want to work on (or - have a look at the [contributor board](https://github.com/orgs/RasaHQ/projects/23)) -2. Write your code, tests and documentation, and format them with ``black`` -3. Create a pull request describing your changes - -For more detailed instructions on how to contribute code, check out these [code contributor guidelines](CONTRIBUTING.md). - -You can find more information about how to contribute to Rasa (in lots of -different ways!) [on our website.](http://rasa.community). - -Your pull request will be reviewed by a maintainer, who will get -back to you about any necessary changes or questions. You will -also be asked to sign a -[Contributor License Agreement](https://cla-assistant.io/RasaHQ/rasa). - - -## Development Internals - -### Installing Poetry -Rasa uses Poetry for packaging and dependency management. If you want to build it from source, -you have to install Poetry first. Please follow -[the official guide](https://python-poetry.org/docs/#installation) to see all possible options. +## 🧠 Join the Agent Engineering Community -To update an existing poetry version to the [version](.github/poetry_version.txt), currently used in rasa, run: -```shell - poetry self update -``` - -### Managing environments - -The official [Poetry guide](https://python-poetry.org/docs/managing-environments/) suggests to use -[pyenv](https://github.com/pyenv/pyenv) or any other similar tool to easily switch between Python versions. -This is how it can be done: - -```bash -pyenv install 3.10.10 -pyenv local 3.10.10 # Activate Python 3.10.10 for the current project -``` -*Note*: If you have trouble installing a specific version of python on your system -it might be worth trying other supported versions. - -By default, Poetry will try to use the currently activated Python version to create the virtual environment -for the current project automatically. You can also create and activate a virtual environment manually — in this -case, Poetry should pick it up and use it to install the dependencies. For example: - -```bash -python -m venv .venv -source .venv/bin/activate -``` - -You can make sure that the environment is picked up by executing - -```bash -poetry env info -``` +We are building a home for people shipping real-world AI agents. -### Building from source - -To install dependencies and `rasa` itself in editable mode execute - -```bash -make install -``` - -*Note for macOS users*: under macOS Big Sur we've seen some compiler issues for -dependencies. Using `export SYSTEM_VERSION_COMPAT=1` before the installation helped. - - -#### Installing optional dependencies - -In order to install rasa's optional dependencies, you need to run: - -```bash -make install-full -``` - -*Note for macOS users*: The command `make install-full` could result in a failure while installing `tokenizers` -(issue described in depth [here](https://github.com/huggingface/tokenizers/issues/1050)). - -In order to resolve it, you must follow these steps to install a Rust compiler: -```bash -brew install rustup -rustup-init -``` - -After initialising the Rust compiler, you should restart the console and check its installation: -```bash -rustc --version -``` - -In case the PATH variable had not been automatically setup, run: -```bash -export PATH="$HOME/.cargo/bin:$PATH" -``` +Agent Engineering is evolving faster than any single framework. This is a vendor-neutral space to discuss architectures, memory, orchestration, and safety with builders across the industry. +### What you get: +* **Network:** Meet engineers building production agents +* **Learn:** Discuss practical patterns, not just theory +* **Access:** Direct influence on the Hello Rasa roadmap and early access to features -### Running and changing the documentation +| Channel | Purpose | +| :--- | :--- | +| **#agent-design** | Architectures, reasoning, memory, testing | +| **#showcase** | Show your builds, demos, and repos | +| **#ask-anything** | Debugging and workflow questions | -First of all, install all the required dependencies: +👉 **[Join the Community](https://info.rasa.com/community?utm_source=github&utm_medium=website&utm_campaign=)** -```bash -make install install-docs -``` - -After the installation has finished, you can run and view the documentation -locally using: - -```bash -make livedocs -``` - -It should open a new tab with the local version of the docs in your browser; -if not, visit http://localhost:3000 in your browser. -You can now change the docs locally and the web page will automatically reload -and apply your changes. - -### Running the Tests - -In order to run the tests, make sure that you have the development requirements installed: - -```bash -make prepare-tests-ubuntu # Only on Ubuntu and Debian based systems -make prepare-tests-macos # Only on macOS -``` - -Then, run the tests: - -```bash -make test -``` - -They can also be run at multiple jobs to save some time: - -```bash -JOBS=[n] make test -``` - -Where `[n]` is the number of jobs desired. If omitted, `[n]` will be automatically chosen by pytest. - - -### Running the Integration Tests - -In order to run the integration tests, make sure that you have the development requirements installed: - -```bash -make prepare-tests-ubuntu # Only on Ubuntu and Debian based systems -make prepare-tests-macos # Only on macOS -``` - -Then, you'll need to start services with the following command which uses -[Docker Compose](https://docs.docker.com/compose/install/): - -```bash -make run-integration-containers -``` - -Finally, you can run the integration tests like this: - -```bash -make test-integration -``` - - -### Resolving merge conflicts - -Poetry doesn't include any solution that can help to resolve merge conflicts in -the lock file `poetry.lock` by default. -However, there is a great tool called [poetry-merge-lock](https://poetry-merge-lock.readthedocs.io/en/latest/). -Here is how you can install it: - -```bash -pip install poetry-merge-lock -``` - -Just execute this command to resolve merge conflicts in `poetry.lock` automatically: - -```bash -poetry-merge-lock -``` - -### Build a Docker image locally +--- -In order to build a Docker image on your local machine execute the following command: +
+
-```bash -make build-docker -``` +# Rasa Open Source (Legacy) -The Docker image is available on your local machine as `rasa:localdev`. +> **Note:** The documentation and installation instructions below apply to the classic Rasa Open Source framework. For the latest CALM-based experience, see the [Hello Rasa](#-the-future-of-rasa-hello-rasa) section above. -### Code Style +Rasa is an open source machine learning framework for automating text and voice-based conversations. With Rasa, you can build contextual assistants on: -To ensure a standardized code style we use the formatter [black](https://github.com/ambv/black). -To ensure our type annotations are correct we use the type checker [pytype](https://github.com/google/pytype). -If your code is not formatted properly or doesn't type check, GitHub will fail to build. +- Facebook Messenger +- Slack +- Google Hangouts +- Webex Teams +- Microsoft Bot Framework +- Rocket.Chat +- Mattermost +- Telegram +- Twilio +- Your own custom conversational channels -#### Formatting +Rasa helps you build contextual assistants that can handle layered conversations with lots of back-and-forth. -If you want to automatically format your code on every commit, you can use [pre-commit](https://pre-commit.com/). -Just install it via `pip install pre-commit` and execute `pre-commit install` in the root folder. -This will add a hook to the repository, which reformats files on every commit. +### 📚 Resources +- 🤓 [Read the docs](https://rasa.com/docs/rasa/) +- 😁 [Install Rasa](https://rasa.com/docs/rasa/installation/environment-set-up) +- 🚀 [Learn all about Conversational AI](https://learning.rasa.com/) +- 🏢 [Explore the enterprise platform](https://rasa.com/product/rasa-platform/) -If you want to set it up manually, install black via `poetry install`. -To reformat files execute -``` -make formatter -``` +## Development Internals & Contributing -#### Type Checking +We are happy to receive contributions. Please review our [Contribution Guidelines](CONTRIBUTING.md) before getting started. -If you want to check types on the codebase, install `mypy` using `poetry install`. -To check the types execute -``` -make types -``` +### Installation for Development +Rasa uses **Poetry** for packaging and dependency management. -### Deploying documentation updates - -We use `Docusaurus v2` to build docs for tagged versions and for the `main` branch. -To run Docusaurus, install `Node.js 12.x`. -The static site that gets built is pushed to the `documentation` branch of this repo. - -We host the site on netlify. On `main` branch builds (see `.github/workflows/documentation.yml`), we push the built docs to -the `documentation` branch. Netlify automatically re-deploys the docs pages whenever there is a change to that branch. - -## Releases -Rasa has implemented robust policies governing version naming, as well as release pace for major, minor, and patch releases. - -The values for a given version number (MAJOR.MINOR.PATCH) are incremented as follows: -- MAJOR version for incompatible API changes or other breaking changes. -- MINOR version for functionality added in a backward compatible manner. -- PATCH version for backward compatible bug fixes. - -The following table describes the version types and their expected *release cadence*: - -| Version Type | Description | Target Cadence | -|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------|-----------------| -| Major | For significant changes, or when any backward-incompatible changes are introduced to the API or data model. | Every 1 - 2 yrs | -| Minor | For when new backward-compatible functionality is introduced, a minor feature is introduced, or when a set of smaller features is rolled out. | +/- Quarterly | -| Patch | For backward-compatible bug fixes that fix incorrect behavior. | As needed | - -While this table represents our target release frequency, we reserve the right to modify it based on changing market conditions and technical requirements. - -### Maintenance Policy -Our End of Life policy defines how long a given release is considered supported, as well as how long a release is -considered to be still in active development or maintenance. - -The maintentance duration and end of life for every release are shown on our website as part of the [Product Release and Maintenance Policy](https://rasa.com/rasa-product-release-and-maintenance-policy/). - -### Cutting a Major / Minor release -#### A week before release day - -1. **Make sure the [milestone](https://github.com/RasaHQ/rasa/milestones) already exists and is scheduled for the -correct date.** -2. **Take a look at the issues & PRs that are in the milestone**: does it look about right for the release highlights -we are planning to ship? Does it look like anything is missing? Don't worry about being aware of every PR that should -be in, but it's useful to take a moment to evaluate what's assigned to the milestone. -3. **Post a message on the engineering Slack channel**, letting the team know you'll be the one cutting the upcoming -release, as well as: - 1. Providing the link to the appropriate milestone - 2. Reminding everyone to go over their issues and PRs and please assign them to the milestone - 3. Reminding everyone of the scheduled date for the release - -#### A day before release day - -1. **Go over the milestone and evaluate the status of any PR merging that's happening. Follow up with people on their -bugs and fixes.** If the release introduces new bugs or regressions that can't be fixed in time, we should discuss on -Slack about this and take a decision on how to move forward. If the issue is not ready to be merged in time, we remove the issue / PR from the milestone and notify the PR owner and the product manager on Slack about it. The PR / issue owners are responsible for -communicating any issues which might be release relevant. Postponing the release should be considered as an edge case scenario. - -#### Release day! 🚀 - -1. **At the start of the day, post a small message on slack announcing release day!** Communicate you'll be handling -the release, and the time you're aiming to start releasing (again, no later than 4pm, as issues may arise and -cause delays). This message should be posted early in the morning and before moving forward with any of the steps of the release, - in order to give enough time to people to check their PRs and issues. That way they can plan any remaining work. A template of the slack message can be found [here](https://rasa-hq.slack.com/archives/C36SS4N8M/p1613032208137500?thread_ts=1612876410.068400&cid=C36SS4N8M). - The release time should be communicated transparently so that others can plan potentially necessary steps accordingly. If there are bigger changes this should be communicated. -2. Make sure the milestone is empty (everything has been either merged or moved to the next milestone) -3. Once everything in the milestone is taken care of, post a small message on Slack communicating you are about to -start the release process (in case anything is missing). -4. **You may now do the release by following the instructions outlined in the -[Rasa Open Source README](#steps-to-release-a-new-version) !** - -#### After a Major release - -After a Major release has been completed, please follow [these instructions to complete the documentation update](./docs/README.md#manual-steps-after-a-new-version). - -### Steps to release a new version -Releasing a new version is quite simple, as the packages are build and distributed by GitHub Actions. - -*Release steps*: -1. Make sure all dependencies are up to date (**especially Rasa SDK**) - - For Rasa SDK, except in the case of a patch release, that means first creating a [new Rasa SDK release](https://github.com/RasaHQ/rasa-sdk#steps-to-release-a-new-version) (make sure the version numbers between the new Rasa and Rasa SDK releases match) - - Once the tag with the new Rasa SDK release is pushed and the package appears on [pypi](https://pypi.org/project/rasa-sdk/), the dependency in the rasa repository can be resolved (see below). -2. If this is a minor / major release: Make sure all fixes from currently supported minor versions have been merged from their respective release branches (e.g. 3.3.x) back into main. -3. In case of a minor release, create a new branch that corresponds to the new release, e.g. - ```bash - git checkout -b 1.2.x - git push origin 1.2.x - ``` -4. Switch to the branch you want to cut the release from (`main` in case of a major, the `..x` branch for minors and patches) - - Update the `rasa-sdk` entry in `pyproject.toml` with the new release version and run `poetry update`. This creates a new `poetry.lock` file with all dependencies resolved. - - Commit the changes with `git commit -am "bump rasa-sdk dependency"` but do not push them. They will be automatically picked up by the following step. -5. If this is a major release, update the list of actively maintained versions [in the README](#actively-maintained-versions) and in [the docs](./docs/docs/actively-maintained-versions.mdx). -6. Run `make release` -7. Create a PR against the release branch (e.g. `1.2.x`) -8. Once your PR is merged, tag a new release (this SHOULD always happen on the release branch), e.g. using +1. **Install Poetry**: Follow the [official guide](https://python-poetry.org/docs/#installation). +2. **Build from source**: ```bash - git checkout 1.2.x - git pull origin 1.2.x - git tag 1.2.0 -m "next release" - git push origin 1.2.0 --tags + make install ``` - GitHub will build this tag and publish the build artifacts. -9. After all the steps are completed and if everything goes well then we should see a message automatically posted in the company's Slack (`product` channel) like this [one](https://rasa-hq.slack.com/archives/C7B08Q5FX/p1614354499046600) -10. If no message appears in the channel then you can do the following checks: - - Check the workflows in [Github Actions](https://github.com/RasaHQ/rasa/actions) and make sure that the merged PR of the current release is completed successfully. To easily find your PR you can use the filters `event: push` and `branch: ` (example on release 2.4 you can see [here](https://github.com/RasaHQ/rasa/actions/runs/643344876)) - - If the workflow is not completed, then try to re run the workflow in case that solves the problem - - If the problem persists, check also the log files and try to find the root cause of the issue - - If you still cannot resolve the error, contact the infrastructure team by providing any helpful information from your investigation -11. After the message is posted correctly in the `product` channel, check also in the `product-engineering-alerts` channel if there are any alerts related to the Rasa Open Source release like this [one](https://rasa-hq.slack.com/archives/C01585AN2NP/p1615486087001000) + *Note for macOS users*: If you run into compiler issues, try `export SYSTEM_VERSION_COMPAT=1` before installation. -### Cutting a Patch release +### Running Tests +Make sure you have development requirements installed: -Patch releases are simpler to cut, since they are meant to contain only bugfixes. - -**The only things you need to do to cut a patch release are:** - -1. Notify the engineering team on Slack that you are planning to cut a patch, in case someone has an important fix -to add. -2. Make sure the bugfix(es) are in the release branch you will use (p.e if you are cutting a `2.0.4` patch, you will -need your fixes to be on the `2.0.x` release branch). All patch releases must come from a `.x` branch! -3. Once you're ready to release the Rasa Open Source patch, checkout the branch, run `make release` and follow the -steps + get the PR merged. -4. Once the PR is in, pull the `.x` branch again and push the tag! +```bash +make prepare-tests-ubuntu # Ubuntu/Debian +make prepare-tests-macos # macOS +make test # Run tests +``` -### Additional Release Tasks -**Note: This is only required if the released version is the highest version available. -For instance, perform the following steps when version > [version](https://github.com/RasaHQ/rasa/blob/main/rasa/version.py) on main.** +### Releases -In order to check compatibility between the new released Rasa version to the latest version of Rasa X/Enterprise, we perform the following steps: -1. Following a new Rasa release, an automated pull request is created in [Rasa-X-Demo](https://github.com/RasaHQ/rasa-x-demo/pulls). -2. Once the above PR is merged, follow instructions [here](https://github.com/RasaHQ/rasa-x-demo/blob/master/.github/VERSION_BUMPER_PR_COMMENT.md), to release a version. -3. Update the new version in the Rasa X/Enterprise [env file](https://github.com/RasaHQ/rasa-x/blob/main/.env). -The [Rasa-X-Demo](https://github.com/RasaHQ/rasa-x-demo) project uses the new updated Rasa version to train and test a model which in turn is used by our CI to run tests in the Rasa X/Enterprise repository, -thus validating compatibility between Rasa and Rasa X/Enterprise. +Rasa follows Semantic Versioning. -### Actively maintained versions + * **Major**: Incompatible API changes + * **Minor**: Backward-compatible functionality + * **Patch**: Backward-compatible bug fixes -Please refer to the [Rasa Product Release and Maintenance Policy](https://rasa.com/rasa-product-release-and-maintenance-policy/) page. +For full details on our release cadence and maintenance policy, visit our [Product Release and Maintenance Policy](https://rasa.com/rasa-product-release-and-maintenance-policy/). ## License -Licensed under the Apache License, Version 2.0. -Copyright 2022 Rasa Technologies GmbH. [Copy of the license](LICENSE.txt). -A list of the Licenses of the dependencies of the project can be found at -the bottom of the -[Libraries Summary](https://libraries.io/github/RasaHQ/rasa). +Licensed under the Apache License, Version 2.0. Copyright 2022 Rasa Technologies GmbH. [Copy of the license](https://www.google.com/search?q=LICENSE.txt). From 91870153e27831885cf38563ad29bad4fd03405d Mon Sep 17 00:00:00 2001 From: ErnestHysa Date: Sun, 31 May 2026 00:16:09 +0100 Subject: [PATCH 41/41] await async enter_room in session_request handler --- rasa/core/channels/socketio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/channels/socketio.py b/rasa/core/channels/socketio.py index c268447f020a..52f93dc46bb3 100644 --- a/rasa/core/channels/socketio.py +++ b/rasa/core/channels/socketio.py @@ -233,7 +233,7 @@ async def session_request(sid: Text, data: Optional[Dict]) -> None: if "session_id" not in data or data["session_id"] is None: data["session_id"] = uuid.uuid4().hex if self.session_persistence: - sio.enter_room(sid, data["session_id"]) + await sio.enter_room(sid, data["session_id"]) await sio.emit("session_confirm", data["session_id"], room=sid) logger.debug(f"User {sid} connected to socketIO endpoint.")