Merge branch '3.6.x' into ATO-1959-patch-security-vulnerability-3.6.x

.github/workflows/security-scans.yml

@@ -129,24 +129,3 @@ jobs:
       - name: Run Bandit 🔪
         if: needs.changes.outputs.backend == 'true'
         run: make lint-security
-
-  snyk:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
-      - name: Run Snyk Open Source to check for Python vulnerabilities
-        uses: snyk/actions/python-3.8@master
-        continue-on-error: true
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          command: monitor
-          args: --all-projects --org=rasa --skip-unresolved
-      - name: Run Snyk Open Source to check for JS vulnerabilities
-        uses: snyk/actions/node@master
-        continue-on-error: true
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          command: monitor
-          args: --org=rasa --yarn-workspaces --strict-out-of-sync=false --prune-repeated-subdependencies

CHANGELOG.mdx

@@ -16,6 +16,27 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . -->
 
 <!-- TOWNCRIER -->
 
+## [3.6.15] - 2023-11-30
+
+Rasa 3.6.15 (2023-11-30)                          
+### Bugfixes
+- [#12965](https://github.com/rasahq/rasa/issues/12965): Fixed connection timeout to action server by setting KEEP_ALIVE_TIMEOUT to 120, and reverting changes introduced in #12886.
+
+
+## [3.6.14] - 2023-11-17
+
+Rasa 3.6.14 (2023-11-17)                         
+### Bugfixes
+- [#12948](https://github.com/rasahq/rasa/issues/12948): Fixed UnexpecTEDIntentlessPolicy training errors that resulted from a change to batching behavior. Changed the batching behavior back to the original for all components. Made the changed batching behavior accessible in DietClassifier using `drop_small_last_batch: True`.
+
+
+## [3.6.13] - 2023-10-23
+
+Rasa 3.6.13 (2023-10-23)                         
+### Bugfixes
+- [#12927](https://github.com/rasahq/rasa/issues/12927): Fix wrong conflicts that occur when rasa validate stories is run with slots that have active_loop set to null in mapping conditions.
+
+
 ## [3.6.12] - 2023-10-10
 
 Rasa 3.6.12 (2023-10-10)                         

pyproject.toml

@@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))"
 
 [tool.poetry]
 name = "rasa"
-version = "3.6.12"
+version = "3.6.15"
 description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants"
 authors = [ "Rasa Technologies GmbH <[email protected]>",]
 maintainers = [ "Tom Bocklisch <[email protected]>",]

rasa/core/agent.py

@@ -112,53 +112,59 @@ async def _pull_model_and_fingerprint(
 
     logger.debug(f"Requesting model from server {model_server.url}...")
 
-    try:
-        params = model_server.combine_parameters()
-        async with model_server.session.request(
-            "GET",
-            model_server.url,
-            timeout=DEFAULT_REQUEST_TIMEOUT,
-            headers=headers,
-            params=params,
-        ) as resp:
-            if resp.status in [204, 304]:
-                logger.debug(
-                    "Model server returned {} status code, "
-                    "indicating that no new model is available. "
-                    "Current fingerprint: {}"
-                    "".format(resp.status, fingerprint)
-                )
-                return None
-            elif resp.status == 404:
-                logger.debug(
-                    "Model server could not find a model at the requested "
-                    "endpoint '{}'. It's possible that no model has been "
-                    "trained, or that the requested tag hasn't been "
-                    "assigned.".format(model_server.url)
-                )
-                return None
-            elif resp.status != 200:
-                logger.debug(
-                    "Tried to fetch model from server, but server response "
-                    "status code is {}. We'll retry later..."
-                    "".format(resp.status)
+    async with model_server.session() as session:
+        try:
+            params = model_server.combine_parameters()
+            async with session.request(
+                "GET",
+                model_server.url,
+                timeout=DEFAULT_REQUEST_TIMEOUT,
+                headers=headers,
+                params=params,
+            ) as resp:
+
+                if resp.status in [204, 304]:
+                    logger.debug(
+                        "Model server returned {} status code, "
+                        "indicating that no new model is available. "
+                        "Current fingerprint: {}"
+                        "".format(resp.status, fingerprint)
+                    )
+                    return None
+                elif resp.status == 404:
+                    logger.debug(
+                        "Model server could not find a model at the requested "
+                        "endpoint '{}'. It's possible that no model has been "
+                        "trained, or that the requested tag hasn't been "
+                        "assigned.".format(model_server.url)
+                    )
+                    return None
+                elif resp.status != 200:
+                    logger.debug(
+                        "Tried to fetch model from server, but server response "
+                        "status code is {}. We'll retry later..."
+                        "".format(resp.status)
+                    )
+                    return None
+
+                model_path = Path(model_directory) / resp.headers.get(
+                    "filename", "model.tar.gz"
                 )
-                return None
-            model_path = Path(model_directory) / resp.headers.get(
-                "filename", "model.tar.gz"
+                with open(model_path, "wb") as file:
+                    file.write(await resp.read())
+
+                logger.debug("Saved model to '{}'".format(os.path.abspath(model_path)))
+
+                # return the new fingerprint
+                return resp.headers.get("ETag")
+
+        except aiohttp.ClientError as e:
+            logger.debug(
+                "Tried to fetch model from server, but "
+                "couldn't reach server. We'll retry later... "
+                "Error: {}.".format(e)
             )
-            with open(model_path, "wb") as file:
-                file.write(await resp.read())
-            logger.debug("Saved model to '{}'".format(os.path.abspath(model_path)))
-            # return the new fingerprint
-            return resp.headers.get("ETag")
-    except aiohttp.ClientError as e:
-        logger.debug(
-            "Tried to fetch model from server, but "
-            "couldn't reach server. We'll retry later... "
-            "Error: {}.".format(e)
-        )
-        return None
+            return None
 
 
 async def _run_model_pulling_worker(model_server: EndpointConfig, agent: Agent) -> None:

rasa/core/constants.py

@@ -24,6 +24,8 @@
 
 DEFAULT_LOCK_LIFETIME = 60  # in seconds
 
+DEFAULT_KEEP_ALIVE_TIMEOUT = 120  # in seconds
+
 BEARER_TOKEN_PREFIX = "Bearer "
 
 # The lowest priority is intended to be used by machine learning policies.

rasa/core/run.py

@@ -1,9 +1,19 @@
 import asyncio
 import logging
 import uuid
+import platform
 import os
 from functools import partial
-from typing import Any, List, Optional, TYPE_CHECKING, Text, Union, Dict
+from typing import (
+    Any,
+    Callable,
+    List,
+    Optional,
+    Text,
+    Tuple,
+    Union,
+    Dict,
+)
 
 import rasa.core.utils
 from rasa.plugin import plugin_manager
@@ -23,8 +33,6 @@
 from sanic import Sanic
 from asyncio import AbstractEventLoop
 
-if TYPE_CHECKING:
-    from aiohttp import ClientSession
 
 logger = logging.getLogger()  # get the root logger
 
@@ -80,6 +88,14 @@ def _create_app_without_api(cors: Optional[Union[Text, List[Text]]] = None) -> S
     return app
 
 
+def _is_apple_silicon_system() -> bool:
+    # check if the system is MacOS
+    if platform.system().lower() != "darwin":
+        return False
+    # check for arm architecture, indicating apple silicon
+    return platform.machine().startswith("arm") or os.uname().machine.startswith("arm")
+
+
 def configure_app(
     input_channels: Optional[List["InputChannel"]] = None,
     cors: Optional[Union[Text, List[Text], None]] = None,
@@ -99,6 +115,9 @@ def configure_app(
     syslog_port: Optional[int] = None,
     syslog_protocol: Optional[Text] = None,
     request_timeout: Optional[int] = None,
+    server_listeners: Optional[List[Tuple[Callable, Text]]] = None,
+    use_uvloop: Optional[bool] = True,
+    keep_alive_timeout: int = constants.DEFAULT_KEEP_ALIVE_TIMEOUT,
 ) -> Sanic:
     """Run the agent."""
     rasa.core.utils.configure_file_logging(
@@ -118,6 +137,14 @@ def configure_app(
     else:
         app = _create_app_without_api(cors)
 
+    app.config.KEEP_ALIVE_TIMEOUT = keep_alive_timeout
+    if _is_apple_silicon_system() or not use_uvloop:
+        app.config.USE_UVLOOP = False
+        # some library still sets the loop to uvloop, even if disabled for sanic
+        # using uvloop leads to breakingio errors, see
+        # https://rasahq.atlassian.net/browse/ENG-667
+        asyncio.set_event_loop_policy(None)
+
     if input_channels:
         channels.channel.register(input_channels, app, route=route)
     else:
@@ -150,6 +177,10 @@ async def run_cmdline_io(running_app: Sanic) -> None:
 
         app.add_task(run_cmdline_io)
 
+    if server_listeners:
+        for (listener, event) in server_listeners:
+            app.register_listener(listener, event)
+
     return app
 
 
@@ -179,6 +210,7 @@ def serve_application(
     syslog_port: Optional[int] = None,
     syslog_protocol: Optional[Text] = None,
     request_timeout: Optional[int] = None,
+    server_listeners: Optional[List[Tuple[Callable, Text]]] = None,
 ) -> None:
     """Run the API entrypoint."""
     if not channel and not credentials:
@@ -204,6 +236,7 @@ def serve_application(
         syslog_port=syslog_port,
         syslog_protocol=syslog_protocol,
         request_timeout=request_timeout,
+        server_listeners=server_listeners,
     )
 
     ssl_context = server.create_ssl_context(
@@ -217,7 +250,7 @@ def serve_application(
         partial(load_agent_on_start, model_path, endpoints, remote_storage),
         "before_server_start",
     )
-    app.register_listener(create_connection_pools, "after_server_start")
+
     app.register_listener(close_resources, "after_server_stop")
 
     number_of_workers = rasa.core.utils.number_of_sanic_workers(
@@ -279,44 +312,3 @@ async def close_resources(app: Sanic, _: AbstractEventLoop) -> None:
     event_broker = current_agent.tracker_store.event_broker
     if event_broker:
         await event_broker.close()
-
-    action_endpoint = current_agent.action_endpoint
-    if action_endpoint:
-        await action_endpoint.session.close()
-
-    model_server = current_agent.model_server
-    if model_server:
-        await model_server.session.close()
-
-
-async def create_connection_pools(app: Sanic, _: AbstractEventLoop) -> None:
-    """Create connection pools for the agent's action server and model server."""
-    current_agent = getattr(app.ctx, "agent", None)
-    if not current_agent:
-        logger.debug("No agent found after server start.")
-        return None
-
-    create_action_endpoint_connection_pool(current_agent)
-    create_model_server_connection_pool(current_agent)
-
-    return None
-
-
-def create_action_endpoint_connection_pool(agent: Agent) -> Optional["ClientSession"]:
-    """Create a connection pool for the action endpoint."""
-    action_endpoint = agent.action_endpoint
-    if not action_endpoint:
-        logger.debug("No action endpoint found after server start.")
-        return None
-
-    return action_endpoint.session
-
-
-def create_model_server_connection_pool(agent: Agent) -> Optional["ClientSession"]:
-    """Create a connection pool for the model server."""
-    model_server = agent.model_server
-    if not model_server:
-        logger.debug("No model server endpoint found after server start.")
-        return None
-
-    return model_server.session

rasa/nlu/classifiers/diet_classifier.py

@@ -50,6 +50,7 @@
 from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.shared.nlu.training_data.message import Message
 from rasa.utils.tensorflow.constants import (
+    DROP_SMALL_LAST_BATCH,
     LABEL,
     IDS,
     HIDDEN_LAYERS_SIZES,
@@ -288,6 +289,9 @@ def get_default_config() -> Dict[Text, Any]:
             # a few steps, as the compilation of the graph tends to take more time than
             # running it. It is recommended to not adjust the optimization parameter.
             RUN_EAGERLY: False,
+            # Determines whether the last batch should be dropped if it contains fewer
+            # than half a batch size of examples
+            DROP_SMALL_LAST_BATCH: False,
         }
 
     def __init__(
@@ -931,6 +935,7 @@ def train(self, training_data: TrainingData) -> Resource:
             self.component_config[BATCH_STRATEGY],
             self.component_config[EVAL_NUM_EXAMPLES],
             self.component_config[RANDOM_SEED],
+            drop_small_last_batch=self.component_config[DROP_SMALL_LAST_BATCH],
         )
         callbacks = train_utils.create_common_callbacks(
             self.component_config[EPOCHS],