From 39f5176877e234d713198f95d89bb666a2642e0b Mon Sep 17 00:00:00 2001 From: m-vdb Date: Mon, 18 Sep 2023 14:50:54 +0200 Subject: [PATCH 1/8] cleanup unnecessary styling in doc theme --- .../theme/RasaDiscoveryBanner/index.jsx | 4 +--- .../RasaDiscoveryBanner/styles.module.css | 15 --------------- .../theme/RasaLabsBanner/styles.module.css | 16 ---------------- .../theme-custom/theme/RasaProBanner/index.jsx | 3 --- .../theme/RasaProBanner/styles.module.css | 18 ------------------ 5 files changed, 1 insertion(+), 55 deletions(-) delete mode 100644 docs/themes/theme-custom/theme/RasaDiscoveryBanner/styles.module.css delete mode 100644 docs/themes/theme-custom/theme/RasaProBanner/styles.module.css diff --git a/docs/themes/theme-custom/theme/RasaDiscoveryBanner/index.jsx b/docs/themes/theme-custom/theme/RasaDiscoveryBanner/index.jsx index 4630c5cbe3fc..a97d5f6f74ad 100644 --- a/docs/themes/theme-custom/theme/RasaDiscoveryBanner/index.jsx +++ b/docs/themes/theme-custom/theme/RasaDiscoveryBanner/index.jsx @@ -1,7 +1,5 @@ import * as React from 'react'; -import clsx from 'clsx'; -import styles from './styles.module.css'; function RasaDiscoveryBanner({isLoading, ...props}) { return ( @@ -19,7 +17,7 @@ function RasaDiscoveryBanner({isLoading, ...props}) {

- Discovery features are highly unstable. We introduce + Discovery features are highly unstable. We introduce them for product exploration. These features will change and should only be used for testing purposes. diff --git a/docs/themes/theme-custom/theme/RasaDiscoveryBanner/styles.module.css b/docs/themes/theme-custom/theme/RasaDiscoveryBanner/styles.module.css deleted file mode 100644 index 4a6e8636517a..000000000000 --- a/docs/themes/theme-custom/theme/RasaDiscoveryBanner/styles.module.css +++ /dev/null @@ -1,15 +0,0 @@ -.label { - background-color: #F6D261; - border: 1px solid transparent; - border-radius: 8px; - padding: 2px 12px; - font-size: 15px !important; - font-weight: 600; - - display: inline-block; -} - -.label[disabled] { - background-color: var(--ifm-color-gray-500); - cursor: default; -} diff --git a/docs/themes/theme-custom/theme/RasaLabsBanner/styles.module.css b/docs/themes/theme-custom/theme/RasaLabsBanner/styles.module.css index eafbc6bd315e..fed55d974a30 100644 --- a/docs/themes/theme-custom/theme/RasaLabsBanner/styles.module.css +++ b/docs/themes/theme-custom/theme/RasaLabsBanner/styles.module.css @@ -1,19 +1,3 @@ -.label { - background-color: #F6D261; - border: 1px solid transparent; - border-radius: 8px; - padding: 2px 12px; - font-size: 15px !important; - font-weight: 600; - - display: inline-block; -} - -.label[disabled] { - background-color: var(--ifm-color-gray-500); - cursor: default; -} - .titleExtension { text-transform: none !important; } diff --git a/docs/themes/theme-custom/theme/RasaProBanner/index.jsx b/docs/themes/theme-custom/theme/RasaProBanner/index.jsx index ef975c61942f..ae87d08ac5e3 100644 --- a/docs/themes/theme-custom/theme/RasaProBanner/index.jsx +++ b/docs/themes/theme-custom/theme/RasaProBanner/index.jsx @@ -1,7 +1,4 @@ import * as React from 'react'; -import clsx from 'clsx'; - -import styles from './styles.module.css'; function RasaProBanner({isLoading, ...props}) { return ( diff --git a/docs/themes/theme-custom/theme/RasaProBanner/styles.module.css b/docs/themes/theme-custom/theme/RasaProBanner/styles.module.css deleted file mode 100644 index 55cfcd71e39b..000000000000 --- a/docs/themes/theme-custom/theme/RasaProBanner/styles.module.css +++ /dev/null @@ -1,18 +0,0 @@ - - -.label { - background-color:#F6D261; - border: 1px solid transparent; - border-radius: 8px; - padding: 2px 12px; - font-size: 15px !important; - font-weight: 600; - - display: inline-block; -} - -.label[disabled] { - background-color: var(--ifm-color-gray-500); - cursor: default; -} - From c0832d0808af4d31550b74f5172124e28b47fd24 Mon Sep 17 00:00:00 2001 From: m-vdb Date: Tue, 19 Sep 2023 15:48:40 +0200 Subject: [PATCH 2/8] for docs --- .../building-classic-assistants/forms.mdx | 4 +++ .../theme/RasaCALMBanner/index.jsx | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx diff --git a/docs/docs/building-classic-assistants/forms.mdx b/docs/docs/building-classic-assistants/forms.mdx index 74158d28367e..854f0468bb14 100644 --- a/docs/docs/building-classic-assistants/forms.mdx +++ b/docs/docs/building-classic-assistants/forms.mdx @@ -6,6 +6,10 @@ description: Follow a rule-based process of information gathering using forms in abstract: One of the most common conversation patterns is to collect a few pieces of information from a user in order to do something (book a restaurant, call an API, search a database, etc.). This is also called **slot filling**. --- +import RasaCALMBanner from "@theme/RasaCALMBanner"; + + + ## Usage To use forms with Rasa you need to make sure that the diff --git a/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx b/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx new file mode 100644 index 000000000000..dabe4a0633a2 --- /dev/null +++ b/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx @@ -0,0 +1,28 @@ +import * as React from 'react'; + +function RasaCALMBanner() { + return ( + <> +

+
+
+ + {/* SVG taken from https://primer.style/design/foundations/icons/rocket-24 */} + + + + Conversational AI with Language Models +
+
+
+

+ This page refers to building NLU-based assistants. Want to supercharge your assistant? + Check out Conversation AI with Language Models (CALM). +

+
+
+ + ) +} + +export default RasaCALMBanner; From 8ffa9ea2308a8aee2ea5b1ffe1b42e4b3368a4df Mon Sep 17 00:00:00 2001 From: m-vdb Date: Wed, 20 Sep 2023 09:36:01 +0200 Subject: [PATCH 3/8] content iteration on banner + url --- .../theme-custom/theme/RasaCALMBanner/index.jsx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx b/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx index dabe4a0633a2..e3be1bdcb8e4 100644 --- a/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx +++ b/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx @@ -1,4 +1,5 @@ import * as React from 'react'; +import useBaseUrl from "@docusaurus/useBaseUrl"; function RasaCALMBanner() { return ( @@ -7,17 +8,18 @@ function RasaCALMBanner() {
- {/* SVG taken from https://primer.style/design/foundations/icons/rocket-24 */} - - + + + - Conversational AI with Language Models + NLU-based assistants

- This page refers to building NLU-based assistants. Want to supercharge your assistant? - Check out Conversation AI with Language Models (CALM). + This page refers to building NLU-based assistants. + If you are working with Conversation AI with Language Models (CALM), + this content may not apply to you.

From ac375eb57af79c84e5dbe63e8fbcefe8bbab4402 Mon Sep 17 00:00:00 2001 From: m-vdb Date: Wed, 20 Sep 2023 09:38:51 +0200 Subject: [PATCH 4/8] rename banner --- docs/docs/building-classic-assistants/forms.mdx | 4 ++-- .../theme/{RasaCALMBanner => RasaNLUBasedBanner}/index.jsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename docs/themes/theme-custom/theme/{RasaCALMBanner => RasaNLUBasedBanner}/index.jsx (95%) diff --git a/docs/docs/building-classic-assistants/forms.mdx b/docs/docs/building-classic-assistants/forms.mdx index 854f0468bb14..31fee414812b 100644 --- a/docs/docs/building-classic-assistants/forms.mdx +++ b/docs/docs/building-classic-assistants/forms.mdx @@ -6,9 +6,9 @@ description: Follow a rule-based process of information gathering using forms in abstract: One of the most common conversation patterns is to collect a few pieces of information from a user in order to do something (book a restaurant, call an API, search a database, etc.). This is also called **slot filling**. --- -import RasaCALMBanner from "@theme/RasaCALMBanner"; +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; - + ## Usage diff --git a/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx b/docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx similarity index 95% rename from docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx rename to docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx index e3be1bdcb8e4..9daad8b96dee 100644 --- a/docs/themes/theme-custom/theme/RasaCALMBanner/index.jsx +++ b/docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx @@ -1,7 +1,7 @@ import * as React from 'react'; import useBaseUrl from "@docusaurus/useBaseUrl"; -function RasaCALMBanner() { +function RasaNLUBasedBanner() { return ( <>
@@ -27,4 +27,4 @@ function RasaCALMBanner() { ) } -export default RasaCALMBanner; +export default RasaNLUBasedBanner; From 9ac47463d2725a50f035f9e2aa3d64eff683dc07 Mon Sep 17 00:00:00 2001 From: m-vdb Date: Wed, 20 Sep 2023 09:42:23 +0200 Subject: [PATCH 5/8] add banners on all DM1 pages --- docs/docs/building-classic-assistants/business-logic.mdx | 4 ++++ docs/docs/building-classic-assistants/chitchat-faqs.mdx | 4 ++++ docs/docs/building-classic-assistants/components.mdx | 4 ++++ .../building-classic-assistants/contextual-conversations.mdx | 4 ++++ docs/docs/building-classic-assistants/fallback-handoff.mdx | 3 +++ docs/docs/building-classic-assistants/language-support.mdx | 4 ++++ docs/docs/building-classic-assistants/model-configuration.mdx | 4 ++++ docs/docs/building-classic-assistants/nlu-only.mdx | 4 ++++ docs/docs/building-classic-assistants/nlu-training-data.mdx | 4 ++++ docs/docs/building-classic-assistants/policies.mdx | 4 ++++ .../docs/building-classic-assistants/reaching-out-to-user.mdx | 4 ++++ docs/docs/building-classic-assistants/rules.mdx | 4 ++++ docs/docs/building-classic-assistants/stories.mdx | 4 ++++ .../docs/building-classic-assistants/training-data-format.mdx | 4 ++++ docs/docs/building-classic-assistants/tuning-your-model.mdx | 3 +++ docs/docs/building-classic-assistants/unexpected-input.mdx | 4 ++++ 16 files changed, 62 insertions(+) diff --git a/docs/docs/building-classic-assistants/business-logic.mdx b/docs/docs/building-classic-assistants/business-logic.mdx index 88b6f0a3a9cf..2f302ac52c74 100644 --- a/docs/docs/building-classic-assistants/business-logic.mdx +++ b/docs/docs/building-classic-assistants/business-logic.mdx @@ -5,6 +5,10 @@ title: Handling Business Logic abstract: Conversational assistants often need to ask users for information in order to help them. You can use Forms to collect the required user information and fulfill a request. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + Conversational assistants often support user goals that involve collecting required information from the user before doing something for them. For example, a restaurant search bot would need to gather a few pieces of information about the user's preferences to find them a suitable restaurant: diff --git a/docs/docs/building-classic-assistants/chitchat-faqs.mdx b/docs/docs/building-classic-assistants/chitchat-faqs.mdx index dafcd71a3de6..a8c2409eb706 100644 --- a/docs/docs/building-classic-assistants/chitchat-faqs.mdx +++ b/docs/docs/building-classic-assistants/chitchat-faqs.mdx @@ -5,6 +5,10 @@ title: Chitchat and FAQs abstract: FAQ assistants are the simplest assistants to build and typically the first kind of assistant anyone builds. This page is a guide to the concepts and training data you need to handle non-contextual questions like FAQs and chitchat. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + [FAQs](../concepts/glossary.mdx#faqs) and [chitchat](../concepts/glossary.mdx#chitchat) are two cases where the conversational assistant responds with a fixed set of messages, and the assistant should always answer the same way, no matter what has happened previously in the conversation. For example, in the following conversation, every question can be asked at any point in the conversation, diff --git a/docs/docs/building-classic-assistants/components.mdx b/docs/docs/building-classic-assistants/components.mdx index 4af79159ab55..2ab06320c874 100644 --- a/docs/docs/building-classic-assistants/components.mdx +++ b/docs/docs/building-classic-assistants/components.mdx @@ -8,6 +8,10 @@ abstract: pre-processing, and more. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + ## Language Models The following components load pre-trained models that are needed if you want to use pre-trained diff --git a/docs/docs/building-classic-assistants/contextual-conversations.mdx b/docs/docs/building-classic-assistants/contextual-conversations.mdx index a91293fd2655..f2b94f1274ef 100644 --- a/docs/docs/building-classic-assistants/contextual-conversations.mdx +++ b/docs/docs/building-classic-assistants/contextual-conversations.mdx @@ -5,6 +5,10 @@ title: Contextual Conversations abstract: Taking context into account is often key to providing a good user experience. This page is a guide to creating contextual conversation patterns. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + In a contextual conversation, something beyond the previous step in the conversation plays a role in what should happen next. For example, if a user asks "How many?", it's not clear from the message alone what the user is asking about. In the context of the assistant saying, diff --git a/docs/docs/building-classic-assistants/fallback-handoff.mdx b/docs/docs/building-classic-assistants/fallback-handoff.mdx index ce76e5e5ffab..b2f3c190febb 100644 --- a/docs/docs/building-classic-assistants/fallback-handoff.mdx +++ b/docs/docs/building-classic-assistants/fallback-handoff.mdx @@ -6,6 +6,9 @@ abstract: This is a guide on how to handle various failures of your assistant. --- import useBaseUrl from "@docusaurus/useBaseUrl"; +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + Even if you design your bot perfectly, users will inevitably say things to your assistant that you did not anticipate. In these cases, your assistant will fail, diff --git a/docs/docs/building-classic-assistants/language-support.mdx b/docs/docs/building-classic-assistants/language-support.mdx index 35e2006b689f..49407b3b39cd 100644 --- a/docs/docs/building-classic-assistants/language-support.mdx +++ b/docs/docs/building-classic-assistants/language-support.mdx @@ -5,6 +5,10 @@ title: Language Support abstract: You can use Rasa to build assistants in any language you want. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + Your Rasa assistant can be used on training data in **any language**. If there are no word embeddings for your language, you can train your featurizers from scratch with the data you provide. diff --git a/docs/docs/building-classic-assistants/model-configuration.mdx b/docs/docs/building-classic-assistants/model-configuration.mdx index 0f378b29ec4d..0e93da2a0b81 100644 --- a/docs/docs/building-classic-assistants/model-configuration.mdx +++ b/docs/docs/building-classic-assistants/model-configuration.mdx @@ -6,6 +6,10 @@ description: Learn about model configuration for Rasa. abstract: The configuration file defines the components and policies that your model will use to make predictions based on user input. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + The recipe key allows for different types of config and model architecture. Currently, "default.v1" and the experimental "graph.v1" recipes are supported. diff --git a/docs/docs/building-classic-assistants/nlu-only.mdx b/docs/docs/building-classic-assistants/nlu-only.mdx index a42b1aa6f5dd..50544c0bc09f 100644 --- a/docs/docs/building-classic-assistants/nlu-only.mdx +++ b/docs/docs/building-classic-assistants/nlu-only.mdx @@ -5,6 +5,10 @@ title: Using NLU Only abstract: Find out how to use only Rasa NLU as a standalone NLU service for your chatbot or virtual assistant. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + If you want to use Rasa only as an NLU component, you can! ## Training NLU-only models diff --git a/docs/docs/building-classic-assistants/nlu-training-data.mdx b/docs/docs/building-classic-assistants/nlu-training-data.mdx index 150e80be3ddc..90e7f58b96d8 100644 --- a/docs/docs/building-classic-assistants/nlu-training-data.mdx +++ b/docs/docs/building-classic-assistants/nlu-training-data.mdx @@ -6,6 +6,10 @@ description: Read more about how to format training data with Rasa NLU for open abstract: NLU training data stores structured information about user messages. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + The goal of NLU (Natural Language Understanding) is to extract structured information from user messages. This usually includes the user's [intent](../concepts/glossary.mdx#intent) and any [entities](../concepts/glossary.mdx#entity) their message contains. You can add extra information such as [regular expressions](#regular-expressions) and [lookup tables](#lookup-tables) to your diff --git a/docs/docs/building-classic-assistants/policies.mdx b/docs/docs/building-classic-assistants/policies.mdx index 95ae2fa8dad6..588999bd0457 100644 --- a/docs/docs/building-classic-assistants/policies.mdx +++ b/docs/docs/building-classic-assistants/policies.mdx @@ -5,6 +5,10 @@ title: Policies abstract: Your assistant uses policies to decide which action to take at each step in a conversation. There are machine-learning and rule-based policies that your assistant can use in tandem. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + You can customize the policies your assistant uses by specifying the `policies` key in your project's `config.yml`. There are different policies to choose from, and you can include diff --git a/docs/docs/building-classic-assistants/reaching-out-to-user.mdx b/docs/docs/building-classic-assistants/reaching-out-to-user.mdx index f3f9512ea5ad..c065826fd7f0 100644 --- a/docs/docs/building-classic-assistants/reaching-out-to-user.mdx +++ b/docs/docs/building-classic-assistants/reaching-out-to-user.mdx @@ -5,6 +5,10 @@ title: Reaching Out to the User abstract: Sometimes you want your assistant to reach out to the user without the user's prompting. For example, you might want the assistant to send a message when the user opens the chat window, or you might want to prompt the user if they haven't sent a message for a while. This page is a guide to enabling your assistant to reach out to the user proactively. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + ## Reaching out first In most use cases, when the user opens the chat window with your assistant, you will want the diff --git a/docs/docs/building-classic-assistants/rules.mdx b/docs/docs/building-classic-assistants/rules.mdx index a854d91ad9a4..3669b60fb71c 100644 --- a/docs/docs/building-classic-assistants/rules.mdx +++ b/docs/docs/building-classic-assistants/rules.mdx @@ -8,6 +8,10 @@ abstract: Rules describe short pieces of conversations that should always follow the same path. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + **Don't overuse rules**. Rules are great to handle small specific conversation patterns, but unlike [stories](./stories.mdx), rules don't have the power to generalize to unseen conversation paths. Combine rules and stories to make your assistant robust and able to handle diff --git a/docs/docs/building-classic-assistants/stories.mdx b/docs/docs/building-classic-assistants/stories.mdx index 2937e9d7278a..5563d3ff45c9 100644 --- a/docs/docs/building-classic-assistants/stories.mdx +++ b/docs/docs/building-classic-assistants/stories.mdx @@ -8,6 +8,10 @@ abstract: model. Stories can be used to train models that are able to generalize to unseen conversation paths. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + ## Format A story is a representation of a conversation between a user and an AI assistant, diff --git a/docs/docs/building-classic-assistants/training-data-format.mdx b/docs/docs/building-classic-assistants/training-data-format.mdx index f4bf5980354e..769c932313ea 100644 --- a/docs/docs/building-classic-assistants/training-data-format.mdx +++ b/docs/docs/building-classic-assistants/training-data-format.mdx @@ -7,6 +7,10 @@ abstract: This page describes the different types of training data that go into a Rasa assistant and how this training data is structured. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + ## Overview Rasa uses [YAML](https://yaml.org/spec/1.2/spec.html) as diff --git a/docs/docs/building-classic-assistants/tuning-your-model.mdx b/docs/docs/building-classic-assistants/tuning-your-model.mdx index 96b9f6996a06..eb9254ca4c33 100644 --- a/docs/docs/building-classic-assistants/tuning-your-model.mdx +++ b/docs/docs/building-classic-assistants/tuning-your-model.mdx @@ -6,6 +6,9 @@ abstract: Rasa will provide you with a suggested NLU config on initialization of --- import useBaseUrl from "@docusaurus/useBaseUrl"; +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + ## How to Choose a Pipeline diff --git a/docs/docs/building-classic-assistants/unexpected-input.mdx b/docs/docs/building-classic-assistants/unexpected-input.mdx index 6de964a93f17..06f0d7a66284 100644 --- a/docs/docs/building-classic-assistants/unexpected-input.mdx +++ b/docs/docs/building-classic-assistants/unexpected-input.mdx @@ -7,6 +7,10 @@ abstract: will say unexpected things. This page is a guide on handling unexpected input. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + Unexpected input is a deviation from the [happy path](../concepts/glossary.mdx#happy--unhappy-paths) that you have defined. For example: From 23c34698ff4962e766242ba00ca902a2872f6683 Mon Sep 17 00:00:00 2001 From: m-vdb Date: Wed, 20 Sep 2023 11:42:46 +0200 Subject: [PATCH 6/8] split content of testing-your-assistant.mdx docs --- CHANGELOG.mdx | 6 +- .../building-classic-assistants/glossary.mdx | 2 +- .../building-classic-assistants/stories.mdx | 2 +- .../testing-your-assistant.mdx | 554 ++++++++++++++++++ .../training-data-format.mdx | 2 +- .../tuning-your-model.mdx | 2 +- docs/docs/command-line-interface.mdx | 8 +- docs/docs/production/setting-up-ci-cd.mdx | 2 +- .../production/testing-your-assistant.mdx | 509 +--------------- docs/sidebars.js | 17 +- 10 files changed, 577 insertions(+), 527 deletions(-) create mode 100644 docs/docs/building-classic-assistants/testing-your-assistant.mdx diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index 887085775907..8b4f6e9c9c41 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -2788,9 +2788,9 @@ Upgrade dependent libraries with security vulnerabilities (Pillow, TensorFlow, u - [#6285](https://github.com/rasahq/rasa/issues/6285): Predictions of the [`FallbackClassifier`](./building-classic-assistants/components.mdx#fallbackclassifier) are ignored when - [evaluating the NLU model](./production/testing-your-assistant.mdx#evaluating-an-nlu-model) + [evaluating the NLU model](./building-classic-assistants/testing-your-assistant.mdx#evaluating-an-nlu-model) Note that the `FallbackClassifier` predictions still apply to - [test stories](./production/testing-your-assistant.mdx#writing-test-stories). + [test stories](./building-classic-assistants/testing-your-assistant.mdx#writing-test-stories). - [#6474](https://github.com/rasahq/rasa/issues/6474): Adapt the training data reader and emulator for wit.ai to their latest format. - [#6498](https://github.com/rasahq/rasa/issues/6498): Adding configurable prefixes to Redis [Tracker](./production/tracker-stores.mdx) and [Lock Stores](./production/lock-stores.mdx) so that a single Redis instance (and logical DB) can support multiple conversation trackers and locks. By default, conversations will be prefixed with `tracker:...` and all locks prefixed with `lock:...`. Additionally, you can add an alphanumeric-only `prefix: value` in `endpoints.yml` such that keys in redis will take the form `value:tracker:...` and `value:lock:...` respectively. @@ -4164,7 +4164,7 @@ This can help to fix problems when using `rasa shell` with custom actions which ### Improved Documentation -- [#2237](https://github.com/rasahq/rasa/issues/2237): Restructure the “Evaluating models” documentation page and rename this page to [Testing Your Assistant](./production/testing-your-assistant.mdx). +- [#2237](https://github.com/rasahq/rasa/issues/2237): Restructure the “Evaluating models” documentation page and rename this page to [Testing Your Assistant](./building-classic-assistants/testing-your-assistant.mdx). - [#5302](https://github.com/rasahq/rasa/issues/5302): Improved documentation on how to build and deploy an action server image for use on other servers such as Rasa X deployments. diff --git a/docs/docs/building-classic-assistants/glossary.mdx b/docs/docs/building-classic-assistants/glossary.mdx index a7a9df1af10f..83047bbf2a53 100644 --- a/docs/docs/building-classic-assistants/glossary.mdx +++ b/docs/docs/building-classic-assistants/glossary.mdx @@ -40,7 +40,7 @@ A way to store bot responses externally instead of including them directly in th The process of using user messages and conversation data to influence the design of an assistant and train the model, combined with engineering best practices. There are 6 steps that make up CDD: Share, Review, Annotate, Fix, Track, and Test. -## [Conversation Tests](../production/testing-your-assistant.mdx) +## [Conversation Tests](./testing-your-assistant.mdx) Modified story format that includes the full text of the user message in addition to the intent label. Test conversations are saved to a test set file (conversation_tests.md), which is used to evaluate the model’s predictions across an entire conversation. diff --git a/docs/docs/building-classic-assistants/stories.mdx b/docs/docs/building-classic-assistants/stories.mdx index 97cc72f1c8a5..d38926b9b346 100644 --- a/docs/docs/building-classic-assistants/stories.mdx +++ b/docs/docs/building-classic-assistants/stories.mdx @@ -200,7 +200,7 @@ Overusing OR statements will slow down training. ## Test Conversation Format The test conversation format is a format that combines both NLU data and stories -into a single file for evaluation. Read more about this format in [Testing Your Assistant](../production/testing-your-assistant.mdx). +into a single file for evaluation. Read more about this format in [Testing Your Assistant](./testing-your-assistant.mdx). :::caution testing only This format is only used for testing and cannot be used for training. diff --git a/docs/docs/building-classic-assistants/testing-your-assistant.mdx b/docs/docs/building-classic-assistants/testing-your-assistant.mdx new file mode 100644 index 000000000000..8cb6001c5c81 --- /dev/null +++ b/docs/docs/building-classic-assistants/testing-your-assistant.mdx @@ -0,0 +1,554 @@ +--- +id: testing-your-assistant +sidebar_label: Testing Your Assistant +title: Testing Your Assistant +abstract: + Rasa lets you validate and test dialogues end-to-end by running through + test stories. In addition, you can + also test the dialogue management and the message processing (NLU) + separately. +--- + +import useBaseUrl from "@docusaurus/useBaseUrl"; + +import RasaProLabel from "@theme/RasaProLabel"; + +import RasaProBanner from "@theme/RasaProBanner"; + +## Validating Data and Stories + +Data validation verifies that no mistakes or major inconsistencies appear in your domain, NLU +data, or story data. To validate your data, have your CI run this command: + +```bash +rasa data validate +``` + +If you pass a `max_history` value to one or more policies in your `config.yml` file, provide the +smallest of those values as + +```bash +rasa data validate --max-history +``` + +If data validation results in errors, training a model can also fail or yield bad performance, so it's +always good to run this check before training a model. By including the +`--fail-on-warnings` flag, this step will fail on warnings indicating more minor issues. + +:::note +Running `rasa data validate` does **not** test if your [rules](../building-classic-assistants/rules.mdx) are consistent with your stories. +However, during training, the `RulePolicy` checks for conflicts between rules and stories. Any such conflict will abort training. +::: + +To read more about the validator and all of the available options, see [the documentation for +`rasa data validate`](../command-line-interface.mdx#rasa-data-validate). + +## Writing Test Stories + +Testing your trained model on test stories is the best way to have confidence in how your assistant +will act in certain situations. Written in a modified story +format, test stories allow you to provide entire conversations and test that, given certain +user input, your model will behave in the expected manner. This is especially +important as you start introducing more complicated stories from user +conversations. + +Test stories are like +the stories in your training data, but include the user message as well. + +Here are some examples: + + + + +```yaml-rasa title="tests/test_stories.yml" {5,9,13} +stories: +- story: A basic story test + steps: + - user: | + hello + intent: greet + - action: utter_ask_howcanhelp + - user: | + show me [chinese]{"entity": "cuisine"} restaurants + intent: inform + - action: utter_ask_location + - user: | + in [Paris]{"entity": "location"} + intent: inform + - action: utter_ask_price +``` + + + + +```yaml-rasa title="tests/test_stories.yml" {8,11} +stories: +- story: A test where a user clicks on a button with payload + steps: + - user: | + hello + intent: greet + - action: utter_ask_howcanhelp + - user: /inform{{"cuisine":"chinese"}} + intent: inform + - action: utter_ask_location + - user: /inform{{"location":"Paris"}} + intent: inform + - action: utter_ask_price +``` + + + + +```yaml-rasa title="tests/test_stories.yml" {5,12} +stories: +- story: A test where a custom action returns events + steps: + - user: | + hey + intent: greet + - action: my_custom_action + - slot_was_set: + - my_slot: "value added by custom action" + - action: utter_ask_age + - user: | + thanks + intent: thankyou + - action: utter_no_worries +``` + + + + +```yaml-rasa title="tests/test_stories.yml" {5,9,14,20} +stories: +- story: A test story with a form + steps: + - user: | + hi + intent: greet + - action: utter_greet + - user: | + im looking for a restaurant + intent: request_restaurant + - action: restaurant_form + - active_loop: restaurant_form + - user: | + [afghan](cuisine) food + intent: inform + - action: restaurant_form + - active_loop: null + - action: utter_slots_values + - user: | + thanks + intent: thankyou + - action: utter_no_worries +``` + + + + +```yaml-rasa title="tests/test_stories.yml" {5,9,14,21} +stories: +- story: A test story with unexpected input during a form + steps: + - user: | + hi + intent: greet + - action: utter_greet + - user: | + im looking for a restaurant + intent: request_restaurant + - action: restaurant_form + - active_loop: restaurant_form + - user: | + How's the weather? + intent: chitchat + - action: utter_chitchat + - action: restaurant_form + - active_loop: null + - action: utter_slots_values + - user: | + thanks + intent: thankyou + - action: utter_no_worries +``` + + + + +```yaml-rasa title="tests/test_stories.yml" {5,9,13} +stories: +- story: A basic test story with multiple entities for a single token + steps: + - user: | + hello + intent: greet + - action: utter_ask_howcanhelp + - user: | + show me [chinese]{"entity": "cuisine"} restaurants + intent: inform + - action: utter_ask_location + - user: | + in [Paris][{"entity": "location"}, {"entity": "city"}] + intent: inform + - action: utter_ask_price +``` + + + + +By default, the command will run tests on stories from any files with names starting with `test_`. You can also provide +a specific test stories file or directory with the `--stories` argument. +You can test your assistant against them by running: + +```bash +rasa test +``` + +Conversation testing is only as thorough and accurate as the test +cases you include, so you should continue to grow your set of test stories +as you make improvements to your assistant. A good rule of thumb to follow is that you should aim for your test stories +to be representative of the true distribution of real conversations. + +See the [CLI documentation on `rasa test`](../command-line-interface.mdx#rasa-test) for +more configuration options. + +:::caution Testing Custom Actions +[Custom Actions](../concepts/custom-actions.mdx) are not executed as part of test stories. If your custom +actions append any events to the conversation, this has to be reflected in your test story +(e.g. by adding `slot_was_set` events to your test story). + +To test the code of your custom actions, you should write unit tests +for them and include these tests in your [CI/CD pipeline](../production/setting-up-ci-cd.mdx). + +::: + +## Evaluating an NLU Model + +In addition to testing stories, you can also test the natural language understanding (NLU) model separately. +Once your assistant is deployed in the real world, it will be processing messages that it hasn't seen +in the training data. To simulate this, you should always set aside some part of your data for testing. +You can either: + +1. [use a held out test set](#using-a-held-out-test-set) by shuffling and splitting your NLU data + +2. [use cross-validation](#using-cross-validation), which automatically creates + multiple train/test splits + +### Using a Held-Out Test Set + +If you use the train-test +set approach, it is best to [shuffle and split your data](../command-line-interface.mdx#rasa-data-split) +using `rasa data split` every time you evaluate your model, as +opposed to using a static NLU test set, which can easily become outdated. + +You can split your NLU data into train and test sets using: + +```bash +rasa data split nlu +``` + +Next, you can see how well your trained NLU model predicts the +data from the test set you generated, using: + +```bash {2} +rasa test nlu + --nlu train_test_split/test_data.yml +``` + +### Using Cross-Validation + +If you've made significant changes to your NLU training data (e.g. +splitting an intent into two intents or adding a lot of training examples), you should run a +full NLU evaluation using cross-validation. Cross-validation automatically creates +multiple train/test splits and averages the results of evaluations on each train/test split. +This means all your data is evaluated during cross-validation, making cross-validation the most +thorough way to automatically test your NLU model. + +To run NLU testing in cross-validation mode run: + +```bash {3} +rasa test nlu + --nlu data/nlu + --cross-validation +``` + +You can specify the number of test/train splits used with the `-f/--folds` flag: + +```bash {4} +rasa test nlu + --nlu data/nlu + --cross-validation + --folds 5 +``` + +Note that during cross-validation, the NLU model will be trained for each fold, +so cross-validation with a large data set and a high number of folds can be time-consuming. +On a small data set, a high number of folds can result in too few examples per intent being available for each test split. + +On the other hand, if you specify a low number of folds, your data will be split into much larger chunks, +and there will be proportionally less data to train on for each fold. + +Choose a number of folds that balances both considerations for your dataset size. + +:::tip hyperparameter tuning +To further improve your model check out this +[tutorial on hyperparameter tuning](https://blog.rasa.com/rasa-nlu-in-depth-part-3-hyperparameters/). +::: + +### Comparing NLU Pipelines + +To get the most out of your training data, you should train and evaluate your model on different pipelines +and different amounts of training data. + +To do so, pass multiple configuration files to the `rasa test` command: + +```bash {2} +rasa test nlu --nlu data/nlu.yml + --config config_1.yml config_2.yml +``` + +This performs several steps: + +1. Create a global 80% train / 20% test split from `data/nlu.yml`. +2. Exclude a certain percentage of data from the global train split. +3. Train models for each configuration on remaining training data. +4. Evaluate each model on the global test split. + +The above process is repeated with different percentages of training data in step 2 +to give you an idea of how each pipeline will behave if you increase the amount of training data. +Since training is not completely deterministic, the whole process is repeated +three times for each configuration specified. + +A graph with the mean and standard deviations of +[f1-scores](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) +across all runs is plotted. +The f1-score graph, along with all train/test sets, the trained models, classification and error reports, +will be saved into a folder called `nlu_comparison_results`. + +Inspecting the f1-score graph can help you understand if you have enough data for your NLU model. +If the graph shows that f1-score is still improving when all of the training data is used, +it may improve further with more data. But if f1-score has plateaued when all training data is used, +adding more data may not help. + +If you want to change the number of runs or exclusion percentages, you can: + +```bash {3} +rasa test nlu --nlu data/nlu.yml + --config config_1.yml config_2.yml + --runs 4 --percentages 0 25 50 70 90 +``` + +### Interpreting the Output + +#### Intent Classifiers + +The `rasa test` script will produce a report (`intent_report.json`), confusion matrix (`intent_confusion_matrix.png`) +and confidence histogram (`intent_histogram.png`) for your intent classification model. + +The report logs [precision](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html), +[recall](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html) and +[f1-score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) for each intent, +as well as providing an overall average. You can save these reports as JSON files using the `--report` argument. + +The confusion matrix shows which intents are mistaken for others. +Any samples which have been incorrectly predicted are logged and saved to a file called `errors.json` for easier debugging. + +
+ image +
+ +The histogram allows you to visualize the confidence for all predictions, +with the correct and incorrect predictions being displayed by blue and red bars respectively. +Improving the quality of your training data will move the blue histogram bars up the plot and the +red histogram bars down the plot. It should also help in reducing the number of red histogram bars itself. + +
+ image +
+ +#### Response Selectors + +`rasa test` evaluates response selectors in the same way that it evaluates intent classifiers, producing a +report (`response_selection_report.json`), confusion matrix (`response_selection_confusion_matrix.png`), +confidence histogram (`response_selection_histogram.png`) and errors (`response_selection_errors.json`). +If your pipeline includes multiple response selectors, they are evaluated in a single report. + +The report logs precision, recall and f1 measure for +each sub-intent of a [retrieval intent](../building-classic-assistants/glossary.mdx#retrieval-intent) and provides an overall average. +You can save these reports as JSON files using the `--report` argument. + +#### Entity Extraction + +`rasa test` reports recall, precision, and f1-score for each entity type that +your trainable entity extractors are trained to recognize. + +Only trainable entity extractors, such as the `DIETClassifier` and `CRFEntityExtractor` are +evaluated by `rasa test`. Pretrained extractors like the `DucklingHTTPExtractor` are not evaluated. + +If you have multiple entity extractors in your pipeline, or use some custom extractors, +multiple entities might be associated with the same token. In this case, +you can use a list notation in the test files, such as + +```yaml +stories: + - story: A basic test story with multiple entities for a single token + steps: + - user: | + I like [ice cream][{\"entity\": \"food\"}, {\"entity\": \"desert\"}] + intent: inform + # ... +``` + +:::caution incorrect entity annotations +If any of your entities are incorrectly annotated, your evaluation may fail. One common problem +is that an entity cannot stop or start inside a token. +For example, if you have an example for a `name` entity +like `[Brian](name)'s house`, this is only valid if your tokenizer splits `Brian's` into +multiple tokens. + +::: + +#### Entity Scoring + +To evaluate entity extraction we apply a simple tag-based approach. We don't consider +[BILOU tags](../building-classic-assistants/nlu-training-data.mdx#bilou-entity-tagging) exactly, but only the +entity type tags on a per token basis. For location entity like “near Alexanderplatz” we +expect the labels `LOC LOC` instead of the BILOU-based `B-LOC L-LOC`. + +Our approach is more lenient when it comes to evaluation, as it rewards +partial extraction and does not penalize the splitting of entities. +For example, given the aforementioned entity “near Alexanderplatz” and a system that extracts +“Alexanderplatz”, our approach rewards the extraction of “Alexanderplatz” and penalizes the missed out word “near”. + +The BILOU-based approach, however, would label this as a complete failure since it expects Alexanderplatz +to be labeled as a last token in an entity (`L-LOC`) instead of a single token entity (`U-LOC`). Note also that +a split extraction of “near” and “Alexanderplatz” would get full scores on our approach and zero on the +BILOU-based one. + +Here's a comparison between the two scoring mechanisms for the phrase “near Alexanderplatz tonight”: + +| extracted | Simple tags (score) | BILOU tags (score) | +| --------------------------------------------------- | ------------------- | ---------------------- | +| `[near Alexanderplatz](loc) [tonight](time)` | loc loc time (3) | B-loc L-loc U-time (3) | +| `[near](loc) [Alexanderplatz](loc) [tonight](time)` | loc loc time (3) | U-loc U-loc U-time (1) | +| `near [Alexanderplatz](loc) [tonight](time)` | O loc time (2) | O U-loc U-time (1) | +| `[near](loc) Alexanderplatz [tonight](time)` | loc O time (2) | U-loc O U-time (1) | +| `[near Alexanderplatz tonight](loc)` | loc loc loc (2) | B-loc I-loc L-loc (1) | + +## Evaluating a Dialogue Model + +You can evaluate your trained dialogue model on a set of test stories +by using the test script: + +```bash +rasa test core --stories test_stories.yml --out results +``` + +This will print any failed stories to `results/failed_test_stories.yml`. +A story fails if at least one of the actions was predicted incorrectly. + +The test script will also save a confusion matrix to a file called +`results/story_confmat.pdf`. For each action in your domain, the confusion +matrix shows how often the action was correctly predicted and how often an +incorrect action was predicted instead. + +### Interpreting the generated warnings + +The test script will also generate a warnings file called `results/stories_with_warnings.yml`. +This file contains all test stories for which [`action_unlikely_intent`](../concepts/default-actions.mdx#action_unlikely_intent) +was predicted at any conversation turn but all actions from the original story were predicted correctly. +However, if a test story originally included an `action_unlikely_intent`, for example to ensure [a rule is designed to +trigger the conversation path after an `action_unlikely_intent`](../concepts/default-actions.mdx#customization-1) but the ensemble of +policies failed to do so, then the corresponding story will end up in `results/failed_test_stories.yml` as +a failed story. + +The stories are sorted by the severity of `action_unlikely_intent`'s prediction. +This severity is calculated by [`UnexpecTEDIntentPolicy`](../building-classic-assistants/policies.mdx#unexpected-intent-policy) itself at prediction time. +The higher the severity, the more unlikely is the intent and hence reviewing that particular +conversation path becomes more critical. + +Note, that `action_unlikely_intent` is predicted by +`UnexpecTEDIntentPolicy` which employs a machine learning based model +under the hood and hence can result in false warnings as well. You can choose to ignore such warnings +if the conversation paths in these stories are already present in the training stories. + +### Comparing Policy Configurations + +To choose a configuration for your dialogue model, or to choose hyperparameters for a +specific policy, you want to measure how well your dialogue model will generalize +to conversations it hasn't seen before. Especially in the beginning +of a project, when you don't have a lot of real conversations to train +your bot on, you may not want to exclude some to use as a test set. + +Rasa has some scripts to help you choose and fine-tune your policy configuration. +Once you are happy with it, you can then train your final configuration on your +full data set. + +To do this, you first have to train models for your different +configurations. Create two (or more) config files including the policies you want to +compare, and then provide them to the train script to train your models: + +```bash +rasa train core -c config_1.yml config_2.yml \ + --out comparison_models --runs 3 --percentages 0 5 25 50 70 95 +``` + +Similar to how the [NLU model was evaluated](#comparing-nlu-pipelines), the above +command trains the dialogue model on multiple configurations and different amounts of training data. +For each config file provided, Rasa will train dialogue models +with 0, 5, 25, 50, 70 and 95% of your training stories excluded from the training +data. This is repeated three times to ensure consistent results. + +Once this script has finished, you can pass multiple models to the test script +to compare the models you just trained: + +```bash +rasa test core -m comparison_models --stories stories_folder + --out comparison_results --evaluate-model-directory +``` + +This will evaluate each model on the stories in `stories_folder` +(can be either training or test set) and plot some graphs +to show you which policy performs best. Since the previous train command +excluded some amount of training data to train each model, +the above test command can measure how well your model predicts the held-out stories. +To compare single policies, create config files containing only one policy each. + +:::note +This training process can take a long time, so we'd suggest letting it run +somewhere in the background where it can't be interrupted. + +::: + +### Testing Action Code + +The approach used to test your action code will depend on how it is +implemented. For example, if you connect to external APIs, you should write integration tests to ensure +that those APIs respond as expected to common inputs. However you test your action code, you should +include these tests in your CI pipeline so that they run each time you make changes. + +If you have any questions or problems, please share them with us in the dedicated +[testing section on our forum](https://forum.rasa.com/tags/testing)! + +## End-To-End Testing + + + + + +:::info New in 3.5 + +You can now use [end-to-end testing](../production/testing-your-assistant.mdx#end-to-end-testing) +to test your assistant as a whole, including dialogue management and custom actions. + +::: diff --git a/docs/docs/building-classic-assistants/training-data-format.mdx b/docs/docs/building-classic-assistants/training-data-format.mdx index 28546971dbbf..6683b7ed521e 100644 --- a/docs/docs/building-classic-assistants/training-data-format.mdx +++ b/docs/docs/building-classic-assistants/training-data-format.mdx @@ -706,7 +706,7 @@ rasa test ``` If you want to know more about testing head over to -[Testing Your Assistant](../production/testing-your-assistant.mdx). +[Testing Your Assistant](./testing-your-assistant.mdx). ## End-to-end Training diff --git a/docs/docs/building-classic-assistants/tuning-your-model.mdx b/docs/docs/building-classic-assistants/tuning-your-model.mdx index 0e5eb2aabd64..7199be4f1e76 100644 --- a/docs/docs/building-classic-assistants/tuning-your-model.mdx +++ b/docs/docs/building-classic-assistants/tuning-your-model.mdx @@ -246,7 +246,7 @@ When using a multi-intent, the intent is featurized for machine learning policie ### Comparing Pipelines Rasa gives you the tools to compare the performance of multiple pipelines on your data directly. -See [Comparing NLU Pipelines](../production/testing-your-assistant.mdx#comparing-nlu-pipelines) for more information. +See [Comparing NLU Pipelines](./testing-your-assistant.mdx#comparing-nlu-pipelines) for more information. ## Choosing the Right Components diff --git a/docs/docs/command-line-interface.mdx b/docs/docs/command-line-interface.mdx index 5324bcccc2f7..9a7c928b7be8 100644 --- a/docs/docs/command-line-interface.mdx +++ b/docs/docs/command-line-interface.mdx @@ -12,7 +12,7 @@ import RasaProBanner from "@theme/RasaProBanner"; ## Cheat Sheet -The following commands are relevant to all assistants built with Rasa. +The following commands are relevant to all assistants built with Rasa. | Command | Effect | | ------------------------- | -------------------------------------------------------------------------------------------------- | @@ -31,7 +31,7 @@ The following commands are relevant to all assistants built with Rasa. | `rasa -h` | Shows all available commands. | -The following commands are only relevant if you are building NLU-based "classic" assistants. +The following commands are only relevant if you are building NLU-based "classic" assistants. | Command | Effect | | ------------------------- | -------------------------------------------------------------------------------------------------- | | @@ -401,8 +401,8 @@ rasa test nlu ``` You can find more details on specific arguments for each testing type in -[Evaluating an NLU Model](./production/testing-your-assistant.mdx#evaluating-an-nlu-model) and -[Evaluating a Dialogue Management Model](./production/testing-your-assistant.mdx#evaluating-a-dialogue-model). +[Evaluating an NLU Model](./building-classic-assistants/testing-your-assistant.mdx#evaluating-an-nlu-model) and +[Evaluating a Dialogue Management Model](./building-classic-assistants/testing-your-assistant.mdx#evaluating-a-dialogue-model). The following arguments are available for `rasa test`: diff --git a/docs/docs/production/setting-up-ci-cd.mdx b/docs/docs/production/setting-up-ci-cd.mdx index 82856ba5ab5f..dc603796e70b 100644 --- a/docs/docs/production/setting-up-ci-cd.mdx +++ b/docs/docs/production/setting-up-ci-cd.mdx @@ -109,7 +109,7 @@ succeeded. ### Deploying Your Rasa Model -If you ran [test stories](../production/testing-your-assistant.mdx) in your CI pipeline, +If you ran [test stories](../building-classic-assistants/testing-your-assistant.mdx) in your CI pipeline, you'll already have a trained model. You can set up your CD pipeline to upload the trained model to your Rasa server if the CI results are satisfactory. For example, to upload a model to Rasa X/Enterprise: diff --git a/docs/docs/production/testing-your-assistant.mdx b/docs/docs/production/testing-your-assistant.mdx index 70d052472fe4..aa968273de7d 100644 --- a/docs/docs/production/testing-your-assistant.mdx +++ b/docs/docs/production/testing-your-assistant.mdx @@ -3,10 +3,7 @@ id: testing-your-assistant sidebar_label: Testing Your Assistant title: Testing Your Assistant abstract: - Rasa lets you validate and test dialogues end-to-end by running through - test stories. In addition, you can - also test the dialogue management and the message processing (NLU) - separately. + Rasa lets you validate and test dialogues end-to-end. --- import useBaseUrl from "@docusaurus/useBaseUrl"; @@ -15,7 +12,7 @@ import RasaProLabel from "@theme/RasaProLabel"; import RasaProBanner from "@theme/RasaProBanner"; -## Validating Data and Stories +## Validating Data Data validation verifies that no mistakes or major inconsistencies appear in your domain, NLU data, or story data. To validate your data, have your CI run this command: @@ -35,511 +32,9 @@ If data validation results in errors, training a model can also fail or yield ba always good to run this check before training a model. By including the `--fail-on-warnings` flag, this step will fail on warnings indicating more minor issues. -:::note -Running `rasa data validate` does **not** test if your [rules](../building-classic-assistants/rules.mdx) are consistent with your stories. -However, during training, the `RulePolicy` checks for conflicts between rules and stories. Any such conflict will abort training. -::: - To read more about the validator and all of the available options, see [the documentation for `rasa data validate`](../command-line-interface.mdx#rasa-data-validate). -## Writing Test Stories - -Testing your trained model on test stories is the best way to have confidence in how your assistant -will act in certain situations. Written in a modified story -format, test stories allow you to provide entire conversations and test that, given certain -user input, your model will behave in the expected manner. This is especially -important as you start introducing more complicated stories from user -conversations. - -Test stories are like -the stories in your training data, but include the user message as well. - -Here are some examples: - - - - -```yaml-rasa title="tests/test_stories.yml" {5,9,13} -stories: -- story: A basic story test - steps: - - user: | - hello - intent: greet - - action: utter_ask_howcanhelp - - user: | - show me [chinese]{"entity": "cuisine"} restaurants - intent: inform - - action: utter_ask_location - - user: | - in [Paris]{"entity": "location"} - intent: inform - - action: utter_ask_price -``` - - - - -```yaml-rasa title="tests/test_stories.yml" {8,11} -stories: -- story: A test where a user clicks on a button with payload - steps: - - user: | - hello - intent: greet - - action: utter_ask_howcanhelp - - user: /inform{{"cuisine":"chinese"}} - intent: inform - - action: utter_ask_location - - user: /inform{{"location":"Paris"}} - intent: inform - - action: utter_ask_price -``` - - - - -```yaml-rasa title="tests/test_stories.yml" {5,12} -stories: -- story: A test where a custom action returns events - steps: - - user: | - hey - intent: greet - - action: my_custom_action - - slot_was_set: - - my_slot: "value added by custom action" - - action: utter_ask_age - - user: | - thanks - intent: thankyou - - action: utter_no_worries -``` - - - - -```yaml-rasa title="tests/test_stories.yml" {5,9,14,20} -stories: -- story: A test story with a form - steps: - - user: | - hi - intent: greet - - action: utter_greet - - user: | - im looking for a restaurant - intent: request_restaurant - - action: restaurant_form - - active_loop: restaurant_form - - user: | - [afghan](cuisine) food - intent: inform - - action: restaurant_form - - active_loop: null - - action: utter_slots_values - - user: | - thanks - intent: thankyou - - action: utter_no_worries -``` - - - - -```yaml-rasa title="tests/test_stories.yml" {5,9,14,21} -stories: -- story: A test story with unexpected input during a form - steps: - - user: | - hi - intent: greet - - action: utter_greet - - user: | - im looking for a restaurant - intent: request_restaurant - - action: restaurant_form - - active_loop: restaurant_form - - user: | - How's the weather? - intent: chitchat - - action: utter_chitchat - - action: restaurant_form - - active_loop: null - - action: utter_slots_values - - user: | - thanks - intent: thankyou - - action: utter_no_worries -``` - - - - -```yaml-rasa title="tests/test_stories.yml" {5,9,13} -stories: -- story: A basic test story with multiple entities for a single token - steps: - - user: | - hello - intent: greet - - action: utter_ask_howcanhelp - - user: | - show me [chinese]{"entity": "cuisine"} restaurants - intent: inform - - action: utter_ask_location - - user: | - in [Paris][{"entity": "location"}, {"entity": "city"}] - intent: inform - - action: utter_ask_price -``` - - - - -By default, the command will run tests on stories from any files with names starting with `test_`. You can also provide -a specific test stories file or directory with the `--stories` argument. -You can test your assistant against them by running: - -```bash -rasa test -``` - -Conversation testing is only as thorough and accurate as the test -cases you include, so you should continue to grow your set of test stories -as you make improvements to your assistant. A good rule of thumb to follow is that you should aim for your test stories -to be representative of the true distribution of real conversations. - -See the [CLI documentation on `rasa test`](../command-line-interface.mdx#rasa-test) for -more configuration options. - -:::caution Testing Custom Actions -[Custom Actions](../concepts/custom-actions.mdx) are not executed as part of test stories. If your custom -actions append any events to the conversation, this has to be reflected in your test story -(e.g. by adding `slot_was_set` events to your test story). - -To test the code of your custom actions, you should write unit tests -for them and include these tests in your [CI/CD pipeline](./setting-up-ci-cd.mdx). - -::: - -## Evaluating an NLU Model - -In addition to testing stories, you can also test the natural language understanding (NLU) model separately. -Once your assistant is deployed in the real world, it will be processing messages that it hasn't seen -in the training data. To simulate this, you should always set aside some part of your data for testing. -You can either: - -1. [use a held out test set](#using-a-held-out-test-set) by shuffling and splitting your NLU data - -2. [use cross-validation](#using-cross-validation), which automatically creates - multiple train/test splits - -### Using a Held-Out Test Set - -If you use the train-test -set approach, it is best to [shuffle and split your data](../command-line-interface.mdx#rasa-data-split) -using `rasa data split` every time you evaluate your model, as -opposed to using a static NLU test set, which can easily become outdated. - -You can split your NLU data into train and test sets using: - -```bash -rasa data split nlu -``` - -Next, you can see how well your trained NLU model predicts the -data from the test set you generated, using: - -```bash {2} -rasa test nlu - --nlu train_test_split/test_data.yml -``` - -### Using Cross-Validation - -If you've made significant changes to your NLU training data (e.g. -splitting an intent into two intents or adding a lot of training examples), you should run a -full NLU evaluation using cross-validation. Cross-validation automatically creates -multiple train/test splits and averages the results of evaluations on each train/test split. -This means all your data is evaluated during cross-validation, making cross-validation the most -thorough way to automatically test your NLU model. - -To run NLU testing in cross-validation mode run: - -```bash {3} -rasa test nlu - --nlu data/nlu - --cross-validation -``` - -You can specify the number of test/train splits used with the `-f/--folds` flag: - -```bash {4} -rasa test nlu - --nlu data/nlu - --cross-validation - --folds 5 -``` - -Note that during cross-validation, the NLU model will be trained for each fold, -so cross-validation with a large data set and a high number of folds can be time-consuming. -On a small data set, a high number of folds can result in too few examples per intent being available for each test split. - -On the other hand, if you specify a low number of folds, your data will be split into much larger chunks, -and there will be proportionally less data to train on for each fold. - -Choose a number of folds that balances both considerations for your dataset size. - -:::tip hyperparameter tuning -To further improve your model check out this -[tutorial on hyperparameter tuning](https://blog.rasa.com/rasa-nlu-in-depth-part-3-hyperparameters/). -::: - -### Comparing NLU Pipelines - -To get the most out of your training data, you should train and evaluate your model on different pipelines -and different amounts of training data. - -To do so, pass multiple configuration files to the `rasa test` command: - -```bash {2} -rasa test nlu --nlu data/nlu.yml - --config config_1.yml config_2.yml -``` - -This performs several steps: - -1. Create a global 80% train / 20% test split from `data/nlu.yml`. -2. Exclude a certain percentage of data from the global train split. -3. Train models for each configuration on remaining training data. -4. Evaluate each model on the global test split. - -The above process is repeated with different percentages of training data in step 2 -to give you an idea of how each pipeline will behave if you increase the amount of training data. -Since training is not completely deterministic, the whole process is repeated -three times for each configuration specified. - -A graph with the mean and standard deviations of -[f1-scores](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) -across all runs is plotted. -The f1-score graph, along with all train/test sets, the trained models, classification and error reports, -will be saved into a folder called `nlu_comparison_results`. - -Inspecting the f1-score graph can help you understand if you have enough data for your NLU model. -If the graph shows that f1-score is still improving when all of the training data is used, -it may improve further with more data. But if f1-score has plateaued when all training data is used, -adding more data may not help. - -If you want to change the number of runs or exclusion percentages, you can: - -```bash {3} -rasa test nlu --nlu data/nlu.yml - --config config_1.yml config_2.yml - --runs 4 --percentages 0 25 50 70 90 -``` - -### Interpreting the Output - -#### Intent Classifiers - -The `rasa test` script will produce a report (`intent_report.json`), confusion matrix (`intent_confusion_matrix.png`) -and confidence histogram (`intent_histogram.png`) for your intent classification model. - -The report logs [precision](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html), -[recall](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html) and -[f1-score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html) for each intent, -as well as providing an overall average. You can save these reports as JSON files using the `--report` argument. - -The confusion matrix shows which intents are mistaken for others. -Any samples which have been incorrectly predicted are logged and saved to a file called `errors.json` for easier debugging. - -
- image -
- -The histogram allows you to visualize the confidence for all predictions, -with the correct and incorrect predictions being displayed by blue and red bars respectively. -Improving the quality of your training data will move the blue histogram bars up the plot and the -red histogram bars down the plot. It should also help in reducing the number of red histogram bars itself. - -
- image -
- -#### Response Selectors - -`rasa test` evaluates response selectors in the same way that it evaluates intent classifiers, producing a -report (`response_selection_report.json`), confusion matrix (`response_selection_confusion_matrix.png`), -confidence histogram (`response_selection_histogram.png`) and errors (`response_selection_errors.json`). -If your pipeline includes multiple response selectors, they are evaluated in a single report. - -The report logs precision, recall and f1 measure for -each sub-intent of a [retrieval intent](../building-classic-assistants/glossary.mdx#retrieval-intent) and provides an overall average. -You can save these reports as JSON files using the `--report` argument. - -#### Entity Extraction - -`rasa test` reports recall, precision, and f1-score for each entity type that -your trainable entity extractors are trained to recognize. - -Only trainable entity extractors, such as the `DIETClassifier` and `CRFEntityExtractor` are -evaluated by `rasa test`. Pretrained extractors like the `DucklingHTTPExtractor` are not evaluated. - -If you have multiple entity extractors in your pipeline, or use some custom extractors, -multiple entities might be associated with the same token. In this case, -you can use a list notation in the test files, such as - -```yaml -stories: - - story: A basic test story with multiple entities for a single token - steps: - - user: | - I like [ice cream][{\"entity\": \"food\"}, {\"entity\": \"desert\"}] - intent: inform - # ... -``` - -:::caution incorrect entity annotations -If any of your entities are incorrectly annotated, your evaluation may fail. One common problem -is that an entity cannot stop or start inside a token. -For example, if you have an example for a `name` entity -like `[Brian](name)'s house`, this is only valid if your tokenizer splits `Brian's` into -multiple tokens. - -::: - -#### Entity Scoring - -To evaluate entity extraction we apply a simple tag-based approach. We don't consider -[BILOU tags](../building-classic-assistants/nlu-training-data.mdx#bilou-entity-tagging) exactly, but only the -entity type tags on a per token basis. For location entity like “near Alexanderplatz” we -expect the labels `LOC LOC` instead of the BILOU-based `B-LOC L-LOC`. - -Our approach is more lenient when it comes to evaluation, as it rewards -partial extraction and does not penalize the splitting of entities. -For example, given the aforementioned entity “near Alexanderplatz” and a system that extracts -“Alexanderplatz”, our approach rewards the extraction of “Alexanderplatz” and penalizes the missed out word “near”. - -The BILOU-based approach, however, would label this as a complete failure since it expects Alexanderplatz -to be labeled as a last token in an entity (`L-LOC`) instead of a single token entity (`U-LOC`). Note also that -a split extraction of “near” and “Alexanderplatz” would get full scores on our approach and zero on the -BILOU-based one. - -Here's a comparison between the two scoring mechanisms for the phrase “near Alexanderplatz tonight”: - -| extracted | Simple tags (score) | BILOU tags (score) | -| --------------------------------------------------- | ------------------- | ---------------------- | -| `[near Alexanderplatz](loc) [tonight](time)` | loc loc time (3) | B-loc L-loc U-time (3) | -| `[near](loc) [Alexanderplatz](loc) [tonight](time)` | loc loc time (3) | U-loc U-loc U-time (1) | -| `near [Alexanderplatz](loc) [tonight](time)` | O loc time (2) | O U-loc U-time (1) | -| `[near](loc) Alexanderplatz [tonight](time)` | loc O time (2) | U-loc O U-time (1) | -| `[near Alexanderplatz tonight](loc)` | loc loc loc (2) | B-loc I-loc L-loc (1) | - -## Evaluating a Dialogue Model - -You can evaluate your trained dialogue model on a set of test stories -by using the test script: - -```bash -rasa test core --stories test_stories.yml --out results -``` - -This will print any failed stories to `results/failed_test_stories.yml`. -A story fails if at least one of the actions was predicted incorrectly. - -The test script will also save a confusion matrix to a file called -`results/story_confmat.pdf`. For each action in your domain, the confusion -matrix shows how often the action was correctly predicted and how often an -incorrect action was predicted instead. - -### Interpreting the generated warnings - -The test script will also generate a warnings file called `results/stories_with_warnings.yml`. -This file contains all test stories for which [`action_unlikely_intent`](../concepts/default-actions.mdx#action_unlikely_intent) -was predicted at any conversation turn but all actions from the original story were predicted correctly. -However, if a test story originally included an `action_unlikely_intent`, for example to ensure [a rule is designed to -trigger the conversation path after an `action_unlikely_intent`](../concepts/default-actions.mdx#customization-1) but the ensemble of -policies failed to do so, then the corresponding story will end up in `results/failed_test_stories.yml` as -a failed story. - -The stories are sorted by the severity of `action_unlikely_intent`'s prediction. -This severity is calculated by [`UnexpecTEDIntentPolicy`](../building-classic-assistants/policies.mdx#unexpected-intent-policy) itself at prediction time. -The higher the severity, the more unlikely is the intent and hence reviewing that particular -conversation path becomes more critical. - -Note, that `action_unlikely_intent` is predicted by -`UnexpecTEDIntentPolicy` which employs a machine learning based model -under the hood and hence can result in false warnings as well. You can choose to ignore such warnings -if the conversation paths in these stories are already present in the training stories. - -### Comparing Policy Configurations - -To choose a configuration for your dialogue model, or to choose hyperparameters for a -specific policy, you want to measure how well your dialogue model will generalize -to conversations it hasn't seen before. Especially in the beginning -of a project, when you don't have a lot of real conversations to train -your bot on, you may not want to exclude some to use as a test set. - -Rasa has some scripts to help you choose and fine-tune your policy configuration. -Once you are happy with it, you can then train your final configuration on your -full data set. - -To do this, you first have to train models for your different -configurations. Create two (or more) config files including the policies you want to -compare, and then provide them to the train script to train your models: - -```bash -rasa train core -c config_1.yml config_2.yml \ - --out comparison_models --runs 3 --percentages 0 5 25 50 70 95 -``` - -Similar to how the [NLU model was evaluated](../production/testing-your-assistant.mdx#comparing-nlu-pipelines), the above -command trains the dialogue model on multiple configurations and different amounts of training data. -For each config file provided, Rasa will train dialogue models -with 0, 5, 25, 50, 70 and 95% of your training stories excluded from the training -data. This is repeated three times to ensure consistent results. - -Once this script has finished, you can pass multiple models to the test script -to compare the models you just trained: - -```bash -rasa test core -m comparison_models --stories stories_folder - --out comparison_results --evaluate-model-directory -``` - -This will evaluate each model on the stories in `stories_folder` -(can be either training or test set) and plot some graphs -to show you which policy performs best. Since the previous train command -excluded some amount of training data to train each model, -the above test command can measure how well your model predicts the held-out stories. -To compare single policies, create config files containing only one policy each. - -:::note -This training process can take a long time, so we'd suggest letting it run -somewhere in the background where it can't be interrupted. - -::: - -### Testing Action Code - -The approach used to test your action code will depend on how it is -implemented. For example, if you connect to external APIs, you should write integration tests to ensure -that those APIs respond as expected to common inputs. However you test your action code, you should -include these tests in your CI pipeline so that they run each time you make changes. - -If you have any questions or problems, please share them with us in the dedicated -[testing section on our forum](https://forum.rasa.com/tags/testing)! - ## End-To-End Testing diff --git a/docs/sidebars.js b/docs/sidebars.js index e7b1831130ea..1a3917fa5779 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -26,7 +26,7 @@ module.exports = { ], }, "tutorial", - "command-line-interface", + "command-line-interface", ], }, { @@ -36,7 +36,7 @@ module.exports = { items: [ // TODO: ENG-537 "concepts/flows", - "concepts/dialogue-understanding", + "concepts/dialogue-understanding", "concepts/domain", "concepts/unhappy-paths", { @@ -54,9 +54,9 @@ module.exports = { label: "Responses", items: [ "concepts/responses", - "concepts/contextual-response-rephraser", + "concepts/contextual-response-rephraser", ], - }, + }, { type: "category", label: "Components", @@ -105,7 +105,8 @@ module.exports = { "building-classic-assistants/training-data-format", "building-classic-assistants/nlu-training-data", "building-classic-assistants/tuning-your-model", - "building-classic-assistants/nlu-only", + "building-classic-assistants/nlu-only", + "building-classic-assistants/testing-your-assistant", ], }, ], @@ -246,7 +247,7 @@ module.exports = { ], }, "operating/tracing", - "operating/spaces", + "operating/spaces", ], }, { @@ -260,7 +261,7 @@ module.exports = { label: "Reference", collapsed: true, items: [ - "building-classic-assistants/glossary", + "building-classic-assistants/glossary", "telemetry/telemetry", "telemetry/reference", require("./docs/reference/sidebar.json"), @@ -283,5 +284,5 @@ module.exports = { }, ], }, - ] + ], }; From f2803e6a80ae1a5e62e07fd6ef80eb0c9cc23a8b Mon Sep 17 00:00:00 2001 From: m-vdb Date: Wed, 20 Sep 2023 14:44:21 +0200 Subject: [PATCH 7/8] update banner url + fix typo --- docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx b/docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx index 9daad8b96dee..a12defd5d87a 100644 --- a/docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx +++ b/docs/themes/theme-custom/theme/RasaNLUBasedBanner/index.jsx @@ -18,7 +18,7 @@ function RasaNLUBasedBanner() {

This page refers to building NLU-based assistants. - If you are working with Conversation AI with Language Models (CALM), + If you are working with Conversational AI with Language Models (CALM), this content may not apply to you.

From a87e9a365af7b767687e6097d211110bfbac808e Mon Sep 17 00:00:00 2001 From: m-vdb Date: Wed, 20 Sep 2023 14:46:47 +0200 Subject: [PATCH 8/8] add banner to new pages --- docs/docs/building-classic-assistants/domain.mdx | 4 ++++ docs/docs/building-classic-assistants/glossary.mdx | 4 ++++ .../building-classic-assistants/testing-your-assistant.mdx | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/docs/building-classic-assistants/domain.mdx b/docs/docs/building-classic-assistants/domain.mdx index b5aad8fe3d27..95e399b28887 100644 --- a/docs/docs/building-classic-assistants/domain.mdx +++ b/docs/docs/building-classic-assistants/domain.mdx @@ -5,6 +5,10 @@ title: Domain abstract: The domain defines the universe in which your assistant operates. It specifies the intents, entities, slots, responses, forms, and actions your bot should know about. It also defines a configuration for conversation sessions. --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + Here is a full example of a domain, taken from the [concertbot](https://github.com/RasaHQ/rasa/tree/main/examples/concertbot) example: diff --git a/docs/docs/building-classic-assistants/glossary.mdx b/docs/docs/building-classic-assistants/glossary.mdx index 83047bbf2a53..d365c0251585 100644 --- a/docs/docs/building-classic-assistants/glossary.mdx +++ b/docs/docs/building-classic-assistants/glossary.mdx @@ -5,6 +5,10 @@ title: Rasa Glossary description: Glossary for all Rasa-related terms --- +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + + ## [Action](../concepts/custom-actions.mdx) A single step that a bot takes in a conversation (e.g. calling an API or sending a response back to the user). diff --git a/docs/docs/building-classic-assistants/testing-your-assistant.mdx b/docs/docs/building-classic-assistants/testing-your-assistant.mdx index 8cb6001c5c81..29a9095617a5 100644 --- a/docs/docs/building-classic-assistants/testing-your-assistant.mdx +++ b/docs/docs/building-classic-assistants/testing-your-assistant.mdx @@ -12,8 +12,10 @@ abstract: import useBaseUrl from "@docusaurus/useBaseUrl"; import RasaProLabel from "@theme/RasaProLabel"; - import RasaProBanner from "@theme/RasaProBanner"; +import RasaNLUBasedBanner from "@theme/RasaNLUBasedBanner"; + + ## Validating Data and Stories