From a2a1c581c0ef9485d95b5f9d432b25e6587b8680 Mon Sep 17 00:00:00 2001 From: Roman Zlobin Date: Mon, 13 Nov 2023 14:07:02 +0300 Subject: [PATCH 1/6] Reduce max line length for tutorials (#241) * set max line length to 80 for tutorials * increase flake8 line limit && fix noqa on some files Flake was also complaining about the length of comments and strings which are displayed properly in the docs. And blake is enough to ensure that code fits in the docs. Line limit decreased because some strings were still not fitting. And also `flake: noqa: E402` doesn't work -- that line disables all quality assurance for that file so instead `per-file-ignores` are added. * cosmetic fixes * add per-file-ignore for `__init__` files --- dff/__init__.py | 1 - dff/context_storages/__init__.py | 1 - dff/messengers/__init__.py | 1 - dff/messengers/common/__init__.py | 1 - dff/messengers/telegram/__init__.py | 1 - dff/pipeline/__init__.py | 1 - dff/pipeline/pipeline/__init__.py | 1 - dff/pipeline/service/__init__.py | 1 - dff/script/__init__.py | 1 - dff/script/conditions/__init__.py | 1 - dff/script/core/__init__.py | 1 - dff/script/extras/__init__.py | 1 - dff/script/extras/conditions/__init__.py | 1 - dff/script/extras/slots/__init__.py | 1 - dff/script/labels/__init__.py | 1 - dff/script/responses/__init__.py | 1 - dff/stats/__init__.py | 1 - dff/utils/__init__.py | 1 - dff/utils/db_benchmark/__init__.py | 1 - dff/utils/parser/__init__.py | 1 - dff/utils/testing/__init__.py | 1 - dff/utils/turn_caching/__init__.py | 1 - dff/utils/viewer/__init__.py | 1 - makefile | 8 +- tutorials/context_storages/1_basics.py | 2 +- tutorials/context_storages/4_redis.py | 4 +- .../context_storages/8_db_benchmarking.py | 4 +- tutorials/messengers/telegram/1_basic.py | 14 ++- tutorials/messengers/telegram/2_buttons.py | 35 ++++++-- .../telegram/3_buttons_with_callback.py | 29 ++++-- tutorials/messengers/telegram/4_conditions.py | 33 +++++-- .../telegram/5_conditions_with_media.py | 52 ++++++++--- .../telegram/6_conditions_extras.py | 15 +++- .../messengers/telegram/7_polling_setup.py | 6 +- .../messengers/telegram/8_webhook_setup.py | 6 +- .../messengers/web_api_interface/1_fastapi.py | 2 +- .../web_api_interface/2_websocket_chat.py | 4 +- .../3_load_testing_with_locust.py | 26 ++++-- .../web_api_interface/4_streamlit_chat.py | 18 ++-- tutorials/pipeline/1_basics.py | 7 +- .../pipeline/2_pre_and_post_processors.py | 15 +++- .../3_pipeline_dict_with_services_basic.py | 9 +- .../3_pipeline_dict_with_services_full.py | 17 +++- .../pipeline/4_groups_and_conditions_full.py | 12 ++- ...5_asynchronous_groups_and_services_full.py | 8 +- .../pipeline/6_custom_messenger_interface.py | 7 +- tutorials/pipeline/7_extra_handlers_basic.py | 7 +- tutorials/pipeline/7_extra_handlers_full.py | 22 ++++- .../8_extra_handlers_and_extensions.py | 8 +- tutorials/script/core/1_basics.py | 36 ++++++-- tutorials/script/core/2_conditions.py | 45 ++++++++-- tutorials/script/core/3_responses.py | 42 +++++++-- tutorials/script/core/4_transitions.py | 90 ++++++++++++++----- tutorials/script/core/5_global_transitions.py | 66 ++++++++++---- .../script/core/7_pre_response_processing.py | 23 +++-- tutorials/script/core/8_misc.py | 34 +++++-- .../core/9_pre_transitions_processing.py | 9 +- tutorials/script/responses/1_basics.py | 24 +++-- tutorials/script/responses/2_buttons.py | 32 +++++-- tutorials/script/responses/3_media.py | 39 ++++++-- tutorials/script/responses/4_multi_message.py | 18 +++- tutorials/stats/1_extractor_functions.py | 13 ++- tutorials/stats/2_pipeline_integration.py | 19 +++- tutorials/utils/1_cache.py | 4 +- tutorials/utils/2_lru_cache.py | 4 +- 65 files changed, 654 insertions(+), 237 deletions(-) diff --git a/dff/__init__.py b/dff/__init__.py index e4265731b..091c40440 100644 --- a/dff/__init__.py +++ b/dff/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 __author__ = "Denis Kuznetsov" diff --git a/dff/context_storages/__init__.py b/dff/context_storages/__init__.py index 0a03a4bf5..e41618440 100644 --- a/dff/context_storages/__init__.py +++ b/dff/context_storages/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .database import DBContextStorage, threadsafe_method, context_storage_factory from .json import JSONContextStorage, json_available diff --git a/dff/messengers/__init__.py b/dff/messengers/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/messengers/__init__.py +++ b/dff/messengers/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/messengers/common/__init__.py b/dff/messengers/common/__init__.py index cbd121e55..ceac90c63 100644 --- a/dff/messengers/common/__init__.py +++ b/dff/messengers/common/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .interface import MessengerInterface, PollingMessengerInterface, CallbackMessengerInterface, CLIMessengerInterface from .types import PipelineRunnerFunction, PollingInterfaceLoopFunction diff --git a/dff/messengers/telegram/__init__.py b/dff/messengers/telegram/__init__.py index 9f67af2dd..cb7e38305 100644 --- a/dff/messengers/telegram/__init__.py +++ b/dff/messengers/telegram/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 try: import telebot diff --git a/dff/pipeline/__init__.py b/dff/pipeline/__init__.py index 322be0795..95f85e82b 100644 --- a/dff/pipeline/__init__.py +++ b/dff/pipeline/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .conditions import ( diff --git a/dff/pipeline/pipeline/__init__.py b/dff/pipeline/pipeline/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/pipeline/pipeline/__init__.py +++ b/dff/pipeline/pipeline/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/pipeline/service/__init__.py b/dff/pipeline/service/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/pipeline/service/__init__.py +++ b/dff/pipeline/service/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/script/__init__.py b/dff/script/__init__.py index b9779c1d6..04afef572 100644 --- a/dff/script/__init__.py +++ b/dff/script/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .core.context import Context from .core.keywords import ( diff --git a/dff/script/conditions/__init__.py b/dff/script/conditions/__init__.py index b9a515d99..49f17e8c3 100644 --- a/dff/script/conditions/__init__.py +++ b/dff/script/conditions/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .std_conditions import ( exact_match, diff --git a/dff/script/core/__init__.py b/dff/script/core/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/script/core/__init__.py +++ b/dff/script/core/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/script/extras/__init__.py b/dff/script/extras/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/script/extras/__init__.py +++ b/dff/script/extras/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/script/extras/conditions/__init__.py b/dff/script/extras/conditions/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/script/extras/conditions/__init__.py +++ b/dff/script/extras/conditions/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/script/extras/slots/__init__.py b/dff/script/extras/slots/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/script/extras/slots/__init__.py +++ b/dff/script/extras/slots/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/script/labels/__init__.py b/dff/script/labels/__init__.py index 901f9af88..a99fb0803 100644 --- a/dff/script/labels/__init__.py +++ b/dff/script/labels/__init__.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .std_labels import repeat, previous, to_start, to_fallback, forward, backward diff --git a/dff/script/responses/__init__.py b/dff/script/responses/__init__.py index d852bbdbf..fe2f294ea 100644 --- a/dff/script/responses/__init__.py +++ b/dff/script/responses/__init__.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .std_responses import choice diff --git a/dff/stats/__init__.py b/dff/stats/__init__.py index 993c75fa9..a93d7dbf8 100644 --- a/dff/stats/__init__.py +++ b/dff/stats/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.sdk.trace.export import ConsoleSpanExporter diff --git a/dff/utils/__init__.py b/dff/utils/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/utils/__init__.py +++ b/dff/utils/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/utils/db_benchmark/__init__.py b/dff/utils/db_benchmark/__init__.py index 1b994d21a..6d02f7a8d 100644 --- a/dff/utils/db_benchmark/__init__.py +++ b/dff/utils/db_benchmark/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from dff.utils.db_benchmark.benchmark import ( time_context_read_write, DBFactory, diff --git a/dff/utils/parser/__init__.py b/dff/utils/parser/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/utils/parser/__init__.py +++ b/dff/utils/parser/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/dff/utils/testing/__init__.py b/dff/utils/testing/__init__.py index 9aa9a773f..4e1de7c35 100644 --- a/dff/utils/testing/__init__.py +++ b/dff/utils/testing/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .common import is_interactive_mode, check_happy_path, run_interactive_mode from .toy_script import TOY_SCRIPT, TOY_SCRIPT_ARGS, HAPPY_PATH from .response_comparers import default_comparer diff --git a/dff/utils/turn_caching/__init__.py b/dff/utils/turn_caching/__init__.py index 6220a9a36..ed53579a7 100644 --- a/dff/utils/turn_caching/__init__.py +++ b/dff/utils/turn_caching/__init__.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 from .singleton_turn_caching import cache_clear, lru_cache, cache diff --git a/dff/utils/viewer/__init__.py b/dff/utils/viewer/__init__.py index 973e6e2d5..40a96afc6 100644 --- a/dff/utils/viewer/__init__.py +++ b/dff/utils/viewer/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# flake8: noqa: F401 diff --git a/makefile b/makefile index ba4275cfa..ca568c1b2 100644 --- a/makefile +++ b/makefile @@ -34,13 +34,13 @@ venv: format: venv black --line-length=120 --exclude='venv|build|tutorials' . - black --line-length=100 tutorials + black --line-length=80 tutorials .PHONY: format lint: venv - flake8 --max-line-length=120 --exclude venv,build,tutorials . - flake8 --max-line-length=100 tutorials - @set -e && black --line-length=120 --check --exclude='venv|build|tutorials' . && black --line-length=100 --check tutorials || ( \ + flake8 --max-line-length=120 --exclude venv,build,tutorials --per-file-ignores='**/__init__.py:F401' . + flake8 --max-line-length=100 --per-file-ignores='**/3_load_testing_with_locust.py:E402 **/4_streamlit_chat.py:E402' tutorials + @set -e && black --line-length=120 --check --exclude='venv|build|tutorials' . && black --line-length=80 --check tutorials || ( \ echo "================================"; \ echo "Bad formatting? Run: make format"; \ echo "================================"; \ diff --git a/tutorials/context_storages/1_basics.py b/tutorials/context_storages/1_basics.py index 08cb4aee4..449f93ef2 100644 --- a/tutorials/context_storages/1_basics.py +++ b/tutorials/context_storages/1_basics.py @@ -34,7 +34,7 @@ if __name__ == "__main__": check_happy_path(pipeline, HAPPY_PATH) - # This is a function for automatic tutorial running (testing) with HAPPY_PATH + # a function for automatic tutorial running (testing) with HAPPY_PATH # This runs tutorial in interactive mode if not in IPython env # and if `DISABLE_INTERACTIVE_MODE` is not set diff --git a/tutorials/context_storages/4_redis.py b/tutorials/context_storages/4_redis.py index 99c5457e0..918b5d127 100644 --- a/tutorials/context_storages/4_redis.py +++ b/tutorials/context_storages/4_redis.py @@ -28,7 +28,9 @@ # %% -db_uri = "redis://{}:{}@localhost:6379/{}".format("", os.environ["REDIS_PASSWORD"], "0") +db_uri = "redis://{}:{}@localhost:6379/{}".format( + "", os.environ["REDIS_PASSWORD"], "0" +) db = context_storage_factory(db_uri) diff --git a/tutorials/context_storages/8_db_benchmarking.py b/tutorials/context_storages/8_db_benchmarking.py index 2c88de3ee..4ccba4901 100644 --- a/tutorials/context_storages/8_db_benchmarking.py +++ b/tutorials/context_storages/8_db_benchmarking.py @@ -136,7 +136,9 @@ """ # %% -benchmark.report(file=tutorial_dir / "Shelve.json", display={"name", "config", "metrics"}) +benchmark.report( + file=tutorial_dir / "Shelve.json", display={"name", "config", "metrics"} +) # %% [markdown] """ diff --git a/tutorials/messengers/telegram/1_basic.py b/tutorials/messengers/telegram/1_basic.py index c209e54c7..45fccc272 100644 --- a/tutorials/messengers/telegram/1_basic.py +++ b/tutorials/messengers/telegram/1_basic.py @@ -44,7 +44,9 @@ class and [telebot](https://pytba.readthedocs.io/en/latest/index.html) script = { "greeting_flow": { "start_node": { - TRANSITIONS: {"greeting_node": cnd.exact_match(Message(text="/start"))}, + TRANSITIONS: { + "greeting_node": cnd.exact_match(Message(text="/start")) + }, }, "greeting_node": { RESPONSE: Message(text="Hi"), @@ -52,7 +54,9 @@ class and [telebot](https://pytba.readthedocs.io/en/latest/index.html) }, "fallback_node": { RESPONSE: Message(text="Please, repeat the request"), - TRANSITIONS: {"greeting_node": cnd.exact_match(Message(text="/start"))}, + TRANSITIONS: { + "greeting_node": cnd.exact_match(Message(text="/start")) + }, }, } } @@ -74,7 +78,8 @@ class and [telebot](https://pytba.readthedocs.io/en/latest/index.html) script=script, start_label=("greeting_flow", "start_node"), fallback_label=("greeting_flow", "fallback_node"), - messenger_interface=interface, # The interface can be passed as a pipeline argument. + messenger_interface=interface, + # The interface can be passed as a pipeline argument. ) @@ -82,5 +87,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/telegram/2_buttons.py b/tutorials/messengers/telegram/2_buttons.py index b803d02e2..efbbf2ab4 100644 --- a/tutorials/messengers/telegram/2_buttons.py +++ b/tutorials/messengers/telegram/2_buttons.py @@ -50,15 +50,19 @@ class is used to represent telegram message, "start": { TRANSITIONS: { ("general", "native_keyboard"): ( - lambda ctx, _: ctx.last_request.text in ("/start", "/restart") + lambda ctx, _: ctx.last_request.text + in ("/start", "/restart") ), }, }, "fallback": { - RESPONSE: TelegramMessage(text="Finishing test, send /restart command to restart"), + RESPONSE: TelegramMessage( + text="Finishing test, send /restart command to restart" + ), TRANSITIONS: { ("general", "native_keyboard"): ( - lambda ctx, _: ctx.last_request.text in ("/start", "/restart") + lambda ctx, _: ctx.last_request.text + in ("/start", "/restart") ), }, }, @@ -80,12 +84,16 @@ class is used to represent telegram message, ), ), TRANSITIONS: { - ("general", "success"): cnd.exact_match(TelegramMessage(text="4")), + ("general", "success"): cnd.exact_match( + TelegramMessage(text="4") + ), ("general", "fail"): cnd.true(), }, }, "success": { - RESPONSE: TelegramMessage(**{"text": "Success!", "ui": RemoveKeyboard()}), + RESPONSE: TelegramMessage( + **{"text": "Success!", "ui": RemoveKeyboard()} + ), TRANSITIONS: {("root", "fallback"): cnd.true()}, }, "fail": { @@ -120,7 +128,10 @@ class is used to represent telegram message, ), ( TelegramMessage(text="5"), - TelegramMessage(text="Incorrect answer, type anything to try again", ui=RemoveKeyboard()), + TelegramMessage( + text="Incorrect answer, type anything to try again", + ui=RemoveKeyboard(), + ), ), ( TelegramMessage(text="ok"), @@ -136,10 +147,15 @@ class is used to represent telegram message, ), ), ), - (TelegramMessage(text="4"), TelegramMessage(text="Success!", ui=RemoveKeyboard())), + ( + TelegramMessage(text="4"), + TelegramMessage(text="Success!", ui=RemoveKeyboard()), + ), ( TelegramMessage(text="Yay!"), - TelegramMessage(text="Finishing test, send /restart command to restart"), + TelegramMessage( + text="Finishing test, send /restart command to restart" + ), ), ( TelegramMessage(text="/start"), @@ -171,5 +187,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/telegram/3_buttons_with_callback.py b/tutorials/messengers/telegram/3_buttons_with_callback.py index ac55d0085..10d54927f 100644 --- a/tutorials/messengers/telegram/3_buttons_with_callback.py +++ b/tutorials/messengers/telegram/3_buttons_with_callback.py @@ -53,15 +53,19 @@ class is used to represent telegram message, "start": { TRANSITIONS: { ("general", "keyboard"): ( - lambda ctx, _: ctx.last_request.text in ("/start", "/restart") + lambda ctx, _: ctx.last_request.text + in ("/start", "/restart") ), }, }, "fallback": { - RESPONSE: TelegramMessage(text="Finishing test, send /restart command to restart"), + RESPONSE: TelegramMessage( + text="Finishing test, send /restart command to restart" + ), TRANSITIONS: { ("general", "keyboard"): ( - lambda ctx, _: ctx.last_request.text in ("/start", "/restart") + lambda ctx, _: ctx.last_request.text + in ("/start", "/restart") ) }, }, @@ -81,8 +85,12 @@ class is used to represent telegram message, } ), TRANSITIONS: { - ("general", "success"): cnd.exact_match(TelegramMessage(callback_query="correct")), - ("general", "fail"): cnd.exact_match(TelegramMessage(callback_query="wrong")), + ("general", "success"): cnd.exact_match( + TelegramMessage(callback_query="correct") + ), + ("general", "fail"): cnd.exact_match( + TelegramMessage(callback_query="wrong") + ), }, }, "success": { @@ -90,7 +98,9 @@ class is used to represent telegram message, TRANSITIONS: {("root", "fallback"): cnd.true()}, }, "fail": { - RESPONSE: TelegramMessage(text="Incorrect answer, type anything to try again"), + RESPONSE: TelegramMessage( + text="Incorrect answer, type anything to try again" + ), TRANSITIONS: {("general", "keyboard"): cnd.true()}, }, }, @@ -132,7 +142,9 @@ class is used to represent telegram message, ), ( TelegramMessage(text="Yay!"), - TelegramMessage(text="Finishing test, send /restart command to restart"), + TelegramMessage( + text="Finishing test, send /restart command to restart" + ), ), ( TelegramMessage(text="/restart"), @@ -164,5 +176,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/telegram/4_conditions.py b/tutorials/messengers/telegram/4_conditions.py index 81d4dc830..d6ba4f4c5 100644 --- a/tutorials/messengers/telegram/4_conditions.py +++ b/tutorials/messengers/telegram/4_conditions.py @@ -58,23 +58,37 @@ script = { "greeting_flow": { "start_node": { - TRANSITIONS: {"node1": telegram_condition(commands=["start", "restart"])}, + TRANSITIONS: { + "node1": telegram_condition(commands=["start", "restart"]) + }, }, "node1": { RESPONSE: TelegramMessage(text="Hi, how are you?"), TRANSITIONS: { - "node2": telegram_condition(update_type=UpdateType.MESSAGE, regexp="fine") + "node2": telegram_condition( + update_type=UpdateType.MESSAGE, regexp="fine" + ) }, # this is the same as # TRANSITIONS: {"node2": telegram_condition(regexp="fine")}, }, "node2": { - RESPONSE: TelegramMessage(text="Good. What do you want to talk about?"), - TRANSITIONS: {"node3": telegram_condition(func=lambda msg: "music" in msg.text)}, + RESPONSE: TelegramMessage( + text="Good. What do you want to talk about?" + ), + TRANSITIONS: { + "node3": telegram_condition( + func=lambda msg: "music" in msg.text + ) + }, }, "node3": { - RESPONSE: TelegramMessage(text="Sorry, I can not talk about music now."), - TRANSITIONS: {"node4": telegram_condition(update_type=UpdateType.ALL)}, + RESPONSE: TelegramMessage( + text="Sorry, I can not talk about music now." + ), + TRANSITIONS: { + "node4": telegram_condition(update_type=UpdateType.ALL) + }, # This condition is true for any type of update }, "node4": { @@ -84,7 +98,9 @@ }, "fallback_node": { RESPONSE: TelegramMessage(text="Ooops"), - TRANSITIONS: {"node1": telegram_condition(commands=["start", "restart"])}, + TRANSITIONS: { + "node1": telegram_condition(commands=["start", "restart"]) + }, }, } } @@ -122,5 +138,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/telegram/5_conditions_with_media.py b/tutorials/messengers/telegram/5_conditions_with_media.py index 08fc4eb94..25b555138 100644 --- a/tutorials/messengers/telegram/5_conditions_with_media.py +++ b/tutorials/messengers/telegram/5_conditions_with_media.py @@ -57,13 +57,19 @@ "root": { "start": { TRANSITIONS: { - ("pics", "ask_picture"): telegram_condition(commands=["start", "restart"]) + ("pics", "ask_picture"): telegram_condition( + commands=["start", "restart"] + ) }, }, "fallback": { - RESPONSE: TelegramMessage(text="Finishing test, send /restart command to restart"), + RESPONSE: TelegramMessage( + text="Finishing test, send /restart command to restart" + ), TRANSITIONS: { - ("pics", "ask_picture"): telegram_condition(commands=["start", "restart"]) + ("pics", "ask_picture"): telegram_condition( + commands=["start", "restart"] + ) }, }, }, @@ -73,8 +79,10 @@ TRANSITIONS: { ("pics", "send_one"): cnd.any( [ - # Telegram can put photos both in 'photo' and 'document' fields. - # We should consider both cases when we check the message for media. + # Telegram can put photos + # both in 'photo' and 'document' fields. + # We should consider both cases + # when we check the message for media. telegram_condition(content_types=["photo"]), telegram_condition( func=lambda message: ( @@ -86,7 +94,9 @@ ), ] ), - ("pics", "send_many"): telegram_condition(content_types=["text"]), + ("pics", "send_many"): telegram_condition( + content_types=["text"] + ), ("pics", "ask_picture"): cnd.true(), }, }, @@ -112,9 +122,14 @@ # testing happy_path = ( - (TelegramMessage(text="/start"), TelegramMessage(text="Send me a picture")), ( - TelegramMessage(attachments=Attachments(files=[Image(source=picture_url)])), + TelegramMessage(text="/start"), + TelegramMessage(text="Send me a picture"), + ), + ( + TelegramMessage( + attachments=Attachments(files=[Image(source=picture_url)]) + ), TelegramMessage( text="Here's my picture!", attachments=Attachments(files=[Image(source=picture_url)]), @@ -122,9 +137,14 @@ ), ( TelegramMessage(text="ok"), - TelegramMessage(text="Finishing test, send /restart command to restart"), + TelegramMessage( + text="Finishing test, send /restart command to restart" + ), + ), + ( + TelegramMessage(text="/restart"), + TelegramMessage(text="Send me a picture"), ), - (TelegramMessage(text="/restart"), TelegramMessage(text="Send me a picture")), ( TelegramMessage(text="No"), TelegramMessage( @@ -134,9 +154,14 @@ ), ( TelegramMessage(text="ok"), - TelegramMessage(text="Finishing test, send /restart command to restart"), + TelegramMessage( + text="Finishing test, send /restart command to restart" + ), + ), + ( + TelegramMessage(text="/restart"), + TelegramMessage(text="Send me a picture"), ), - (TelegramMessage(text="/restart"), TelegramMessage(text="Send me a picture")), ) @@ -178,5 +203,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/telegram/6_conditions_extras.py b/tutorials/messengers/telegram/6_conditions_extras.py index aca4131ed..eee19866f 100644 --- a/tutorials/messengers/telegram/6_conditions_extras.py +++ b/tutorials/messengers/telegram/6_conditions_extras.py @@ -65,10 +65,14 @@ [ # say hi when invited to a chat telegram_condition( - update_type=UpdateType.CHAT_JOIN_REQUEST, func=lambda x: True + update_type=UpdateType.CHAT_JOIN_REQUEST, + func=lambda x: True, ), # say hi when someone enters the chat - telegram_condition(update_type=UpdateType.MY_CHAT_MEMBER, func=lambda x: True), + telegram_condition( + update_type=UpdateType.MY_CHAT_MEMBER, + func=lambda x: True, + ), ] ), # send a message when inline query is received @@ -80,7 +84,9 @@ "greeting_flow": { "start_node": { RESPONSE: TelegramMessage(text="Bot running"), - TRANSITIONS: {"node1": telegram_condition(commands=["start", "restart"])}, + TRANSITIONS: { + "node1": telegram_condition(commands=["start", "restart"]) + }, }, "node1": { RESPONSE: TelegramMessage(text="Hi"), @@ -114,5 +120,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/telegram/7_polling_setup.py b/tutorials/messengers/telegram/7_polling_setup.py index f513a2438..d070e4728 100644 --- a/tutorials/messengers/telegram/7_polling_setup.py +++ b/tutorials/messengers/telegram/7_polling_setup.py @@ -50,7 +50,8 @@ # %% pipeline = Pipeline.from_script( *TOY_SCRIPT_ARGS, - messenger_interface=interface, # The interface can be passed as a pipeline argument. + messenger_interface=interface, + # The interface can be passed as a pipeline argument ) @@ -58,5 +59,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/telegram/8_webhook_setup.py b/tutorials/messengers/telegram/8_webhook_setup.py index 2590e2224..a7f4fd68f 100644 --- a/tutorials/messengers/telegram/8_webhook_setup.py +++ b/tutorials/messengers/telegram/8_webhook_setup.py @@ -44,7 +44,8 @@ # %% pipeline = Pipeline.from_script( *TOY_SCRIPT_ARGS, - messenger_interface=interface, # The interface can be passed as a pipeline argument. + messenger_interface=interface, + # The interface can be passed as a pipeline argument ) # testing @@ -55,5 +56,6 @@ def main(): pipeline.run() -if __name__ == "__main__" and is_interactive_mode(): # prevent run during doc building +if __name__ == "__main__" and is_interactive_mode(): + # prevent run during doc building main() diff --git a/tutorials/messengers/web_api_interface/1_fastapi.py b/tutorials/messengers/web_api_interface/1_fastapi.py index 73df867a1..44e87831c 100644 --- a/tutorials/messengers/web_api_interface/1_fastapi.py +++ b/tutorials/messengers/web_api_interface/1_fastapi.py @@ -42,7 +42,7 @@ async def respond( user_id: str, user_message: Message, ): - context = await pipeline._run_pipeline(user_message, user_id) # run in async + context = await pipeline._run_pipeline(user_message, user_id) return {"user_id": user_id, "response": context.last_response} diff --git a/tutorials/messengers/web_api_interface/2_websocket_chat.py b/tutorials/messengers/web_api_interface/2_websocket_chat.py index 3153d77ce..ce40928ca 100644 --- a/tutorials/messengers/web_api_interface/2_websocket_chat.py +++ b/tutorials/messengers/web_api_interface/2_websocket_chat.py @@ -35,7 +35,9 @@ # %% pipeline = Pipeline.from_script( - TOY_SCRIPT, ("greeting_flow", "start_node"), ("greeting_flow", "fallback_node") + TOY_SCRIPT, + ("greeting_flow", "start_node"), + ("greeting_flow", "fallback_node"), ) diff --git a/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py b/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py index f986187a4..8afc6ed3a 100644 --- a/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py +++ b/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py @@ -32,14 +32,13 @@ # %% -######################################################################################## -# this patch is only needed to run this file in IPython kernel and can be safely removed +################################################################################ +# this patch is only needed to run this file in IPython kernel +# and can be safely removed import gevent.monkey gevent.monkey.patch_all() - -# flake8: noqa: E402 -######################################################################################## +################################################################################ # %% @@ -87,13 +86,18 @@ def check_happy_path(self, happy_path): for request, response in happy_path: with self.client.post( f"/chat?user_id={user_id}", - headers={"accept": "application/json", "Content-Type": "application/json"}, + headers={ + "accept": "application/json", + "Content-Type": "application/json", + }, # Name is the displayed name of the request. name=f"/chat?user_message={request.json()}", data=request.json(), catch_response=True, ) as candidate_response: - text_response = Message.model_validate(candidate_response.json().get("response")) + text_response = Message.model_validate( + candidate_response.json().get("response") + ) if response is not None: if callable(response): @@ -102,7 +106,8 @@ def check_happy_path(self, happy_path): candidate_response.failure(error_message) elif text_response != response: candidate_response.failure( - f"Expected: {response.model_dump_json()}\nGot: {text_response.model_dump_json()}" + f"Expected: {response.model_dump_json()}\n" + f"Got: {text_response.model_dump_json()}" ) time.sleep(self.wait_time()) @@ -117,7 +122,10 @@ def check_first_message(msg: Message) -> str | None: if msg.text is None: return f"Message does not contain text: {msg.model_dump_json()}" if "Hi" not in msg.text: - return f'"Hi" is not in the response message: {msg.model_dump_json()}' + return ( + f'"Hi" is not in the response message: ' + f"{msg.model_dump_json()}" + ) return None self.check_happy_path( diff --git a/tutorials/messengers/web_api_interface/4_streamlit_chat.py b/tutorials/messengers/web_api_interface/4_streamlit_chat.py index 6a64df2a1..90dd195dd 100644 --- a/tutorials/messengers/web_api_interface/4_streamlit_chat.py +++ b/tutorials/messengers/web_api_interface/4_streamlit_chat.py @@ -29,8 +29,6 @@ loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - -# flake8: noqa: E402 ########################################################### @@ -58,7 +56,10 @@ def query(payload, user_id) -> requests.Response: response = requests.post( API_URL + f"?user_id={user_id}", - headers={"accept": "application/json", "Content-Type": "application/json"}, + headers={ + "accept": "application/json", + "Content-Type": "application/json", + }, json=payload, ) return response @@ -107,7 +108,8 @@ def send_and_receive(): Add both the request and response to `user_requests` and `bot_responses`. We do not call this function inside the `text_input.on_change` because then - we'd call it whenever the text field loses focus (e.g. when a browser tab is switched). + we'd call it whenever the text field loses focus + (e.g. when a browser tab is switched). """ user_request = st.session_state["input"] @@ -117,14 +119,18 @@ def send_and_receive(): st.session_state["user_requests"].append(user_request) bot_response = query( - Message(text=user_request).model_dump(), user_id=st.session_state["user_id"] + Message(text=user_request).model_dump(), + user_id=st.session_state["user_id"], ) bot_response.raise_for_status() bot_message = Message.model_validate(bot_response.json()["response"]).text # # Implementation without using Message: - # bot_response = query({"text": user_request}, user_id=st.session_state["user_id"]) + # bot_response = query( + # {"text": user_request}, + # user_id=st.session_state["user_id"] + # ) # bot_response.raise_for_status() # # bot_message = bot_response.json()["response"]["text"] diff --git a/tutorials/pipeline/1_basics.py b/tutorials/pipeline/1_basics.py index d2919b27e..91d0dbbcd 100644 --- a/tutorials/pipeline/1_basics.py +++ b/tutorials/pipeline/1_basics.py @@ -49,7 +49,8 @@ # %% pipeline = Pipeline.from_script( - TOY_SCRIPT, # Pipeline script object, defined in `dff.utils.testing.toy_script`. + TOY_SCRIPT, + # Pipeline script object, defined in `dff.utils.testing.toy_script` start_label=("greeting_flow", "start_node"), fallback_label=("greeting_flow", "fallback_node"), ) @@ -70,8 +71,8 @@ # %% if __name__ == "__main__": - check_happy_path(pipeline, HAPPY_PATH) # This is a function for automatic tutorial running - # (testing) with HAPPY_PATH + check_happy_path(pipeline, HAPPY_PATH) + # a function for automatic tutorial running (testing) with HAPPY_PATH # This runs tutorial in interactive mode if not in IPython env # and if `DISABLE_INTERACTIVE_MODE` is not set diff --git a/tutorials/pipeline/2_pre_and_post_processors.py b/tutorials/pipeline/2_pre_and_post_processors.py index eb335feb3..f26fafa54 100644 --- a/tutorials/pipeline/2_pre_and_post_processors.py +++ b/tutorials/pipeline/2_pre_and_post_processors.py @@ -19,7 +19,12 @@ from dff.pipeline import Pipeline -from dff.utils.testing import check_happy_path, is_interactive_mode, HAPPY_PATH, TOY_SCRIPT_ARGS +from dff.utils.testing import ( + check_happy_path, + is_interactive_mode, + HAPPY_PATH, + TOY_SCRIPT_ARGS, +) logger = logging.getLogger(__name__) @@ -81,6 +86,10 @@ def pong_processor(ctx: Context): message = Message(text=input("Send request: ")) ctx: Context = pipeline(message, ctx_id) print(f"Response: {ctx.last_response}") - ping_pong = ctx.misc.get("ping", False) and ctx.misc.get("pong", False) - print(f"Ping-pong exchange: {'completed' if ping_pong else 'failed'}.") + ping_pong = ctx.misc.get("ping", False) and ctx.misc.get( + "pong", False + ) + print( + f"Ping-pong exchange: {'completed' if ping_pong else 'failed'}." + ) logger.info(f"Context misc: {ctx.misc}") diff --git a/tutorials/pipeline/3_pipeline_dict_with_services_basic.py b/tutorials/pipeline/3_pipeline_dict_with_services_basic.py index df3c0c36d..dbe7fb1e5 100644 --- a/tutorials/pipeline/3_pipeline_dict_with_services_basic.py +++ b/tutorials/pipeline/3_pipeline_dict_with_services_basic.py @@ -55,11 +55,16 @@ # %% def prepreprocess(_): - logger.info("preprocession intent-detection Service running (defined as a dict)") + logger.info( + "preprocession intent-detection Service running (defined as a dict)" + ) def preprocess(_): - logger.info("another preprocession web-based annotator Service (defined as a callable)") + logger.info( + "another preprocession web-based annotator Service " + "(defined as a callable)" + ) def postprocess(_): diff --git a/tutorials/pipeline/3_pipeline_dict_with_services_full.py b/tutorials/pipeline/3_pipeline_dict_with_services_full.py index b085fa2ff..91c29dbde 100644 --- a/tutorials/pipeline/3_pipeline_dict_with_services_full.py +++ b/tutorials/pipeline/3_pipeline_dict_with_services_full.py @@ -87,7 +87,9 @@ # %% def prepreprocess(ctx: Context): - logger.info("preprocession intent-detection Service running (defined as a dict)") + logger.info( + "preprocession intent-detection Service running (defined as a dict)" + ) ctx.misc["preprocess_detection"] = { ctx.last_request.text: "some_intent" } # Similar syntax can be used to access @@ -100,15 +102,22 @@ def preprocess(ctx: Context, _, info: ServiceRuntimeInfo): f"(defined as a callable), named '{info.name}'" ) with urllib.request.urlopen("https://example.com/") as webpage: - web_content = webpage.read().decode(webpage.headers.get_content_charset()) + web_content = webpage.read().decode( + webpage.headers.get_content_charset() + ) ctx.misc["another_detection"] = { - ctx.last_request.text: "online" if "Example Domain" in web_content else "offline" + ctx.last_request.text: "online" + if "Example Domain" in web_content + else "offline" } def postprocess(ctx: Context, pl: Pipeline): logger.info("postprocession Service (defined as an object)") - logger.info(f"resulting misc looks like:" f"{json.dumps(ctx.misc, indent=4, default=str)}") + logger.info( + f"resulting misc looks like:" + f"{json.dumps(ctx.misc, indent=4, default=str)}" + ) fallback_flow, fallback_node, _ = pl.actor.fallback_label received_response = pl.script[fallback_flow][fallback_node].response responses_match = received_response == ctx.last_response diff --git a/tutorials/pipeline/4_groups_and_conditions_full.py b/tutorials/pipeline/4_groups_and_conditions_full.py index e363fda2d..fcd6ef5ae 100644 --- a/tutorials/pipeline/4_groups_and_conditions_full.py +++ b/tutorials/pipeline/4_groups_and_conditions_full.py @@ -179,8 +179,12 @@ def runtime_info_printing_service(_, __, info: ServiceRuntimeInfo): Service( handler=simple_service, start_condition=all_condition( - service_successful_condition(".pipeline.service_group_0.simple_service_0"), - service_successful_condition(".pipeline.service_group_0.simple_service_1"), + service_successful_condition( + ".pipeline.service_group_0.simple_service_0" + ), + service_successful_condition( + ".pipeline.service_group_0.simple_service_1" + ), ), # Alternative: # service_successful_condition(".pipeline.service_group_0") name="running_service", @@ -189,7 +193,9 @@ def runtime_info_printing_service(_, __, info: ServiceRuntimeInfo): Service( handler=never_running_service, start_condition=not_condition( - service_successful_condition(".pipeline.named_group.running_service") + service_successful_condition( + ".pipeline.named_group.running_service" + ) ), ), ], diff --git a/tutorials/pipeline/5_asynchronous_groups_and_services_full.py b/tutorials/pipeline/5_asynchronous_groups_and_services_full.py index dc1d585ca..137c02989 100644 --- a/tutorials/pipeline/5_asynchronous_groups_and_services_full.py +++ b/tutorials/pipeline/5_asynchronous_groups_and_services_full.py @@ -98,11 +98,15 @@ async def web_querying_service(ctx: Context, _, info: ServiceRuntimeInfo): with urllib.request.urlopen( f"https://jsonplaceholder.typicode.com/photos/{photo_number}" ) as webpage: - web_content = webpage.read().decode(webpage.headers.get_content_charset()) + web_content = webpage.read().decode( + webpage.headers.get_content_charset() + ) ctx.misc["web_query"].update( { f"{ctx.last_request}" - f":photo_number_{photo_number}": json.loads(web_content)["title"] + f":photo_number_{photo_number}": json.loads(web_content)[ + "title" + ] } ) logger.info(f"Service '{info.name}' has completed HTTPS request") diff --git a/tutorials/pipeline/6_custom_messenger_interface.py b/tutorials/pipeline/6_custom_messenger_interface.py index c20a38b20..99f2d096e 100644 --- a/tutorials/pipeline/6_custom_messenger_interface.py +++ b/tutorials/pipeline/6_custom_messenger_interface.py @@ -74,7 +74,8 @@ # %% app = Flask(__name__) -messenger_interface = CallbackMessengerInterface() # For this simple case of Flask, +messenger_interface = CallbackMessengerInterface() +# For this simple case of Flask, # CallbackMessengerInterface may not be overridden @@ -112,7 +113,9 @@ def purify_request(ctx: Context): elif isinstance(last_request, Message): logger.info("Capturing request from CLI") else: - raise Exception(f"Request of type {type(last_request)} can not be purified!") + raise Exception( + f"Request of type {type(last_request)} can not be purified!" + ) def cat_response2webpage(ctx: Context): diff --git a/tutorials/pipeline/7_extra_handlers_basic.py b/tutorials/pipeline/7_extra_handlers_basic.py index 2f72955b6..bea7c9317 100644 --- a/tutorials/pipeline/7_extra_handlers_basic.py +++ b/tutorials/pipeline/7_extra_handlers_basic.py @@ -56,7 +56,12 @@ def collect_timestamp_before(ctx: Context, _, info: ExtraHandlerRuntimeInfo): def collect_timestamp_after(ctx: Context, _, info: ExtraHandlerRuntimeInfo): - ctx.misc.update({f"{info.component.name}": datetime.now() - ctx.misc[f"{info.component.name}"]}) + ctx.misc.update( + { + f"{info.component.name}": datetime.now() + - ctx.misc[f"{info.component.name}"] + } + ) async def heavy_service(_): diff --git a/tutorials/pipeline/7_extra_handlers_full.py b/tutorials/pipeline/7_extra_handlers_full.py index 0027c8ca9..ad89f3230 100644 --- a/tutorials/pipeline/7_extra_handlers_full.py +++ b/tutorials/pipeline/7_extra_handlers_full.py @@ -95,7 +95,9 @@ def get_extra_handler_misc_field( def time_measure_before_handler(ctx, _, info): - ctx.misc.update({get_extra_handler_misc_field(info, "time"): datetime.now()}) + ctx.misc.update( + {get_extra_handler_misc_field(info, "time"): datetime.now()} + ) def time_measure_after_handler(ctx, _, info): @@ -108,7 +110,13 @@ def time_measure_after_handler(ctx, _, info): def ram_measure_before_handler(ctx, _, info): - ctx.misc.update({get_extra_handler_misc_field(info, "ram"): psutil.virtual_memory().available}) + ctx.misc.update( + { + get_extra_handler_misc_field( + info, "ram" + ): psutil.virtual_memory().available + } + ) def ram_measure_after_handler(ctx, _, info): @@ -124,7 +132,11 @@ def ram_measure_after_handler(ctx, _, info): def json_converter_before_handler(ctx, _, info): ctx.misc.update( - {get_extra_handler_misc_field(info, "str"): json.dumps(ctx.misc, indent=4, default=str)} + { + get_extra_handler_misc_field(info, "str"): json.dumps( + ctx.misc, indent=4, default=str + ) + } ) @@ -141,7 +153,9 @@ def json_converter_after_handler(ctx, _, info): after_handler=[time_measure_after_handler, ram_measure_after_handler], ) def heavy_service(ctx: Context): - memory_heap[ctx.last_request.text] = [random.randint(0, num) for num in range(0, 1000)] + memory_heap[ctx.last_request.text] = [ + random.randint(0, num) for num in range(0, 1000) + ] @to_service( diff --git a/tutorials/pipeline/8_extra_handlers_and_extensions.py b/tutorials/pipeline/8_extra_handlers_and_extensions.py index 8b610af5f..d4930d2e5 100644 --- a/tutorials/pipeline/8_extra_handlers_and_extensions.py +++ b/tutorials/pipeline/8_extra_handlers_and_extensions.py @@ -105,8 +105,12 @@ def after(_, __, info: ExtraHandlerRuntimeInfo): def after_all(_, __, info: ExtraHandlerRuntimeInfo): - pipeline_info.update({"total_time": datetime.now() - start_times[info.component.path]}) - logger.info(f"Pipeline stats: {json.dumps(pipeline_info, indent=4, default=str)}") + pipeline_info.update( + {"total_time": datetime.now() - start_times[info.component.path]} + ) + logger.info( + f"Pipeline stats: {json.dumps(pipeline_info, indent=4, default=str)}" + ) async def long_service(_, __, info: ServiceRuntimeInfo): diff --git a/tutorials/script/core/1_basics.py b/tutorials/script/core/1_basics.py index a5ebb2d99..a06ffb96d 100644 --- a/tutorials/script/core/1_basics.py +++ b/tutorials/script/core/1_basics.py @@ -61,17 +61,27 @@ # If "Hi" == request of the user then we make the transition. }, "node1": { - RESPONSE: Message(text="Hi, how are you?"), # When the agent enters node1, + RESPONSE: Message( + text="Hi, how are you?" + ), # When the agent enters node1, # return "Hi, how are you?". - TRANSITIONS: {"node2": cnd.exact_match(Message(text="I'm fine, how are you?"))}, + TRANSITIONS: { + "node2": cnd.exact_match(Message(text="I'm fine, how are you?")) + }, }, "node2": { RESPONSE: Message(text="Good. What do you want to talk about?"), - TRANSITIONS: {"node3": cnd.exact_match(Message(text="Let's talk about music."))}, + TRANSITIONS: { + "node3": cnd.exact_match( + Message(text="Let's talk about music.") + ) + }, }, "node3": { RESPONSE: Message(text="Sorry, I can not talk about music now."), - TRANSITIONS: {"node4": cnd.exact_match(Message(text="Ok, goodbye."))}, + TRANSITIONS: { + "node4": cnd.exact_match(Message(text="Ok, goodbye.")) + }, }, "node4": { RESPONSE: Message(text="Bye"), @@ -88,7 +98,10 @@ happy_path = ( - (Message(text="Hi"), Message(text="Hi, how are you?")), # start_node -> node1 + ( + Message(text="Hi"), + Message(text="Hi, how are you?"), + ), # start_node -> node1 ( Message(text="I'm fine, how are you?"), Message(text="Good. What do you want to talk about?"), @@ -100,8 +113,14 @@ (Message(text="Ok, goodbye."), Message(text="Bye")), # node3 -> node4 (Message(text="Hi"), Message(text="Hi, how are you?")), # node4 -> node1 (Message(text="stop"), Message(text="Ooops")), # node1 -> fallback_node - (Message(text="stop"), Message(text="Ooops")), # fallback_node -> fallback_node - (Message(text="Hi"), Message(text="Hi, how are you?")), # fallback_node -> node1 + ( + Message(text="stop"), + Message(text="Ooops"), + ), # fallback_node -> fallback_node + ( + Message(text="Hi"), + Message(text="Hi, how are you?"), + ), # fallback_node -> node1 ( Message(text="I'm fine, how are you?"), Message(text="Good. What do you want to talk about?"), @@ -144,4 +163,5 @@ # Run tutorial in interactive mode if not in IPython env # and if `DISABLE_INTERACTIVE_MODE` is not set. if is_interactive_mode(): - run_interactive_mode(pipeline) # This runs tutorial in interactive mode. + run_interactive_mode(pipeline) + # This runs tutorial in interactive mode. diff --git a/tutorials/script/core/2_conditions.py b/tutorials/script/core/2_conditions.py index 35b862d4b..d8838b9f4 100644 --- a/tutorials/script/core/2_conditions.py +++ b/tutorials/script/core/2_conditions.py @@ -84,7 +84,9 @@ def hi_lower_case_condition(ctx: Context, _: Pipeline, *args, **kwargs) -> bool: return "hi" in request.text.lower() -def complex_user_answer_condition(ctx: Context, _: Pipeline, *args, **kwargs) -> bool: +def complex_user_answer_condition( + ctx: Context, _: Pipeline, *args, **kwargs +) -> bool: request = ctx.last_request # The user request can be anything. if request is None or request.misc is None: @@ -94,7 +96,9 @@ def complex_user_answer_condition(ctx: Context, _: Pipeline, *args, **kwargs) -> def predetermined_condition(condition: bool): # Wrapper for internal condition function. - def internal_condition_function(ctx: Context, _: Pipeline, *args, **kwargs) -> bool: + def internal_condition_function( + ctx: Context, _: Pipeline, *args, **kwargs + ) -> bool: # It always returns `condition`. return condition @@ -117,7 +121,11 @@ def internal_condition_function(ctx: Context, _: Pipeline, *args, **kwargs) -> b }, "node2": { RESPONSE: Message(text="Good. What do you want to talk about?"), - TRANSITIONS: {"node3": cnd.all([cnd.regexp(r"talk"), cnd.regexp(r"about.*music")])}, + TRANSITIONS: { + "node3": cnd.all( + [cnd.regexp(r"talk"), cnd.regexp(r"about.*music")] + ) + }, # Mix sequence of conditions by `cnd.all`. # `all` is alias `aggregate` with # `aggregate_func` == `all`. @@ -130,7 +138,12 @@ def internal_condition_function(ctx: Context, _: Pipeline, *args, **kwargs) -> b "node4": { RESPONSE: Message(text="bye"), TRANSITIONS: { - "node1": cnd.any([hi_lower_case_condition, cnd.exact_match(Message(text="hello"))]) + "node1": cnd.any( + [ + hi_lower_case_condition, + cnd.exact_match(Message(text="hello")), + ] + ) }, # Mix sequence of conditions by `cnd.any`. # `any` is alias `aggregate` with @@ -147,7 +160,9 @@ def internal_condition_function(ctx: Context, _: Pipeline, *args, **kwargs) -> b # If the value is `True` then we will go to `node1`. # If the value is `False` then we will check a result of # `predetermined_condition(True)` for `fallback_node`. - "fallback_node": predetermined_condition(True), # or you can use `cnd.true()` + "fallback_node": predetermined_condition( + True + ), # or you can use `cnd.true()` # Last condition function will return # `true` and will repeat `fallback_node` # if `complex_user_answer_condition` return `false`. @@ -158,7 +173,10 @@ def internal_condition_function(ctx: Context, _: Pipeline, *args, **kwargs) -> b # testing happy_path = ( - (Message(text="Hi"), Message(text="Hi, how are you?")), # start_node -> node1 + ( + Message(text="Hi"), + Message(text="Hi, how are you?"), + ), # start_node -> node1 ( Message(text="i'm fine, how are you?"), Message(text="Good. What do you want to talk about?"), @@ -170,9 +188,18 @@ def internal_condition_function(ctx: Context, _: Pipeline, *args, **kwargs) -> b (Message(text="Ok, goodbye."), Message(text="bye")), # node3 -> node4 (Message(text="Hi"), Message(text="Hi, how are you?")), # node4 -> node1 (Message(text="stop"), Message(text="Ooops")), # node1 -> fallback_node - (Message(text="one"), Message(text="Ooops")), # fallback_node -> fallback_node - (Message(text="help"), Message(text="Ooops")), # fallback_node -> fallback_node - (Message(text="nope"), Message(text="Ooops")), # fallback_node -> fallback_node + ( + Message(text="one"), + Message(text="Ooops"), + ), # fallback_node -> fallback_node + ( + Message(text="help"), + Message(text="Ooops"), + ), # fallback_node -> fallback_node + ( + Message(text="nope"), + Message(text="Ooops"), + ), # fallback_node -> fallback_node ( Message(misc={"some_key": "some_value"}), Message(text="Hi, how are you?"), diff --git a/tutorials/script/core/3_responses.py b/tutorials/script/core/3_responses.py index 3d12a75ce..6d42b3b2b 100644 --- a/tutorials/script/core/3_responses.py +++ b/tutorials/script/core/3_responses.py @@ -45,7 +45,9 @@ # %% -def cannot_talk_about_topic_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Message: +def cannot_talk_about_topic_response( + ctx: Context, _: Pipeline, *args, **kwargs +) -> Message: request = ctx.last_request if request is None or request.text is None: topic = None @@ -69,7 +71,9 @@ def func(_: Context, __: Pipeline, *args, **kwargs) -> Message: return func -def fallback_trace_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Message: +def fallback_trace_response( + ctx: Context, _: Pipeline, *args, **kwargs +) -> Message: return Message( misc={ "previous_node": list(ctx.labels.values())[-2], @@ -89,18 +93,29 @@ def fallback_trace_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Messa }, "node1": { RESPONSE: rsp.choice( - [Message(text="Hi, what is up?"), Message(text="Hello, how are you?")] + [ + Message(text="Hi, what is up?"), + Message(text="Hello, how are you?"), + ] ), # Random choice from candidate list. - TRANSITIONS: {"node2": cnd.exact_match(Message(text="I'm fine, how are you?"))}, + TRANSITIONS: { + "node2": cnd.exact_match(Message(text="I'm fine, how are you?")) + }, }, "node2": { RESPONSE: Message(text="Good. What do you want to talk about?"), - TRANSITIONS: {"node3": cnd.exact_match(Message(text="Let's talk about music."))}, + TRANSITIONS: { + "node3": cnd.exact_match( + Message(text="Let's talk about music.") + ) + }, }, "node3": { RESPONSE: cannot_talk_about_topic_response, - TRANSITIONS: {"node4": cnd.exact_match(Message(text="Ok, goodbye."))}, + TRANSITIONS: { + "node4": cnd.exact_match(Message(text="Ok, goodbye.")) + }, }, "node4": { RESPONSE: upper_case_response(Message(text="bye")), @@ -116,7 +131,10 @@ def fallback_trace_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Messa # testing happy_path = ( - (Message(text="Hi"), Message(text="Hello, how are you?")), # start_node -> node1 + ( + Message(text="Hi"), + Message(text="Hello, how are you?"), + ), # start_node -> node1 ( Message(text="I'm fine, how are you?"), Message(text="Good. What do you want to talk about?"), @@ -130,7 +148,10 @@ def fallback_trace_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Messa ( Message(text="stop"), Message( - misc={"previous_node": ("greeting_flow", "node1"), "last_request": Message(text="stop")} + misc={ + "previous_node": ("greeting_flow", "node1"), + "last_request": Message(text="stop"), + } ), ), # node1 -> fallback_node @@ -161,7 +182,10 @@ def fallback_trace_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Messa } ), ), # f_n->f_n - (Message(text="Hi"), Message(text="Hello, how are you?")), # fallback_node -> node1 + ( + Message(text="Hi"), + Message(text="Hello, how are you?"), + ), # fallback_node -> node1 ( Message(text="I'm fine, how are you?"), Message(text="Good. What do you want to talk about?"), diff --git a/tutorials/script/core/4_transitions.py b/tutorials/script/core/4_transitions.py index 86a6f9e99..1a1ece538 100644 --- a/tutorials/script/core/4_transitions.py +++ b/tutorials/script/core/4_transitions.py @@ -41,7 +41,9 @@ # %% -def greeting_flow_n2_transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: +def greeting_flow_n2_transition( + _: Context, __: Pipeline, *args, **kwargs +) -> NodeLabel3Type: return ("greeting_flow", "node2", 1.0) @@ -93,8 +95,12 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: # it doesn't need a `RESPONSE`. RESPONSE: Message(), TRANSITIONS: { - ("music_flow", "node1"): cnd.regexp(r"talk about music"), # first check - ("greeting_flow", "node1"): cnd.regexp(r"hi|hello", re.IGNORECASE), # second check + ("music_flow", "node1"): cnd.regexp( + r"talk about music" + ), # first check + ("greeting_flow", "node1"): cnd.regexp( + r"hi|hello", re.IGNORECASE + ), # second check "fallback_node": cnd.true(), # third check # "fallback_node" is equivalent to # ("global_flow", "fallback_node"). @@ -104,9 +110,15 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: # an error occurred while the agent was running. RESPONSE: Message(text="Ooops"), TRANSITIONS: { - ("music_flow", "node1"): cnd.regexp(r"talk about music"), # first check - ("greeting_flow", "node1"): cnd.regexp(r"hi|hello", re.IGNORECASE), # second check - lbl.previous(): cnd.regexp(r"previous", re.IGNORECASE), # third check + ("music_flow", "node1"): cnd.regexp( + r"talk about music" + ), # first check + ("greeting_flow", "node1"): cnd.regexp( + r"hi|hello", re.IGNORECASE + ), # second check + lbl.previous(): cnd.regexp( + r"previous", re.IGNORECASE + ), # third check # lbl.previous() is equivalent # to ("previous_flow", "previous_node") lbl.repeat(): cnd.true(), # fourth check @@ -137,10 +149,14 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: lbl.forward(0.5): cnd.regexp(r"talk about"), # second check # lbl.forward(0.5) is equivalent # to ("greeting_flow", "node3", 0.5) - ("music_flow", "node1"): cnd.regexp(r"talk about music"), # first check + ("music_flow", "node1"): cnd.regexp( + r"talk about music" + ), # first check # ("music_flow", "node1") is equivalent # to ("music_flow", "node1", 1.0) - lbl.previous(): cnd.regexp(r"previous", re.IGNORECASE), # third check + lbl.previous(): cnd.regexp( + r"previous", re.IGNORECASE + ), # third check }, }, "node3": { @@ -158,7 +174,8 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: "music_flow": { "node1": { RESPONSE: Message( - text="I love `System of a Down` group, would you like to talk about it?" + text="I love `System of a Down` group, " + "would you like to talk about it?" ), TRANSITIONS: { lbl.forward(): cnd.regexp(r"yes|yep|ok", re.IGNORECASE), @@ -167,7 +184,8 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: }, "node2": { RESPONSE: Message( - text="System of a Down is an Armenian-American heavy metal band formed in 1994." + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." ), TRANSITIONS: { lbl.forward(): cnd.regexp(r"next", re.IGNORECASE), @@ -177,7 +195,8 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: }, "node3": { RESPONSE: Message( - text="The band achieved commercial success with the release of five studio albums." + text="The band achieved commercial success " + "with the release of five studio albums." ), TRANSITIONS: { lbl.forward(): cnd.regexp(r"next", re.IGNORECASE), @@ -189,8 +208,12 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: "node4": { RESPONSE: Message(text="That's all what I know."), TRANSITIONS: { - greeting_flow_n2_transition: cnd.regexp(r"next", re.IGNORECASE), # second check - high_priority_node_transition("greeting_flow", "node4"): cnd.regexp( + greeting_flow_n2_transition: cnd.regexp( + r"next", re.IGNORECASE + ), # second check + high_priority_node_transition( + "greeting_flow", "node4" + ): cnd.regexp( r"next time", re.IGNORECASE ), # first check lbl.to_fallback(): cnd.true(), # third check @@ -202,37 +225,57 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: # testing happy_path = ( (Message(text="hi"), Message(text="Hi, how are you?")), - (Message(text="i'm fine, how are you?"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="i'm fine, how are you?"), + Message(text="Good. What do you want to talk about?"), + ), ( Message(text="talk about music."), - Message(text="I love `System of a Down` group, would you like to talk about it?"), + Message( + text="I love `System of a Down` group, " + "would you like to talk about it?" + ), ), ( Message(text="yes"), - Message(text="System of a Down is an Armenian-American heavy metal band formed in 1994."), + Message( + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." + ), ), ( Message(text="next"), Message( - text="The band achieved commercial success with the release of five studio albums." + text="The band achieved commercial success " + "with the release of five studio albums." ), ), ( Message(text="back"), - Message(text="System of a Down is an Armenian-American heavy metal band formed in 1994."), + Message( + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." + ), ), ( Message(text="repeat"), - Message(text="System of a Down is an Armenian-American heavy metal band formed in 1994."), + Message( + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." + ), ), ( Message(text="next"), Message( - text="The band achieved commercial success with the release of five studio albums." + text="The band achieved commercial success " + "with the release of five studio albums." ), ), (Message(text="next"), Message(text="That's all what I know.")), - (Message(text="next"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="next"), + Message(text="Good. What do you want to talk about?"), + ), (Message(text="previous"), Message(text="That's all what I know.")), (Message(text="next time"), Message(text="Bye")), (Message(text="stop"), Message(text="Ooops")), @@ -242,7 +285,10 @@ def transition(_: Context, __: Pipeline, *args, **kwargs) -> NodeLabel3Type: (Message(text="hi"), Message(text="Hi, how are you?")), (Message(text="stop"), Message(text="Ooops")), (Message(text="previous"), Message(text="Hi, how are you?")), - (Message(text="i'm fine, how are you?"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="i'm fine, how are you?"), + Message(text="Good. What do you want to talk about?"), + ), ( Message(text="let's talk about something."), Message(text="Sorry, I can not talk about that now."), diff --git a/tutorials/script/core/5_global_transitions.py b/tutorials/script/core/5_global_transitions.py index 8d555d0f1..f0dc5a0fc 100644 --- a/tutorials/script/core/5_global_transitions.py +++ b/tutorials/script/core/5_global_transitions.py @@ -43,19 +43,27 @@ toy_script = { GLOBAL: { TRANSITIONS: { - ("greeting_flow", "node1", 1.1): cnd.regexp(r"\b(hi|hello)\b", re.I), # first check - ("music_flow", "node1", 1.1): cnd.regexp(r"talk about music"), # second check + ("greeting_flow", "node1", 1.1): cnd.regexp( + r"\b(hi|hello)\b", re.I + ), # first check + ("music_flow", "node1", 1.1): cnd.regexp( + r"talk about music" + ), # second check lbl.to_fallback(0.1): cnd.true(), # fifth check lbl.forward(): cnd.all( [ cnd.regexp(r"next\b"), - cnd.has_last_labels(labels=[("music_flow", i) for i in ["node2", "node3"]]), + cnd.has_last_labels( + labels=[("music_flow", i) for i in ["node2", "node3"]] + ), ] # third check ), lbl.repeat(0.2): cnd.all( [ cnd.regexp(r"repeat", re.I), - cnd.negation(cnd.has_last_labels(flow_labels=["global_flow"])), + cnd.negation( + cnd.has_last_labels(flow_labels=["global_flow"]) + ), ] # fourth check ), } @@ -97,19 +105,22 @@ "music_flow": { "node1": { RESPONSE: Message( - text="I love `System of a Down` group, would you like to talk about it?" + text="I love `System of a Down` group, " + "would you like to talk about it?" ), TRANSITIONS: {lbl.forward(): cnd.regexp(r"yes|yep|ok", re.I)}, }, "node2": { RESPONSE: Message( - text="System of a Down is an Armenian-American heavy metal band formed in 1994." + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." ) # Only the global transitions setting are used in this node. }, "node3": { RESPONSE: Message( - text="The band achieved commercial success with the release of five studio albums." + text="The band achieved commercial success " + "with the release of five studio albums." ), TRANSITIONS: {lbl.backward(): cnd.regexp(r"back", re.I)}, }, @@ -126,37 +137,57 @@ # testing happy_path = ( (Message(text="hi"), Message(text="Hi, how are you?")), - (Message(text="i'm fine, how are you?"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="i'm fine, how are you?"), + Message(text="Good. What do you want to talk about?"), + ), ( Message(text="talk about music."), - Message(text="I love `System of a Down` group, would you like to talk about it?"), + Message( + text="I love `System of a Down` group, " + "would you like to talk about it?" + ), ), ( Message(text="yes"), - Message(text="System of a Down is an Armenian-American heavy metal band formed in 1994."), + Message( + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." + ), ), ( Message(text="next"), Message( - text="The band achieved commercial success with the release of five studio albums." + text="The band achieved commercial success " + "with the release of five studio albums." ), ), ( Message(text="back"), - Message(text="System of a Down is an Armenian-American heavy metal band formed in 1994."), + Message( + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." + ), ), ( Message(text="repeat"), - Message(text="System of a Down is an Armenian-American heavy metal band formed in 1994."), + Message( + text="System of a Down is " + "an Armenian-American heavy metal band formed in 1994." + ), ), ( Message(text="next"), Message( - text="The band achieved commercial success with the release of five studio albums." + text="The band achieved commercial success " + "with the release of five studio albums." ), ), (Message(text="next"), Message(text="That's all what I know.")), - (Message(text="next"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="next"), + Message(text="Good. What do you want to talk about?"), + ), (Message(text="previous"), Message(text="That's all what I know.")), (Message(text="next time"), Message(text="bye")), (Message(text="stop"), Message(text="Ooops")), @@ -166,7 +197,10 @@ (Message(text="hi"), Message(text="Hi, how are you?")), (Message(text="stop"), Message(text="Ooops")), (Message(text="previous"), Message(text="Hi, how are you?")), - (Message(text="i'm fine, how are you?"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="i'm fine, how are you?"), + Message(text="Good. What do you want to talk about?"), + ), ( Message(text="let's talk about something."), Message(text="Sorry, I can not talk about that now."), diff --git a/tutorials/script/core/7_pre_response_processing.py b/tutorials/script/core/7_pre_response_processing.py index 233d62f75..38269b6a6 100644 --- a/tutorials/script/core/7_pre_response_processing.py +++ b/tutorials/script/core/7_pre_response_processing.py @@ -38,9 +38,13 @@ # %% def add_prefix(prefix): - def add_prefix_processing(ctx: Context, _: Pipeline, *args, **kwargs) -> Context: + def add_prefix_processing( + ctx: Context, _: Pipeline, *args, **kwargs + ) -> Context: processed_node = ctx.current_node - processed_node.response = Message(text=f"{prefix}: {processed_node.response.text}") + processed_node.response = Message( + text=f"{prefix}: {processed_node.response.text}" + ) ctx.overwrite_current_node_in_processing(processed_node) return ctx @@ -57,7 +61,10 @@ def add_prefix_processing(ctx: Context, _: Pipeline, *args, **kwargs) -> Context # %% toy_script = { "root": { - "start": {RESPONSE: Message(), TRANSITIONS: {("flow", "step_0"): cnd.true()}}, + "start": { + RESPONSE: Message(), + TRANSITIONS: {("flow", "step_0"): cnd.true()}, + }, "fallback": {RESPONSE: Message(text="the end")}, }, GLOBAL: { @@ -73,7 +80,10 @@ def add_prefix_processing(ctx: Context, _: Pipeline, *args, **kwargs) -> Context "proc_name_3": add_prefix("l3_local"), } }, - "step_0": {RESPONSE: Message(text="first"), TRANSITIONS: {lbl.forward(): cnd.true()}}, + "step_0": { + RESPONSE: Message(text="first"), + TRANSITIONS: {lbl.forward(): cnd.true()}, + }, "step_1": { PRE_RESPONSE_PROCESSING: {"proc_name_1": add_prefix("l1_step_1")}, RESPONSE: Message(text="second"), @@ -104,7 +114,10 @@ def add_prefix_processing(ctx: Context, _: Pipeline, *args, **kwargs) -> Context (Message(), Message(text="l3_local: l2_local: l1_step_1: second")), (Message(), Message(text="l3_local: l2_step_2: l1_global: third")), (Message(), Message(text="l3_step_3: l2_local: l1_global: fourth")), - (Message(), Message(text="l4_step_4: l3_local: l2_local: l1_global: fifth")), + ( + Message(), + Message(text="l4_step_4: l3_local: l2_local: l1_global: fifth"), + ), (Message(), Message(text="l3_local: l2_local: l1_global: first")), ) diff --git a/tutorials/script/core/8_misc.py b/tutorials/script/core/8_misc.py index 7756eca12..ac5eb813d 100644 --- a/tutorials/script/core/8_misc.py +++ b/tutorials/script/core/8_misc.py @@ -37,13 +37,19 @@ def custom_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Message: if ctx.validation: return Message() current_node = ctx.current_node - return Message(text=f"ctx.last_label={ctx.last_label}: current_node.misc={current_node.misc}") + return Message( + text=f"ctx.last_label={ctx.last_label}: " + f"current_node.misc={current_node.misc}" + ) # %% toy_script = { "root": { - "start": {RESPONSE: Message(), TRANSITIONS: {("flow", "step_0"): cnd.true()}}, + "start": { + RESPONSE: Message(), + TRANSITIONS: {("flow", "step_0"): cnd.true()}, + }, "fallback": {RESPONSE: Message(text="the end")}, }, GLOBAL: { @@ -95,42 +101,54 @@ def custom_response(ctx: Context, _: Pipeline, *args, **kwargs) -> Message: Message(), Message( text="ctx.last_label=('flow', 'step_0'): current_node.misc=" - "{'var1': 'global_data', 'var2': 'rewrite_by_local', 'var3': 'info_of_step_0'}" + "{'var1': 'global_data', " + "'var2': 'rewrite_by_local', " + "'var3': 'info_of_step_0'}" ), ), ( Message(), Message( text="ctx.last_label=('flow', 'step_1'): current_node.misc=" - "{'var1': 'global_data', 'var2': 'rewrite_by_local', 'var3': 'info_of_step_1'}" + "{'var1': 'global_data', " + "'var2': 'rewrite_by_local', " + "'var3': 'info_of_step_1'}" ), ), ( Message(), Message( text="ctx.last_label=('flow', 'step_2'): current_node.misc=" - "{'var1': 'global_data', 'var2': 'rewrite_by_local', 'var3': 'info_of_step_2'}" + "{'var1': 'global_data', " + "'var2': 'rewrite_by_local', " + "'var3': 'info_of_step_2'}" ), ), ( Message(), Message( text="ctx.last_label=('flow', 'step_3'): current_node.misc=" - "{'var1': 'global_data', 'var2': 'rewrite_by_local', 'var3': 'info_of_step_3'}" + "{'var1': 'global_data', " + "'var2': 'rewrite_by_local', " + "'var3': 'info_of_step_3'}" ), ), ( Message(), Message( text="ctx.last_label=('flow', 'step_4'): current_node.misc=" - "{'var1': 'global_data', 'var2': 'rewrite_by_local', 'var3': 'info_of_step_4'}" + "{'var1': 'global_data', " + "'var2': 'rewrite_by_local', " + "'var3': 'info_of_step_4'}" ), ), ( Message(), Message( text="ctx.last_label=('flow', 'step_0'): current_node.misc=" - "{'var1': 'global_data', 'var2': 'rewrite_by_local', 'var3': 'info_of_step_0'}" + "{'var1': 'global_data', " + "'var2': 'rewrite_by_local', " + "'var3': 'info_of_step_0'}" ), ), ) diff --git a/tutorials/script/core/9_pre_transitions_processing.py b/tutorials/script/core/9_pre_transitions_processing.py index 859a1cd5f..b875b4eb0 100644 --- a/tutorials/script/core/9_pre_transitions_processing.py +++ b/tutorials/script/core/9_pre_transitions_processing.py @@ -58,14 +58,19 @@ def get_previous_node_response_for_response_processing( # a dialog script toy_script = { "root": { - "start": {RESPONSE: Message(), TRANSITIONS: {("flow", "step_0"): cnd.true()}}, + "start": { + RESPONSE: Message(), + TRANSITIONS: {("flow", "step_0"): cnd.true()}, + }, "fallback": {RESPONSE: Message(text="the end")}, }, GLOBAL: { PRE_RESPONSE_PROCESSING: { "proc_name_1": get_previous_node_response_for_response_processing }, - PRE_TRANSITIONS_PROCESSING: {"proc_name_1": save_previous_node_response_to_ctx_processing}, + PRE_TRANSITIONS_PROCESSING: { + "proc_name_1": save_previous_node_response_to_ctx_processing + }, TRANSITIONS: {lbl.forward(0.1): cnd.true()}, }, "flow": { diff --git a/tutorials/script/responses/1_basics.py b/tutorials/script/responses/1_basics.py index c737b05f5..e021a4a67 100644 --- a/tutorials/script/responses/1_basics.py +++ b/tutorials/script/responses/1_basics.py @@ -17,7 +17,11 @@ from dff.script.conditions import exact_match from dff.script import RESPONSE, TRANSITIONS from dff.pipeline import Pipeline -from dff.utils.testing import check_happy_path, is_interactive_mode, run_interactive_mode +from dff.utils.testing import ( + check_happy_path, + is_interactive_mode, + run_interactive_mode, +) # %% @@ -29,11 +33,15 @@ }, "node1": { RESPONSE: Message(text="Hi, how are you?"), - TRANSITIONS: {"node2": exact_match(Message(text="i'm fine, how are you?"))}, + TRANSITIONS: { + "node2": exact_match(Message(text="i'm fine, how are you?")) + }, }, "node2": { RESPONSE: Message(text="Good. What do you want to talk about?"), - TRANSITIONS: {"node3": exact_match(Message(text="Let's talk about music."))}, + TRANSITIONS: { + "node3": exact_match(Message(text="Let's talk about music.")) + }, }, "node3": { RESPONSE: Message(text="Sorry, I can not talk about music now."), @@ -52,7 +60,10 @@ happy_path = ( (Message(text="Hi"), Message(text="Hi, how are you?")), - (Message(text="i'm fine, how are you?"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="i'm fine, how are you?"), + Message(text="Good. What do you want to talk about?"), + ), ( Message(text="Let's talk about music."), Message(text="Sorry, I can not talk about music now."), @@ -62,7 +73,10 @@ (Message(text="stop"), Message(text="Ooops")), (Message(text="stop"), Message(text="Ooops")), (Message(text="Hi"), Message(text="Hi, how are you?")), - (Message(text="i'm fine, how are you?"), Message(text="Good. What do you want to talk about?")), + ( + Message(text="i'm fine, how are you?"), + Message(text="Good. What do you want to talk about?"), + ), ( Message(text="Let's talk about music."), Message(text="Sorry, I can not talk about music now."), diff --git a/tutorials/script/responses/2_buttons.py b/tutorials/script/responses/2_buttons.py index 08cec46db..ff6373c0c 100644 --- a/tutorials/script/responses/2_buttons.py +++ b/tutorials/script/responses/2_buttons.py @@ -92,7 +92,8 @@ def payload_check_inner(ctx: Context, _: Pipeline): "question_3": { RESPONSE: Message( **{ - "text": "What's 114 + 115? (type in the index of the correct option)", + "text": "What's 114 + 115? " + "(type in the index of the correct option)", "misc": { "ui": Keyboard( buttons=[ @@ -120,7 +121,8 @@ def payload_check_inner(ctx: Context, _: Pipeline): Message(text="Hi"), Message( **{ - "text": "Starting test! What's 2 + 2? (type in the index of the correct option)", + "text": "Starting test! What's 2 + 2? " + "(type in the index of the correct option)", "misc": { "ui": Keyboard( buttons=[ @@ -136,7 +138,8 @@ def payload_check_inner(ctx: Context, _: Pipeline): Message(text="0"), Message( **{ - "text": "Starting test! What's 2 + 2? (type in the index of the correct option)", + "text": "Starting test! What's 2 + 2? " + "(type in the index of the correct option)", "misc": { "ui": Keyboard( buttons=[ @@ -152,7 +155,8 @@ def payload_check_inner(ctx: Context, _: Pipeline): Message(text="1"), Message( **{ - "text": "Next question: what's 6 * 8? (type in the index of the correct option)", + "text": "Next question: what's 6 * 8? " + "(type in the index of the correct option)", "misc": { "ui": Keyboard( buttons=[ @@ -168,7 +172,8 @@ def payload_check_inner(ctx: Context, _: Pipeline): Message(text="0"), Message( **{ - "text": "Next question: what's 6 * 8? (type in the index of the correct option)", + "text": "Next question: what's 6 * 8? " + "(type in the index of the correct option)", "misc": { "ui": Keyboard( buttons=[ @@ -184,7 +189,8 @@ def payload_check_inner(ctx: Context, _: Pipeline): Message(text="1"), Message( **{ - "text": "What's 114 + 115? (type in the index of the correct option)", + "text": "What's 114 + 115? " + "(type in the index of the correct option)", "misc": { "ui": Keyboard( buttons=[ @@ -200,7 +206,8 @@ def payload_check_inner(ctx: Context, _: Pipeline): Message(text="1"), Message( **{ - "text": "What's 114 + 115? (type in the index of the correct option)", + "text": "What's 114 + 115? " + "(type in the index of the correct option)", "misc": { "ui": Keyboard( buttons=[ @@ -218,12 +225,19 @@ def payload_check_inner(ctx: Context, _: Pipeline): def process_request(ctx: Context): - ui = ctx.last_response and ctx.last_response.misc and ctx.last_response.misc.get("ui") + ui = ( + ctx.last_response + and ctx.last_response.misc + and ctx.last_response.misc.get("ui") + ) if ui and ui.buttons: try: chosen_button = ui.buttons[int(ctx.last_request.text)] except (IndexError, ValueError): - raise ValueError("Type in the index of the correct option to choose from the buttons.") + raise ValueError( + "Type in the index of the correct option " + "to choose from the buttons." + ) ctx.last_request = Message(misc={"payload": chosen_button.payload}) diff --git a/tutorials/script/responses/3_media.py b/tutorials/script/responses/3_media.py index 8b43d58f9..ad29dd054 100644 --- a/tutorials/script/responses/3_media.py +++ b/tutorials/script/responses/3_media.py @@ -36,7 +36,9 @@ TRANSITIONS: {("pics", "ask_picture"): cnd.true()}, }, "fallback": { - RESPONSE: Message(text="Final node reached, send any message to restart."), + RESPONSE: Message( + text="Final node reached, send any message to restart." + ), TRANSITIONS: {("pics", "ask_picture"): cnd.true()}, }, }, @@ -45,13 +47,16 @@ RESPONSE: Message(text="Please, send me a picture url"), TRANSITIONS: { ("pics", "send_one", 1.1): cnd.regexp(r"^http.+\.png$"), - ("pics", "send_many", 1.0): cnd.regexp(f"{img_url} repeat 10 times"), + ("pics", "send_many", 1.0): cnd.regexp( + f"{img_url} repeat 10 times" + ), ("pics", "repeat", 0.9): cnd.true(), }, }, "send_one": { RESPONSE: Message( - text="here's my picture!", attachments=Attachments(files=[Image(source=img_url)]) + text="here's my picture!", + attachments=Attachments(files=[Image(source=img_url)]), ), TRANSITIONS: {("root", "fallback"): cnd.true()}, }, @@ -63,10 +68,14 @@ TRANSITIONS: {("root", "fallback"): cnd.true()}, }, "repeat": { - RESPONSE: Message(text="I cannot find the picture. Please, try again."), + RESPONSE: Message( + text="I cannot find the picture. Please, try again." + ), TRANSITIONS: { ("pics", "send_one", 1.1): cnd.regexp(r"^http.+\.png$"), - ("pics", "send_many", 1.0): cnd.regexp(r"^http.+\.png repeat 10 times"), + ("pics", "send_many", 1.0): cnd.regexp( + r"^http.+\.png repeat 10 times" + ), ("pics", "repeat", 0.9): cnd.true(), }, }, @@ -75,12 +84,21 @@ happy_path = ( (Message(text="Hi"), Message(text="Please, send me a picture url")), - (Message(text="no"), Message(text="I cannot find the picture. Please, try again.")), + ( + Message(text="no"), + Message(text="I cannot find the picture. Please, try again."), + ), ( Message(text=img_url), - Message(text="here's my picture!", attachments=Attachments(files=[Image(source=img_url)])), + Message( + text="here's my picture!", + attachments=Attachments(files=[Image(source=img_url)]), + ), + ), + ( + Message(text="ok"), + Message(text="Final node reached, send any message to restart."), ), - (Message(text="ok"), Message(text="Final node reached, send any message to restart.")), (Message(text="ok"), Message(text="Please, send me a picture url")), ( Message(text=f"{img_url} repeat 10 times"), @@ -89,7 +107,10 @@ attachments=Attachments(files=[Image(source=img_url)] * 10), ), ), - (Message(text="ok"), Message(text="Final node reached, send any message to restart.")), + ( + Message(text="ok"), + Message(text="Final node reached, send any message to restart."), + ), ) diff --git a/tutorials/script/responses/4_multi_message.py b/tutorials/script/responses/4_multi_message.py index d6dca4b1f..e2b783675 100644 --- a/tutorials/script/responses/4_multi_message.py +++ b/tutorials/script/responses/4_multi_message.py @@ -34,18 +34,28 @@ RESPONSE: MultiMessage( messages=[ Message(text="Hi, what is up?", misc={"confidences": 0.85}), - Message(text="Hello, how are you?", misc={"confidences": 0.9}), + Message( + text="Hello, how are you?", misc={"confidences": 0.9} + ), ] ), - TRANSITIONS: {"node2": cnd.exact_match(Message(text="I'm fine, how are you?"))}, + TRANSITIONS: { + "node2": cnd.exact_match(Message(text="I'm fine, how are you?")) + }, }, "node2": { RESPONSE: Message(text="Good. What do you want to talk about?"), - TRANSITIONS: {"node3": cnd.exact_match(Message(text="Let's talk about music."))}, + TRANSITIONS: { + "node3": cnd.exact_match( + Message(text="Let's talk about music.") + ) + }, }, "node3": { RESPONSE: Message(text="Sorry, I can not talk about that now."), - TRANSITIONS: {"node4": cnd.exact_match(Message(text="Ok, goodbye."))}, + TRANSITIONS: { + "node4": cnd.exact_match(Message(text="Ok, goodbye.")) + }, }, "node4": { RESPONSE: Message(text="bye"), diff --git a/tutorials/stats/1_extractor_functions.py b/tutorials/stats/1_extractor_functions.py index f8ba327c0..14f9371fd 100644 --- a/tutorials/stats/1_extractor_functions.py +++ b/tutorials/stats/1_extractor_functions.py @@ -34,7 +34,13 @@ import asyncio from dff.script import Context -from dff.pipeline import Pipeline, ACTOR, Service, ExtraHandlerRuntimeInfo, to_service +from dff.pipeline import ( + Pipeline, + ACTOR, + Service, + ExtraHandlerRuntimeInfo, + to_service, +) from dff.utils.testing.toy_script import TOY_SCRIPT, HAPPY_PATH from dff.stats import OtelInstrumentor, default_extractors from dff.utils.testing import is_interactive_mode, check_happy_path @@ -104,7 +110,10 @@ async def heavy_service(ctx: Context): "fallback_label": ("greeting_flow", "fallback_node"), "components": [ heavy_service, - Service(handler=ACTOR, after_handler=[default_extractors.get_current_label]), + Service( + handler=ACTOR, + after_handler=[default_extractors.get_current_label], + ), ], } ) diff --git a/tutorials/stats/2_pipeline_integration.py b/tutorials/stats/2_pipeline_integration.py index 53c3b4b3a..c00c6a2f2 100644 --- a/tutorials/stats/2_pipeline_integration.py +++ b/tutorials/stats/2_pipeline_integration.py @@ -27,7 +27,11 @@ GlobalExtraHandlerType, ) from dff.utils.testing.toy_script import TOY_SCRIPT, HAPPY_PATH -from dff.stats import OtelInstrumentor, set_logger_destination, set_tracer_destination +from dff.stats import ( + OtelInstrumentor, + set_logger_destination, + set_tracer_destination, +) from dff.stats import OTLPLogExporter, OTLPSpanExporter from dff.stats import default_extractors from dff.utils.testing import is_interactive_mode, check_happy_path @@ -90,7 +94,10 @@ async def heavy_service(ctx: Context): get_service_state, default_extractors.get_timing_after, ], - components=[{"handler": heavy_service}, {"handler": heavy_service}], + components=[ + {"handler": heavy_service}, + {"handler": heavy_service}, + ], ), Service( handler=ACTOR, @@ -106,8 +113,12 @@ async def heavy_service(ctx: Context): ], } ) -pipeline.add_global_handler(GlobalExtraHandlerType.BEFORE_ALL, default_extractors.get_timing_before) -pipeline.add_global_handler(GlobalExtraHandlerType.AFTER_ALL, default_extractors.get_timing_after) +pipeline.add_global_handler( + GlobalExtraHandlerType.BEFORE_ALL, default_extractors.get_timing_before +) +pipeline.add_global_handler( + GlobalExtraHandlerType.AFTER_ALL, default_extractors.get_timing_after +) pipeline.add_global_handler(GlobalExtraHandlerType.AFTER_ALL, get_service_state) if __name__ == "__main__": diff --git a/tutorials/utils/1_cache.py b/tutorials/utils/1_cache.py index bc1235551..676f2f9e5 100644 --- a/tutorials/utils/1_cache.py +++ b/tutorials/utils/1_cache.py @@ -57,7 +57,9 @@ def response(ctx: Context, _, *__, **___) -> Message: # %% -toy_script = {"flow": {"node1": {TRANSITIONS: {repeat(): true()}, RESPONSE: response}}} +toy_script = { + "flow": {"node1": {TRANSITIONS: {repeat(): true()}, RESPONSE: response}} +} happy_path = ( (Message(), Message(text="1-2-1-2")), diff --git a/tutorials/utils/2_lru_cache.py b/tutorials/utils/2_lru_cache.py index 0aaf326bc..b0fa4f028 100644 --- a/tutorials/utils/2_lru_cache.py +++ b/tutorials/utils/2_lru_cache.py @@ -56,7 +56,9 @@ def response(ctx: Context, _, *__, **___) -> Message: # %% -toy_script = {"flow": {"node1": {TRANSITIONS: {repeat(): true()}, RESPONSE: response}}} +toy_script = { + "flow": {"node1": {TRANSITIONS: {repeat(): true()}, RESPONSE: response}} +} happy_path = ( (Message(), Message(text="1-2-3-2-4")), From 14e6f1ee20d821c0ba3f3fcbb1eedd61a314cb23 Mon Sep 17 00:00:00 2001 From: ruthenian8 Date: Thu, 16 Nov 2023 18:04:59 +0300 Subject: [PATCH 2/6] fix custom messenger interface (#206) * move custom messenger interface to web_api; add form; modify application * Update formatting * back up streamlit chat tutorial * Update web_api_interface * update statsu tutorial * fix indentation * rollback tutorial numeration * improvements to the fastapi tutorial * split lines to fit in docs * add on_request_async && doc fixes * fix merge conflict * lint * mention on_request_async in tutorial * Update doc line for interfaces --------- Co-authored-by: Roman Zlobin --- dff/messengers/common/interface.py | 27 ++- dff/messengers/telegram/interface.py | 2 +- tests/pipeline/test_tutorials.py | 17 +- .../messengers/web_api_interface/1_fastapi.py | 67 +++++++- .../web_api_interface/2_websocket_chat.py | 17 +- .../pipeline/6_custom_messenger_interface.py | 162 ------------------ ...ers_basic.py => 6_extra_handlers_basic.py} | 2 +- ...dlers_full.py => 6_extra_handlers_full.py} | 4 +- ....py => 7_extra_handlers_and_extensions.py} | 2 +- tutorials/stats/2_pipeline_integration.py | 2 +- 10 files changed, 103 insertions(+), 199 deletions(-) delete mode 100644 tutorials/pipeline/6_custom_messenger_interface.py rename tutorials/pipeline/{7_extra_handlers_basic.py => 6_extra_handlers_basic.py} (99%) rename tutorials/pipeline/{7_extra_handlers_full.py => 6_extra_handlers_full.py} (98%) rename tutorials/pipeline/{8_extra_handlers_and_extensions.py => 7_extra_handlers_and_extensions.py} (99%) diff --git a/dff/messengers/common/interface.py b/dff/messengers/common/interface.py index c91fc94af..96ffc288b 100644 --- a/dff/messengers/common/interface.py +++ b/dff/messengers/common/interface.py @@ -30,7 +30,7 @@ async def connect(self, pipeline_runner: PipelineRunnerFunction): May be used for sending an introduction message or displaying general bot information. :param pipeline_runner: A function that should return pipeline response to user request; - usually it's a :py:meth:`~Pipeline._run_pipeline(request, ctx_id)` function. + usually it's a :py:meth:`~dff.pipeline.pipeline.pipeline.Pipeline._run_pipeline` function. :type pipeline_runner: PipelineRunnerFunction """ raise NotImplementedError @@ -97,7 +97,7 @@ async def connect( for most cases the loop itself shouldn't be overridden. :param pipeline_runner: A function that should return pipeline response to user request; - usually it's a :py:meth:`~Pipeline._run_pipeline(request, ctx_id)` function. + usually it's a :py:meth:`~dff.pipeline.pipeline.pipeline.Pipeline._run_pipeline` function. :type pipeline_runner: PipelineRunnerFunction :param loop: a function that determines whether polling should be continued; called in each cycle, should return `True` to continue polling or `False` to stop. @@ -124,18 +124,33 @@ def __init__(self): async def connect(self, pipeline_runner: PipelineRunnerFunction): self._pipeline_runner = pipeline_runner + async def on_request_async(self, request: Any, ctx_id: Hashable) -> Context: + """ + Method invoked on user input. This method works just like + :py:meth:`~dff.pipeline.pipeline.pipeline.Pipeline._run_pipeline`, + however callback message interface may contain additional functionality (e.g. for external API accessing). + Return context that represents dialog with the user; + `last_response`, `id` and some dialog info can be extracted from there. + + :param request: User input. + :param ctx_id: Any unique id that will be associated with dialog between this user and pipeline. + :return: Context that represents dialog with the user. + """ + return await self._pipeline_runner(request, ctx_id) + def on_request(self, request: Any, ctx_id: Hashable) -> Context: """ - Method invoked on user input. This method works just like :py:meth:`.__call__(request, ctx_id)`, + Method invoked on user input. This method works just like + :py:meth:`~dff.pipeline.pipeline.pipeline.Pipeline._run_pipeline`, however callback message interface may contain additional functionality (e.g. for external API accessing). - Returns context that represents dialog with the user; + Return context that represents dialog with the user; `last_response`, `id` and some dialog info can be extracted from there. :param request: User input. :param ctx_id: Any unique id that will be associated with dialog between this user and pipeline. :return: Context that represents dialog with the user. """ - return asyncio.run(self._pipeline_runner(request, ctx_id)) + return asyncio.run(self.on_request_async(request, ctx_id)) class CLIMessengerInterface(PollingMessengerInterface): @@ -169,7 +184,7 @@ async def connect(self, pipeline_runner: PipelineRunnerFunction, **kwargs): The CLIProvider generates new dialog id used to user identification on each `connect` call. :param pipeline_runner: A function that should return pipeline response to user request; - usually it's a :py:meth:`~Pipeline._run_pipeline(request, ctx_id)` function. + usually it's a :py:meth:`~dff.pipeline.pipeline.pipeline.Pipeline._run_pipeline` function. :type pipeline_runner: PipelineRunnerFunction :param \\**kwargs: argument, added for compatibility with super class, it shouldn't be used normally. """ diff --git a/dff/messengers/telegram/interface.py b/dff/messengers/telegram/interface.py index 55dc6e289..96b86102f 100644 --- a/dff/messengers/telegram/interface.py +++ b/dff/messengers/telegram/interface.py @@ -235,7 +235,7 @@ async def endpoint(): json_string = request.get_data().decode("utf-8") update = types.Update.de_json(json_string) - resp = self.on_request(*extract_telegram_request_and_id(update, self.messenger)) + resp = await self.on_request_async(*extract_telegram_request_and_id(update, self.messenger)) self.messenger.send_response(resp.id, resp.last_response) return "" diff --git a/tests/pipeline/test_tutorials.py b/tests/pipeline/test_tutorials.py index 422999240..cf0ed4468 100644 --- a/tests/pipeline/test_tutorials.py +++ b/tests/pipeline/test_tutorials.py @@ -3,7 +3,6 @@ from tests.test_utils import get_path_from_tests_to_current_dir from dff.utils.testing import check_happy_path, HAPPY_PATH -from dff.script import Message dot_path_to_addon = get_path_from_tests_to_current_dir(__file__, separator=".") @@ -19,10 +18,9 @@ "4_groups_and_conditions_full", "5_asynchronous_groups_and_services_basic", "5_asynchronous_groups_and_services_full", - "6_custom_messenger_interface", - "7_extra_handlers_basic", - "7_extra_handlers_full", - "8_extra_handlers_and_extensions", + "6_extra_handlers_basic", + "6_extra_handlers_full", + "7_extra_handlers_and_extensions", ], ) def test_tutorials(tutorial_module_name: str): @@ -30,11 +28,4 @@ def test_tutorials(tutorial_module_name: str): tutorial_module = importlib.import_module(f"tutorials.{dot_path_to_addon}.{tutorial_module_name}") except ModuleNotFoundError as e: pytest.skip(f"dependencies unavailable: {e.msg}") - if tutorial_module_name == "6_custom_messenger_interface": - happy_path = tuple( - (req, Message(misc={"webpage": tutorial_module.construct_webpage_by_response(res.text)})) - for req, res in HAPPY_PATH - ) - check_happy_path(tutorial_module.pipeline, happy_path) - else: - check_happy_path(tutorial_module.pipeline, HAPPY_PATH) + check_happy_path(tutorial_module.pipeline, HAPPY_PATH) diff --git a/tutorials/messengers/web_api_interface/1_fastapi.py b/tutorials/messengers/web_api_interface/1_fastapi.py index 44e87831c..199d1eb28 100644 --- a/tutorials/messengers/web_api_interface/1_fastapi.py +++ b/tutorials/messengers/web_api_interface/1_fastapi.py @@ -2,12 +2,13 @@ """ # Web API: 1. FastAPI -This tutorial shows how to create an API for DFF using FastAPI. +This tutorial shows how to create an API for DFF using FastAPI and +introduces messenger interfaces. You can see the result at http://127.0.0.1:8000/docs. -Here, `_run_pipeline` (same as %mddoclink(api,pipeline.pipeline.pipeline,Pipeline.run)) -method is used to execute pipeline once. +Here, %mddoclink(api,messengers.common.interface,CallbackMessengerInterface) +is used to process requests. %mddoclink(api,script.core.message,Message) is used in creating a JSON Schema for the endpoint. """ @@ -15,6 +16,7 @@ # %pip install dff uvicorn fastapi # %% +from dff.messengers.common.interface import CallbackMessengerInterface from dff.script import Message from dff.pipeline import Pipeline from dff.utils.testing import TOY_SCRIPT_ARGS, is_interactive_mode @@ -23,9 +25,63 @@ from pydantic import BaseModel from fastapi import FastAPI +# %% [markdown] +""" +Messenger interfaces establish communication between users and the pipeline. +They manage message channel initialization and termination +as well as pipeline execution on every user request. +There are two built-in messenger interface types that can be extended +through inheritance: + +* `PollingMessengerInterface` - Starts polling for user requests + in a loop upon initialization, + it has following methods: + + * `_request()` - Method that is used to retrieve user requests from a messenger, + should return list of tuples: (user request, unique dialog id). + * `_respond(responses)` - Method that sends responses generated by pipeline + to users through a messenger, + accepts list of dialog `Contexts`. + * `_on_exception(e)` - Method that is called when a critical exception occurs + i.e. exception from context storage or messenger interface, not a service exception. + + Such exceptions lead to the termination of the loop. + * `connect(pipeline_runner, loop, timeout)` - + Method that starts the polling loop. + + This method is called inside `pipeline.run` method. + + It accepts 3 arguments: + + * a callback that runs pipeline, + * a function that should return True to continue polling, + * and time to wait between loop executions. + +* `CallbackMessengerInterface` - Creates message channel + and provides a callback for pipeline execution, + it has following methods: + + * `on_request(request, ctx_id)` or `on_request_async(request, ctx_id)` - + Method that should be called each time + user provides new input to pipeline, + returns dialog Context. + * `connect(pipeline_runner)` - Method that sets `pipeline_runner` as + a function to be called inside `on_request`. + + This method is called inside `pipeline.run` method. + +You can find API reference for these classes [here](%doclink(api,messengers.common.interface)). + +Here the default `CallbackMessengerInterface` is used to setup +communication between the pipeline on the server side and the messenger client. +""" # %% -pipeline = Pipeline.from_script(*TOY_SCRIPT_ARGS) +messenger_interface = CallbackMessengerInterface() +# CallbackMessengerInterface instantiating the dedicated messenger interface +pipeline = Pipeline.from_script( + *TOY_SCRIPT_ARGS, messenger_interface=messenger_interface +) # %% @@ -42,13 +98,14 @@ async def respond( user_id: str, user_message: Message, ): - context = await pipeline._run_pipeline(user_message, user_id) + context = await messenger_interface.on_request_async(user_message, user_id) return {"user_id": user_id, "response": context.last_response} # %% if __name__ == "__main__": if is_interactive_mode(): # do not run this during doc building + pipeline.run() # runs the messenger_interface.connect method uvicorn.run( app, host="127.0.0.1", diff --git a/tutorials/messengers/web_api_interface/2_websocket_chat.py b/tutorials/messengers/web_api_interface/2_websocket_chat.py index ce40928ca..4719d14f3 100644 --- a/tutorials/messengers/web_api_interface/2_websocket_chat.py +++ b/tutorials/messengers/web_api_interface/2_websocket_chat.py @@ -15,8 +15,8 @@ > all inside a long string. > This, of course, is not optimal and you wouldn't use it for production. -Here, `_run_pipeline` (same as %mddoclink(api,pipeline.pipeline.pipeline,Pipeline.run)) -method is used to execute pipeline once. +Here, %mddoclink(api,messengers.common.interface,CallbackMessengerInterface) +is used to process requests. %mddoclink(api,script.core.message,Message) is used to represent text messages. """ @@ -24,9 +24,10 @@ # %pip install dff uvicorn fastapi # %% +from dff.messengers.common.interface import CallbackMessengerInterface from dff.script import Message from dff.pipeline import Pipeline -from dff.utils.testing import TOY_SCRIPT, is_interactive_mode +from dff.utils.testing import TOY_SCRIPT_ARGS, is_interactive_mode import uvicorn from fastapi import FastAPI, WebSocket, WebSocketDisconnect @@ -34,10 +35,9 @@ # %% +messenger_interface = CallbackMessengerInterface() pipeline = Pipeline.from_script( - TOY_SCRIPT, - ("greeting_flow", "start_node"), - ("greeting_flow", "fallback_node"), + *TOY_SCRIPT_ARGS, messenger_interface=messenger_interface ) @@ -93,7 +93,9 @@ async def websocket_endpoint(websocket: WebSocket, client_id: int): data = await websocket.receive_text() await websocket.send_text(f"User: {data}") request = Message(text=data) - context = await pipeline._run_pipeline(request, client_id) + context = await messenger_interface.on_request_async( + request, client_id + ) response = context.last_response.text if response is not None: await websocket.send_text(f"Bot: {response}") @@ -106,6 +108,7 @@ async def websocket_endpoint(websocket: WebSocket, client_id: int): # %% if __name__ == "__main__": if is_interactive_mode(): # do not run this during doc building + pipeline.run() uvicorn.run( app, host="127.0.0.1", diff --git a/tutorials/pipeline/6_custom_messenger_interface.py b/tutorials/pipeline/6_custom_messenger_interface.py deleted file mode 100644 index 99f2d096e..000000000 --- a/tutorials/pipeline/6_custom_messenger_interface.py +++ /dev/null @@ -1,162 +0,0 @@ -# %% [markdown] -""" -# 6. Custom messenger interface - -The following tutorial shows messenger interfaces usage. - -Here, %mddoclink(api,messengers.common.interface,CallbackMessengerInterface) -and %mddoclink(api,messengers.common.interface,PollingMessengerInterface) -are shown as alternatives for connection to custom user messenger backends. -""" - -# %pip install dff flask - -# %% -import logging - -from dff.messengers.common.interface import CallbackMessengerInterface -from dff.script import Context, Message -from flask import Flask, request, Request - -from dff.pipeline import Pipeline, ACTOR -from dff.utils.testing import is_interactive_mode, TOY_SCRIPT - -logger = logging.getLogger(__name__) - - -# %% [markdown] -""" -Messenger interfaces are used for providing -a way for communication between user and `pipeline`. -They manage message channel initialization and termination -as well as pipeline execution on every user request. -There are two built-in messenger interface types (that may be overridden): - -* `PollingMessengerInterface` - Starts polling for user request - in a loop upon initialization, - it has following methods: - * `_request()` - Method that is executed in a loop, - should return list of tuples: (user request, unique dialog id). - * `_respond(responses)` - Method that is executed in a loop - after all user requests processing, - accepts list of dialog `Contexts`. - * `_on_exception(e)` - Method that is called on - exception that happens in the loop, - should catch the exception - (it is also called on pipeline termination). - * `connect(pipeline_runner, loop, timeout)` - - Method that is called on connection to message channel, - accepts pipeline_runner (a callback, running pipeline). - * loop - A function to be called on each loop - execution (should return True to continue polling). - * timeout - Time in seconds to wait between loop executions. - -* `CallbackMessengerInterface` - Creates message channel - and provides a callback for pipeline execution, - it has following method: - * `on_request(request, ctx_id)` - Method that should be called each time - user provides new input to pipeline, - returns dialog Context. - -`CLIMessengerInterface` is also - a messenger interface that overrides `PollingMessengerInterface` and - provides default message channel between pipeline and console/file IO. - -Here a default `CallbackMessengerInterface` is used to setup -communication between pipeline and Flask server. -Two services are used to process request: - -* `purify_request` - Extracts user request from Flask HTTP request. -* `construct_webpage_by_response` - Wraps actor response in webpage and - adds response-based image to it. -""" - -# %% -app = Flask(__name__) - -messenger_interface = CallbackMessengerInterface() -# For this simple case of Flask, -# CallbackMessengerInterface may not be overridden - - -def construct_webpage_by_response(response: str) -> str: - return f""" - - - - - - - -

{response}

- Response picture - - - """ - - -def purify_request(ctx: Context): - last_request = ctx.last_request - if isinstance(last_request, Request): - logger.info(f"Capturing request from: {last_request.base_url}") - ctx.last_request = Message(text=last_request.args.get("request")) - elif isinstance(last_request, Message): - logger.info("Capturing request from CLI") - else: - raise Exception( - f"Request of type {type(last_request)} can not be purified!" - ) - - -def cat_response2webpage(ctx: Context): - ctx.last_response = Message( - misc={"webpage": construct_webpage_by_response(ctx.last_response.text)} - ) - - -# %% -pipeline_dict = { - "script": TOY_SCRIPT, - "start_label": ("greeting_flow", "start_node"), - "fallback_label": ("greeting_flow", "fallback_node"), - "messenger_interface": messenger_interface, - "components": [ - purify_request, - { - "handler": ACTOR, - "name": "encapsulated-actor", - }, # Actor here is encapsulated in another service with specific name - cat_response2webpage, - ], -} - - -@app.route("/pipeline_web_interface") -async def route(): - ctx_id = 0 # 0 will be current dialog (context) identification. - return messenger_interface.on_request(request, ctx_id).last_response.text - - -# %% -pipeline = Pipeline(**pipeline_dict) - -if ( - __name__ == "__main__" and is_interactive_mode() -): # This tutorial will be run in interactive mode only - pipeline.run() - app.run() - # Navigate to - # http://127.0.0.1:5000/pipeline_web_interface?request={REQUEST} - # to receive response - # e.g. http://127.0.0.1:5000/pipeline_web_interface?request=Hi - # will bring you to actor start node diff --git a/tutorials/pipeline/7_extra_handlers_basic.py b/tutorials/pipeline/6_extra_handlers_basic.py similarity index 99% rename from tutorials/pipeline/7_extra_handlers_basic.py rename to tutorials/pipeline/6_extra_handlers_basic.py index bea7c9317..386dd2885 100644 --- a/tutorials/pipeline/7_extra_handlers_basic.py +++ b/tutorials/pipeline/6_extra_handlers_basic.py @@ -1,6 +1,6 @@ # %% [markdown] """ -# 7. Extra Handlers (basic) +# 6. Extra Handlers (basic) The following tutorial shows extra handlers possibilities and use cases. diff --git a/tutorials/pipeline/7_extra_handlers_full.py b/tutorials/pipeline/6_extra_handlers_full.py similarity index 98% rename from tutorials/pipeline/7_extra_handlers_full.py rename to tutorials/pipeline/6_extra_handlers_full.py index ad89f3230..3672f0f82 100644 --- a/tutorials/pipeline/7_extra_handlers_full.py +++ b/tutorials/pipeline/6_extra_handlers_full.py @@ -1,11 +1,11 @@ # %% [markdown] """ -# 7. Extra Handlers (full) +# 6. Extra Handlers (full) The following tutorial shows extra handlers possibilities and use cases. This tutorial is a more advanced version of the -[previous tutorial](%doclink(tutorial,pipeline.7_extra_handlers_basic)). +[previous tutorial](%doclink(tutorial,pipeline.6_extra_handlers_basic)). """ # %pip install dff psutil diff --git a/tutorials/pipeline/8_extra_handlers_and_extensions.py b/tutorials/pipeline/7_extra_handlers_and_extensions.py similarity index 99% rename from tutorials/pipeline/8_extra_handlers_and_extensions.py rename to tutorials/pipeline/7_extra_handlers_and_extensions.py index d4930d2e5..ca1d9a652 100644 --- a/tutorials/pipeline/8_extra_handlers_and_extensions.py +++ b/tutorials/pipeline/7_extra_handlers_and_extensions.py @@ -1,6 +1,6 @@ # %% [markdown] """ -# 8. Extra Handlers and Extensions +# 7. Extra Handlers and Extensions The following tutorial shows how pipeline can be extended by global extra handlers and custom functions. diff --git a/tutorials/stats/2_pipeline_integration.py b/tutorials/stats/2_pipeline_integration.py index c00c6a2f2..88b7c263e 100644 --- a/tutorials/stats/2_pipeline_integration.py +++ b/tutorials/stats/2_pipeline_integration.py @@ -3,7 +3,7 @@ # 2. Pipeline Integration In the DFF ecosystem, extractor functions act as regular extra handlers ( -[see the pipeline module documentation](%doclink(tutorial,pipeline.7_extra_handlers_basic)) +[see the pipeline module documentation](%doclink(tutorial,pipeline.6_extra_handlers_basic)) ). Hence, you can decorate any part of your pipeline, including services, service groups and the pipeline as a whole, to obtain the statistics From 979a7dde0f196058fb55047679b4451c97c4eee7 Mon Sep 17 00:00:00 2001 From: ruthenian8 Date: Mon, 20 Nov 2023 11:34:39 +0300 Subject: [PATCH 3/6] Feat/readme (#281) * update index * Update readme.md * update readme && index * update readme with why section * fix tutorials and example comparison in tutorials index * move badges to the top of the page * remove unnecessary extras * improve basic example * change doc links * add license * remove unnecessary extras from get_started * Update README.md Refer to index.html --------- Co-authored-by: Roman Zlobin --- README.md | 124 +++++++++++++----------------------- docs/source/get_started.rst | 8 --- docs/source/index.rst | 57 +++++++++++++++++ docs/source/tutorials.rst | 4 +- 4 files changed, 105 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index ebab43d72..43c1bf13e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,5 @@ - # Dialog Flow Framework -The Dialog Flow Framework (DFF) allows you to write conversational services. -The service is written by defining a special dialog graph that describes the behavior of the dialog service. -The dialog graph contains the dialog script. DFF offers a specialized language (DSL) for quickly writing dialog graphs. -You can use it in services such as writing skills for Amazon Alexa, etc., chatbots for social networks, website call centers, etc. - [![Documentation Status](https://github.com/deeppavlov/dialog_flow_framework/workflows/build_and_publish_docs/badge.svg)](https://deeppavlov.github.io/dialog_flow_framework) [![Codestyle](https://github.com/deeppavlov/dialog_flow_framework/workflows/codestyle/badge.svg)](https://github.com/deeppavlov/dialog_flow_framework/actions/workflows/codestyle.yml) [![Tests](https://github.com/deeppavlov/dialog_flow_framework/workflows/test_coverage/badge.svg)](https://github.com/deeppavlov/dialog_flow_framework/actions/workflows/test_coverage.yml) @@ -14,6 +8,17 @@ You can use it in services such as writing skills for Amazon Alexa, etc., chatbo [![PyPI](https://img.shields.io/pypi/v/dff)](https://pypi.org/project/dff/) [![Downloads](https://pepy.tech/badge/dff)](https://pepy.tech/project/dff) +The Dialog Flow Framework (DFF) allows you to develop conversational services. +DFF offers a specialized domain-specific language (DSL) for quickly writing dialogs in pure Python. The service is created by defining a special dialog graph that determines the behavior of the dialog agent. The latter is then leveraged in the DFF pipeline. +You can use the framework in various services, such as Amazon Alexa skills, chatbots for social networks, call centers, etc. + +## Why choose DFF + +* Written in pure Python, the framework is easily accessible for both beginners and experienced developers. +* For the same reason, all the abstractions used in DFF can be easily customized and extended using regular language synthax. +* DFF offers easy and straightforward tools for state management which is as easy as setting values of a Python dictionary. +* The framework is being actively maintained and thoroughly tested. The team is open to suggestions and quickly reacts to bug reports. + # Quick Start ## Installation @@ -26,7 +31,6 @@ pip install dff The above command will set the minimum dependencies to start working with DFF. The installation process allows the user to choose from different packages based on their dependencies, which are: ```bash -pip install dff[core] # minimal dependencies (by default) pip install dff[json] # dependencies for using JSON pip install dff[pickle] # dependencies for using Pickle pip install dff[redis] # dependencies for using Redis @@ -37,13 +41,6 @@ pip install dff[sqlite] # dependencies for using SQLite pip install dff[ydb] # dependencies for using Yandex Database pip install dff[telegram] # dependencies for using Telegram pip install dff[benchmark] # dependencies for benchmarking -pip install dff[full] # full dependencies including all options above -pip install dff[tests] # dependencies for running tests -pip install dff[test_full] # full dependencies for running all tests (all options above) -pip install dff[tutorials] # dependencies for running tutorials (all options above) -pip install dff[devel] # dependencies for development -pip install dff[doc] # dependencies for documentation -pip install dff[devel_full] # full dependencies for development (all options above) ``` For example, if you are going to use one of the database backends, @@ -54,11 +51,15 @@ pip install dff[postgresql, mysql] ## Basic example +The following code snippet builds a simplistic chat bot that replies with messages +``Hi!`` and ``OK`` depending on user input, which only takes a few lines of code. +All the abstractions used in this example are thoroughly explained in the dedicated +[user guide](https://deeppavlov.github.io/dialog_flow_framework/user_guides/basic_conceptions.html). + ```python -from dff.script import GLOBAL, TRANSITIONS, RESPONSE, Context, Message +from dff.script import GLOBAL, TRANSITIONS, RESPONSE, Message from dff.pipeline import Pipeline import dff.script.conditions.std_conditions as cnd -from typing import Tuple # create a dialog script script = { @@ -69,8 +70,8 @@ script = { } }, "flow": { - "node_hi": {RESPONSE: Message(text="Hi!!!")}, - "node_ok": {RESPONSE: Message(text="Okey")}, + "node_hi": {RESPONSE: Message(text="Hi!")}, + "node_ok": {RESPONSE: Message(text="OK")}, }, } @@ -78,80 +79,47 @@ script = { pipeline = Pipeline.from_script(script, start_label=("flow", "node_hi")) -# handler requests -def turn_handler(in_request: Message, pipeline: Pipeline) -> Tuple[Message, Context]: - # Pass the next request of user into pipeline and it returns updated context with actor response +def turn_handler(in_request: Message, pipeline: Pipeline) -> Message: + # Pass user request into pipeline and get dialog context (message history) + # The pipeline will automatically choose the correct response using script ctx = pipeline(in_request, 0) - # Get last actor response from the context + # Get last response from the context out_response = ctx.last_response - # The next condition branching needs for testing - return out_response, ctx + return out_response while True: - in_request = input("type your answer: ") - out_response, ctx = turn_handler(Message(text=in_request), pipeline) - print(out_response.text) + in_request = input("Your message: ") + out_response = turn_handler(Message(text=in_request), pipeline) + print("Response: ", out_response.text) ``` When you run this code, you get similar output: ``` -type your answer: hi -Okey -type your answer: Hi -Hi!!! -type your answer: ok -Okey -type your answer: ok -Okey +Your message: hi +Response: OK +Your message: Hi +Response: Hi! +Your message: ok +Response: OK +Your message: ok +Response: OK ``` -To get more advanced examples, take a look at -[tutorials](https://github.com/deeppavlov/dialog_flow_framework/tree/master/tutorials) on GitHub. - -# Context Storages -## Description - -Context Storages allow you to save and retrieve user dialogue states -(in the form of a `Context` object) using various database backends. - -The following backends are currently supported: -* [JSON](https://www.json.org/json-en.html) -* [pickle](https://docs.python.org/3/library/pickle.html) -* [shelve](https://docs.python.org/3/library/shelve.html) -* [SQLite](https://www.sqlite.org/index.html) -* [PostgreSQL](https://www.postgresql.org/) -* [MySQL](https://www.mysql.com/) -* [MongoDB](https://www.mongodb.com/) -* [Redis](https://redis.io/) -* [Yandex DataBase](https://ydb.tech/) - -Aside from this, we offer some interfaces for saving data to your local file system. -These are not meant to be used in production, but can be helpful for prototyping your application. - -## Basic example - -```python -from dff.script import Context -from dff.pipeline import Pipeline -from dff.context_storages import SQLContextStorage -from .script import some_df_script - -db = SQLContextStorage("postgresql+asyncpg://user:password@host:port/dbname") - -pipeline = Pipeline.from_script(some_df_script, start_label=("root", "start"), fallback_label=("root", "fallback")) +More advanced examples are available as a part of documentation: +[tutorials](https://deeppavlov.github.io/dialog_flow_framework/tutorials.html). +## Further steps -def handle_request(request): - user_id = request.args["user_id"] - new_context = pipeline(request, user_id) - return new_context.last_response - -``` - -To get more advanced examples, take a look at -[tutorials](https://github.com/deeppavlov/dialog_flow_framework/tree/master/tutorials/context_storages) on GitHub. +To further explore the API of the framework, you can make use of the [detailed documentation](https://deeppavlov.github.io/dialog_flow_framework/index.html). +Broken down into several sections to highlight all the aspects of development with DFF, +the documentation for the library is constantly available online. # Contributing to the Dialog Flow Framework +We are open to accepting pull requests and bug reports. Please refer to [CONTRIBUTING.md](https://github.com/deeppavlov/dialog_flow_framework/blob/master/CONTRIBUTING.md). + +# License + +DFF is distributed under the terms of the [Apache License 2.0](https://github.com/deeppavlov/dialog_flow_framework/blob/master/LICENSE). diff --git a/docs/source/get_started.rst b/docs/source/get_started.rst index 5c38ab967..1c5ee7a3c 100644 --- a/docs/source/get_started.rst +++ b/docs/source/get_started.rst @@ -17,7 +17,6 @@ The installation process allows the user to choose from different packages based .. code-block:: console - pip install dff[core] # minimal dependencies (by default) pip install dff[json] # dependencies for using JSON pip install dff[pickle] # dependencies for using Pickle pip install dff[redis] # dependencies for using Redis @@ -28,13 +27,6 @@ The installation process allows the user to choose from different packages based pip install dff[ydb] # dependencies for using Yandex Database pip install dff[telegram] # dependencies for using Telegram pip install dff[benchmark] # dependencies for benchmarking - pip install dff[full] # full dependencies including all options above - pip install dff[tests] # dependencies for running tests - pip install dff[test_full] # full dependencies for running all tests (all options above) - pip install dff[tutorials] # dependencies for running tutorials (all options above) - pip install dff[devel] # dependencies for development - pip install dff[doc] # dependencies for documentation - pip install dff[devel_full] # full dependencies for development (all options above) For example, if you are going to use one of the database backends, you can specify the corresponding requirements yourself. diff --git a/docs/source/index.rst b/docs/source/index.rst index 45b951ab2..42b71d3f3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -21,7 +21,64 @@ It allows developers to easily create and manage complex dialog flows, integrate and handle user input in a flexible and efficient manner. Additionally, the framework is highly customizable, allowing developers to easily adapt it to their specific needs and requirements. +DFF documentation includes the following sections: + +:doc:`Getting started <./get_started>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Essential information about installing and using the library +aimed at beginners can be found in the ``Getting started`` part +of the documentation. This section also introduces the basic terms +that form the principles of the framework. +For deeper understanding of the API, consult the documentation sections described below. + +:doc:`User guides <./user_guides>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``User guides`` section provides comprehensive human-readable explanations +of how your conversational service should be set up and function. +It specifically highlights the aspects that are not covered by the API reference, +e.g. deployment, optimization techniques, etc. + +:doc:`Tutorials <./tutorials>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most capabilities of DFF can also be explored in the ``Tutorials`` +section. These interactive files dynamically showcase how different +modules and classes of the framework interact. + +:doc:`Examples <./examples>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Links to demonsration projects that leverage the library +integrating it with external services and models and serve as examples +can be found in this section. + +:doc:`API reference <./reference>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The API reference contains documentation for classes and abstractions +used in the library which can be used to determine the exact typing +and behavior of all the functions involved. + +:doc:`Development <./development>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``Development`` section shows the library's current development status and specifies the contribution rules. + +:doc:`Community <./community>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``Community`` section links you to useful resources where you can find supplemental information +about the framework and ask questions. + +:doc:`About us <./about_us>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can get more info about the development team in the ``About us`` section. + .. toctree:: + :hidden: :maxdepth: 1 get_started diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index 6241abdcc..ba9b2e098 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -14,8 +14,8 @@ The Script section covers the basics of the script concept, including conditions and serialization. It also includes tutorials on pre-response and pre-transitions processing. Finally, the Utils section covers the cache and LRU cache utilities in DFF. -The main difference between Tutorials and Examples is that Examples typically show how to implement -a specific feature or solve a particular problem, whereas Tutorials provide a more +The main difference between Tutorials and Examples is that Tutorials typically show how to implement +a specific feature or solve a particular problem, whereas Examples provide a more comprehensive overview of how to build a complete application. | To understand the basics of DFF, read the following tutorials: From 2215496cf2680913b2153a4a570303c2f2f1375d Mon Sep 17 00:00:00 2001 From: Roman Zlobin Date: Mon, 20 Nov 2023 14:10:04 +0300 Subject: [PATCH 4/6] replace "Amazon Alexa" with "personal assistants" --- README.md | 2 +- docs/source/get_started.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 43c1bf13e..1c5bc1986 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The Dialog Flow Framework (DFF) allows you to develop conversational services. DFF offers a specialized domain-specific language (DSL) for quickly writing dialogs in pure Python. The service is created by defining a special dialog graph that determines the behavior of the dialog agent. The latter is then leveraged in the DFF pipeline. -You can use the framework in various services, such as Amazon Alexa skills, chatbots for social networks, call centers, etc. +You can use the framework in various services such as social networks, call centers, websites, personal assistants, etc. ## Why choose DFF diff --git a/docs/source/get_started.rst b/docs/source/get_started.rst index 1c5ee7a3c..b0749793e 100644 --- a/docs/source/get_started.rst +++ b/docs/source/get_started.rst @@ -50,7 +50,7 @@ It allows developers to easily write and manage dialog systems by defining a spe dialog graph that describes the behavior of the service. DFF offers a specialized language (DSL) for quickly writing dialog graphs, making it easy for developers to create chatbots for a wide -range of applications such as social networks, call centers, websites, skills for Amazon Alexa, etc. +range of applications, such as social networks, call centers, websites, personal assistants, etc. DFF has several important concepts: From d4fce7553fda9bc4562f9053724c3c5fb0e0368c Mon Sep 17 00:00:00 2001 From: ruthenian8 Date: Tue, 21 Nov 2023 14:55:22 +0300 Subject: [PATCH 5/6] Feat/locust guide (#236) * Add profiling guide introduction * Update profiling guide * Update profiling guide * include the guide in the doctree * update guide; add warnings to stats tutors * Update guide with links to tutorials * update opt guide * fix bulletpoint list in opt guide * minor doc fixes * Update optimization guide * fix and add section links --------- Co-authored-by: Roman Zlobin --- docs/source/user_guides.rst | 7 ++ .../source/user_guides/optimization_guide.rst | 107 ++++++++++++++++++ .../context_storages/8_db_benchmarking.py | 25 +++- .../3_load_testing_with_locust.py | 5 +- tutorials/stats/1_extractor_functions.py | 7 ++ tutorials/stats/2_pipeline_integration.py | 5 + 6 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 docs/source/user_guides/optimization_guide.rst diff --git a/docs/source/user_guides.rst b/docs/source/user_guides.rst index 024c84ecc..0cb1a1531 100644 --- a/docs/source/user_guides.rst +++ b/docs/source/user_guides.rst @@ -23,6 +23,12 @@ for exploring the telemetry data collected from your conversational services. We show how to plug in the telemetry collection and configure the pre-built Superset dashboard shipped with DFF. +:doc:`Optimization guide <./user_guides/optimization_guide>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``optimization guide`` demonstrates various tools provided by the library +that you can use to profile your conversational service, +and to locate and remove performance bottlenecks. .. toctree:: :hidden: @@ -30,3 +36,4 @@ Superset dashboard shipped with DFF. user_guides/basic_conceptions user_guides/context_guide user_guides/superset_guide + user_guides/optimization_guide diff --git a/docs/source/user_guides/optimization_guide.rst b/docs/source/user_guides/optimization_guide.rst new file mode 100644 index 000000000..198e70402 --- /dev/null +++ b/docs/source/user_guides/optimization_guide.rst @@ -0,0 +1,107 @@ +Optimization Guide +------------------ + +Introduction +~~~~~~~~~~~~ + +When optimizing a dialog service to provide the best possible user experience, +it's essential to identify and address performance issues. +Similar to any complex system, a dialog service can have performance bottlenecks at various levels. +These bottlenecks can occur during I/O operations like receiving and sending messages, +as well as when synchronizing service states with a database. +As the number of callbacks in the script and pipeline increases, +the performance of DFF classes can degrade leading to longer response time. + +As a result, it becomes necessary to locate the part of the pipeline that is causing issues, so that +further optimization steps can be taken. DFF provides several tools that address the need for +profiling individual system components. This guide will walk you through the process +of using these tools in practice and optimizing the profiled application. + +Profiling with Locust testing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`Locust `__ is a tool for load testing web applications that +simultaneously spawns several user agents that execute a pre-determined behavior +against the target application. Assuming that your pipeline is integrated into a web +server application, like Flask or FastAPI, that is not strongly impacted by the load, +the load testing reveals how well your pipeline would scale to a highly loaded environment. +Using this approach, you can also measure the scalability of each component in your pipeline, +if you take advantage of the Opentelemetry package bundled with the library (`stats` extra required) +as described below. + +Since Locust testing can only target web apps, +this approach only applies if you integrate your dialog pipeline into a web application. +The `FastAPI integration tutorial <../tutorials/tutorials.messengers.web_api_interface.1_fastapi.py>`_ +shows the most straightforward way to do this. +At this stage, you will also need to instrument the pipeline components that you want to additionally profile +using `extractor functions`. Put simply, you are decorating the components of the pipeline +with functions that can report their performance, e.g. their execution time or the CPU load. + +.. note:: + + You can get more info on how instrumentation is done and statistics are collected + in the `stats tutorial <../tutorials/tutorials.stats.1_extractor_functions.py>`__. + +When you are done setting up the instrumentation, you can launch the web server to accept connections from locust. + +The final step is to run a Locust file which will result in artificial load traffic being generated and sent to your server. +A Locust file is a script that implements the behavior of artificial users, +i.e. the requests to the server that will be made during testing. + +.. note:: + + An example Locust script along with instructions on how to run it can be found in the + `load testing tutorial <../tutorials/tutorials.messengers.web_api_interface.3_load_testing_with_locust.py>`_. + The simplest way, however, is to pass a locust file to the Python interpreter. + +Once Locust is running, you can access its GUI, where you can set the number of users to emulate. +After configuring this parameter, the active phase of testing will begin, +and the results will become accessible on an interactive dashboard. +These reported values include timing data, such as the average response time of your service, +allowing you to assess the performance's reasonableness and impact on user experience. + +The data provided by extractor functions will be available in the Clickhouse database; +you can view it using the Superset dashboard (see `instructions <./superset_guide.html>`__) +or analyze it with your own queries using the Clickhouse client. + +Profiling context storages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Benchmarking the performance of context storage is crucial to understanding +how different storage methods impact your dialog service's efficiency. +This process involves running tests to measure the speed and reliability of various context storage solutions. +Given the exact configuration of your system, one or the other database type may be performing more efficiently, +so you may prefer to change your database depending on the testing results. + +.. note:: + The exact instructions of how the testing can be carried out are available in the + `DB benchmarking tutorial <../tutorials/tutorials.context_storages.8_db_benchmarking.py>`__. + +Optimization techniques +~~~~~~~~~~~~~~~~~~~~~~~ + +Aside from choosing an appropriate database type, there exists a number of other recommendations +that may help you improve the efficiency of your service. + +* Firstly, follow the DRY principle not only with regard to your code, but also with regard to + computational operations. In other words, you have to make sure that your callback functions work only once + during a dialog turn and only when needed. E.g. you can take note of the `conditions` api available as a part + of the `Pipeline` module: while normally a pipeline service runs every turn, you can restrict it + to only run on turns when a particular condition is satisfied, greatly reducing + the number of performed actions (see the + `Groups and Conditions tutorial <../tutorials/tutorials.pipeline.4_groups_and_conditions_full.py>`__). + +* Using caching for resource-consuming callbacks and actions may also prove to be a helpful strategy. + In this manner, you can improve the computational efficiency of your pipeline, + while making very few changes to the code itself. DFF includes a caching mechanism + for response functions. However, the simplicity + of the DFF API makes it easy to integrate any custom caching solutions that you may come up with. + See the `Cache tutorial <../tutorials/tutorials.utils.1_cache.py>`__. + +* Finally, be mindful about the use of computationally expensive algorithms, like NLU classifiers + or LLM-based generative networks, since those require a great deal of time and resources + to produce an answer. In case you need to use one, take full advantage of caching along with + other means to relieve the computational load imposed by neural networks such as message queueing. + +.. + todo: add a link to a user guide about using message queueing. diff --git a/tutorials/context_storages/8_db_benchmarking.py b/tutorials/context_storages/8_db_benchmarking.py index 4ccba4901..1b56414b4 100644 --- a/tutorials/context_storages/8_db_benchmarking.py +++ b/tutorials/context_storages/8_db_benchmarking.py @@ -56,6 +56,21 @@ Note: context storages passed into these functions will be cleared. +Once the benchmark results are saved to a file, you can view and analyze them using two methods: + +* [Using the Report Function](#Using-the-report-function): This function + can display specified information from a given file. + By default, it prints the name and average metrics for each benchmark case. + +* [Using the Streamlit App](#Using-Streamlit-app): A Streamlit app + is available for viewing and comparing benchmark results. + You can upload benchmark result files using the app's "Benchmark sets" tab, + inspect individual results in the "View" tab, and compare metrics in the "Compare" tab. + +Benchmark results are saved according to a specific schema, +which can be found in the benchmark schema documentation. +Each database being benchmarked will have its own result file. + ### Configuration The first one is a higher-level wrapper of the second one. @@ -73,7 +88,15 @@ Its most basic implementation is %mddoclink(api,utils.db_benchmark.basic_config,BasicBenchmarkConfig). -It has several parameters: +DFF provides configuration presets in the +%mddoclink(api,utils.db_benchmark.basic_config) module, +covering various contexts, messages, and edge cases. +You can use these presets by passing them to the benchmark functions or create +your own configuration. + +To learn more about using presets see [Configuration presets](#Configuration-presets) + +Benchmark configs have several parameters: Setting `context_num` to 50 means that we'll run fifty cycles of writing and reading context. This way we'll be able to get a more accurate average read/write time as well as diff --git a/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py b/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py index 8afc6ed3a..a2ae9a647 100644 --- a/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py +++ b/tutorials/messengers/web_api_interface/3_load_testing_with_locust.py @@ -19,8 +19,11 @@ ```bash locust -f {file_name} ``` -3. Run in interactive mode: +3. Run from python: ```python + import sys + from locust import main + sys.argv = ["locust", "-f", {file_name}] main.main() ``` diff --git a/tutorials/stats/1_extractor_functions.py b/tutorials/stats/1_extractor_functions.py index 14f9371fd..f0e8baaf8 100644 --- a/tutorials/stats/1_extractor_functions.py +++ b/tutorials/stats/1_extractor_functions.py @@ -21,6 +21,13 @@ The output of these functions will be captured by an OpenTelemetry instrumentor and directed to the Opentelemetry collector server which in its turn batches and persists data to Clickhouse or other OLAP storages. + +:warning: Both the Opentelemetry collector and the Clickhouse instance must be running +during statistics collection. If you cloned the DFF repo, launch them using `docker-compose`: +```bash +docker-compose up otelcol clickhouse +``` + For more information on OpenTelemetry instrumentation, refer to the body of this tutorial as well as [OpenTelemetry documentation]( https://opentelemetry.io/docs/instrumentation/python/manual/ diff --git a/tutorials/stats/2_pipeline_integration.py b/tutorials/stats/2_pipeline_integration.py index 88b7c263e..bbd9766f8 100644 --- a/tutorials/stats/2_pipeline_integration.py +++ b/tutorials/stats/2_pipeline_integration.py @@ -10,6 +10,11 @@ specific for that component. Some examples of this functionality are showcased in this tutorial. +:warning: Both the Opentelemetry collector and the Clickhouse instance must be running +during statistics collection. If you cloned the DFF repo, launch them using `docker-compose`: +```bash +docker-compose up otelcol clickhouse +``` """ # %pip install dff[stats] From 873792d8eb565bcbe34b46b8fb5f8eb33add60e5 Mon Sep 17 00:00:00 2001 From: Roman Zlobin Date: Tue, 21 Nov 2023 19:37:44 +0300 Subject: [PATCH 6/6] fix stats tutorials warnings --- tutorials/stats/1_extractor_functions.py | 10 +++++++--- tutorials/stats/2_pipeline_integration.py | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tutorials/stats/1_extractor_functions.py b/tutorials/stats/1_extractor_functions.py index f0e8baaf8..981333a1d 100644 --- a/tutorials/stats/1_extractor_functions.py +++ b/tutorials/stats/1_extractor_functions.py @@ -22,12 +22,16 @@ the Opentelemetry collector server which in its turn batches and persists data to Clickhouse or other OLAP storages. -:warning: Both the Opentelemetry collector and the Clickhouse instance must be running -during statistics collection. If you cloned the DFF repo, launch them using `docker-compose`: +
+ +Both the Opentelemetry collector and the Clickhouse instance must be running +during statistics collection. If you cloned the DFF repo, launch them using `docker compose`: ```bash -docker-compose up otelcol clickhouse +docker compose --profile stats up ``` +
+ For more information on OpenTelemetry instrumentation, refer to the body of this tutorial as well as [OpenTelemetry documentation]( https://opentelemetry.io/docs/instrumentation/python/manual/ diff --git a/tutorials/stats/2_pipeline_integration.py b/tutorials/stats/2_pipeline_integration.py index bbd9766f8..cd042d11e 100644 --- a/tutorials/stats/2_pipeline_integration.py +++ b/tutorials/stats/2_pipeline_integration.py @@ -10,11 +10,15 @@ specific for that component. Some examples of this functionality are showcased in this tutorial. -:warning: Both the Opentelemetry collector and the Clickhouse instance must be running -during statistics collection. If you cloned the DFF repo, launch them using `docker-compose`: +
+ +Both the Opentelemetry collector and the Clickhouse instance must be running +during statistics collection. If you cloned the DFF repo, launch them using `docker compose`: ```bash -docker-compose up otelcol clickhouse +docker compose --profile stats up ``` + +
""" # %pip install dff[stats]