From f764627b0f9aee96c0404d4fac52671a1cd50aca Mon Sep 17 00:00:00 2001 From: optk2k Date: Mon, 23 Oct 2023 14:59:11 +0300 Subject: [PATCH 01/10] add rule to .gitignore for asdf --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c021e7d41d..0ca16b47aa 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,9 @@ target/ # pyenv .python-version +# asdf +.tool-versions + # celery beat schedule file celerybeat-schedule From 055601f7e6e3ae02abb2d23db5b52b21cd0aa141 Mon Sep 17 00:00:00 2001 From: optk2k Date: Thu, 26 Oct 2023 15:05:19 +0300 Subject: [PATCH 02/10] processing KeyboardInterrupt, EOFError and whitespace --- deeppavlov/core/commands/infer.py | 104 +++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 31 deletions(-) diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index 5a6ed65023..e6cf19405e 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -16,7 +16,7 @@ from itertools import islice from logging import getLogger from pathlib import Path -from typing import Optional, Union +from typing import Any, Callable, Optional, Union from deeppavlov.core.commands.utils import import_packages, parse_config from deeppavlov.core.common.chainer import Chainer @@ -28,8 +28,13 @@ log = getLogger(__name__) -def build_model(config: Union[str, Path, dict], mode: str = 'infer', - load_trained: bool = False, install: bool = False, download: bool = False) -> Chainer: +def build_model( + config: Union[str, Path, dict], + mode: str = "infer", + load_trained: bool = False, + install: bool = False, + download: bool = False, +) -> Chainer: """Build and return the model described in corresponding configuration file.""" config = parse_config(config) @@ -38,66 +43,103 @@ def build_model(config: Union[str, Path, dict], mode: str = 'infer', if download: deep_download(config) - import_packages(config.get('metadata', {}).get('imports', [])) + import_packages(config.get("metadata", {}).get("imports", [])) - model_config = config['chainer'] + model_config = config["chainer"] - model = Chainer(model_config['in'], model_config['out'], model_config.get('in_y')) + model = Chainer(model_config["in"], model_config["out"], model_config.get("in_y")) - for component_config in model_config['pipe']: - if load_trained and ('fit_on' in component_config or 'in_y' in component_config): + for component_config in model_config["pipe"]: + if load_trained and ( + "fit_on" in component_config or "in_y" in component_config + ): try: - component_config['load_path'] = component_config['save_path'] + component_config["load_path"] = component_config["save_path"] except KeyError: - log.warning('No "save_path" parameter for the {} component, so "load_path" will not be renewed' - .format(component_config.get('class_name', component_config.get('ref', 'UNKNOWN')))) + log.warning( + 'No "save_path" parameter for the {} component, so "load_path" will not be renewed'.format( + component_config.get( + "class_name", component_config.get("ref", "UNKNOWN") + ) + ) + ) component = from_params(component_config, mode=mode) - if 'id' in component_config: - model._components_dict[component_config['id']] = component + if "id" in component_config: + model._components_dict[component_config["id"]] = component - if 'in' in component_config: - c_in = component_config['in'] - c_out = component_config['out'] - in_y = component_config.get('in_y', None) - main = component_config.get('main', False) + if "in" in component_config: + c_in = component_config["in"] + c_out = component_config["out"] + in_y = component_config.get("in_y", None) + main = component_config.get("main", False) model.append(component, c_in, c_out, in_y, main) return model +def end_repl_mode(function: Callable[..., Any]) -> Callable[..., Any]: + """Decorator for processing ctrl-c, ctrl-d pressing.""" + + def wrapper(*args: Any, **kwargs: Any): + try: + return function(*args, **kwargs) + except (KeyboardInterrupt, EOFError): + print("\nExit repl mode.") + sys.exit(0) + + return wrapper + + +@end_repl_mode def interact_model(config: Union[str, Path, dict]) -> None: """Start interaction with the model described in corresponding configuration file.""" model = build_model(config) + print("\nExit repl - type q and press enter, or press ctrl-c, or ctrl-d.") + + def input_data(prompt: str) -> tuple[str]: + """Filter and processing input data.""" + while True: + data: str = input(f"\033[34m\033[107m{prompt}:\033[0m ") + if data == "": + continue + if data.isspace(): + continue + if data.strip() == "q": + print("\nExit repl mode.") + sys.exit(0) + + return (data,) + while True: - args = [] + arguments: list[tuple[str]] = [] for in_x in model.in_x: - args.append((input('{}::'.format(in_x)),)) - # check for exit command - if args[-1][0] in {'exit', 'stop', 'quit', 'q'}: - return + data = input_data(in_x) + arguments.append(data) - pred = model(*args) + pred = model(*arguments) if len(model.out_params) > 1: pred = zip(*pred) - print('>>', *pred) + print("==> ", *pred) -def predict_on_stream(config: Union[str, Path, dict], - batch_size: Optional[int] = None, - file_path: Optional[str] = None) -> None: +def predict_on_stream( + config: Union[str, Path, dict], + batch_size: Optional[int] = None, + file_path: Optional[str] = None, +) -> None: """Make a prediction with the component described in corresponding configuration file.""" batch_size = batch_size or 1 - if file_path is None or file_path == '-': + if file_path is None or file_path == "-": if sys.stdin.isatty(): - raise RuntimeError('To process data from terminal please use interact mode') + raise RuntimeError("To process data from terminal please use interact mode") f = sys.stdin else: - f = open(file_path, encoding='utf8') + f = open(file_path, encoding="utf8") model: Chainer = build_model(config) From f3f1872f114719d7cd99ead3cfda675f8b03f293 Mon Sep 17 00:00:00 2001 From: optk2k Date: Fri, 3 Nov 2023 15:32:30 +0300 Subject: [PATCH 03/10] add rules for black --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..6b90104c10 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 120 +target-version = ["py36", "py37", "py38", "py39", "py310"] From 995cfcc35d18d2520a49f5a8d844b4c923c4cdf5 Mon Sep 17 00:00:00 2001 From: optk2k Date: Tue, 7 Nov 2023 11:18:14 +0300 Subject: [PATCH 04/10] uppercase Ctrl-C, Ctrl-D --- deeppavlov/core/commands/infer.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index e6cf19405e..8a3c9575b2 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -50,17 +50,13 @@ def build_model( model = Chainer(model_config["in"], model_config["out"], model_config.get("in_y")) for component_config in model_config["pipe"]: - if load_trained and ( - "fit_on" in component_config or "in_y" in component_config - ): + if load_trained and ("fit_on" in component_config or "in_y" in component_config): try: component_config["load_path"] = component_config["save_path"] except KeyError: log.warning( 'No "save_path" parameter for the {} component, so "load_path" will not be renewed'.format( - component_config.get( - "class_name", component_config.get("ref", "UNKNOWN") - ) + component_config.get("class_name", component_config.get("ref", "UNKNOWN")) ) ) @@ -80,7 +76,7 @@ def build_model( def end_repl_mode(function: Callable[..., Any]) -> Callable[..., Any]: - """Decorator for processing ctrl-c, ctrl-d pressing.""" + """Decorator for processing Ctrl-C, Ctrl-D pressing.""" def wrapper(*args: Any, **kwargs: Any): try: @@ -97,7 +93,7 @@ def interact_model(config: Union[str, Path, dict]) -> None: """Start interaction with the model described in corresponding configuration file.""" model = build_model(config) - print("\nExit repl - type q and press enter, or press ctrl-c, or ctrl-d.") + print("\nExit repl - type q and press Enter, or press Ctrl-C, or Ctrl-D.") def input_data(prompt: str) -> tuple[str]: """Filter and processing input data.""" From 89decfd7cc72fe0270a64234e6391a0832ffb15c Mon Sep 17 00:00:00 2001 From: optk2k Date: Tue, 7 Nov 2023 11:22:19 +0300 Subject: [PATCH 05/10] return output >> --- deeppavlov/core/commands/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index 8a3c9575b2..718c0a5518 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -119,7 +119,7 @@ def input_data(prompt: str) -> tuple[str]: if len(model.out_params) > 1: pred = zip(*pred) - print("==> ", *pred) + print(">> ", *pred) def predict_on_stream( From 4edf52f45d348c7125ed3d8f815d1c53ce643f4b Mon Sep 17 00:00:00 2001 From: optk2k Date: Tue, 7 Nov 2023 11:27:56 +0300 Subject: [PATCH 06/10] delete whitespace in repl --- deeppavlov/core/commands/infer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index 718c0a5518..f92c79c6e2 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -99,10 +99,6 @@ def input_data(prompt: str) -> tuple[str]: """Filter and processing input data.""" while True: data: str = input(f"\033[34m\033[107m{prompt}:\033[0m ") - if data == "": - continue - if data.isspace(): - continue if data.strip() == "q": print("\nExit repl mode.") sys.exit(0) From cba4e3f87fd2c209a19862832db29d87f020b9e1 Mon Sep 17 00:00:00 2001 From: optk2k Date: Tue, 7 Nov 2023 11:30:28 +0300 Subject: [PATCH 07/10] delete repl in comment --- deeppavlov/core/commands/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index f92c79c6e2..dea8619156 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -93,7 +93,7 @@ def interact_model(config: Union[str, Path, dict]) -> None: """Start interaction with the model described in corresponding configuration file.""" model = build_model(config) - print("\nExit repl - type q and press Enter, or press Ctrl-C, or Ctrl-D.") + print("\nExit - type q and press Enter, or press Ctrl-C, or Ctrl-D.") def input_data(prompt: str) -> tuple[str]: """Filter and processing input data.""" From 4d49cff0759398ba7af8fe75169bcbc0dafcf5e5 Mon Sep 17 00:00:00 2001 From: optk2k Date: Tue, 7 Nov 2023 11:33:51 +0300 Subject: [PATCH 08/10] remove type hint --- deeppavlov/core/commands/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index dea8619156..ae25457543 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -95,7 +95,7 @@ def interact_model(config: Union[str, Path, dict]) -> None: print("\nExit - type q and press Enter, or press Ctrl-C, or Ctrl-D.") - def input_data(prompt: str) -> tuple[str]: + def input_data(prompt: str): """Filter and processing input data.""" while True: data: str = input(f"\033[34m\033[107m{prompt}:\033[0m ") From cc72cecec4b749e13a4274fa276b00458c38c94f Mon Sep 17 00:00:00 2001 From: optk2k Date: Tue, 7 Nov 2023 14:17:13 +0300 Subject: [PATCH 09/10] add tests KeyboardInterrupt and EOFError --- tests/test_core_commands/test_infer.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/test_core_commands/test_infer.py diff --git a/tests/test_core_commands/test_infer.py b/tests/test_core_commands/test_infer.py new file mode 100644 index 0000000000..258f119570 --- /dev/null +++ b/tests/test_core_commands/test_infer.py @@ -0,0 +1,26 @@ +import pytest +from deeppavlov.core.commands.infer import end_repl_mode + + +def test_end_repl_mode_decorator_keyboard_interrupt(): + """Check Ctrl-C.""" + + def error_keyboard_interrupt(): + raise KeyboardInterrupt + + with pytest.raises(SystemExit) as ex: + function = end_repl_mode(error_keyboard_interrupt) + function() + assert ex.value.code == 0 + + +def test_end_repl_mode_decorator_eoferror(): + """Check Ctrl-D.""" + + def error_eoferror(): + raise EOFError + + with pytest.raises(SystemExit) as ex: + function = end_repl_mode(error_eoferror) + function() + assert ex.value.code == 0 From d7f91cae56d81c18d06d80c99368c8f0bb394778 Mon Sep 17 00:00:00 2001 From: optk2k Date: Wed, 8 Nov 2023 12:35:37 +0300 Subject: [PATCH 10/10] refactoring and tests --- deeppavlov/core/commands/infer.py | 29 +++++++++++++------------- tests/test_core_commands/test_infer.py | 24 ++++++++++++++++++++- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/deeppavlov/core/commands/infer.py b/deeppavlov/core/commands/infer.py index ae25457543..46f4094f82 100644 --- a/deeppavlov/core/commands/infer.py +++ b/deeppavlov/core/commands/infer.py @@ -82,12 +82,24 @@ def wrapper(*args: Any, **kwargs: Any): try: return function(*args, **kwargs) except (KeyboardInterrupt, EOFError): - print("\nExit repl mode.") + print("\nExit.") sys.exit(0) return wrapper +def preparing_arguments(model): + """Prepares arguments.""" + arguments = [] + for in_x in model: + data: str = input(f"\033[34m\033[107m{in_x}:\033[0m ") + if data.strip() == "q": + print("\nExit.") + sys.exit(0) + arguments.append((data,)) + return arguments + + @end_repl_mode def interact_model(config: Union[str, Path, dict]) -> None: """Start interaction with the model described in corresponding configuration file.""" @@ -95,21 +107,8 @@ def interact_model(config: Union[str, Path, dict]) -> None: print("\nExit - type q and press Enter, or press Ctrl-C, or Ctrl-D.") - def input_data(prompt: str): - """Filter and processing input data.""" - while True: - data: str = input(f"\033[34m\033[107m{prompt}:\033[0m ") - if data.strip() == "q": - print("\nExit repl mode.") - sys.exit(0) - - return (data,) - while True: - arguments: list[tuple[str]] = [] - for in_x in model.in_x: - data = input_data(in_x) - arguments.append(data) + arguments = preparing_arguments(model.in_x) pred = model(*arguments) if len(model.out_params) > 1: diff --git a/tests/test_core_commands/test_infer.py b/tests/test_core_commands/test_infer.py index 258f119570..fe1d9cd507 100644 --- a/tests/test_core_commands/test_infer.py +++ b/tests/test_core_commands/test_infer.py @@ -1,5 +1,5 @@ import pytest -from deeppavlov.core.commands.infer import end_repl_mode +from deeppavlov.core.commands.infer import end_repl_mode, interact_model, preparing_arguments def test_end_repl_mode_decorator_keyboard_interrupt(): @@ -24,3 +24,25 @@ def error_eoferror(): function = end_repl_mode(error_eoferror) function() assert ex.value.code == 0 + + +def test_preparing_arguments(monkeypatch): + """Check format arguments.""" + + def input_data(data: str): + return "data" + + monkeypatch.setattr("builtins.input", input_data) + assert preparing_arguments([1, 2]) == [("data",), ("data",)] + + +def test_preparing_arguments_exit(monkeypatch): + """Check exit by `q`""" + + def input_data(data: str): + return "q" + + monkeypatch.setattr("builtins.input", input_data) + with pytest.raises(SystemExit) as ex: + preparing_arguments([1, 2]) + assert ex.value.code == 0