From ff459bd1bc29e3c9d36510a3c599a8a458b80289 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 14:51:22 +0200 Subject: [PATCH 01/47] Add task builder --- pyzeebe/task/task_config.py | 23 +++++++ pyzeebe/worker/task_builder.py | 44 +++++++++++++ tests/conftest.py | 22 +++++++ tests/unit/worker/task_builder_test.py | 90 ++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 pyzeebe/task/task_config.py create mode 100644 pyzeebe/worker/task_builder.py create mode 100644 tests/unit/worker/task_builder_test.py diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py new file mode 100644 index 00000000..54512026 --- /dev/null +++ b/pyzeebe/task/task_config.py @@ -0,0 +1,23 @@ +import logging +from dataclasses import dataclass, field +from typing import List, Optional + +from pyzeebe import ExceptionHandler, TaskDecorator, Job + +logger = logging.getLogger(__name__) + + +def default_exception_handler(e: Exception, job: Job) -> None: + logger.warning(f"Task type: {job.type} - failed job {job}. Error: {e}.") + job.set_failure_status(f"Failed job. Error: {e}") + + +@dataclass +class TaskConfig: + type: str + exception_handler: ExceptionHandler = default_exception_handler + timeout: int = 10000 + max_jobs_to_activate: int = 32 + variables_to_fetch: Optional[List[str]] = None + before: List[TaskDecorator] = field(default_factory=list) + after: List[TaskDecorator] = field(default_factory=list) diff --git a/pyzeebe/worker/task_builder.py b/pyzeebe/worker/task_builder.py new file mode 100644 index 00000000..004ba1c8 --- /dev/null +++ b/pyzeebe/worker/task_builder.py @@ -0,0 +1,44 @@ +import logging +from typing import List, Callable + +from pyzeebe import Job, TaskDecorator +from pyzeebe.task.task_config import TaskConfig + +logger = logging.getLogger(__name__) +DecoratorRunner = Callable[[Job], Job] +JobHandler = Callable[[Job], Job] + + +def build_task(task_function, task_config: TaskConfig) -> JobHandler: + before_decorator_runner = create_decorator_runner(task_config.before) + after_decorator_runner = create_decorator_runner(task_config.after) + + def job_handler(job: Job) -> Job: + try: + job = before_decorator_runner(job) + job.variables = task_function(**job.variables) + job = after_decorator_runner(job) + job.set_success_status() + except Exception as e: + task_config.exception_handler(e, job) + finally: + return job + + return job_handler + + +def create_decorator_runner(decorators: List[TaskDecorator]) -> DecoratorRunner: + def decorator_runner(job: Job): + for decorator in decorators: + job = run_decorator(decorator, job) + return job + + return decorator_runner + + +def run_decorator(decorator: TaskDecorator, job: Job) -> Job: + try: + return decorator(job) + except Exception as e: + logger.warning(f"Failed to run decorator {decorator}. Exception: {e}") + return job diff --git a/tests/conftest.py b/tests/conftest.py index b1f43ef6..d1c3b12a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ from pyzeebe import ZeebeClient, ZeebeWorker, ZeebeTaskRouter, Job from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.task.task import Task +from pyzeebe.task.task_config import TaskConfig from pyzeebe.worker.task_handler import ZeebeTaskHandler from tests.unit.utils.gateway_mock import GatewayMock from tests.unit.utils.random_utils import random_job @@ -18,6 +19,14 @@ def job_with_adapter(zeebe_adapter): return random_job(zeebe_adapter=zeebe_adapter) +@pytest.fixture +def mocked_job_with_adapter(job_with_adapter): + job_with_adapter.set_success_status = MagicMock() + job_with_adapter.set_failure_status = MagicMock() + job_with_adapter.set_error_status = MagicMock() + return job_with_adapter + + @pytest.fixture def job_without_adapter(): return random_job() @@ -52,11 +61,24 @@ def task(task_type): return Task(task_type, MagicMock(wraps=lambda x: dict(x=x)), MagicMock(wraps=lambda x, y, z: x)) +@pytest.fixture +def task_config(task_type): + return TaskConfig(task_type, MagicMock()) + + @pytest.fixture def task_type(): return str(uuid4()) +@pytest.fixture +def original_task_function(): + def original_function(): + pass + + return MagicMock(wraps=original_function) + + @pytest.fixture def stop_after_test(): stop_test = Event() diff --git a/tests/unit/worker/task_builder_test.py b/tests/unit/worker/task_builder_test.py new file mode 100644 index 00000000..7af3f233 --- /dev/null +++ b/tests/unit/worker/task_builder_test.py @@ -0,0 +1,90 @@ +from collections import Callable + +from pyzeebe import Job, TaskDecorator +from pyzeebe.task.task_config import TaskConfig +from pyzeebe.worker.task_builder import build_task + + +def test_returned_task_is_callable(original_task_function: Callable, task_config: TaskConfig): + task = build_task(original_task_function, task_config) + assert callable(task) + + +def test_exception_handler_called(original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + exception = Exception() + original_task_function.side_effect = exception + + build_task(original_task_function, task_config)(mocked_job_with_adapter) + + task_config.exception_handler.assert_called_with(exception, mocked_job_with_adapter) + + +def test_parameters_are_provided_to_task(original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + mocked_job_with_adapter.variables = {"x": 1} + + build_task(original_task_function, task_config)(mocked_job_with_adapter) + + original_task_function.assert_called_with(x=1) + + +def test_variables_are_added_to_result(original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + original_task_function.return_value = {"x": 1} + + job = build_task(original_task_function, task_config)(mocked_job_with_adapter) + + assert job.variables.pop("x") == 1 + + +def test_complete_job_called(original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): + build_task(original_task_function, task_config)(mocked_job_with_adapter) + mocked_job_with_adapter.set_success_status.assert_called_once() + + +def test_returned_task_runs_original_function(original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + build_task(original_task_function, task_config)(mocked_job_with_adapter) + + original_task_function.assert_called_once() + + +def test_before_decorator_called(original_task_function: Callable, decorator: TaskDecorator, task_config: TaskConfig, + mocked_job_with_adapter: Job): + task_config.before.append(decorator) + + build_task(original_task_function, task_config)(mocked_job_with_adapter) + + task_config.before.pop().assert_called_once() + + +def test_after_decorator_called(original_task_function: Callable, decorator: TaskDecorator, task_config: TaskConfig, + mocked_job_with_adapter: Job): + task_config.after.append(decorator) + + build_task(original_task_function, task_config)(mocked_job_with_adapter) + + task_config.after.pop().assert_called_once() + + +def test_failing_decorator_continues(original_task_function: Callable, decorator: TaskDecorator, + task_config: TaskConfig, mocked_job_with_adapter: Job): + decorator.side_effect = Exception() + task_config.before.append(decorator) + + # Assert no exception is raised + build_task(original_task_function, task_config)(mocked_job_with_adapter) + decorator.assert_called_once() + task_config.exception_handler.assert_not_called() + + +def test_decorator_variables_are_added(original_task_function: Callable, decorator: TaskDecorator, + task_config: TaskConfig, mocked_job_with_adapter: Job): + decorator_return_value = mocked_job_with_adapter + decorator_return_value.variables = {"x": 2} + decorator.return_value = decorator_return_value + + job = build_task(original_task_function, task_config)(mocked_job_with_adapter) + + assert "x" in job.variables From bd11a09c667f9665e51b0488bf1d457fd3e8e63e Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 17:18:49 +0200 Subject: [PATCH 02/47] Add single_value function to dict conversion to task_builder --- pyzeebe/task/task_config.py | 7 ++ pyzeebe/worker/task_builder.py | 31 +++++- tests/unit/worker/task_builder_test.py | 144 ++++++++++++++++--------- 3 files changed, 128 insertions(+), 54 deletions(-) diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index 54512026..b49650cb 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -3,6 +3,7 @@ from typing import List, Optional from pyzeebe import ExceptionHandler, TaskDecorator, Job +from pyzeebe.exceptions import NoVariableNameGiven logger = logging.getLogger(__name__) @@ -19,5 +20,11 @@ class TaskConfig: timeout: int = 10000 max_jobs_to_activate: int = 32 variables_to_fetch: Optional[List[str]] = None + single_value: bool = False + variable_name: Optional[str] = None before: List[TaskDecorator] = field(default_factory=list) after: List[TaskDecorator] = field(default_factory=list) + + def __post_init__(self): + if self.single_value and not self.variable_name: + raise NoVariableNameGiven(self.type) diff --git a/pyzeebe/worker/task_builder.py b/pyzeebe/worker/task_builder.py index 004ba1c8..26f705f7 100644 --- a/pyzeebe/worker/task_builder.py +++ b/pyzeebe/worker/task_builder.py @@ -1,5 +1,5 @@ import logging -from typing import List, Callable +from typing import List, Callable, Dict from pyzeebe import Job, TaskDecorator from pyzeebe.task.task_config import TaskConfig @@ -9,7 +9,17 @@ JobHandler = Callable[[Job], Job] -def build_task(task_function, task_config: TaskConfig) -> JobHandler: +def build_task(task_function, task_config: TaskConfig): + if not task_config.variables_to_fetch: + task_config.variables_to_fetch = get_parameters_from_function(task_function) + + if task_config.single_value: + task_function = convert_to_dict_function(task_function, task_config.variable_name) + + return build_job_handler(task_function, task_config), task_config + + +def build_job_handler(task_function, task_config: TaskConfig) -> JobHandler: before_decorator_runner = create_decorator_runner(task_config.before) after_decorator_runner = create_decorator_runner(task_config.after) @@ -42,3 +52,20 @@ def run_decorator(decorator: TaskDecorator, job: Job) -> Job: except Exception as e: logger.warning(f"Failed to run decorator {decorator}. Exception: {e}") return job + + +def convert_to_dict_function(single_value_function: Callable, variable_name: str) -> Callable[..., Dict]: + def inner_fn(*args, **kwargs): + return {variable_name: single_value_function(*args, **kwargs)} + + return inner_fn + + +def get_parameters_from_function(fn: Callable) -> List[str]: + parameters = fn.__code__.co_varnames + if "args" in parameters: + return [] + elif "kwargs" in parameters: + return [] + else: + return list(parameters) diff --git a/tests/unit/worker/task_builder_test.py b/tests/unit/worker/task_builder_test.py index 7af3f233..ab1e20f9 100644 --- a/tests/unit/worker/task_builder_test.py +++ b/tests/unit/worker/task_builder_test.py @@ -2,89 +2,129 @@ from pyzeebe import Job, TaskDecorator from pyzeebe.task.task_config import TaskConfig -from pyzeebe.worker.task_builder import build_task +from pyzeebe.worker import task_builder -def test_returned_task_is_callable(original_task_function: Callable, task_config: TaskConfig): - task = build_task(original_task_function, task_config) - assert callable(task) +class TestBuildTask: + def test_single_value_func(self, task_config: TaskConfig, mocked_job_with_adapter: Job): + task_config.single_value = True + task_config.variable_name = "y" + mocked_job_with_adapter.variables = {"x": 1} + job = task_builder.build_task(lambda x: x, task_config)[0](mocked_job_with_adapter) -def test_exception_handler_called(original_task_function: Callable, task_config: TaskConfig, - mocked_job_with_adapter: Job): - exception = Exception() - original_task_function.side_effect = exception + assert job.variables.pop("y") == 1 - build_task(original_task_function, task_config)(mocked_job_with_adapter) + def test_variables_to_fetch_added_to_task_config(self, task_config: TaskConfig): + expected_variables_to_fetch = ["x", "y"] - task_config.exception_handler.assert_called_with(exception, mocked_job_with_adapter) + def dummy_fn(x, y): + pass + _, updated_task_config = task_builder.build_task(dummy_fn, task_config) -def test_parameters_are_provided_to_task(original_task_function: Callable, task_config: TaskConfig, - mocked_job_with_adapter: Job): - mocked_job_with_adapter.variables = {"x": 1} + assert updated_task_config.variables_to_fetch == expected_variables_to_fetch - build_task(original_task_function, task_config)(mocked_job_with_adapter) - original_task_function.assert_called_with(x=1) +class TestBuildJobHandler: + def test_returned_task_is_callable(self, original_task_function: Callable, task_config: TaskConfig): + task = task_builder.build_job_handler(original_task_function, task_config) + assert callable(task) + def test_exception_handler_called(self, original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + exception = Exception() + original_task_function.side_effect = exception -def test_variables_are_added_to_result(original_task_function: Callable, task_config: TaskConfig, - mocked_job_with_adapter: Job): - original_task_function.return_value = {"x": 1} + task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) - job = build_task(original_task_function, task_config)(mocked_job_with_adapter) + task_config.exception_handler.assert_called_with(exception, mocked_job_with_adapter) - assert job.variables.pop("x") == 1 + def test_parameters_are_provided_to_task(self, original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + mocked_job_with_adapter.variables = {"x": 1} + task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) -def test_complete_job_called(original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): - build_task(original_task_function, task_config)(mocked_job_with_adapter) - mocked_job_with_adapter.set_success_status.assert_called_once() + original_task_function.assert_called_with(x=1) + def test_variables_are_added_to_result(self, original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + original_task_function.return_value = {"x": 1} -def test_returned_task_runs_original_function(original_task_function: Callable, task_config: TaskConfig, - mocked_job_with_adapter: Job): - build_task(original_task_function, task_config)(mocked_job_with_adapter) + job = task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) - original_task_function.assert_called_once() + assert job.variables.pop("x") == 1 - -def test_before_decorator_called(original_task_function: Callable, decorator: TaskDecorator, task_config: TaskConfig, + def test_complete_job_called(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): - task_config.before.append(decorator) + task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + mocked_job_with_adapter.set_success_status.assert_called_once() + + def test_returned_task_runs_original_function(self, original_task_function: Callable, task_config: TaskConfig, + mocked_job_with_adapter: Job): + task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + + original_task_function.assert_called_once() + + def test_before_decorator_called(self, original_task_function: Callable, decorator: TaskDecorator, + task_config: TaskConfig, + mocked_job_with_adapter: Job): + task_config.before.append(decorator) + + task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + + task_config.before.pop().assert_called_once() + + def test_after_decorator_called(self, original_task_function: Callable, decorator: TaskDecorator, + task_config: TaskConfig, + mocked_job_with_adapter: Job): + task_config.after.append(decorator) + + task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + + task_config.after.pop().assert_called_once() + + def test_failing_decorator_continues(self, original_task_function: Callable, decorator: TaskDecorator, + task_config: TaskConfig, mocked_job_with_adapter: Job): + decorator.side_effect = Exception() + task_config.before.append(decorator) - build_task(original_task_function, task_config)(mocked_job_with_adapter) + # Assert no exception is raised + task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + decorator.assert_called_once() + task_config.exception_handler.assert_not_called() - task_config.before.pop().assert_called_once() + def test_decorator_variables_are_added(self, original_task_function: Callable, decorator: TaskDecorator, + task_config: TaskConfig, mocked_job_with_adapter: Job): + decorator_return_value = mocked_job_with_adapter + decorator_return_value.variables = {"x": 2} + decorator.return_value = decorator_return_value + job = task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) -def test_after_decorator_called(original_task_function: Callable, decorator: TaskDecorator, task_config: TaskConfig, - mocked_job_with_adapter: Job): - task_config.after.append(decorator) + assert "x" in job.variables - build_task(original_task_function, task_config)(mocked_job_with_adapter) - task_config.after.pop().assert_called_once() +class TestConvertToDictFunction: + def test_converting_to_dict(self): + dict_function = task_builder.convert_to_dict_function(lambda x: x, "x") + assert {"x": 1} == dict_function(1) -def test_failing_decorator_continues(original_task_function: Callable, decorator: TaskDecorator, - task_config: TaskConfig, mocked_job_with_adapter: Job): - decorator.side_effect = Exception() - task_config.before.append(decorator) - # Assert no exception is raised - build_task(original_task_function, task_config)(mocked_job_with_adapter) - decorator.assert_called_once() - task_config.exception_handler.assert_not_called() +class TestGetFunctionParameters: + def test_get_single_param(self): + def dummy_function(x): + pass + assert task_builder.get_parameters_from_function(dummy_function) == ["x"] -def test_decorator_variables_are_added(original_task_function: Callable, decorator: TaskDecorator, - task_config: TaskConfig, mocked_job_with_adapter: Job): - decorator_return_value = mocked_job_with_adapter - decorator_return_value.variables = {"x": 2} - decorator.return_value = decorator_return_value + def test_get_multiple_params(self): + def dummy_function(x, y, z): + pass - job = build_task(original_task_function, task_config)(mocked_job_with_adapter) + assert task_builder.get_parameters_from_function(dummy_function) == ["x", "y", "z"] - assert "x" in job.variables + def test_get_param_from_lambda(self): + assert task_builder.get_parameters_from_function(lambda x: None) == ["x"] From 7095d9eb5189e6c600b1c87e2235ee8f157fe938 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 17:19:59 +0200 Subject: [PATCH 03/47] Move task_builder from pyzeebe/worker to pyzeebe/task --- pyzeebe/{worker => task}/task_builder.py | 0 tests/unit/{worker => task}/task_builder_test.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename pyzeebe/{worker => task}/task_builder.py (100%) rename tests/unit/{worker => task}/task_builder_test.py (99%) diff --git a/pyzeebe/worker/task_builder.py b/pyzeebe/task/task_builder.py similarity index 100% rename from pyzeebe/worker/task_builder.py rename to pyzeebe/task/task_builder.py diff --git a/tests/unit/worker/task_builder_test.py b/tests/unit/task/task_builder_test.py similarity index 99% rename from tests/unit/worker/task_builder_test.py rename to tests/unit/task/task_builder_test.py index ab1e20f9..af2043ed 100644 --- a/tests/unit/worker/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -2,7 +2,7 @@ from pyzeebe import Job, TaskDecorator from pyzeebe.task.task_config import TaskConfig -from pyzeebe.worker import task_builder +from pyzeebe.task import task_builder class TestBuildTask: From 0f659fdf1ca6c05c9f61380a59640ed7ea3a0fe0 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 17:21:57 +0200 Subject: [PATCH 04/47] Add type annotations to build_task --- pyzeebe/task/task_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index 26f705f7..3313f496 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -1,5 +1,5 @@ import logging -from typing import List, Callable, Dict +from typing import List, Callable, Dict, Tuple from pyzeebe import Job, TaskDecorator from pyzeebe.task.task_config import TaskConfig @@ -9,7 +9,7 @@ JobHandler = Callable[[Job], Job] -def build_task(task_function, task_config: TaskConfig): +def build_task(task_function, task_config: TaskConfig) -> Tuple[JobHandler, TaskConfig]: if not task_config.variables_to_fetch: task_config.variables_to_fetch = get_parameters_from_function(task_function) From dc57eff6ff7422251c56f7c21c46d969a9cdef97 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 17:22:12 +0200 Subject: [PATCH 05/47] Add type annotations to build_task --- pyzeebe/task/task_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index 3313f496..f00ae988 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -9,7 +9,7 @@ JobHandler = Callable[[Job], Job] -def build_task(task_function, task_config: TaskConfig) -> Tuple[JobHandler, TaskConfig]: +def build_task(task_function: Callable, task_config: TaskConfig) -> Tuple[JobHandler, TaskConfig]: if not task_config.variables_to_fetch: task_config.variables_to_fetch = get_parameters_from_function(task_function) @@ -19,7 +19,7 @@ def build_task(task_function, task_config: TaskConfig) -> Tuple[JobHandler, Task return build_job_handler(task_function, task_config), task_config -def build_job_handler(task_function, task_config: TaskConfig) -> JobHandler: +def build_job_handler(task_function: Callable, task_config: TaskConfig) -> JobHandler: before_decorator_runner = create_decorator_runner(task_config.before) after_decorator_runner = create_decorator_runner(task_config.after) From 0ab044549c7b22a8aeebf11f37b316e9e9c6b03e Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 17:34:33 +0200 Subject: [PATCH 06/47] Add log when failing to execute job --- pyzeebe/task/task_builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index f00ae988..0a5982d5 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -30,6 +30,7 @@ def job_handler(job: Job) -> Job: job = after_decorator_runner(job) job.set_success_status() except Exception as e: + logger.debug(f"Failed job: {job}. Error: {e}.") task_config.exception_handler(e, job) finally: return job From 89b1667fb76cf5acd277ce8cb304e36857ec7fa8 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 21:49:17 +0200 Subject: [PATCH 07/47] Refactor Task to be a dataclass containing the job_handler and the configuration --- pyzeebe/task/task.py | 30 +++----- pyzeebe/task/task_builder.py | 10 +-- tests/conftest.py | 16 +++- .../grpc_internals/zeebe_job_adapter_test.py | 76 +++++++------------ tests/unit/task/task_builder_test.py | 14 +++- tests/unit/task/task_test.py | 43 ----------- tests/unit/utils/random_utils.py | 6 +- tests/unit/worker/task_handler_test.py | 3 +- tests/unit/worker/worker_test.py | 10 +-- 9 files changed, 75 insertions(+), 133 deletions(-) delete mode 100644 tests/unit/task/task_test.py diff --git a/pyzeebe/task/task.py b/pyzeebe/task/task.py index 9780c210..367ff285 100644 --- a/pyzeebe/task/task.py +++ b/pyzeebe/task/task.py @@ -1,24 +1,14 @@ -from typing import Callable, List, Dict +from dataclasses import dataclass -from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase -from pyzeebe.job.job import Job -from pyzeebe.task.exception_handler import ExceptionHandler +from pyzeebe.task.types import JobHandler +from pyzeebe.task.task_config import TaskConfig -class Task(ZeebeDecoratorBase): - def __init__(self, task_type: str, task_handler: Callable[..., Dict], exception_handler: ExceptionHandler, - timeout: int = 10000, max_jobs_to_activate: int = 32, variables_to_fetch: List[str] = None, - before: List = None, after: List = None): - super().__init__(before=before, after=after) +@dataclass +class Task: + job_handler: JobHandler + config: TaskConfig - self.type = task_type - self.inner_function = task_handler - self.exception_handler = exception_handler - self.timeout = timeout - self.max_jobs_to_activate = max_jobs_to_activate - self.variables_to_fetch = variables_to_fetch or [] - self.handler: Callable[[Job], Job] = None - - def __repr__(self) -> str: - return str({"type": self.type, "timeout": self.timeout, "max_jobs_to_activate": self.max_jobs_to_activate, - "variables_to_fetch": self.variables_to_fetch}) + @property + def type(self): + return self.config.type diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index 0a5982d5..2f085b2f 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -1,22 +1,22 @@ import logging -from typing import List, Callable, Dict, Tuple +from typing import List, Callable, Dict from pyzeebe import Job, TaskDecorator +from pyzeebe.task.task import Task from pyzeebe.task.task_config import TaskConfig +from pyzeebe.task.types import DecoratorRunner, JobHandler logger = logging.getLogger(__name__) -DecoratorRunner = Callable[[Job], Job] -JobHandler = Callable[[Job], Job] -def build_task(task_function: Callable, task_config: TaskConfig) -> Tuple[JobHandler, TaskConfig]: +def build_task(task_function: Callable, task_config: TaskConfig) -> Task: if not task_config.variables_to_fetch: task_config.variables_to_fetch = get_parameters_from_function(task_function) if task_config.single_value: task_function = convert_to_dict_function(task_function, task_config.variable_name) - return build_job_handler(task_function, task_config), task_config + return Task(build_job_handler(task_function, task_config), task_config) def build_job_handler(task_function: Callable, task_config: TaskConfig) -> JobHandler: diff --git a/tests/conftest.py b/tests/conftest.py index d1c3b12a..5bfd1176 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ from pyzeebe import ZeebeClient, ZeebeWorker, ZeebeTaskRouter, Job from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter -from pyzeebe.task.task import Task +from pyzeebe.task import task_builder from pyzeebe.task.task_config import TaskConfig from pyzeebe.worker.task_handler import ZeebeTaskHandler from tests.unit.utils.gateway_mock import GatewayMock @@ -57,8 +57,14 @@ def zeebe_worker(zeebe_adapter): @pytest.fixture -def task(task_type): - return Task(task_type, MagicMock(wraps=lambda x: dict(x=x)), MagicMock(wraps=lambda x, y, z: x)) +def task(original_task_function, task_config): + return task_builder.build_task(original_task_function, task_config) + + +@pytest.fixture +def first_active_job(task, job_from_task, grpc_servicer) -> str: + grpc_servicer.active_jobs[job_from_task.key] = job_from_task + return job_from_task @pytest.fixture @@ -76,7 +82,9 @@ def original_task_function(): def original_function(): pass - return MagicMock(wraps=original_function) + mock = MagicMock(wraps=original_function) + mock.__code__ = original_function.__code__ + return mock @pytest.fixture diff --git a/tests/unit/grpc_internals/zeebe_job_adapter_test.py b/tests/unit/grpc_internals/zeebe_job_adapter_test.py index 1551f26e..fcd770c9 100644 --- a/tests/unit/grpc_internals/zeebe_job_adapter_test.py +++ b/tests/unit/grpc_internals/zeebe_job_adapter_test.py @@ -13,15 +13,9 @@ from tests.unit.utils.random_utils import RANDOM_RANGE, random_job -def create_random_task_and_activate(grpc_servicer, task_type: str = None) -> str: - if task_type: - mock_task_type = task_type - else: - mock_task_type = str(uuid4()) - task = Task(task_type=mock_task_type, task_handler=lambda x: x, exception_handler=lambda x: x) +def activate_task(grpc_servicer, task: Task): job = random_job(task) grpc_servicer.active_jobs[job.key] = job - return mock_task_type def get_first_active_job(task_type, zeebe_adapter) -> Job: @@ -29,14 +23,14 @@ def get_first_active_job(task_type, zeebe_adapter) -> Job: timeout=100, variables_to_fetch=[], worker=str(uuid4()))) -def test_activate_jobs(zeebe_adapter, grpc_servicer): - task_type = create_random_task_and_activate(grpc_servicer) +def test_activate_jobs(zeebe_adapter, grpc_servicer, task): + activate_task(grpc_servicer, task) active_jobs_count = randint(4, 100) counter = 0 for i in range(0, active_jobs_count): - create_random_task_and_activate(grpc_servicer, task_type) + activate_task(grpc_servicer, task) - for job in zeebe_adapter.activate_jobs(task_type=task_type, worker=str(uuid4()), timeout=randint(10, 100), + for job in zeebe_adapter.activate_jobs(task_type=task.type, worker=str(uuid4()), timeout=randint(10, 100), request_timeout=100, max_jobs_to_activate=1, variables_to_fetch=[]): counter += 1 assert isinstance(job, Job) @@ -82,10 +76,8 @@ def test_activate_jobs_common_errors_called(zeebe_adapter): zeebe_adapter._common_zeebe_grpc_errors.assert_called() -def test_complete_job(zeebe_adapter, grpc_servicer): - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - response = zeebe_adapter.complete_job(job_key=job.key, variables={}) +def test_complete_job(zeebe_adapter, first_active_job: Job): + response = zeebe_adapter.complete_job(job_key=first_active_job.key, variables={}) assert isinstance(response, CompleteJobResponse) @@ -94,32 +86,26 @@ def test_complete_job_not_found(zeebe_adapter): zeebe_adapter.complete_job(job_key=randint(0, RANDOM_RANGE), variables={}) -def test_complete_job_already_completed(zeebe_adapter, grpc_servicer): - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - zeebe_adapter.complete_job(job_key=job.key, variables={}) +def test_complete_job_already_completed(zeebe_adapter, first_active_job: Job): + zeebe_adapter.complete_job(job_key=first_active_job.key, variables={}) with pytest.raises(JobAlreadyDeactivated): - zeebe_adapter.complete_job(job_key=job.key, variables={}) + zeebe_adapter.complete_job(job_key=first_active_job.key, variables={}) -def test_complete_job_common_errors_called(zeebe_adapter, grpc_servicer): +def test_complete_job_common_errors_called(zeebe_adapter, first_active_job: Job): zeebe_adapter._common_zeebe_grpc_errors = MagicMock() error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.INTERNAL) zeebe_adapter._gateway_stub.CompleteJob = MagicMock(side_effect=error) - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - zeebe_adapter.complete_job(job_key=job.key, variables={}) + zeebe_adapter.complete_job(job_key=first_active_job.key, variables={}) zeebe_adapter._common_zeebe_grpc_errors.assert_called() -def test_fail_job(zeebe_adapter, grpc_servicer): - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - response = zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) +def test_fail_job(zeebe_adapter, first_active_job: Job): + response = zeebe_adapter.fail_job(job_key=first_active_job.key, message=str(uuid4())) assert isinstance(response, FailJobResponse) @@ -128,32 +114,26 @@ def test_fail_job_not_found(zeebe_adapter): zeebe_adapter.fail_job(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) -def test_fail_job_already_failed(zeebe_adapter, grpc_servicer): - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) +def test_fail_job_already_failed(zeebe_adapter, first_active_job: Job): + zeebe_adapter.fail_job(job_key=first_active_job.key, message=str(uuid4())) with pytest.raises(JobAlreadyDeactivated): - zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) + zeebe_adapter.fail_job(job_key=first_active_job.key, message=str(uuid4())) -def test_fail_job_common_errors_called(zeebe_adapter, grpc_servicer): +def test_fail_job_common_errors_called(zeebe_adapter, first_active_job: Job): zeebe_adapter._common_zeebe_grpc_errors = MagicMock() error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.INTERNAL) zeebe_adapter._gateway_stub.FailJob = MagicMock(side_effect=error) - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - zeebe_adapter.fail_job(job_key=job.key, message=str(uuid4())) + zeebe_adapter.fail_job(job_key=first_active_job.key, message=str(uuid4())) zeebe_adapter._common_zeebe_grpc_errors.assert_called() -def test_throw_error(zeebe_adapter, grpc_servicer): - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - response = zeebe_adapter.throw_error(job_key=job.key, message=str(uuid4())) +def test_throw_error(zeebe_adapter, first_active_job: Job): + response = zeebe_adapter.throw_error(job_key=first_active_job.key, message=str(uuid4())) assert isinstance(response, ThrowErrorResponse) @@ -162,23 +142,21 @@ def test_throw_error_job_not_found(zeebe_adapter): zeebe_adapter.throw_error(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) -def test_throw_error_already_thrown(zeebe_adapter, grpc_servicer): - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) - zeebe_adapter.throw_error(job_key=job.key, message=str(uuid4())) +def test_throw_error_already_thrown(zeebe_adapter, first_active_job: Job): + zeebe_adapter.throw_error(job_key=first_active_job.key, message=str(uuid4())) with pytest.raises(JobAlreadyDeactivated): - zeebe_adapter.throw_error(job_key=job.key, message=str(uuid4())) + zeebe_adapter.throw_error(job_key=first_active_job.key, message=str(uuid4())) -def test_throw_error_common_errors_called(zeebe_adapter, grpc_servicer): +def test_throw_error_common_errors_called(zeebe_adapter, grpc_servicer, task): zeebe_adapter._common_zeebe_grpc_errors = MagicMock() error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.INTERNAL) zeebe_adapter._gateway_stub.ThrowError = MagicMock(side_effect=error) - task_type = create_random_task_and_activate(grpc_servicer) - job = get_first_active_job(task_type, zeebe_adapter) + activate_task(grpc_servicer, task) + job = get_first_active_job(task.type, zeebe_adapter) zeebe_adapter.throw_error(job_key=job.key, message=str(uuid4())) zeebe_adapter._common_zeebe_grpc_errors.assert_called() diff --git a/tests/unit/task/task_builder_test.py b/tests/unit/task/task_builder_test.py index af2043ed..4878bc97 100644 --- a/tests/unit/task/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -1,17 +1,23 @@ from collections import Callable from pyzeebe import Job, TaskDecorator -from pyzeebe.task.task_config import TaskConfig from pyzeebe.task import task_builder +from pyzeebe.task.task import Task +from pyzeebe.task.task_config import TaskConfig class TestBuildTask: + def test_returns_task(self, original_task_function: Callable, task_config: TaskConfig): + task = task_builder.build_task(original_task_function, task_config) + + assert isinstance(task, Task) + def test_single_value_func(self, task_config: TaskConfig, mocked_job_with_adapter: Job): task_config.single_value = True task_config.variable_name = "y" mocked_job_with_adapter.variables = {"x": 1} - job = task_builder.build_task(lambda x: x, task_config)[0](mocked_job_with_adapter) + job = task_builder.build_task(lambda x: x, task_config).job_handler(mocked_job_with_adapter) assert job.variables.pop("y") == 1 @@ -21,9 +27,9 @@ def test_variables_to_fetch_added_to_task_config(self, task_config: TaskConfig): def dummy_fn(x, y): pass - _, updated_task_config = task_builder.build_task(dummy_fn, task_config) + task = task_builder.build_task(dummy_fn, task_config) - assert updated_task_config.variables_to_fetch == expected_variables_to_fetch + assert task.config.variables_to_fetch == expected_variables_to_fetch class TestBuildJobHandler: diff --git a/tests/unit/task/task_test.py b/tests/unit/task/task_test.py deleted file mode 100644 index a3a7a6f1..00000000 --- a/tests/unit/task/task_test.py +++ /dev/null @@ -1,43 +0,0 @@ -import uuid - -from pyzeebe.task.task import Task - - -def test_add_before(): - base_decorator = Task(task_type=str(uuid.uuid4()), task_handler=lambda x: x, exception_handler=lambda x: x) - base_decorator.before(lambda x: x) - assert len(base_decorator._before) == 1 - - -def test_add_after(): - base_decorator = Task(task_type=str(uuid.uuid4()), task_handler=lambda x: x, exception_handler=lambda x: x) - base_decorator.after(lambda x: x) - assert len(base_decorator._after) == 1 - - -def test_add_before_plus_constructor(): - def constructor_decorator(x): - return x + 1 - - def function_decorator(x): - return x - - base_decorator = Task(task_type=str(uuid.uuid4()), task_handler=lambda x: x, exception_handler=lambda x: x, - before=[constructor_decorator]) - base_decorator.before(function_decorator) - assert len(base_decorator._before) == 2 - assert base_decorator._before == [constructor_decorator, function_decorator] - - -def test_add_after_plus_constructor(): - def constructor_decorator(x): - return x + 1 - - def function_decorator(x): - return x - - base_decorator = Task(task_type=str(uuid.uuid4()), task_handler=lambda x: x, exception_handler=lambda x: x, - after=[constructor_decorator]) - base_decorator.after(function_decorator) - assert len(base_decorator._after) == 2 - assert base_decorator._after == [constructor_decorator, function_decorator] diff --git a/tests/unit/utils/random_utils.py b/tests/unit/utils/random_utils.py index b0730717..0a5988fb 100644 --- a/tests/unit/utils/random_utils.py +++ b/tests/unit/utils/random_utils.py @@ -3,13 +3,15 @@ from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.job.job import Job +from pyzeebe.task import task_builder from pyzeebe.task.task import Task +from pyzeebe.task.task_config import TaskConfig RANDOM_RANGE = 1000000000 -def random_job(task: Task = Task(task_type="test", task_handler=lambda x: {"x": x}, - exception_handler=lambda x, y, z: x), zeebe_adapter: ZeebeAdapter = None) -> Job: +def random_job(task: Task = task_builder.build_task(lambda x: {"x": x}, TaskConfig("test")), + zeebe_adapter: ZeebeAdapter = None) -> Job: return Job(_type=task.type, key=randint(0, RANDOM_RANGE), worker=str(uuid4()), retries=randint(0, 10), workflow_instance_key=randint(0, RANDOM_RANGE), bpmn_process_id=str(uuid4()), workflow_definition_version=randint(0, 100), diff --git a/tests/unit/worker/task_handler_test.py b/tests/unit/worker/task_handler_test.py index e6ae28e7..b36813e6 100644 --- a/tests/unit/worker/task_handler_test.py +++ b/tests/unit/worker/task_handler_test.py @@ -47,7 +47,8 @@ def test_remove_task_from_many(task_handler, task): task_handler.tasks.append(task) for i in range(0, randint(0, 100)): - task_handler.tasks.append(Task(str(uuid4()), lambda x: x, lambda x: x)) + task.config.type = str(uuid4()) + task_handler.tasks.append(task) assert task_handler.remove_task(task.type) is not None assert task not in task_handler.tasks diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index 9711418d..66098133 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -1,7 +1,7 @@ +import time from random import randint from unittest.mock import patch, MagicMock from uuid import uuid4 -import time import pytest @@ -224,10 +224,10 @@ def test_activate_jobs_called(self, zeebe_worker, task): zeebe_worker.zeebe_adapter.activate_jobs = MagicMock() zeebe_worker._get_jobs(task) zeebe_worker.zeebe_adapter.activate_jobs.assert_called_with(task_type=task.type, worker=zeebe_worker.name, - timeout=task.timeout, - max_jobs_to_activate=task.max_jobs_to_activate, - variables_to_fetch=task.variables_to_fetch, - request_timeout=zeebe_worker.request_timeout) + timeout=task.config.timeout, + max_jobs_to_activate=task.config.max_jobs_to_activate, + variables_to_fetch=task.config.variables_to_fetch, + request_timeout=zeebe_worker.config.request_timeout) class TestIncludeRouter: From 67b0a0a09d93ac7e3c2336887cf468d9e427a184 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 21:50:09 +0200 Subject: [PATCH 08/47] Add pyzeebe/task/types --- pyzeebe/task/types.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 pyzeebe/task/types.py diff --git a/pyzeebe/task/types.py b/pyzeebe/task/types.py new file mode 100644 index 00000000..25667ea9 --- /dev/null +++ b/pyzeebe/task/types.py @@ -0,0 +1,6 @@ +from typing import Callable + +from pyzeebe import Job + +DecoratorRunner = Callable[[Job], Job] +JobHandler = Callable[[Job], Job] From 2c3f931d2d0983dc2acca6cb45ce2c58a4473281 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 20 Feb 2021 21:51:03 +0200 Subject: [PATCH 09/47] Move TaskDecorator from pyzeebe/task/task_decorator to pyzeebe/task/types --- pyzeebe/__init__.py | 2 +- pyzeebe/decorators/zeebe_decorator_base.py | 2 +- pyzeebe/task/task_decorator.py | 5 ----- pyzeebe/task/types.py | 1 + pyzeebe/worker/task_handler.py | 2 +- pyzeebe/worker/task_router.py | 2 +- pyzeebe/worker/worker.py | 2 +- 7 files changed, 6 insertions(+), 10 deletions(-) delete mode 100644 pyzeebe/task/task_decorator.py diff --git a/pyzeebe/__init__.py b/pyzeebe/__init__.py index dd3a8349..8480063d 100644 --- a/pyzeebe/__init__.py +++ b/pyzeebe/__init__.py @@ -7,6 +7,6 @@ from pyzeebe.job.job import Job from pyzeebe.job.job_status import JobStatus from pyzeebe.task.exception_handler import ExceptionHandler -from pyzeebe.task.task_decorator import TaskDecorator +from pyzeebe.task.types import TaskDecorator from pyzeebe.worker.task_router import ZeebeTaskRouter from pyzeebe.worker.worker import ZeebeWorker diff --git a/pyzeebe/decorators/zeebe_decorator_base.py b/pyzeebe/decorators/zeebe_decorator_base.py index 3cd39ce7..4447702a 100644 --- a/pyzeebe/decorators/zeebe_decorator_base.py +++ b/pyzeebe/decorators/zeebe_decorator_base.py @@ -1,6 +1,6 @@ from typing import List -from pyzeebe.task.task_decorator import TaskDecorator +from pyzeebe import TaskDecorator class ZeebeDecoratorBase(object): diff --git a/pyzeebe/task/task_decorator.py b/pyzeebe/task/task_decorator.py deleted file mode 100644 index cdc18ed6..00000000 --- a/pyzeebe/task/task_decorator.py +++ /dev/null @@ -1,5 +0,0 @@ -from typing import Callable - -from pyzeebe.job.job import Job - -TaskDecorator = Callable[[Job], Job] diff --git a/pyzeebe/task/types.py b/pyzeebe/task/types.py index 25667ea9..a018c192 100644 --- a/pyzeebe/task/types.py +++ b/pyzeebe/task/types.py @@ -4,3 +4,4 @@ DecoratorRunner = Callable[[Job], Job] JobHandler = Callable[[Job], Job] +TaskDecorator = Callable[[Job], Job] diff --git a/pyzeebe/worker/task_handler.py b/pyzeebe/worker/task_handler.py index a0b9be49..cd90a246 100644 --- a/pyzeebe/worker/task_handler.py +++ b/pyzeebe/worker/task_handler.py @@ -7,7 +7,7 @@ from pyzeebe.job.job import Job from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.task import Task -from pyzeebe.task.task_decorator import TaskDecorator +from pyzeebe import TaskDecorator logger = logging.getLogger(__name__) diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index a5db9673..8cd26c23 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -2,7 +2,7 @@ from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.task import Task -from pyzeebe.task.task_decorator import TaskDecorator +from pyzeebe import TaskDecorator from pyzeebe.worker.task_handler import ZeebeTaskHandler, default_exception_handler diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index d3419594..1cb606d5 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -10,7 +10,7 @@ from pyzeebe.job.job import Job from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.task import Task -from pyzeebe.task.task_decorator import TaskDecorator +from pyzeebe import TaskDecorator from pyzeebe.worker.task_handler import ZeebeTaskHandler, default_exception_handler from pyzeebe.worker.task_router import ZeebeTaskRouter From d3f2aba45323d20622e306a552256497a223c18c Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 19:38:45 +0200 Subject: [PATCH 10/47] Refactor task_handler to accept TaskConfig --- pyzeebe/worker/task_handler.py | 90 ++++++-------------- pyzeebe/worker/task_router.py | 36 -------- pyzeebe/worker/worker.py | 37 --------- tests/unit/task/task_builder_test.py | 48 +++++++++++ tests/unit/task/task_config_test.py | 11 +++ tests/unit/worker/task_handler_test.py | 110 ++----------------------- 6 files changed, 91 insertions(+), 241 deletions(-) create mode 100644 tests/unit/task/task_config_test.py diff --git a/pyzeebe/worker/task_handler.py b/pyzeebe/worker/task_handler.py index cd90a246..aa8d8cd7 100644 --- a/pyzeebe/worker/task_handler.py +++ b/pyzeebe/worker/task_handler.py @@ -1,13 +1,13 @@ import logging -from abc import abstractmethod -from typing import Tuple, List, Callable, Dict +from typing import Tuple, List, Callable, Union +from pyzeebe import TaskDecorator from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase -from pyzeebe.exceptions import NoVariableNameGiven, TaskNotFound, DuplicateTaskType +from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType from pyzeebe.job.job import Job -from pyzeebe.task.exception_handler import ExceptionHandler +from pyzeebe.task import task_builder from pyzeebe.task.task import Task -from pyzeebe import TaskDecorator +from pyzeebe.task.task_config import TaskConfig logger = logging.getLogger(__name__) @@ -27,73 +27,37 @@ def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator super().__init__(before, after) self.tasks: List[Task] = [] - def task(self, task_type: str, exception_handler: ExceptionHandler = default_exception_handler, - variables_to_fetch: List[str] = None, timeout: int = 10000, max_jobs_to_activate: int = 32, - before: List[TaskDecorator] = None, after: List[TaskDecorator] = None, single_value: bool = False, - variable_name: str = None): + def task(self, task_config: Union[TaskConfig, str]): """ Decorator to create a task Args: - before (List[TaskDecorator]): All decorators which should be performed before the task. - after (List[TaskDecorator]): All decorators which should be performed after the task. - timeout (int): How long Zeebe should wait before the job is retried. Default: 10000 milliseconds - single_value (bool): If the function returns a single value (int, string, list) and not a dictionary set - this to True. Default: False - variable_name (str): If single_value then this will be the variable name given to zeebe: - { : } - timeout (int): Maximum duration of the task in milliseconds. If the timeout is surpasses Zeebe will give up - on the job and retry it. Default: 10000 - max_jobs_to_activate (int): Maximum jobs the worker will execute in parallel (of this task). Default: 32 + task_config (Union[str, TaskConfig]): Either the task type or a task configuration object Raises: DuplicateTaskType: If a task from the router already exists in the worker """ - self._is_task_duplicate(task_type) - - if single_value and not variable_name: - raise NoVariableNameGiven(task_type=task_type) - - elif single_value and variable_name: - return self._non_dict_task(task_type=task_type, variable_name=variable_name, timeout=timeout, - max_jobs_to_activate=max_jobs_to_activate, exception_handler=exception_handler, - before=before, after=after, variables_to_fetch=variables_to_fetch) - - else: - return self._dict_task(task_type=task_type, exception_handler=exception_handler, before=before, after=after, - timeout=timeout, max_jobs_to_activate=max_jobs_to_activate, - variables_to_fetch=variables_to_fetch) - - @abstractmethod - def _dict_task(self, task_type: str, exception_handler: ExceptionHandler = default_exception_handler, - timeout: int = 10000, max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None, variables_to_fetch: List[str] = None): - raise NotImplemented() - - @abstractmethod - def _non_dict_task(self, task_type: str, variable_name: str, - exception_handler: ExceptionHandler = default_exception_handler, timeout: int = 10000, - max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None, variables_to_fetch: List[str] = None): - raise NotImplemented() - - @staticmethod - def _single_value_function_to_dict(variable_name: str, fn: Callable) -> Callable[..., Dict]: - def inner_fn(*args, **kwargs): - return {variable_name: fn(*args, **kwargs)} - - return inner_fn - - @staticmethod - def _get_parameters_from_function(fn: Callable) -> List[str]: - parameters = fn.__code__.co_varnames - if "args" in parameters: - return [] - elif "kwargs" in parameters: - return [] - else: - return list(parameters) + if isinstance(task_config, str): + task_config = TaskConfig(task_config) + + self._is_task_duplicate(task_config.type) + + task_config = self._add_decorators_to_config(task_config) + + def wrapper(fn: Callable): + task = task_builder.build_task(fn, task_config) + self.tasks.append(task) + return fn + + return wrapper + + def _add_decorators_to_config(self, config: TaskConfig) -> TaskConfig: + before_decorators = self._before.copy() + before_decorators.extend(config.before) + config.before = before_decorators + config.after.extend(self._after) + return config def _is_task_duplicate(self, task_type: str) -> None: try: diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 8cd26c23..434d3093 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -7,42 +7,6 @@ class ZeebeTaskRouter(ZeebeTaskHandler): - def _dict_task(self, task_type: str, exception_handler: ExceptionHandler = default_exception_handler, - timeout: int = 10000, max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None, variables_to_fetch: List[str] = None): - def wrapper(fn: Callable[..., Dict]): - nonlocal variables_to_fetch - if not variables_to_fetch: - variables_to_fetch = self._get_parameters_from_function(fn) - - task = self._create_task(task_type=task_type, task_handler=fn, exception_handler=exception_handler, - timeout=timeout, max_jobs_to_activate=max_jobs_to_activate, before=before, - after=after, variables_to_fetch=variables_to_fetch) - - self.tasks.append(task) - return fn - - return wrapper - - def _non_dict_task(self, task_type: str, variable_name: str, - exception_handler: ExceptionHandler = default_exception_handler, timeout: int = 10000, - max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None, variables_to_fetch: List[str] = None): - def wrapper(fn: Callable[..., Dict]): - nonlocal variables_to_fetch - if not variables_to_fetch: - variables_to_fetch = self._get_parameters_from_function(fn) - - dict_fn = self._single_value_function_to_dict(variable_name=variable_name, fn=fn) - - task = self._create_task(task_type=task_type, task_handler=dict_fn, exception_handler=exception_handler, - timeout=timeout, max_jobs_to_activate=max_jobs_to_activate, before=before, - after=after, variables_to_fetch=variables_to_fetch) - - self.tasks.append(task) - return fn - - return wrapper def _create_task(self, task_type: str, task_handler: Callable, exception_handler: ExceptionHandler, timeout: int = 10000, max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 1cb606d5..71dbe8a6 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -194,43 +194,6 @@ def include_router(self, *routers: ZeebeTaskRouter) -> None: for task in router.tasks: self._add_task(task) - def _dict_task(self, task_type: str, exception_handler: ExceptionHandler = default_exception_handler, - timeout: int = 10000, max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None, variables_to_fetch: List[str] = None): - def wrapper(fn: Callable[..., Dict]): - nonlocal variables_to_fetch - if not variables_to_fetch: - variables_to_fetch = self._get_parameters_from_function(fn) - - task = Task(task_type=task_type, task_handler=fn, exception_handler=exception_handler, timeout=timeout, - max_jobs_to_activate=max_jobs_to_activate, before=before, after=after, - variables_to_fetch=variables_to_fetch) - self._add_task(task) - - return fn - - return wrapper - - def _non_dict_task(self, task_type: str, variable_name: str, - exception_handler: ExceptionHandler = default_exception_handler, timeout: int = 10000, - max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None, variables_to_fetch: List[str] = None): - def wrapper(fn: Callable[..., Union[str, bool, int, List]]): - nonlocal variables_to_fetch - if not variables_to_fetch: - variables_to_fetch = self._get_parameters_from_function(fn) - - dict_fn = self._single_value_function_to_dict(variable_name=variable_name, fn=fn) - - task = Task(task_type=task_type, task_handler=dict_fn, exception_handler=exception_handler, timeout=timeout, - max_jobs_to_activate=max_jobs_to_activate, before=before, after=after, - variables_to_fetch=variables_to_fetch) - self._add_task(task) - - return fn - - return wrapper - def _add_task(self, task: Task) -> None: self._is_task_duplicate(task.type) task.handler = self._create_task_handler(task) diff --git a/tests/unit/task/task_builder_test.py b/tests/unit/task/task_builder_test.py index 4878bc97..086c40ef 100644 --- a/tests/unit/task/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -120,6 +120,12 @@ def test_converting_to_dict(self): class TestGetFunctionParameters: + def test_get_no_param(self): + def no_parameters(): + pass + + assert task_builder.get_parameters_from_function(no_parameters) == [] + def test_get_single_param(self): def dummy_function(x): pass @@ -134,3 +140,45 @@ def dummy_function(x, y, z): def test_get_param_from_lambda(self): assert task_builder.get_parameters_from_function(lambda x: None) == ["x"] + + def test_get_one_positional_param(self): + def one_pos_func(x): + pass + + assert task_builder.get_parameters_from_function(one_pos_func) == ["x"] + + def test_get_multiple_positional_params(self): + def mul_pos_func(x, y, z): + pass + + assert task_builder.get_parameters_from_function(mul_pos_func) == ["x", "y", "z"] + + def test_get_one_keyword_param(self): + def one_key_func(x=0): + pass + + assert task_builder.get_parameters_from_function(one_key_func) == ["x"] + + def test_get_multiple_keyword_params(self): + def mul_key_func(x=0, y=0, z=0): + pass + + assert task_builder.get_parameters_from_function(mul_key_func) == ["x", "y", "z"] + + def test_get_positional_and_keyword_params(self): + def pos_and_key_func(x, y=0): + pass + + assert task_builder.get_parameters_from_function(pos_and_key_func) == ["x", "y"] + + def test_get_params_from_args(self): + def args_func(*args): + pass + + assert task_builder.get_parameters_from_function(args_func) == [] + + def test_get_params_from_kwargs(self): + def kwargs_func(**kwargs): + pass + + assert task_builder.get_parameters_from_function(kwargs_func) == [] diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py new file mode 100644 index 00000000..b3f3bb7d --- /dev/null +++ b/tests/unit/task/task_config_test.py @@ -0,0 +1,11 @@ +from uuid import uuid4 + +import pytest + +from pyzeebe.exceptions import NoVariableNameGiven +from pyzeebe.task.task_config import TaskConfig + + +def test_add_non_dict_task_without_variable_name(): + with pytest.raises(NoVariableNameGiven): + TaskConfig(str(uuid4()), single_value=True) diff --git a/tests/unit/worker/task_handler_test.py b/tests/unit/worker/task_handler_test.py index b36813e6..6a6b0653 100644 --- a/tests/unit/worker/task_handler_test.py +++ b/tests/unit/worker/task_handler_test.py @@ -1,9 +1,9 @@ -from unittest.mock import patch, MagicMock +from unittest.mock import patch from uuid import uuid4 import pytest -from pyzeebe.exceptions import NoVariableNameGiven, TaskNotFound, DuplicateTaskType +from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType from pyzeebe.task.task import Task from pyzeebe.worker.task_handler import default_exception_handler from tests.unit.utils.random_utils import randint @@ -47,8 +47,9 @@ def test_remove_task_from_many(task_handler, task): task_handler.tasks.append(task) for i in range(0, randint(0, 100)): - task.config.type = str(uuid4()) - task_handler.tasks.append(task) + @task_handler.task(str(uuid4())) + def dummy_function(): + pass assert task_handler.remove_task(task.type) is not None assert task not in task_handler.tasks @@ -58,45 +59,6 @@ def test_remove_fake_task(task_handler): task_handler.remove_task(str(uuid4())) -def test_add_dict_task(task_handler): - task_handler._dict_task = MagicMock() - - @task_handler.task(task_type=str(uuid4())) - def dict_task(): - return {} - - task_handler._dict_task.assert_called() - - -def test_add_non_dict_task(task_handler): - task_handler._non_dict_task = MagicMock() - - @task_handler.task(task_type=str(uuid4()), single_value=True, variable_name=str(uuid4())) - def non_dict_task(): - return True - - task_handler._non_dict_task.assert_called() - - -def test_add_non_dict_task_without_variable_name(task_handler): - with pytest.raises(NoVariableNameGiven): - @task_handler.task(task_type=str(uuid4()), single_value=True) - def non_dict_task(): - return True - - -def test_fn_to_dict(task_handler): - variable_name = str(uuid4()) - - def no_dict_fn(x): - return x + 1 - - dict_fn = task_handler._single_value_function_to_dict(fn=no_dict_fn, variable_name=variable_name) - - variable = randint(0, 1000) - assert dict_fn(variable) == {variable_name: variable + 1} - - def test_default_exception_handler(job_without_adapter): with patch("pyzeebe.worker.task_handler.logger.warning") as logging_mock: with patch("pyzeebe.job.job.Job.set_failure_status") as failure_mock: @@ -107,68 +69,6 @@ def test_default_exception_handler(job_without_adapter): logging_mock.assert_called() -def test_get_parameters_from_function_no_parameters(task_handler): - def no_parameters(): - pass - - assert task_handler._get_parameters_from_function(no_parameters) == [] - - -def test_get_parameters_from_function_one_positional(task_handler): - def one_pos_func(x): - pass - - assert task_handler._get_parameters_from_function(one_pos_func) == ["x"] - - -def test_get_parameters_from_function_multiple_positional(task_handler): - def mul_pos_func(x, y, z): - pass - - assert task_handler._get_parameters_from_function(mul_pos_func) == ["x", "y", "z"] - - -def test_get_parameters_from_function_one_keyword(task_handler): - def one_key_func(x=0): - pass - - assert task_handler._get_parameters_from_function(one_key_func) == ["x"] - - -def test_get_parameters_from_function_multiple_keywords(task_handler): - def mul_key_func(x=0, y=0, z=0): - pass - - assert task_handler._get_parameters_from_function(mul_key_func) == ["x", "y", "z"] - - -def test_get_parameters_from_function_positional_and_keyword(task_handler): - def pos_and_key_func(x, y=0): - pass - - assert task_handler._get_parameters_from_function(pos_and_key_func) == ["x", "y"] - - -def test_get_parameters_from_function_args(task_handler): - def args_func(*args): - pass - - assert task_handler._get_parameters_from_function(args_func) == [] - - -def test_get_parameters_from_function_kwargs(task_handler): - def kwargs_func(**kwargs): - pass - - assert task_handler._get_parameters_from_function(kwargs_func) == [] - - -def test_get_parameters_from_function_lambda(task_handler): - my_func = lambda x: x - - assert task_handler._get_parameters_from_function(my_func) == ["x"] - - def test_check_is_task_duplicate_with_duplicate(task_handler, task): task_handler.tasks.append(task) with pytest.raises(DuplicateTaskType): From b8c3a946d32d26b881d5e02359339b5a7fdfef70 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 19:43:20 +0200 Subject: [PATCH 11/47] Rename TaskHandler to ZeebeTaskRouter --- pyzeebe/worker/task_handler.py | 109 ------------------ pyzeebe/worker/task_router.py | 118 ++++++++++++++++--- pyzeebe/worker/worker.py | 12 +- tests/conftest.py | 4 +- tests/unit/worker/task_handler_test.py | 79 ------------- tests/unit/worker/task_router_test.py | 153 +++++++++---------------- 6 files changed, 160 insertions(+), 315 deletions(-) delete mode 100644 pyzeebe/worker/task_handler.py delete mode 100644 tests/unit/worker/task_handler_test.py diff --git a/pyzeebe/worker/task_handler.py b/pyzeebe/worker/task_handler.py deleted file mode 100644 index aa8d8cd7..00000000 --- a/pyzeebe/worker/task_handler.py +++ /dev/null @@ -1,109 +0,0 @@ -import logging -from typing import Tuple, List, Callable, Union - -from pyzeebe import TaskDecorator -from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase -from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType -from pyzeebe.job.job import Job -from pyzeebe.task import task_builder -from pyzeebe.task.task import Task -from pyzeebe.task.task_config import TaskConfig - -logger = logging.getLogger(__name__) - - -def default_exception_handler(e: Exception, job: Job) -> None: - logger.warning(f"Task type: {job.type} - failed job {job}. Error: {e}.") - job.set_failure_status(f"Failed job. Error: {e}") - - -class ZeebeTaskHandler(ZeebeDecoratorBase): - def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): - """ - Args: - before (List[TaskDecorator]): Decorators to be performed before each task - after (List[TaskDecorator]): Decorators to be performed after each task - """ - super().__init__(before, after) - self.tasks: List[Task] = [] - - def task(self, task_config: Union[TaskConfig, str]): - """ - Decorator to create a task - - Args: - task_config (Union[str, TaskConfig]): Either the task type or a task configuration object - - Raises: - DuplicateTaskType: If a task from the router already exists in the worker - - """ - if isinstance(task_config, str): - task_config = TaskConfig(task_config) - - self._is_task_duplicate(task_config.type) - - task_config = self._add_decorators_to_config(task_config) - - def wrapper(fn: Callable): - task = task_builder.build_task(fn, task_config) - self.tasks.append(task) - return fn - - return wrapper - - def _add_decorators_to_config(self, config: TaskConfig) -> TaskConfig: - before_decorators = self._before.copy() - before_decorators.extend(config.before) - config.before = before_decorators - config.after.extend(self._after) - return config - - def _is_task_duplicate(self, task_type: str) -> None: - try: - self.get_task(task_type) - raise DuplicateTaskType(task_type) - except TaskNotFound: - return - - def remove_task(self, task_type: str) -> Task: - """ - Remove a task - - Args: - task_type (str): The type of the wanted task - - Returns: - Task: The task that was removed - - Raises: - TaskNotFound: If no task with specified type exists - - """ - task_index = self._get_task_index(task_type) - return self.tasks.pop(task_index) - - def get_task(self, task_type: str) -> Task: - """ - Get a task by its type - - Args: - task_type (str): The type of the wanted task - - Returns: - Task: The wanted task - - Raises: - TaskNotFound: If no task with specified type exists - - """ - return self._get_task_and_index(task_type)[0] - - def _get_task_index(self, task_type: str) -> int: - return self._get_task_and_index(task_type)[-1] - - def _get_task_and_index(self, task_type: str) -> Tuple[Task, int]: - for index, task in enumerate(self.tasks): - if task.type == task_type: - return task, index - raise TaskNotFound(f"Could not find task {task_type}") diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 434d3093..e3117583 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -1,27 +1,109 @@ -from typing import List, Callable, Dict +import logging +from typing import Tuple, List, Callable, Union -from pyzeebe.task.exception_handler import ExceptionHandler -from pyzeebe.task.task import Task from pyzeebe import TaskDecorator -from pyzeebe.worker.task_handler import ZeebeTaskHandler, default_exception_handler +from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase +from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType +from pyzeebe.job.job import Job +from pyzeebe.task import task_builder +from pyzeebe.task.task import Task +from pyzeebe.task.task_config import TaskConfig + +logger = logging.getLogger(__name__) + + +def default_exception_handler(e: Exception, job: Job) -> None: + logger.warning(f"Task type: {job.type} - failed job {job}. Error: {e}.") + job.set_failure_status(f"Failed job. Error: {e}") + + +class ZeebeTaskRouter(ZeebeDecoratorBase): + def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): + """ + Args: + before (List[TaskDecorator]): Decorators to be performed before each task + after (List[TaskDecorator]): Decorators to be performed after each task + """ + super().__init__(before, after) + self.tasks: List[Task] = [] + + def task(self, task_config: Union[TaskConfig, str]): + """ + Decorator to create a task + + Args: + task_config (Union[str, TaskConfig]): Either the task type or a task configuration object + + Raises: + DuplicateTaskType: If a task from the router already exists in the worker + + """ + if isinstance(task_config, str): + task_config = TaskConfig(task_config) + + self._is_task_duplicate(task_config.type) + task_config = self._add_decorators_to_config(task_config) -class ZeebeTaskRouter(ZeebeTaskHandler): + def wrapper(fn: Callable): + task = task_builder.build_task(fn, task_config) + self.tasks.append(task) + return fn - def _create_task(self, task_type: str, task_handler: Callable, exception_handler: ExceptionHandler, - timeout: int = 10000, max_jobs_to_activate: int = 32, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None, variables_to_fetch: List[str] = None) -> Task: - task = Task(task_type=task_type, task_handler=task_handler, exception_handler=exception_handler, - timeout=timeout, max_jobs_to_activate=max_jobs_to_activate, variables_to_fetch=variables_to_fetch) - return self._add_decorators_to_task(task, before or [], after or []) + return wrapper - def _add_decorators_to_task(self, task: Task, before: List[TaskDecorator], - after: List[TaskDecorator]) -> Task: + def _add_decorators_to_config(self, config: TaskConfig) -> TaskConfig: before_decorators = self._before.copy() - before_decorators.extend(before) + before_decorators.extend(config.before) + config.before = before_decorators + config.after.extend(self._after) + return config + + def _is_task_duplicate(self, task_type: str) -> None: + try: + self.get_task(task_type) + raise DuplicateTaskType(task_type) + except TaskNotFound: + return + + def remove_task(self, task_type: str) -> Task: + """ + Remove a task + + Args: + task_type (str): The type of the wanted task + + Returns: + Task: The task that was removed + + Raises: + TaskNotFound: If no task with specified type exists + + """ + task_index = self._get_task_index(task_type) + return self.tasks.pop(task_index) + + def get_task(self, task_type: str) -> Task: + """ + Get a task by its type + + Args: + task_type (str): The type of the wanted task + + Returns: + Task: The wanted task + + Raises: + TaskNotFound: If no task with specified type exists + + """ + return self._get_task_and_index(task_type)[0] - after.extend(self._after) + def _get_task_index(self, task_type: str) -> int: + return self._get_task_and_index(task_type)[-1] - task.before(*before_decorators) - task.after(*after) - return task + def _get_task_and_index(self, task_type: str) -> Tuple[Task, int]: + for index, task in enumerate(self.tasks): + if task.type == task_type: + return task, index + raise TaskNotFound(f"Could not find task {task_type}") diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 71dbe8a6..5c4838b2 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -2,22 +2,20 @@ import socket import time from threading import Thread, Event -from typing import List, Callable, Generator, Tuple, Dict, Union +from typing import List, Callable, Generator, Tuple, Dict +from pyzeebe import TaskDecorator from pyzeebe.credentials.base_credentials import BaseCredentials from pyzeebe.exceptions.pyzeebe_exceptions import MaxConsecutiveTaskThreadError from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.job.job import Job -from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.task import Task -from pyzeebe import TaskDecorator -from pyzeebe.worker.task_handler import ZeebeTaskHandler, default_exception_handler from pyzeebe.worker.task_router import ZeebeTaskRouter logger = logging.getLogger(__name__) -class ZeebeWorker(ZeebeTaskHandler): +class ZeebeWorker(ZeebeTaskRouter): """A zeebe worker that can connect to a zeebe instance and perform tasks.""" def __init__(self, name: str = None, request_timeout: int = 0, hostname: str = None, port: int = None, @@ -43,7 +41,7 @@ def __init__(self, name: str = None, request_timeout: int = 0, hostname: str = N self.stop_event = Event() self._task_threads: Dict[str, Thread] = {} self.watcher_max_errors_factor = watcher_max_errors_factor - self._watcher_thread = None + self._watcher_thread = None def work(self, watch: bool = False) -> None: """ @@ -147,7 +145,7 @@ def _check_max_errors(self, consecutive_errors: int): max_errors = self.watcher_max_errors_factor * len(self.tasks) if consecutive_errors >= max_errors: raise MaxConsecutiveTaskThreadError(f"Number of consecutive errors ({consecutive_errors}) exceeded " - f"max allowed number of errors ({max_errors})") + f"max allowed number of errors ({max_errors})") def _restart_task_thread(self, task_type: str) -> None: task = self.get_task(task_type) diff --git a/tests/conftest.py b/tests/conftest.py index 5bfd1176..cc5e2f3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.task import task_builder from pyzeebe.task.task_config import TaskConfig -from pyzeebe.worker.task_handler import ZeebeTaskHandler +from pyzeebe.worker.task_router import ZeebeTaskRouter from tests.unit.utils.gateway_mock import GatewayMock from tests.unit.utils.random_utils import random_job @@ -124,7 +124,7 @@ def routers(): @pytest.fixture def task_handler(): - return ZeebeTaskHandler() + return ZeebeTaskRouter() @pytest.fixture diff --git a/tests/unit/worker/task_handler_test.py b/tests/unit/worker/task_handler_test.py deleted file mode 100644 index 6a6b0653..00000000 --- a/tests/unit/worker/task_handler_test.py +++ /dev/null @@ -1,79 +0,0 @@ -from unittest.mock import patch -from uuid import uuid4 - -import pytest - -from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType -from pyzeebe.task.task import Task -from pyzeebe.worker.task_handler import default_exception_handler -from tests.unit.utils.random_utils import randint - - -def test_get_task(task_handler, task): - task_handler.tasks.append(task) - found_task = task_handler.get_task(task.type) - assert isinstance(found_task, Task) - assert found_task == task - - -def test_get_fake_task(task_handler): - with pytest.raises(TaskNotFound): - task_handler.get_task(str(uuid4())) - - -def test_get_task_index(task_handler, task): - task_handler.tasks.append(task) - index = task_handler._get_task_index(task.type) - assert isinstance(index, int) - assert task_handler.tasks[index] == task - - -def test_get_task_and_index(task_handler, task): - task_handler.tasks.append(task) - found_task, index = task_handler._get_task_and_index(task.type) - assert isinstance(index, int) - assert task_handler.tasks[index] == task - assert isinstance(found_task, Task) - assert found_task == task - - -def test_remove_task(task_handler, task): - task_handler.tasks.append(task) - assert task_handler.remove_task(task.type) is not None - assert task not in task_handler.tasks - - -def test_remove_task_from_many(task_handler, task): - task_handler.tasks.append(task) - - for i in range(0, randint(0, 100)): - @task_handler.task(str(uuid4())) - def dummy_function(): - pass - assert task_handler.remove_task(task.type) is not None - assert task not in task_handler.tasks - - -def test_remove_fake_task(task_handler): - with pytest.raises(TaskNotFound): - task_handler.remove_task(str(uuid4())) - - -def test_default_exception_handler(job_without_adapter): - with patch("pyzeebe.worker.task_handler.logger.warning") as logging_mock: - with patch("pyzeebe.job.job.Job.set_failure_status") as failure_mock: - failure_mock.return_value = None - default_exception_handler(Exception(), job_without_adapter) - - failure_mock.assert_called() - logging_mock.assert_called() - - -def test_check_is_task_duplicate_with_duplicate(task_handler, task): - task_handler.tasks.append(task) - with pytest.raises(DuplicateTaskType): - task_handler._is_task_duplicate(task.type) - - -def test_check_is_task_duplicate_no_duplicate(task_handler, task): - task_handler.tasks.append(task) diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 7fee0e61..2fe4d09c 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -1,126 +1,79 @@ -from random import randint from unittest.mock import patch from uuid import uuid4 -from pyzeebe.job.job import Job +import pytest +from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType +from pyzeebe.task.task import Task +from pyzeebe.worker.task_router import default_exception_handler +from tests.unit.utils.random_utils import randint -def decorator(job: Job) -> Job: - return job +def test_get_task(task_handler, task): + task_handler.tasks.append(task) + found_task = task_handler.get_task(task.type) + assert isinstance(found_task, Task) + assert found_task == task -def test_add_task_through_decorator(router): - task_type = str(uuid4()) - timeout = randint(0, 10000) - max_jobs_to_activate = randint(0, 1000) - @router.task(task_type=task_type, timeout=timeout, max_jobs_to_activate=max_jobs_to_activate) - def example_test_task(x): - return {"x": x} +def test_get_fake_task(task_handler): + with pytest.raises(TaskNotFound): + task_handler.get_task(str(uuid4())) - assert len(router.tasks) == 1 - variable = str(uuid4()) - assert example_test_task(variable) == {"x": variable} +def test_get_task_index(task_handler, task): + task_handler.tasks.append(task) + index = task_handler._get_task_index(task.type) + assert isinstance(index, int) + assert task_handler.tasks[index] == task - task = router.get_task(task_type) - assert task is not None - variable = str(uuid4()) - assert task.inner_function(variable) == {"x": variable} - assert task.variables_to_fetch == ["x"] - assert task.timeout == timeout - assert task.max_jobs_to_activate == max_jobs_to_activate +def test_get_task_and_index(task_handler, task): + task_handler.tasks.append(task) + found_task, index = task_handler._get_task_and_index(task.type) + assert isinstance(index, int) + assert task_handler.tasks[index] == task + assert isinstance(found_task, Task) + assert found_task == task -def test_router_before_decorator(router): - task_type = str(uuid4()) - router.before(decorator) +def test_remove_task(task_handler, task): + task_handler.tasks.append(task) + assert task_handler.remove_task(task.type) is not None + assert task not in task_handler.tasks - @router.task(task_type=task_type) - def task_fn(x): - return {"x": x} - task = router.get_task(task_type) - assert task is not None - assert len(task._before) == 1 - assert len(task._after) == 0 +def test_remove_task_from_many(task_handler, task): + task_handler.tasks.append(task) + for i in range(0, randint(0, 100)): + @task_handler.task(str(uuid4())) + def dummy_function(): + pass + assert task_handler.remove_task(task.type) is not None + assert task not in task_handler.tasks -def test_router_after_before_multiple(router): - task_type = str(uuid4()) - router.before(decorator) - @router.task(task_type=task_type, before=[decorator]) - def task_fn(x): - return {"x": x} +def test_remove_fake_task(task_handler): + with pytest.raises(TaskNotFound): + task_handler.remove_task(str(uuid4())) - task = router.get_task(task_type) - assert task is not None - assert len(task._before) == 2 - assert len(task._after) == 0 +def test_default_exception_handler(job_without_adapter): + with patch("pyzeebe.worker.task_router.logger.warning") as logging_mock: + with patch("pyzeebe.job.job.Job.set_failure_status") as failure_mock: + failure_mock.return_value = None + default_exception_handler(Exception(), job_without_adapter) -def test_router_after_decorator(router): - task_type = str(uuid4()) - router.after(decorator) + failure_mock.assert_called() + logging_mock.assert_called() - @router.task(task_type=task_type) - def task_fn(x): - return {"x": x} - task = router.get_task(task_type) - assert task is not None - assert len(task._after) == 1 - assert len(task._before) == 0 +def test_check_is_task_duplicate_with_duplicate(task_handler, task): + task_handler.tasks.append(task) + with pytest.raises(DuplicateTaskType): + task_handler._is_task_duplicate(task.type) -def test_router_after_decorator_multiple(router): - task_type = str(uuid4()) - router.after(decorator) - - @router.task(task_type=task_type, after=[decorator]) - def task_fn(x): - return {"x": x} - - task = router.get_task(task_type) - assert task is not None - assert len(task._after) == 2 - assert len(task._before) == 0 - - -def test_router_non_dict_task(router): - with patch("pyzeebe.worker.task_handler.ZeebeTaskHandler._single_value_function_to_dict") as single_value_mock: - task_type = str(uuid4()) - variable_name = str(uuid4()) - - @router.task(task_type=task_type, single_value=True, variable_name=variable_name) - def task_fn(x): - return {"x": x} - - single_value_mock.assert_called_with(variable_name=variable_name, fn=task_fn) - assert len(router.tasks) == 1 - - -def test_router_dict_task(router): - task_type = str(uuid4()) - - @router.task(task_type=task_type) - def task_fn(x): - return {"x": x} - - assert len(router.tasks) == 1 - - -def test_add_decorators_to_task(router, task): - router._add_decorators_to_task(task, [decorator], [decorator]) - assert len(task._before) == 1 - assert len(task._after) == 1 - - -def test_add_decorators_to_task_with_router_decorators(router, task): - router.before(decorator) - router.after(decorator) - router._add_decorators_to_task(task, [decorator], [decorator]) - assert len(task._before) == 2 - assert len(task._after) == 2 +def test_check_is_task_duplicate_no_duplicate(task_handler, task): + task_handler.tasks.append(task) From 59131d03d41504b63c4b4228bf5794bbf972a6b2 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 20:28:51 +0200 Subject: [PATCH 12/47] Refactor ZeebeTaskRouter tests --- pyzeebe/worker/task_router.py | 6 -- tests/conftest.py | 7 +-- tests/unit/task/task_config_test.py | 12 +++- tests/unit/worker/task_router_test.py | 83 +++++++++++++-------------- 4 files changed, 51 insertions(+), 57 deletions(-) diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index e3117583..5280b576 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -4,7 +4,6 @@ from pyzeebe import TaskDecorator from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType -from pyzeebe.job.job import Job from pyzeebe.task import task_builder from pyzeebe.task.task import Task from pyzeebe.task.task_config import TaskConfig @@ -12,11 +11,6 @@ logger = logging.getLogger(__name__) -def default_exception_handler(e: Exception, job: Job) -> None: - logger.warning(f"Task type: {job.type} - failed job {job}. Error: {e}.") - job.set_failure_status(f"Failed job. Error: {e}") - - class ZeebeTaskRouter(ZeebeDecoratorBase): def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): """ diff --git a/tests/conftest.py b/tests/conftest.py index cc5e2f3b..5c64086e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import pytest -from pyzeebe import ZeebeClient, ZeebeWorker, ZeebeTaskRouter, Job +from pyzeebe import ZeebeClient, ZeebeWorker, Job from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.task import task_builder from pyzeebe.task.task_config import TaskConfig @@ -122,11 +122,6 @@ def routers(): return [ZeebeTaskRouter() for _ in range(0, randint(2, 100))] -@pytest.fixture -def task_handler(): - return ZeebeTaskRouter() - - @pytest.fixture def decorator(): def simple_decorator(job: Job) -> Job: diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py index b3f3bb7d..e4813d5a 100644 --- a/tests/unit/task/task_config_test.py +++ b/tests/unit/task/task_config_test.py @@ -1,11 +1,21 @@ +from unittest.mock import patch from uuid import uuid4 import pytest +from pyzeebe import Job from pyzeebe.exceptions import NoVariableNameGiven -from pyzeebe.task.task_config import TaskConfig +from pyzeebe.task.task_config import TaskConfig, default_exception_handler def test_add_non_dict_task_without_variable_name(): with pytest.raises(NoVariableNameGiven): TaskConfig(str(uuid4()), single_value=True) + + +def test_default_exception_handler(mocked_job_with_adapter: Job): + with patch("pyzeebe.task.task_config.logger.warning") as logging_mock: + default_exception_handler(Exception(), mocked_job_with_adapter) + + mocked_job_with_adapter.set_failure_status.assert_called() + logging_mock.assert_called() diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 2fe4d09c..7adb2432 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -1,79 +1,74 @@ -from unittest.mock import patch from uuid import uuid4 import pytest from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType from pyzeebe.task.task import Task -from pyzeebe.worker.task_router import default_exception_handler +from pyzeebe.worker.task_router import ZeebeTaskRouter from tests.unit.utils.random_utils import randint -def test_get_task(task_handler, task): - task_handler.tasks.append(task) - found_task = task_handler.get_task(task.type) - assert isinstance(found_task, Task) +def test_get_task(router: ZeebeTaskRouter, task: Task): + router.tasks.append(task) + + found_task = router.get_task(task.type) + assert found_task == task -def test_get_fake_task(task_handler): +def test_get_fake_task(router: ZeebeTaskRouter): with pytest.raises(TaskNotFound): - task_handler.get_task(str(uuid4())) + router.get_task(str(uuid4())) + + +def test_get_task_index(router: ZeebeTaskRouter, task: Task): + router.tasks.append(task) + + index = router._get_task_index(task.type) + + assert router.tasks[index] == task -def test_get_task_index(task_handler, task): - task_handler.tasks.append(task) - index = task_handler._get_task_index(task.type) - assert isinstance(index, int) - assert task_handler.tasks[index] == task +def test_get_task_and_index(router: ZeebeTaskRouter, task: Task): + router.tasks.append(task) + found_task, index = router._get_task_and_index(task.type) -def test_get_task_and_index(task_handler, task): - task_handler.tasks.append(task) - found_task, index = task_handler._get_task_and_index(task.type) - assert isinstance(index, int) - assert task_handler.tasks[index] == task - assert isinstance(found_task, Task) + assert router.tasks[index] == task assert found_task == task -def test_remove_task(task_handler, task): - task_handler.tasks.append(task) - assert task_handler.remove_task(task.type) is not None - assert task not in task_handler.tasks +def test_remove_task(router: ZeebeTaskRouter, task: Task): + router.tasks.append(task) + router.remove_task(task.type) -def test_remove_task_from_many(task_handler, task): - task_handler.tasks.append(task) + assert task not in router.tasks + + +def test_remove_task_from_many(router: ZeebeTaskRouter, task: Task): + router.tasks.append(task) for i in range(0, randint(0, 100)): - @task_handler.task(str(uuid4())) + @router.task(str(uuid4())) def dummy_function(): pass - assert task_handler.remove_task(task.type) is not None - assert task not in task_handler.tasks + router.remove_task(task.type) -def test_remove_fake_task(task_handler): - with pytest.raises(TaskNotFound): - task_handler.remove_task(str(uuid4())) + assert task not in router.tasks -def test_default_exception_handler(job_without_adapter): - with patch("pyzeebe.worker.task_router.logger.warning") as logging_mock: - with patch("pyzeebe.job.job.Job.set_failure_status") as failure_mock: - failure_mock.return_value = None - default_exception_handler(Exception(), job_without_adapter) - - failure_mock.assert_called() - logging_mock.assert_called() +def test_remove_fake_task(router: ZeebeTaskRouter): + with pytest.raises(TaskNotFound): + router.remove_task(str(uuid4())) -def test_check_is_task_duplicate_with_duplicate(task_handler, task): - task_handler.tasks.append(task) +def test_check_is_task_duplicate_with_duplicate(router: ZeebeTaskRouter, task: Task): + router.tasks.append(task) with pytest.raises(DuplicateTaskType): - task_handler._is_task_duplicate(task.type) + router._is_task_duplicate(task.type) -def test_check_is_task_duplicate_no_duplicate(task_handler, task): - task_handler.tasks.append(task) +def test_check_is_task_duplicate_no_duplicate(router: ZeebeTaskRouter, task: Task): + router.tasks.append(task) From 3bb171db11c4768f0eb5e5ab32e1cf4aa7204950 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 20:54:22 +0200 Subject: [PATCH 13/47] Refactor ZeebeWorker to use task_builder --- pyzeebe/task/task.py | 4 +- pyzeebe/task/task_builder.py | 2 +- pyzeebe/worker/task_router.py | 13 +++-- pyzeebe/worker/worker.py | 77 +++---------------------- tests/unit/worker/worker_test.py | 96 ++++++-------------------------- 5 files changed, 35 insertions(+), 157 deletions(-) diff --git a/pyzeebe/task/task.py b/pyzeebe/task/task.py index 367ff285..d0075f5d 100644 --- a/pyzeebe/task/task.py +++ b/pyzeebe/task/task.py @@ -1,11 +1,13 @@ from dataclasses import dataclass +from typing import Callable -from pyzeebe.task.types import JobHandler from pyzeebe.task.task_config import TaskConfig +from pyzeebe.task.types import JobHandler @dataclass class Task: + original_function: Callable job_handler: JobHandler config: TaskConfig diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index 2f085b2f..31ac8977 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -16,7 +16,7 @@ def build_task(task_function: Callable, task_config: TaskConfig) -> Task: if task_config.single_value: task_function = convert_to_dict_function(task_function, task_config.variable_name) - return Task(build_job_handler(task_function, task_config), task_config) + return Task(task_function, build_job_handler(task_function, task_config), task_config) def build_job_handler(task_function: Callable, task_config: TaskConfig) -> JobHandler: diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 5280b576..eeefce7f 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -34,18 +34,19 @@ def task(self, task_config: Union[TaskConfig, str]): """ if isinstance(task_config, str): task_config = TaskConfig(task_config) - - self._is_task_duplicate(task_config.type) - - task_config = self._add_decorators_to_config(task_config) + config_with_decorators = self._add_decorators_to_config(task_config) def wrapper(fn: Callable): - task = task_builder.build_task(fn, task_config) - self.tasks.append(task) + task = task_builder.build_task(fn, config_with_decorators) + self._add_task(task) return fn return wrapper + def _add_task(self, task: Task): + self._is_task_duplicate(task.type) + self.tasks.append(task) + def _add_decorators_to_config(self, config: TaskConfig) -> TaskConfig: before_decorators = self._before.copy() before_decorators.extend(config.before) diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 5c4838b2..7dfb50bd 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -2,13 +2,14 @@ import socket import time from threading import Thread, Event -from typing import List, Callable, Generator, Tuple, Dict +from typing import List, Generator, Dict from pyzeebe import TaskDecorator from pyzeebe.credentials.base_credentials import BaseCredentials from pyzeebe.exceptions.pyzeebe_exceptions import MaxConsecutiveTaskThreadError from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.job.job import Job +from pyzeebe.task import task_builder from pyzeebe.task.task import Task from pyzeebe.worker.task_router import ZeebeTaskRouter @@ -167,7 +168,7 @@ def _handle_task(self, task: Task) -> None: def _handle_jobs(self, task: Task) -> None: for job in self._get_jobs(task): - thread = Thread(target=task.handler, + thread = Thread(target=task.job_handler, args=(job,), name=f"{self.__class__.__name__}-Job-{job.type}") logger.debug(f"Running job: {job}") @@ -175,9 +176,9 @@ def _handle_jobs(self, task: Task) -> None: def _get_jobs(self, task: Task) -> Generator[Job, None, None]: logger.debug(f"Activating jobs for task: {task}") - return self.zeebe_adapter.activate_jobs(task_type=task.type, worker=self.name, timeout=task.timeout, - max_jobs_to_activate=task.max_jobs_to_activate, - variables_to_fetch=task.variables_to_fetch, + return self.zeebe_adapter.activate_jobs(task_type=task.type, worker=self.name, timeout=task.config.timeout, + max_jobs_to_activate=task.config.max_jobs_to_activate, + variables_to_fetch=task.config.variables_to_fetch, request_timeout=self.request_timeout) def include_router(self, *routers: ZeebeTaskRouter) -> None: @@ -190,69 +191,5 @@ def include_router(self, *routers: ZeebeTaskRouter) -> None: """ for router in routers: for task in router.tasks: + task.config = self._add_decorators_to_config(task.config) self._add_task(task) - - def _add_task(self, task: Task) -> None: - self._is_task_duplicate(task.type) - task.handler = self._create_task_handler(task) - self.tasks.append(task) - - def _create_task_handler(self, task: Task) -> Callable[[Job], Job]: - before_decorator_runner = self._create_before_decorator_runner(task) - after_decorator_runner = self._create_after_decorator_runner(task) - - def task_handler(job: Job) -> Job: - job = before_decorator_runner(job) - job, task_succeeded = self._run_task_inner_function(task, job) - job = after_decorator_runner(job) - if task_succeeded: - self._complete_job(job) - return job - - return task_handler - - @staticmethod - def _run_task_inner_function(task: Task, job: Job) -> Tuple[Job, bool]: - task_succeeded = False - try: - job.variables = task.inner_function(**job.variables) - task_succeeded = True - except Exception as e: - logger.debug(f"Failed job: {job}. Error: {e}.") - task.exception_handler(e, job) - finally: - return job, task_succeeded - - def _complete_job(self, job: Job) -> None: - try: - logger.debug(f"Completing job: {job}") - self.zeebe_adapter.complete_job(job_key=job.key, variables=job.variables) - except Exception as e: - logger.warning(f"Failed to complete job: {job}. Error: {e}") - - def _create_before_decorator_runner(self, task: Task) -> Callable[[Job], Job]: - decorators = task._before.copy() - decorators.extend(self._before) - return self._create_decorator_runner(decorators) - - def _create_after_decorator_runner(self, task: Task) -> Callable[[Job], Job]: - decorators = self._after.copy() - decorators.extend(task._after) - return self._create_decorator_runner(decorators) - - @staticmethod - def _create_decorator_runner(decorators: List[TaskDecorator]) -> Callable[[Job], Job]: - def decorator_runner(job: Job): - for decorator in decorators: - job = ZeebeWorker._run_decorator(decorator, job) - return job - - return decorator_runner - - @staticmethod - def _run_decorator(decorator: TaskDecorator, job: Job) -> Job: - try: - return decorator(job) - except Exception as e: - logger.warning(f"Failed to run decorator {decorator}. Error: {e}") - return job diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index 66098133..88a436a6 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -1,5 +1,4 @@ import time -from random import randint from unittest.mock import patch, MagicMock from uuid import uuid4 @@ -7,11 +6,12 @@ from pyzeebe.exceptions import DuplicateTaskType, MaxConsecutiveTaskThreadError from pyzeebe.job.job import Job +from pyzeebe.task.task import Task from pyzeebe.worker.worker import ZeebeWorker class TestAddTask: - def test_task_added(self, zeebe_worker, task): + def test_add_task(self, zeebe_worker, task): zeebe_worker._add_task(task) assert zeebe_worker.get_task(task.type) == task @@ -33,56 +33,14 @@ def test_task_type_saved(self, zeebe_worker, task): assert zeebe_worker.get_task(task.type).type == task.type - def test_original_function_not_changed(self, zeebe_worker, task, job_from_task): - zeebe_worker._add_task(task) - - assert task.inner_function(**job_from_task.variables) == job_from_task.variables - - def test_task_handler_calls_original_function(self, zeebe_worker, task, job_from_task): - zeebe_worker._add_task(task) - - task.handler(job_from_task) - - task.inner_function.assert_called_once() - - def test_task_timeout_saved(self, zeebe_worker, task): - timeout = randint(0, 10000) - task.timeout = timeout - - zeebe_worker._add_task(task) - - assert zeebe_worker.get_task(task.type).timeout == timeout - - def test_task_max_jobs_saved(self, zeebe_worker, task): - max_jobs_to_activate = randint(0, 1000) - task.max_jobs_to_activate = max_jobs_to_activate - - zeebe_worker._add_task(task) - - assert zeebe_worker.get_task(task.type).max_jobs_to_activate == max_jobs_to_activate - - def test_variables_to_fetch_match_function_parameters(self, zeebe_worker, task_type): + def test_variables_to_fetch_match_function_parameters(self, zeebe_worker: ZeebeWorker, task_type: str): expected_variables_to_fetch = ["x"] @zeebe_worker.task(task_type) def _(x): pass - assert zeebe_worker.get_task(task_type).variables_to_fetch == expected_variables_to_fetch - - def test_task_handler_is_callable(self, zeebe_worker, task): - zeebe_worker._add_task(task) - - assert callable(task.handler) - - def test_exception_handler_called(self, zeebe_worker, task, job_from_task): - task.inner_function.side_effect = Exception() - task.exception_handler = MagicMock() - zeebe_worker._add_task(task) - - task.handler(job_from_task) - - task.exception_handler.assert_called() + assert zeebe_worker.get_task(task_type).config.variables_to_fetch == expected_variables_to_fetch class TestDecorator: @@ -106,36 +64,16 @@ def test_add_constructor_after_decorator(self, decorator): assert len(zeebe_worker._after) == 1 assert decorator in zeebe_worker._after - def test_create_before_decorator_runner(self, zeebe_worker, task, decorator, job_from_task): - task.before(decorator) - - decorators = zeebe_worker._create_before_decorator_runner(task) - - assert isinstance(decorators(job_from_task), Job) - - def test_before_task_decorator_called(self, zeebe_worker, task, decorator, job_from_task): - task.before(decorator) - zeebe_worker._add_task(task) - - task.handler(job_from_task) - - decorator.assert_called_with(job_from_task) - - def test_after_task_decorator_called(self, zeebe_worker, task, decorator, job_from_task): - task.after(decorator) - zeebe_worker._add_task(task) - - task.handler(job_from_task) - - decorator.assert_called_with(job_from_task) - def test_decorator_failed(self, zeebe_worker, task, decorator, job_from_task): decorator.side_effect = Exception() zeebe_worker.before(decorator) zeebe_worker.after(decorator) - zeebe_worker._add_task(task) - assert isinstance(task.handler(job_from_task), Job) + @zeebe_worker.task("test") + def dummy_function(): + pass + + assert isinstance(task.job_handler(job_from_task), Job) assert decorator.call_count == 2 @@ -147,28 +85,28 @@ def get_jobs_mock(self, zeebe_worker): @pytest.fixture(autouse=True) def task_handler_mock(self, task): - task.handler = MagicMock(wraps=task.handler) + task.job_handler = MagicMock(wraps=task.job_handler) def test_handle_no_job(self, zeebe_worker, task, get_jobs_mock): get_jobs_mock.return_value = [] zeebe_worker._handle_jobs(task) - task.handler.assert_not_called() + task.job_handler.assert_not_called() def test_handle_one_job(self, zeebe_worker, task, job_from_task, get_jobs_mock): get_jobs_mock.return_value = [job_from_task] zeebe_worker._handle_jobs(task) - task.handler.assert_called_with(job_from_task) + task.job_handler.assert_called_with(job_from_task) def test_handle_many_jobs(self, zeebe_worker, task, job_from_task, get_jobs_mock): get_jobs_mock.return_value = [job_from_task] * 10 zeebe_worker._handle_jobs(task) - assert task.handler.call_count == 10 + assert task.job_handler.call_count == 10 class TestWorkerThreads: @@ -220,14 +158,14 @@ def fake_task_handler_return_immediately(*_args): class TestGetJobs: - def test_activate_jobs_called(self, zeebe_worker, task): + def test_activate_jobs_called(self, zeebe_worker: ZeebeWorker, task: Task): zeebe_worker.zeebe_adapter.activate_jobs = MagicMock() zeebe_worker._get_jobs(task) zeebe_worker.zeebe_adapter.activate_jobs.assert_called_with(task_type=task.type, worker=zeebe_worker.name, timeout=task.config.timeout, max_jobs_to_activate=task.config.max_jobs_to_activate, variables_to_fetch=task.config.variables_to_fetch, - request_timeout=zeebe_worker.config.request_timeout) + request_timeout=zeebe_worker.request_timeout) class TestIncludeRouter: @@ -246,7 +184,7 @@ def test_router_before_decorator(self, zeebe_worker, router, decorator, job_with router.before(decorator) task = self.include_router_with_task(zeebe_worker, router) - task.handler(job_without_adapter) + task.job_handler(job_without_adapter) assert decorator.call_count == 1 @@ -254,7 +192,7 @@ def test_router_after_decorator(self, zeebe_worker, router, decorator, job_witho router.after(decorator) task = self.include_router_with_task(zeebe_worker, router) - task.handler(job_without_adapter) + task.job_handler(job_without_adapter) assert decorator.call_count == 1 From f52ac70737447e7bbbee383273fd3cf88edd74cd Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 21:06:45 +0200 Subject: [PATCH 14/47] Fix worker tests --- pyzeebe/task/task_builder.py | 25 +++++++++++++++---------- tests/unit/worker/worker_test.py | 26 +++++++++++++++----------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index 31ac8977..fa42429b 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -1,5 +1,5 @@ import logging -from typing import List, Callable, Dict +from typing import List, Callable, Dict, Tuple from pyzeebe import Job, TaskDecorator from pyzeebe.task.task import Task @@ -24,20 +24,25 @@ def build_job_handler(task_function: Callable, task_config: TaskConfig) -> JobHa after_decorator_runner = create_decorator_runner(task_config.after) def job_handler(job: Job) -> Job: - try: - job = before_decorator_runner(job) - job.variables = task_function(**job.variables) - job = after_decorator_runner(job) + job = before_decorator_runner(job) + job.variables, succeeded = run_original_task_function(task_function, task_config, job) + job = after_decorator_runner(job) + if succeeded: job.set_success_status() - except Exception as e: - logger.debug(f"Failed job: {job}. Error: {e}.") - task_config.exception_handler(e, job) - finally: - return job + return job return job_handler +def run_original_task_function(task_function: Callable, task_config: TaskConfig, job: Job) -> Tuple[Dict, bool]: + try: + return task_function(**job.variables), True + except Exception as e: + logger.debug(f"Failed job: {job}. Error: {e}.") + task_config.exception_handler(e, job) + return job.variables, False + + def create_decorator_runner(decorators: List[TaskDecorator]) -> DecoratorRunner: def decorator_runner(job: Job): for decorator in decorators: diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index 88a436a6..d0610913 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -4,6 +4,7 @@ import pytest +from pyzeebe import TaskDecorator from pyzeebe.exceptions import DuplicateTaskType, MaxConsecutiveTaskThreadError from pyzeebe.job.job import Job from pyzeebe.task.task import Task @@ -64,16 +65,19 @@ def test_add_constructor_after_decorator(self, decorator): assert len(zeebe_worker._after) == 1 assert decorator in zeebe_worker._after - def test_decorator_failed(self, zeebe_worker, task, decorator, job_from_task): + def test_decorator_failed(self, zeebe_worker: ZeebeWorker, task: Task, decorator: TaskDecorator, + job_from_task: Job): decorator.side_effect = Exception() zeebe_worker.before(decorator) zeebe_worker.after(decorator) - @zeebe_worker.task("test") + @zeebe_worker.task(task.config) def dummy_function(): pass - assert isinstance(task.job_handler(job_from_task), Job) + task = zeebe_worker.get_task(task.type) + task.job_handler(job_from_task) + assert decorator.call_count == 2 @@ -180,29 +184,29 @@ def test_include_multiple_routers(self, zeebe_worker, routers): assert len(zeebe_worker.tasks) == len(routers) - def test_router_before_decorator(self, zeebe_worker, router, decorator, job_without_adapter): + def test_router_before_decorator(self, zeebe_worker, router, decorator, mocked_job_with_adapter): router.before(decorator) task = self.include_router_with_task(zeebe_worker, router) - task.job_handler(job_without_adapter) + task.job_handler(mocked_job_with_adapter) - assert decorator.call_count == 1 + decorator.assert_called_once() - def test_router_after_decorator(self, zeebe_worker, router, decorator, job_without_adapter): + def test_router_after_decorator(self, zeebe_worker, router, decorator, mocked_job_with_adapter): router.after(decorator) task = self.include_router_with_task(zeebe_worker, router) - task.job_handler(job_without_adapter) + task.job_handler(mocked_job_with_adapter) - assert decorator.call_count == 1 + decorator.assert_called_once() @staticmethod def include_router_with_task(zeebe_worker, router, task_type=None): task_type = task_type or str(uuid4()) @router.task(task_type) - def _(x): - return dict(x=x) + def _(): + return {} zeebe_worker.include_router(router) return zeebe_worker.get_task(task_type) From cb03ed449cd27134ec4d64a491a2aa93dc2a4352 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 21:11:43 +0200 Subject: [PATCH 15/47] Rename functions named _ to dummy_function --- tests/unit/worker/worker_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index d0610913..d4d94aa0 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -24,7 +24,7 @@ def test_raises_on_duplicate(self, zeebe_worker, task): def test_only_one_task_added(self, zeebe_worker): @zeebe_worker.task(str(uuid4())) - def _(): + def dummy_function(): pass assert len(zeebe_worker.tasks) == 1 @@ -38,7 +38,7 @@ def test_variables_to_fetch_match_function_parameters(self, zeebe_worker: ZeebeW expected_variables_to_fetch = ["x"] @zeebe_worker.task(task_type) - def _(x): + def dummy_function(x): pass assert zeebe_worker.get_task(task_type).config.variables_to_fetch == expected_variables_to_fetch @@ -205,7 +205,7 @@ def include_router_with_task(zeebe_worker, router, task_type=None): task_type = task_type or str(uuid4()) @router.task(task_type) - def _(): + def dummy_function(): return {} zeebe_worker.include_router(router) From 10574650d44425f894d0ee0ed6801bd11c448f44 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 21:18:09 +0200 Subject: [PATCH 16/47] Add type annotations to ZeebeWorker tests --- tests/unit/worker/worker_test.py | 53 ++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index d4d94aa0..ad8c347a 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -1,10 +1,12 @@ import time +from threading import Event as StopEvent +from typing import List from unittest.mock import patch, MagicMock from uuid import uuid4 import pytest -from pyzeebe import TaskDecorator +from pyzeebe import TaskDecorator, ZeebeTaskRouter from pyzeebe.exceptions import DuplicateTaskType, MaxConsecutiveTaskThreadError from pyzeebe.job.job import Job from pyzeebe.task.task import Task @@ -12,24 +14,24 @@ class TestAddTask: - def test_add_task(self, zeebe_worker, task): + def test_add_task(self, zeebe_worker: ZeebeWorker, task: Task): zeebe_worker._add_task(task) assert zeebe_worker.get_task(task.type) == task - def test_raises_on_duplicate(self, zeebe_worker, task): + def test_raises_on_duplicate(self, zeebe_worker: ZeebeWorker, task: Task): zeebe_worker._add_task(task) with pytest.raises(DuplicateTaskType): zeebe_worker._add_task(task) - def test_only_one_task_added(self, zeebe_worker): + def test_only_one_task_added(self, zeebe_worker: ZeebeWorker): @zeebe_worker.task(str(uuid4())) def dummy_function(): pass assert len(zeebe_worker.tasks) == 1 - def test_task_type_saved(self, zeebe_worker, task): + def test_task_type_saved(self, zeebe_worker: ZeebeWorker, task: Task): zeebe_worker._add_task(task) assert zeebe_worker.get_task(task.type).type == task.type @@ -45,22 +47,22 @@ def dummy_function(x): class TestDecorator: - def test_add_before_decorator(self, zeebe_worker, decorator): + def test_add_before_decorator(self, zeebe_worker: ZeebeWorker, decorator: TaskDecorator): zeebe_worker.before(decorator) assert len(zeebe_worker._before) == 1 assert decorator in zeebe_worker._before - def test_add_after_decorator(self, zeebe_worker, decorator): + def test_add_after_decorator(self, zeebe_worker: ZeebeWorker, decorator: TaskDecorator): zeebe_worker.after(decorator) assert len(zeebe_worker._after) == 1 assert decorator in zeebe_worker._after - def test_add_constructor_before_decorator(self, decorator): + def test_add_constructor_before_decorator(self, decorator: TaskDecorator): zeebe_worker = ZeebeWorker(before=[decorator]) assert len(zeebe_worker._before) == 1 assert decorator in zeebe_worker._before - def test_add_constructor_after_decorator(self, decorator): + def test_add_constructor_after_decorator(self, decorator: TaskDecorator): zeebe_worker = ZeebeWorker(after=[decorator]) assert len(zeebe_worker._after) == 1 assert decorator in zeebe_worker._after @@ -83,29 +85,30 @@ def dummy_function(): class TestHandleJobs: @pytest.fixture(autouse=True) - def get_jobs_mock(self, zeebe_worker): + def get_jobs_mock(self, zeebe_worker: ZeebeWorker): zeebe_worker._get_jobs = MagicMock() return zeebe_worker._get_jobs @pytest.fixture(autouse=True) - def task_handler_mock(self, task): + def task_handler_mock(self, task: Task): task.job_handler = MagicMock(wraps=task.job_handler) - def test_handle_no_job(self, zeebe_worker, task, get_jobs_mock): + def test_handle_no_job(self, zeebe_worker: ZeebeWorker, task: Task, get_jobs_mock: MagicMock): get_jobs_mock.return_value = [] zeebe_worker._handle_jobs(task) task.job_handler.assert_not_called() - def test_handle_one_job(self, zeebe_worker, task, job_from_task, get_jobs_mock): + def test_handle_one_job(self, zeebe_worker: ZeebeWorker, task: Task, job_from_task: Job, get_jobs_mock: MagicMock): get_jobs_mock.return_value = [job_from_task] zeebe_worker._handle_jobs(task) task.job_handler.assert_called_with(job_from_task) - def test_handle_many_jobs(self, zeebe_worker, task, job_from_task, get_jobs_mock): + def test_handle_many_jobs(self, zeebe_worker: ZeebeWorker, task: Task, job_from_task: Job, + get_jobs_mock: MagicMock): get_jobs_mock.return_value = [job_from_task] * 10 zeebe_worker._handle_jobs(task) @@ -114,7 +117,7 @@ def test_handle_many_jobs(self, zeebe_worker, task, job_from_task, get_jobs_mock class TestWorkerThreads: - def test_work_thread_start_called(self, zeebe_worker, task): + def test_work_thread_start_called(self, zeebe_worker: ZeebeWorker, task: Task): with patch("pyzeebe.worker.worker.Thread") as thread_mock: thread_instance_mock = MagicMock() thread_mock.return_value = thread_instance_mock @@ -123,12 +126,13 @@ def test_work_thread_start_called(self, zeebe_worker, task): zeebe_worker.stop() thread_instance_mock.start.assert_called_once() - def test_stop_worker(self, zeebe_worker): + def test_stop_worker(self, zeebe_worker: ZeebeWorker): zeebe_worker.work() zeebe_worker.stop() def test_watch_task_threads_dont_restart_running_threads( - self, zeebe_worker, task, handle_task_mock, stop_event_mock, handle_not_alive_thread_spy, stop_after_test): + self, zeebe_worker: ZeebeWorker, task: Task, handle_task_mock: MagicMock, stop_event_mock: MagicMock, + handle_not_alive_thread_spy: MagicMock, stop_after_test: StopEvent): def fake_task_handler_never_return(*_args): while not stop_after_test.is_set(): time.sleep(0.05) @@ -144,7 +148,8 @@ def fake_task_handler_never_return(*_args): assert handle_not_alive_thread_spy.call_count == 0 def test_watch_task_threads_that_die_get_restarted_then_exit_after_too_many_errors( - self, zeebe_worker, task, handle_task_mock, stop_event_mock, handle_not_alive_thread_spy): + self, zeebe_worker: ZeebeWorker, task: Task, handle_task_mock: MagicMock, stop_event_mock: MagicMock, + handle_not_alive_thread_spy: MagicMock): def fake_task_handler_return_immediately(*_args): pass @@ -173,18 +178,19 @@ def test_activate_jobs_called(self, zeebe_worker: ZeebeWorker, task: Task): class TestIncludeRouter: - def test_include_router_adds_task(self, zeebe_worker, router, task_type): + def test_include_router_adds_task(self, zeebe_worker: ZeebeWorker, router: ZeebeTaskRouter, task_type: str): self.include_router_with_task(zeebe_worker, router, task_type) assert zeebe_worker.get_task(task_type) is not None - def test_include_multiple_routers(self, zeebe_worker, routers): + def test_include_multiple_routers(self, zeebe_worker: ZeebeWorker, routers: List[ZeebeTaskRouter]): for router in routers: self.include_router_with_task(zeebe_worker, router) assert len(zeebe_worker.tasks) == len(routers) - def test_router_before_decorator(self, zeebe_worker, router, decorator, mocked_job_with_adapter): + def test_router_before_decorator(self, zeebe_worker: ZeebeWorker, router: ZeebeTaskRouter, decorator: TaskDecorator, + mocked_job_with_adapter: Job): router.before(decorator) task = self.include_router_with_task(zeebe_worker, router) @@ -192,7 +198,8 @@ def test_router_before_decorator(self, zeebe_worker, router, decorator, mocked_j decorator.assert_called_once() - def test_router_after_decorator(self, zeebe_worker, router, decorator, mocked_job_with_adapter): + def test_router_after_decorator(self, zeebe_worker: ZeebeWorker, router: ZeebeTaskRouter, decorator: TaskDecorator, + mocked_job_with_adapter: Job): router.after(decorator) task = self.include_router_with_task(zeebe_worker, router) @@ -201,7 +208,7 @@ def test_router_after_decorator(self, zeebe_worker, router, decorator, mocked_jo decorator.assert_called_once() @staticmethod - def include_router_with_task(zeebe_worker, router, task_type=None): + def include_router_with_task(zeebe_worker: ZeebeWorker, router: ZeebeTaskRouter, task_type: str = None) -> Task: task_type = task_type or str(uuid4()) @router.task(task_type) From d920ed9a73ce783b49a25eabba100ddc0a219c12 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 21:21:21 +0200 Subject: [PATCH 17/47] Export TaskConfig, default_exception_handler --- pyzeebe/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyzeebe/__init__.py b/pyzeebe/__init__.py index 8480063d..ed7580b1 100644 --- a/pyzeebe/__init__.py +++ b/pyzeebe/__init__.py @@ -7,6 +7,7 @@ from pyzeebe.job.job import Job from pyzeebe.job.job_status import JobStatus from pyzeebe.task.exception_handler import ExceptionHandler +from pyzeebe.task.task_config import TaskConfig, default_exception_handler from pyzeebe.task.types import TaskDecorator from pyzeebe.worker.task_router import ZeebeTaskRouter from pyzeebe.worker.worker import ZeebeWorker From d2c4d0429b74dac9ae57429c37e80ead4a733d56 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 21:22:35 +0200 Subject: [PATCH 18/47] Remove redundant import from zeebe_worker.py --- pyzeebe/worker/worker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 7dfb50bd..9932f33f 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -9,7 +9,6 @@ from pyzeebe.exceptions.pyzeebe_exceptions import MaxConsecutiveTaskThreadError from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.job.job import Job -from pyzeebe.task import task_builder from pyzeebe.task.task import Task from pyzeebe.worker.task_router import ZeebeTaskRouter From 48cb320105991be9a2183bebe6ead5c99d6436f5 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 21:31:01 +0200 Subject: [PATCH 19/47] Remove ZeebeDecoratorBase --- pyzeebe/decorators/__init__.py | 0 pyzeebe/decorators/zeebe_decorator_base.py | 15 ------ pyzeebe/task/task_config.py | 4 +- pyzeebe/worker/task_router.py | 12 +++-- tests/unit/decorators/__init__.py | 0 .../decorators/zeebe_decorator_base_test.py | 50 ------------------- tests/unit/worker/task_router_test.py | 25 ++++++++++ 7 files changed, 37 insertions(+), 69 deletions(-) delete mode 100644 pyzeebe/decorators/__init__.py delete mode 100644 pyzeebe/decorators/zeebe_decorator_base.py delete mode 100644 tests/unit/decorators/__init__.py delete mode 100644 tests/unit/decorators/zeebe_decorator_base_test.py diff --git a/pyzeebe/decorators/__init__.py b/pyzeebe/decorators/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pyzeebe/decorators/zeebe_decorator_base.py b/pyzeebe/decorators/zeebe_decorator_base.py deleted file mode 100644 index 4447702a..00000000 --- a/pyzeebe/decorators/zeebe_decorator_base.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import List - -from pyzeebe import TaskDecorator - - -class ZeebeDecoratorBase(object): - def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): - self._before: List[TaskDecorator] = before or [] - self._after: List[TaskDecorator] = after or [] - - def before(self, *decorators: TaskDecorator) -> None: - self._before.extend(decorators) - - def after(self, *decorators: TaskDecorator) -> None: - self._after.extend(decorators) diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index b49650cb..f1b84fb3 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -2,8 +2,10 @@ from dataclasses import dataclass, field from typing import List, Optional -from pyzeebe import ExceptionHandler, TaskDecorator, Job from pyzeebe.exceptions import NoVariableNameGiven +from pyzeebe.job.job import Job +from pyzeebe.task.exception_handler import ExceptionHandler +from pyzeebe.task.types import TaskDecorator logger = logging.getLogger(__name__) diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index eeefce7f..b25f0049 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -2,7 +2,6 @@ from typing import Tuple, List, Callable, Union from pyzeebe import TaskDecorator -from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType from pyzeebe.task import task_builder from pyzeebe.task.task import Task @@ -11,14 +10,15 @@ logger = logging.getLogger(__name__) -class ZeebeTaskRouter(ZeebeDecoratorBase): +class ZeebeTaskRouter: def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): """ Args: before (List[TaskDecorator]): Decorators to be performed before each task after (List[TaskDecorator]): Decorators to be performed after each task """ - super().__init__(before, after) + self._before: List[TaskDecorator] = before or [] + self._after: List[TaskDecorator] = after or [] self.tasks: List[Task] = [] def task(self, task_config: Union[TaskConfig, str]): @@ -61,6 +61,12 @@ def _is_task_duplicate(self, task_type: str) -> None: except TaskNotFound: return + def before(self, *decorators: TaskDecorator) -> None: + self._before.extend(decorators) + + def after(self, *decorators: TaskDecorator) -> None: + self._after.extend(decorators) + def remove_task(self, task_type: str) -> Task: """ Remove a task diff --git a/tests/unit/decorators/__init__.py b/tests/unit/decorators/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/decorators/zeebe_decorator_base_test.py b/tests/unit/decorators/zeebe_decorator_base_test.py deleted file mode 100644 index 304ed4cd..00000000 --- a/tests/unit/decorators/zeebe_decorator_base_test.py +++ /dev/null @@ -1,50 +0,0 @@ -from pyzeebe.decorators.zeebe_decorator_base import ZeebeDecoratorBase -from tests.unit.utils.random_utils import random_job - - -def test_add_before(): - base_decorator = ZeebeDecoratorBase() - base_decorator.before(lambda x: x) - assert len(base_decorator._before) == 1 - - -def test_add_after(): - base_decorator = ZeebeDecoratorBase() - base_decorator.after(lambda x: x) - assert len(base_decorator._after) == 1 - - -def test_add_before_plus_constructor(): - def constructor_decorator(x): - return x - - def function_decorator(x): - return x - - job = random_job() - - assert constructor_decorator(job) == job - assert function_decorator(job) == job - - base_decorator = ZeebeDecoratorBase(before=[constructor_decorator]) - base_decorator.before(function_decorator) - assert len(base_decorator._before) == 2 - assert base_decorator._before == [constructor_decorator, function_decorator] - - -def test_add_after_plus_constructor(): - def constructor_decorator(x): - return x - - def function_decorator(x): - return x - - job = random_job() - - assert constructor_decorator(job) == job - assert function_decorator(job) == job - - base_decorator = ZeebeDecoratorBase(after=[constructor_decorator]) - base_decorator.after(function_decorator) - assert len(base_decorator._after) == 2 - assert base_decorator._after == [constructor_decorator, function_decorator] diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 7adb2432..ba513e76 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -2,6 +2,7 @@ import pytest +from pyzeebe import TaskDecorator from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType from pyzeebe.task.task import Task from pyzeebe.worker.task_router import ZeebeTaskRouter @@ -72,3 +73,27 @@ def test_check_is_task_duplicate_with_duplicate(router: ZeebeTaskRouter, task: T def test_check_is_task_duplicate_no_duplicate(router: ZeebeTaskRouter, task: Task): router.tasks.append(task) + + +def test_add_before_decorator(router: ZeebeTaskRouter, decorator: TaskDecorator): + router.before(decorator) + + assert len(router._before) == 1 + + +def test_add_after_decorator(router: ZeebeTaskRouter, decorator: TaskDecorator): + router.after(decorator) + + assert len(router._after) == 1 + + +def test_add_before_decorator_through_constructor(decorator: TaskDecorator): + router = ZeebeTaskRouter(before=[decorator]) + + assert len(router._before) == 1 + + +def test_add_after_decorator_through_constructor(decorator: TaskDecorator): + router = ZeebeTaskRouter(after=[decorator]) + + assert len(router._after) == 1 From d02ae2badc6b396606d521eb0c67a7f62a625452 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 22:27:44 +0200 Subject: [PATCH 20/47] Refactor integration tests to use pytest fixtures and use new ZeebeWorker --- tests/integration/integration_test.py | 44 +++++++++++++++------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/tests/integration/integration_test.py b/tests/integration/integration_test.py index 224afd94..f8e0dc1c 100644 --- a/tests/integration/integration_test.py +++ b/tests/integration/integration_test.py @@ -4,60 +4,64 @@ import pytest -from pyzeebe import ZeebeWorker, ZeebeClient, Job +from pyzeebe import ZeebeClient, ZeebeWorker, Job, TaskConfig from pyzeebe.exceptions import WorkflowNotFound -zeebe_client: ZeebeClient -zeebe_worker = ZeebeWorker() +@pytest.fixture(scope="session") +def zeebe_client(): + return ZeebeClient() -def exception_handler(exc: Exception, job: Job) -> None: - job.set_error_status(f"Failed to run task {job.type}. Reason: {exc}") +@pytest.fixture(scope="session") +def zeebe_worker(): + worker = ZeebeWorker() -@zeebe_worker.task(task_type="test", exception_handler=exception_handler) -def task_handler(should_throw: bool, input: str) -> Dict: - if should_throw: - raise Exception("Error thrown") - else: - return {"output": input + str(uuid4())} + def exception_handler(exc: Exception, job: Job) -> None: + job.set_error_status(f"Failed to run task {job.type}. Reason: {exc}") + @worker.task(TaskConfig("test", exception_handler)) + def task_handler(should_throw: bool, input: str) -> Dict: + if should_throw: + raise Exception("Error thrown") + else: + return {"output": input + str(uuid4())} + + return worker -@pytest.fixture(scope="module", autouse=True) -def setup(): - global zeebe_client, task_handler +@pytest.fixture(scope="module", autouse=True) +def setup(zeebe_worker, zeebe_client): zeebe_worker.work(watch=True) - zeebe_client = ZeebeClient() try: integration_tests_path = os.path.join("tests", "integration") zeebe_client.deploy_workflow(os.path.join(integration_tests_path, "test.bpmn")) except FileNotFoundError: zeebe_client.deploy_workflow("test.bpmn") - yield zeebe_client + yield zeebe_worker.stop(wait=True) assert not zeebe_worker._watcher_thread.is_alive() -def test_run_workflow(): +def test_run_workflow(zeebe_client: ZeebeClient): workflow_key = zeebe_client.run_workflow("test", {"input": str(uuid4()), "should_throw": False}) assert isinstance(workflow_key, int) -def test_non_existent_workflow(): +def test_non_existent_workflow(zeebe_client: ZeebeClient): with pytest.raises(WorkflowNotFound): zeebe_client.run_workflow(str(uuid4())) -def test_run_workflow_with_result(): +def test_run_workflow_with_result(zeebe_client: ZeebeClient): input = str(uuid4()) output = zeebe_client.run_workflow_with_result("test", {"input": input, "should_throw": False}) assert isinstance(output["output"], str) assert output["output"].startswith(input) -def test_cancel_workflow(): +def test_cancel_workflow(zeebe_client: ZeebeClient): workflow_key = zeebe_client.run_workflow("test", {"input": str(uuid4()), "should_throw": False}) zeebe_client.cancel_workflow_instance(workflow_key) From 579055d45a9faf3e914bc89a6f6fed38ea38aecd Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 21 Feb 2021 22:47:56 +0200 Subject: [PATCH 21/47] Use inspect to get function parameters This allows for parameters named args or kwargs that are not *args or **kwargs --- pyzeebe/task/task_builder.py | 13 ++++++------- tests/unit/task/task_builder_test.py | 6 ++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index fa42429b..1699a615 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -1,3 +1,4 @@ +import inspect import logging from typing import List, Callable, Dict, Tuple @@ -68,10 +69,8 @@ def inner_fn(*args, **kwargs): def get_parameters_from_function(fn: Callable) -> List[str]: - parameters = fn.__code__.co_varnames - if "args" in parameters: - return [] - elif "kwargs" in parameters: - return [] - else: - return list(parameters) + function_signature = inspect.signature(fn) + for parameter_name, parameter in function_signature.parameters.items(): + if parameter.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + return [] + return list(function_signature.parameters) diff --git a/tests/unit/task/task_builder_test.py b/tests/unit/task/task_builder_test.py index 086c40ef..79ce03ac 100644 --- a/tests/unit/task/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -182,3 +182,9 @@ def kwargs_func(**kwargs): pass assert task_builder.get_parameters_from_function(kwargs_func) == [] + + def test_get_standard_named_params(self): + def func(args, kwargs): + pass + + assert task_builder.get_parameters_from_function(func) == ["args", "kwargs"] From 67710f166084e372280e09da7c3dd138b3287ce5 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Tue, 23 Feb 2021 09:56:02 +0200 Subject: [PATCH 22/47] Import Callable from typing instead of collections --- tests/unit/task/task_builder_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/task/task_builder_test.py b/tests/unit/task/task_builder_test.py index 79ce03ac..b962e28d 100644 --- a/tests/unit/task/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -1,4 +1,4 @@ -from collections import Callable +from typing import Callable from pyzeebe import Job, TaskDecorator from pyzeebe.task import task_builder From 409d1142edd0df6fad72b23e9a7f88cd23ef582d Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 11:41:47 +0200 Subject: [PATCH 23/47] Remove usage of dataclasses The dataclasses library is not supported in python 3.6 --- pyzeebe/task/task.py | 9 ++++----- pyzeebe/task/task_config.py | 29 +++++++++++++++-------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pyzeebe/task/task.py b/pyzeebe/task/task.py index d0075f5d..913ac3a1 100644 --- a/pyzeebe/task/task.py +++ b/pyzeebe/task/task.py @@ -1,15 +1,14 @@ -from dataclasses import dataclass from typing import Callable from pyzeebe.task.task_config import TaskConfig from pyzeebe.task.types import JobHandler -@dataclass class Task: - original_function: Callable - job_handler: JobHandler - config: TaskConfig + def __init__(self, original_function: Callable, job_handler: JobHandler, config: TaskConfig): + self.original_function = original_function + self.job_handler = job_handler + self.config = config @property def type(self): diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index f1b84fb3..c7b16ec4 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -1,5 +1,4 @@ import logging -from dataclasses import dataclass, field from typing import List, Optional from pyzeebe.exceptions import NoVariableNameGiven @@ -15,18 +14,20 @@ def default_exception_handler(e: Exception, job: Job) -> None: job.set_failure_status(f"Failed job. Error: {e}") -@dataclass class TaskConfig: - type: str - exception_handler: ExceptionHandler = default_exception_handler - timeout: int = 10000 - max_jobs_to_activate: int = 32 - variables_to_fetch: Optional[List[str]] = None - single_value: bool = False - variable_name: Optional[str] = None - before: List[TaskDecorator] = field(default_factory=list) - after: List[TaskDecorator] = field(default_factory=list) + def __init__(self, type: str, exception_handler: ExceptionHandler = default_exception_handler, + timeout: int = 10000, max_jobs_to_activate: int = 32, variables_to_fetch: Optional[List[str]] = None, + single_value: bool = False, variable_name: Optional[str] = None, before: List[TaskDecorator] = None, + after: List[TaskDecorator] = None): + if single_value and not variable_name: + raise NoVariableNameGiven(type) - def __post_init__(self): - if self.single_value and not self.variable_name: - raise NoVariableNameGiven(self.type) + self.type = type + self.exception_handler = exception_handler + self.timeout = timeout + self.max_jobs_to_activate = max_jobs_to_activate + self.variables_to_fetch = variables_to_fetch + self.single_value = single_value + self.variable_name = variable_name + self.before = before or [] + self.after = after or [] From 37f1431181ed5dd599ed1cc4d863ecffa814be09 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 11:43:42 +0200 Subject: [PATCH 24/47] Rename NoVariableNameGiven to NoVariableNameGivenError --- docs/exceptions.rst | 2 +- pyzeebe/exceptions/pyzeebe_exceptions.py | 2 +- pyzeebe/task/task_config.py | 4 ++-- tests/unit/task/task_config_test.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/exceptions.rst b/docs/exceptions.rst index eb4cbfc4..03ad6425 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -8,7 +8,7 @@ All ``pyzeebe`` exceptions inherit from :py:class:`PyZeebeException` .. autoexception:: pyzeebe.exceptions.TaskNotFound -.. autoexception:: pyzeebe.exceptions.NoVariableNameGiven +.. autoexception:: pyzeebe.exceptions.NoVariableNameGivenError .. autoexception:: pyzeebe.exceptions.NoZeebeAdapter diff --git a/pyzeebe/exceptions/pyzeebe_exceptions.py b/pyzeebe/exceptions/pyzeebe_exceptions.py index ecc25afb..fcd0dc69 100644 --- a/pyzeebe/exceptions/pyzeebe_exceptions.py +++ b/pyzeebe/exceptions/pyzeebe_exceptions.py @@ -6,7 +6,7 @@ class TaskNotFound(PyZeebeException): pass -class NoVariableNameGiven(PyZeebeException): +class NoVariableNameGivenError(PyZeebeException): def __init__(self, task_type: str): super().__init__(f"No variable name given for single_value task {task_type}") self.task_type = task_type diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index c7b16ec4..2b7196f3 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -1,7 +1,7 @@ import logging from typing import List, Optional -from pyzeebe.exceptions import NoVariableNameGiven +from pyzeebe.exceptions import NoVariableNameGivenError from pyzeebe.job.job import Job from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.types import TaskDecorator @@ -20,7 +20,7 @@ def __init__(self, type: str, exception_handler: ExceptionHandler = default_exce single_value: bool = False, variable_name: Optional[str] = None, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): if single_value and not variable_name: - raise NoVariableNameGiven(type) + raise NoVariableNameGivenError(type) self.type = type self.exception_handler = exception_handler diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py index e4813d5a..bff43109 100644 --- a/tests/unit/task/task_config_test.py +++ b/tests/unit/task/task_config_test.py @@ -4,12 +4,12 @@ import pytest from pyzeebe import Job -from pyzeebe.exceptions import NoVariableNameGiven +from pyzeebe.exceptions import NoVariableNameGivenError from pyzeebe.task.task_config import TaskConfig, default_exception_handler def test_add_non_dict_task_without_variable_name(): - with pytest.raises(NoVariableNameGiven): + with pytest.raises(NoVariableNameGivenError): TaskConfig(str(uuid4()), single_value=True) From 6a35d71ecf5fe393f2c9eacf6b21c35e6a77c18a Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 11:44:41 +0200 Subject: [PATCH 25/47] Get index with 1 instead of -1 --- pyzeebe/worker/task_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index b25f0049..408e0608 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -101,7 +101,7 @@ def get_task(self, task_type: str) -> Task: return self._get_task_and_index(task_type)[0] def _get_task_index(self, task_type: str) -> int: - return self._get_task_and_index(task_type)[-1] + return self._get_task_and_index(task_type)[1] def _get_task_and_index(self, task_type: str) -> Tuple[Task, int]: for index, task in enumerate(self.tasks): From 17978418a8f4e6bec4b8a4fb7d1aa5087d989718 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 11:47:06 +0200 Subject: [PATCH 26/47] Change test_remove_task_from_many to have at least one task --- tests/unit/worker/task_router_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index ba513e76..7048ba49 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -50,7 +50,7 @@ def test_remove_task(router: ZeebeTaskRouter, task: Task): def test_remove_task_from_many(router: ZeebeTaskRouter, task: Task): router.tasks.append(task) - for i in range(0, randint(0, 100)): + for _ in range(1, randint(0, 100)): @router.task(str(uuid4())) def dummy_function(): pass From f7e7987a02554fe98c1317483e22faa4f1000b5a Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 11:48:16 +0200 Subject: [PATCH 27/47] Rename TaskNotFound to TaskNotFoundError --- docs/exceptions.rst | 2 +- pyzeebe/exceptions/pyzeebe_exceptions.py | 2 +- pyzeebe/worker/task_router.py | 10 +++++----- tests/unit/worker/task_router_test.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/exceptions.rst b/docs/exceptions.rst index 03ad6425..3009eacf 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -6,7 +6,7 @@ All ``pyzeebe`` exceptions inherit from :py:class:`PyZeebeException` .. autoexception:: pyzeebe.exceptions.PyZeebeException -.. autoexception:: pyzeebe.exceptions.TaskNotFound +.. autoexception:: pyzeebe.exceptions.TaskNotFoundError .. autoexception:: pyzeebe.exceptions.NoVariableNameGivenError diff --git a/pyzeebe/exceptions/pyzeebe_exceptions.py b/pyzeebe/exceptions/pyzeebe_exceptions.py index fcd0dc69..166d65d6 100644 --- a/pyzeebe/exceptions/pyzeebe_exceptions.py +++ b/pyzeebe/exceptions/pyzeebe_exceptions.py @@ -2,7 +2,7 @@ class PyZeebeException(Exception): pass -class TaskNotFound(PyZeebeException): +class TaskNotFoundError(PyZeebeException): pass diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 408e0608..7018b4cb 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -2,7 +2,7 @@ from typing import Tuple, List, Callable, Union from pyzeebe import TaskDecorator -from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType +from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskType from pyzeebe.task import task_builder from pyzeebe.task.task import Task from pyzeebe.task.task_config import TaskConfig @@ -58,7 +58,7 @@ def _is_task_duplicate(self, task_type: str) -> None: try: self.get_task(task_type) raise DuplicateTaskType(task_type) - except TaskNotFound: + except TaskNotFoundError: return def before(self, *decorators: TaskDecorator) -> None: @@ -78,7 +78,7 @@ def remove_task(self, task_type: str) -> Task: Task: The task that was removed Raises: - TaskNotFound: If no task with specified type exists + TaskNotFoundError: If no task with specified type exists """ task_index = self._get_task_index(task_type) @@ -95,7 +95,7 @@ def get_task(self, task_type: str) -> Task: Task: The wanted task Raises: - TaskNotFound: If no task with specified type exists + TaskNotFoundError: If no task with specified type exists """ return self._get_task_and_index(task_type)[0] @@ -107,4 +107,4 @@ def _get_task_and_index(self, task_type: str) -> Tuple[Task, int]: for index, task in enumerate(self.tasks): if task.type == task_type: return task, index - raise TaskNotFound(f"Could not find task {task_type}") + raise TaskNotFoundError(f"Could not find task {task_type}") diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 7048ba49..58e0da87 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -3,7 +3,7 @@ import pytest from pyzeebe import TaskDecorator -from pyzeebe.exceptions import TaskNotFound, DuplicateTaskType +from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskType from pyzeebe.task.task import Task from pyzeebe.worker.task_router import ZeebeTaskRouter from tests.unit.utils.random_utils import randint @@ -18,7 +18,7 @@ def test_get_task(router: ZeebeTaskRouter, task: Task): def test_get_fake_task(router: ZeebeTaskRouter): - with pytest.raises(TaskNotFound): + with pytest.raises(TaskNotFoundError): router.get_task(str(uuid4())) @@ -61,7 +61,7 @@ def dummy_function(): def test_remove_fake_task(router: ZeebeTaskRouter): - with pytest.raises(TaskNotFound): + with pytest.raises(TaskNotFoundError): router.remove_task(str(uuid4())) From a154a85708446963eeb1c5117b5871b6db2b505c Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 11:55:00 +0200 Subject: [PATCH 28/47] Rename all pyzeebe exceptions to end with Error --- docs/exceptions.rst | 34 ++++++++--------- pyzeebe/client/client.py | 38 +++++++++---------- .../credentials/camunda_cloud_credentials.py | 6 +-- pyzeebe/credentials/oauth_credentials.py | 4 +- pyzeebe/exceptions/credentials_exceptions.py | 4 +- pyzeebe/exceptions/job_exceptions.py | 6 +-- pyzeebe/exceptions/message_exceptions.py | 2 +- pyzeebe/exceptions/pyzeebe_exceptions.py | 5 ++- pyzeebe/exceptions/workflow_exceptions.py | 10 ++--- pyzeebe/exceptions/zeebe_exceptions.py | 4 +- pyzeebe/grpc_internals/zeebe_adapter_base.py | 6 +-- pyzeebe/grpc_internals/zeebe_job_adapter.py | 16 ++++---- .../grpc_internals/zeebe_message_adapter.py | 4 +- .../grpc_internals/zeebe_workflow_adapter.py | 14 +++---- pyzeebe/job/job.py | 8 ++-- pyzeebe/worker/task_router.py | 4 +- tests/integration/integration_test.py | 4 +- tests/unit/client/client_test.py | 6 +-- .../camunda_cloud_credentials_test.py | 6 +-- .../credentials/oauth_credentials_test.py | 4 +- .../grpc_internals/zeebe_adapter_base_test.py | 8 ++-- .../grpc_internals/zeebe_job_adapter_test.py | 22 +++++------ .../zeebe_message_adapter_test.py | 4 +- .../zeebe_workflow_adapter_test.py | 14 +++---- tests/unit/job/job_test.py | 8 ++-- tests/unit/worker/task_router_test.py | 4 +- tests/unit/worker/worker_test.py | 4 +- 27 files changed, 125 insertions(+), 124 deletions(-) diff --git a/docs/exceptions.rst b/docs/exceptions.rst index 3009eacf..838c9176 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -2,42 +2,42 @@ Exceptions ========== -All ``pyzeebe`` exceptions inherit from :py:class:`PyZeebeException` +All ``pyzeebe`` exceptions inherit from :py:class:`PyZeebeError` -.. autoexception:: pyzeebe.exceptions.PyZeebeException +.. autoexception:: pyzeebe.exceptions.PyZeebeError .. autoexception:: pyzeebe.exceptions.TaskNotFoundError .. autoexception:: pyzeebe.exceptions.NoVariableNameGivenError -.. autoexception:: pyzeebe.exceptions.NoZeebeAdapter +.. autoexception:: pyzeebe.exceptions.NoZeebeAdapterError -.. autoexception:: pyzeebe.exceptions.DuplicateTaskType +.. autoexception:: pyzeebe.exceptions.DuplicateTaskTypeError -.. autoexception:: pyzeebe.exceptions.ActivateJobsRequestInvalid +.. autoexception:: pyzeebe.exceptions.ActivateJobsRequestInvalidError -.. autoexception:: pyzeebe.exceptions.JobAlreadyDeactivated +.. autoexception:: pyzeebe.exceptions.JobAlreadyDeactivatedError -.. autoexception:: pyzeebe.exceptions.JobNotFound +.. autoexception:: pyzeebe.exceptions.JobNotFoundError -.. autoexception:: pyzeebe.exceptions.MessageAlreadyExists +.. autoexception:: pyzeebe.exceptions.MessageAlreadyExistsError -.. autoexception:: pyzeebe.exceptions.WorkflowNotFound +.. autoexception:: pyzeebe.exceptions.WorkflowNotFoundError -.. autoexception:: pyzeebe.exceptions.WorkflowInstanceNotFound +.. autoexception:: pyzeebe.exceptions.WorkflowInstanceNotFoundError -.. autoexception:: pyzeebe.exceptions.WorkflowHasNoStartEvent +.. autoexception:: pyzeebe.exceptions.WorkflowHasNoStartEventError -.. autoexception:: pyzeebe.exceptions.WorkflowInvalid +.. autoexception:: pyzeebe.exceptions.WorkflowInvalidError -.. autoexception:: pyzeebe.exceptions.InvalidJSON +.. autoexception:: pyzeebe.exceptions.InvalidJSONError -.. autoexception:: pyzeebe.exceptions.ZeebeBackPressure +.. autoexception:: pyzeebe.exceptions.ZeebeBackPressureError -.. autoexception:: pyzeebe.exceptions.ZeebeGatewayUnavailable +.. autoexception:: pyzeebe.exceptions.ZeebeGatewayUnavailableError .. autoexception:: pyzeebe.exceptions.ZeebeInternalError -.. autoexception:: pyzeebe.exceptions.InvalidOAuthCredentials +.. autoexception:: pyzeebe.exceptions.InvalidOAuthCredentialsError -.. autoexception:: pyzeebe.exceptions.InvalidCamundaCloudCredentials +.. autoexception:: pyzeebe.exceptions.InvalidCamundaCloudCredentialsError diff --git a/pyzeebe/client/client.py b/pyzeebe/client/client.py index f9b2b7dc..ee31fde0 100644 --- a/pyzeebe/client/client.py +++ b/pyzeebe/client/client.py @@ -35,11 +35,11 @@ def run_workflow(self, bpmn_process_id: str, variables: Dict = None, version: in int: workflow_instance_key, the unique id of the running workflow generated by Zeebe. Raises: - WorkflowNotFound: No workflow with bpmn_process_id exists - InvalidJSON: variables is not JSONable - WorkflowHasNoStartEvent: The specified workflow does not have a start event - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + WorkflowNotFoundError: No workflow with bpmn_process_id exists + InvalidJSONError: variables is not JSONable + WorkflowHasNoStartEventError: The specified workflow does not have a start event + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ @@ -62,11 +62,11 @@ def run_workflow_with_result(self, bpmn_process_id: str, variables: Dict = None, dict: A dictionary of the end state of the workflow instance Raises: - WorkflowNotFound: No workflow with bpmn_process_id exists - InvalidJSON: variables is not JSONable - WorkflowHasNoStartEvent: The specified workflow does not have a start event - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + WorkflowNotFoundError: No workflow with bpmn_process_id exists + InvalidJSONError: variables is not JSONable + WorkflowHasNoStartEventError: The specified workflow does not have a start event + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ @@ -86,9 +86,9 @@ def cancel_workflow_instance(self, workflow_instance_key: int) -> int: int: The workflow_instance_key Raises: - WorkflowInstanceNotFound: If no workflow instance with workflow_instance_key exists - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + WorkflowInstanceNotFoundError: If no workflow instance with workflow_instance_key exists + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ @@ -103,9 +103,9 @@ def deploy_workflow(self, *workflow_file_path: str) -> None: workflow_file_path (str): The file path to a workflow definition file (bpmn/yaml) Raises: - WorkflowInvalid: If one of the workflow file definitions is invalid - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + WorkflowInvalidError: If one of the workflow file definitions is invalid + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ @@ -125,9 +125,9 @@ def publish_message(self, name: str, correlation_key: str, variables: Dict = Non active, a MessageAlreadyExists will be raised. Raises: - MessageAlreadyExist: If a message with message_id already exists - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + MessageAlreadyExistError: If a message with message_id already exists + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ diff --git a/pyzeebe/credentials/camunda_cloud_credentials.py b/pyzeebe/credentials/camunda_cloud_credentials.py index 3758b2c8..3a16fdcc 100644 --- a/pyzeebe/credentials/camunda_cloud_credentials.py +++ b/pyzeebe/credentials/camunda_cloud_credentials.py @@ -1,5 +1,5 @@ from pyzeebe.credentials.oauth_credentials import OAuthCredentials -from pyzeebe.exceptions import InvalidOAuthCredentials, InvalidCamundaCloudCredentials +from pyzeebe.exceptions import InvalidOAuthCredentialsError, InvalidCamundaCloudCredentialsError class CamundaCloudCredentials(OAuthCredentials): @@ -7,8 +7,8 @@ def __init__(self, client_id: str, client_secret: str, cluster_id: str): try: super().__init__(url="https://login.cloud.camunda.io/oauth/token", client_id=client_id, client_secret=client_secret, audience=f"{cluster_id}.zeebe.camunda.io") - except InvalidOAuthCredentials: - raise InvalidCamundaCloudCredentials(client_id=client_id, cluster_id=cluster_id) + except InvalidOAuthCredentialsError: + raise InvalidCamundaCloudCredentialsError(client_id=client_id, cluster_id=cluster_id) def get_connection_uri(self) -> str: return f"{self.audience}:443" diff --git a/pyzeebe/credentials/oauth_credentials.py b/pyzeebe/credentials/oauth_credentials.py index 48374635..68b6bd9e 100644 --- a/pyzeebe/credentials/oauth_credentials.py +++ b/pyzeebe/credentials/oauth_credentials.py @@ -4,7 +4,7 @@ from requests_oauthlib import OAuth2Session from pyzeebe.credentials.base_credentials import BaseCredentials -from pyzeebe.exceptions import InvalidOAuthCredentials +from pyzeebe.exceptions import InvalidOAuthCredentialsError class OAuthCredentials(BaseCredentials): @@ -34,7 +34,7 @@ def get_access_token(url: str, client_id: str, client_secret: str, audience: str response.raise_for_status() return response.json()["access_token"] except HTTPError: - raise InvalidOAuthCredentials(url=url, client_id=client_id, audience=audience) + raise InvalidOAuthCredentialsError(url=url, client_id=client_id, audience=audience) def get_connection_uri(self) -> str: return None diff --git a/pyzeebe/exceptions/credentials_exceptions.py b/pyzeebe/exceptions/credentials_exceptions.py index 5fe5a1ab..ba763432 100644 --- a/pyzeebe/exceptions/credentials_exceptions.py +++ b/pyzeebe/exceptions/credentials_exceptions.py @@ -1,12 +1,12 @@ from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException -class InvalidOAuthCredentials(PyZeebeException): +class InvalidOAuthCredentialsError(PyZeebeException): def __init__(self, url: str, client_id: str, audience: str): super().__init__( f"Invalid OAuth credentials supplied for {url} with audience {audience} and client id {client_id}") -class InvalidCamundaCloudCredentials(PyZeebeException): +class InvalidCamundaCloudCredentialsError(PyZeebeException): def __init__(self, client_id: str, cluster_id: str): super().__init__(f"Invalid credentials supplied for cluster {cluster_id} with client {client_id}") diff --git a/pyzeebe/exceptions/job_exceptions.py b/pyzeebe/exceptions/job_exceptions.py index feb27a42..4b9eda39 100644 --- a/pyzeebe/exceptions/job_exceptions.py +++ b/pyzeebe/exceptions/job_exceptions.py @@ -1,7 +1,7 @@ from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException -class ActivateJobsRequestInvalid(PyZeebeException): +class ActivateJobsRequestInvalidError(PyZeebeException): def __init__(self, task_type: str, worker: str, timeout: int, max_jobs_to_activate: int): msg = "Failed to activate jobs. Reasons:" if task_type == "" or task_type is None: @@ -16,13 +16,13 @@ def __init__(self, task_type: str, worker: str, timeout: int, max_jobs_to_activa super().__init__(msg) -class JobAlreadyDeactivated(PyZeebeException): +class JobAlreadyDeactivatedError(PyZeebeException): def __init__(self, job_key: int): super().__init__(f"Job {job_key} was already stopped (Completed/Failed/Error)") self.job_key = job_key -class JobNotFound(PyZeebeException): +class JobNotFoundError(PyZeebeException): def __init__(self, job_key: int): super().__init__(f"Job {job_key} not found") self.job_key = job_key diff --git a/pyzeebe/exceptions/message_exceptions.py b/pyzeebe/exceptions/message_exceptions.py index d93c5068..d204bd67 100644 --- a/pyzeebe/exceptions/message_exceptions.py +++ b/pyzeebe/exceptions/message_exceptions.py @@ -1,5 +1,5 @@ from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException -class MessageAlreadyExists(PyZeebeException): +class MessageAlreadyExistsError(PyZeebeException): pass diff --git a/pyzeebe/exceptions/pyzeebe_exceptions.py b/pyzeebe/exceptions/pyzeebe_exceptions.py index 166d65d6..a43c105e 100644 --- a/pyzeebe/exceptions/pyzeebe_exceptions.py +++ b/pyzeebe/exceptions/pyzeebe_exceptions.py @@ -12,14 +12,15 @@ def __init__(self, task_type: str): self.task_type = task_type -class NoZeebeAdapter(PyZeebeException): +class NoZeebeAdapterError(PyZeebeException): pass -class DuplicateTaskType(PyZeebeException): +class DuplicateTaskTypeError(PyZeebeException): def __init__(self, task_type: str): super().__init__(f"Task with type {task_type} already exists") self.task_type = task_type + class MaxConsecutiveTaskThreadError(PyZeebeException): pass diff --git a/pyzeebe/exceptions/workflow_exceptions.py b/pyzeebe/exceptions/workflow_exceptions.py index 688e6cfd..502717d1 100644 --- a/pyzeebe/exceptions/workflow_exceptions.py +++ b/pyzeebe/exceptions/workflow_exceptions.py @@ -1,7 +1,7 @@ from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException -class WorkflowNotFound(PyZeebeException): +class WorkflowNotFoundError(PyZeebeException): def __init__(self, bpmn_process_id: str, version: int): super().__init__( f"Workflow definition: {bpmn_process_id} with {version} was not found") @@ -9,21 +9,21 @@ def __init__(self, bpmn_process_id: str, version: int): self.version = version -class WorkflowInstanceNotFound(PyZeebeException): +class WorkflowInstanceNotFoundError(PyZeebeException): def __init__(self, workflow_instance_key: int): super().__init__(f"Workflow instance key: {workflow_instance_key} was not found") self.workflow_instance_key = workflow_instance_key -class WorkflowHasNoStartEvent(PyZeebeException): +class WorkflowHasNoStartEventError(PyZeebeException): def __init__(self, bpmn_process_id: str): super().__init__(f"Workflow {bpmn_process_id} has no start event that can be called manually") self.bpmn_process_id = bpmn_process_id -class WorkflowInvalid(PyZeebeException): +class WorkflowInvalidError(PyZeebeException): pass -class InvalidJSON(PyZeebeException): +class InvalidJSONError(PyZeebeException): pass diff --git a/pyzeebe/exceptions/zeebe_exceptions.py b/pyzeebe/exceptions/zeebe_exceptions.py index 7fe37a3d..3b9b39db 100644 --- a/pyzeebe/exceptions/zeebe_exceptions.py +++ b/pyzeebe/exceptions/zeebe_exceptions.py @@ -1,11 +1,11 @@ from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException -class ZeebeBackPressure(PyZeebeException): +class ZeebeBackPressureError(PyZeebeException): pass -class ZeebeGatewayUnavailable(PyZeebeException): +class ZeebeGatewayUnavailableError(PyZeebeException): pass diff --git a/pyzeebe/grpc_internals/zeebe_adapter_base.py b/pyzeebe/grpc_internals/zeebe_adapter_base.py index 28eb6ef2..df5ac638 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_base.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_base.py @@ -5,7 +5,7 @@ from zeebe_grpc.gateway_pb2_grpc import GatewayStub from pyzeebe.credentials.base_credentials import BaseCredentials -from pyzeebe.exceptions import ZeebeBackPressure, ZeebeGatewayUnavailable, ZeebeInternalError +from pyzeebe.exceptions import ZeebeBackPressureError, ZeebeGatewayUnavailableError, ZeebeInternalError logger = logging.getLogger(__name__) @@ -81,12 +81,12 @@ def _should_retry(self): def _common_zeebe_grpc_errors(self, rpc_error: grpc.RpcError): if self.is_error_status(rpc_error, grpc.StatusCode.RESOURCE_EXHAUSTED): - raise ZeebeBackPressure() + raise ZeebeBackPressureError() elif self.is_error_status(rpc_error, grpc.StatusCode.UNAVAILABLE): self._current_connection_retries += 1 if not self._should_retry(): self._close() - raise ZeebeGatewayUnavailable() + raise ZeebeGatewayUnavailableError() elif self.is_error_status(rpc_error, grpc.StatusCode.INTERNAL): self._current_connection_retries += 1 if not self._should_retry(): diff --git a/pyzeebe/grpc_internals/zeebe_job_adapter.py b/pyzeebe/grpc_internals/zeebe_job_adapter.py index 3436ff11..6a30bdc4 100644 --- a/pyzeebe/grpc_internals/zeebe_job_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_job_adapter.py @@ -6,7 +6,7 @@ from zeebe_grpc.gateway_pb2 import ActivateJobsRequest, CompleteJobRequest, CompleteJobResponse, FailJobRequest, \ FailJobResponse, ThrowErrorRequest, ThrowErrorResponse -from pyzeebe.exceptions import ActivateJobsRequestInvalid, JobAlreadyDeactivated, JobNotFound +from pyzeebe.exceptions import ActivateJobsRequestInvalidError, JobAlreadyDeactivatedError, JobNotFoundError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase from pyzeebe.job.job import Job @@ -27,7 +27,7 @@ def activate_jobs(self, task_type: str, worker: str, timeout: int, max_jobs_to_a yield job except grpc.RpcError as rpc_error: if self.is_error_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): - raise ActivateJobsRequestInvalid(task_type, worker, timeout, max_jobs_to_activate) + raise ActivateJobsRequestInvalidError(task_type, worker, timeout, max_jobs_to_activate) else: self._common_zeebe_grpc_errors(rpc_error) @@ -51,9 +51,9 @@ def complete_job(self, job_key: int, variables: Dict) -> CompleteJobResponse: return self._gateway_stub.CompleteJob(CompleteJobRequest(jobKey=job_key, variables=json.dumps(variables))) except grpc.RpcError as rpc_error: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise JobNotFound(job_key=job_key) + raise JobNotFoundError(job_key=job_key) elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise JobAlreadyDeactivated(job_key=job_key) + raise JobAlreadyDeactivatedError(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -62,9 +62,9 @@ def fail_job(self, job_key: int, message: str) -> FailJobResponse: return self._gateway_stub.FailJob(FailJobRequest(jobKey=job_key, errorMessage=message)) except grpc.RpcError as rpc_error: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise JobNotFound(job_key=job_key) + raise JobNotFoundError(job_key=job_key) elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise JobAlreadyDeactivated(job_key=job_key) + raise JobAlreadyDeactivatedError(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -74,8 +74,8 @@ def throw_error(self, job_key: int, message: str) -> ThrowErrorResponse: ThrowErrorRequest(jobKey=job_key, errorMessage=message)) except grpc.RpcError as rpc_error: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise JobNotFound(job_key=job_key) + raise JobNotFoundError(job_key=job_key) elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise JobAlreadyDeactivated(job_key=job_key) + raise JobAlreadyDeactivatedError(job_key=job_key) else: self._common_zeebe_grpc_errors(rpc_error) diff --git a/pyzeebe/grpc_internals/zeebe_message_adapter.py b/pyzeebe/grpc_internals/zeebe_message_adapter.py index 01d9a75e..8c4cc72f 100644 --- a/pyzeebe/grpc_internals/zeebe_message_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_message_adapter.py @@ -4,7 +4,7 @@ import grpc from zeebe_grpc.gateway_pb2 import PublishMessageRequest, PublishMessageResponse -from pyzeebe.exceptions import MessageAlreadyExists +from pyzeebe.exceptions import MessageAlreadyExistsError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase @@ -17,6 +17,6 @@ def publish_message(self, name: str, correlation_key: str, time_to_live_in_milli timeToLive=time_to_live_in_milliseconds, variables=json.dumps(variables))) except grpc.RpcError as rpc_error: if self.is_error_status(rpc_error, grpc.StatusCode.ALREADY_EXISTS): - raise MessageAlreadyExists() + raise MessageAlreadyExistsError() else: self._common_zeebe_grpc_errors(rpc_error) diff --git a/pyzeebe/grpc_internals/zeebe_workflow_adapter.py b/pyzeebe/grpc_internals/zeebe_workflow_adapter.py index ce5c9020..2dbc5d60 100644 --- a/pyzeebe/grpc_internals/zeebe_workflow_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_workflow_adapter.py @@ -6,8 +6,8 @@ from zeebe_grpc.gateway_pb2 import CreateWorkflowInstanceRequest, CreateWorkflowInstanceWithResultRequest, \ CancelWorkflowInstanceRequest, WorkflowRequestObject, DeployWorkflowRequest, DeployWorkflowResponse -from pyzeebe.exceptions import InvalidJSON, WorkflowNotFound, WorkflowInstanceNotFound, WorkflowHasNoStartEvent, \ - WorkflowInvalid +from pyzeebe.exceptions import InvalidJSONError, WorkflowNotFoundError, WorkflowInstanceNotFoundError, WorkflowHasNoStartEventError, \ + WorkflowInvalidError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase @@ -36,12 +36,12 @@ def create_workflow_instance_with_result(self, bpmn_process_id: str, version: in def _create_workflow_errors(self, rpc_error: grpc.RpcError, bpmn_process_id: str, version: int, variables: Dict) -> None: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise WorkflowNotFound(bpmn_process_id=bpmn_process_id, version=version) + raise WorkflowNotFoundError(bpmn_process_id=bpmn_process_id, version=version) elif self.is_error_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): - raise InvalidJSON( + raise InvalidJSONError( f"Cannot start workflow: {bpmn_process_id} with version {version}. Variables: {variables}") elif self.is_error_status(rpc_error, grpc.StatusCode.FAILED_PRECONDITION): - raise WorkflowHasNoStartEvent(bpmn_process_id=bpmn_process_id) + raise WorkflowHasNoStartEventError(bpmn_process_id=bpmn_process_id) else: self._common_zeebe_grpc_errors(rpc_error) @@ -51,7 +51,7 @@ def cancel_workflow_instance(self, workflow_instance_key: int) -> None: CancelWorkflowInstanceRequest(workflowInstanceKey=workflow_instance_key)) except grpc.RpcError as rpc_error: if self.is_error_status(rpc_error, grpc.StatusCode.NOT_FOUND): - raise WorkflowInstanceNotFound(workflow_instance_key=workflow_instance_key) + raise WorkflowInstanceNotFoundError(workflow_instance_key=workflow_instance_key) else: self._common_zeebe_grpc_errors(rpc_error) @@ -61,7 +61,7 @@ def deploy_workflow(self, *workflow_file_path: str) -> DeployWorkflowResponse: DeployWorkflowRequest(workflows=map(self._get_workflow_request_object, workflow_file_path))) except grpc.RpcError as rpc_error: if self.is_error_status(rpc_error, grpc.StatusCode.INVALID_ARGUMENT): - raise WorkflowInvalid() + raise WorkflowInvalidError() else: self._common_zeebe_grpc_errors(rpc_error) diff --git a/pyzeebe/job/job.py b/pyzeebe/job/job.py index 741eddd1..4fd293e7 100644 --- a/pyzeebe/job/job.py +++ b/pyzeebe/job/job.py @@ -1,6 +1,6 @@ from typing import Dict -from pyzeebe.exceptions import NoZeebeAdapter +from pyzeebe.exceptions import NoZeebeAdapterError from pyzeebe.job.job_status import JobStatus @@ -39,7 +39,7 @@ def set_success_status(self) -> None: if self.zeebe_adapter: self.zeebe_adapter.complete_job(job_key=self.key, variables=self.variables) else: - raise NoZeebeAdapter() + raise NoZeebeAdapterError() def set_failure_status(self, message: str) -> None: """ @@ -59,7 +59,7 @@ def set_failure_status(self, message: str) -> None: if self.zeebe_adapter: self.zeebe_adapter.fail_job(job_key=self.key, message=message) else: - raise NoZeebeAdapter() + raise NoZeebeAdapterError() def set_error_status(self, message: str) -> None: """ @@ -79,7 +79,7 @@ def set_error_status(self, message: str) -> None: if self.zeebe_adapter: self.zeebe_adapter.throw_error(job_key=self.key, message=message) else: - raise NoZeebeAdapter() + raise NoZeebeAdapterError() def __repr__(self): return str({"jobKey": self.key, "taskType": self.type, "workflowInstanceKey": self.workflow_instance_key, diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 7018b4cb..1f8fb02d 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -2,7 +2,7 @@ from typing import Tuple, List, Callable, Union from pyzeebe import TaskDecorator -from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskType +from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskTypeError from pyzeebe.task import task_builder from pyzeebe.task.task import Task from pyzeebe.task.task_config import TaskConfig @@ -57,7 +57,7 @@ def _add_decorators_to_config(self, config: TaskConfig) -> TaskConfig: def _is_task_duplicate(self, task_type: str) -> None: try: self.get_task(task_type) - raise DuplicateTaskType(task_type) + raise DuplicateTaskTypeError(task_type) except TaskNotFoundError: return diff --git a/tests/integration/integration_test.py b/tests/integration/integration_test.py index f8e0dc1c..f869d9b3 100644 --- a/tests/integration/integration_test.py +++ b/tests/integration/integration_test.py @@ -5,7 +5,7 @@ import pytest from pyzeebe import ZeebeClient, ZeebeWorker, Job, TaskConfig -from pyzeebe.exceptions import WorkflowNotFound +from pyzeebe.exceptions import WorkflowNotFoundError @pytest.fixture(scope="session") @@ -51,7 +51,7 @@ def test_run_workflow(zeebe_client: ZeebeClient): def test_non_existent_workflow(zeebe_client: ZeebeClient): - with pytest.raises(WorkflowNotFound): + with pytest.raises(WorkflowNotFoundError): zeebe_client.run_workflow(str(uuid4())) diff --git a/tests/unit/client/client_test.py b/tests/unit/client/client_test.py index 97da6353..c20e7873 100644 --- a/tests/unit/client/client_test.py +++ b/tests/unit/client/client_test.py @@ -4,7 +4,7 @@ import pytest -from pyzeebe.exceptions import WorkflowNotFound +from pyzeebe.exceptions import WorkflowNotFoundError def test_run_workflow(zeebe_client, grpc_servicer): @@ -29,12 +29,12 @@ def test_deploy_workflow(zeebe_client): def test_run_non_existent_workflow(zeebe_client): - with pytest.raises(WorkflowNotFound): + with pytest.raises(WorkflowNotFoundError): zeebe_client.run_workflow(bpmn_process_id=str(uuid4())) def test_run_non_existent_workflow_with_result(zeebe_client): - with pytest.raises(WorkflowNotFound): + with pytest.raises(WorkflowNotFoundError): zeebe_client.run_workflow_with_result(bpmn_process_id=str(uuid4())) diff --git a/tests/unit/credentials/camunda_cloud_credentials_test.py b/tests/unit/credentials/camunda_cloud_credentials_test.py index 51d104c5..e79bb610 100644 --- a/tests/unit/credentials/camunda_cloud_credentials_test.py +++ b/tests/unit/credentials/camunda_cloud_credentials_test.py @@ -4,7 +4,7 @@ import pytest from pyzeebe.credentials.camunda_cloud_credentials import CamundaCloudCredentials -from pyzeebe.exceptions import InvalidOAuthCredentials, InvalidCamundaCloudCredentials +from pyzeebe.exceptions import InvalidOAuthCredentialsError, InvalidCamundaCloudCredentialsError def test_init(): @@ -20,7 +20,7 @@ def test_init(): def test_invalid_credentials(): CamundaCloudCredentials.get_access_token = MagicMock( - side_effect=InvalidOAuthCredentials(str(uuid4()), str(uuid4()), str(uuid4()))) + side_effect=InvalidOAuthCredentialsError(str(uuid4()), str(uuid4()), str(uuid4()))) - with pytest.raises(InvalidCamundaCloudCredentials): + with pytest.raises(InvalidCamundaCloudCredentialsError): CamundaCloudCredentials(str(uuid4()), str(uuid4()), str(uuid4())) diff --git a/tests/unit/credentials/oauth_credentials_test.py b/tests/unit/credentials/oauth_credentials_test.py index 23b33bfb..e940facb 100644 --- a/tests/unit/credentials/oauth_credentials_test.py +++ b/tests/unit/credentials/oauth_credentials_test.py @@ -5,7 +5,7 @@ from requests import HTTPError from pyzeebe.credentials.oauth_credentials import OAuthCredentials -from pyzeebe.exceptions import InvalidOAuthCredentials +from pyzeebe.exceptions import InvalidOAuthCredentialsError def test_get_access_token(): @@ -27,6 +27,6 @@ def test_get_invalid_access_token(): with patch("requests_oauthlib.OAuth2Session.post") as post_mock: post_mock.side_effect = HTTPError() - with pytest.raises(InvalidOAuthCredentials): + with pytest.raises(InvalidOAuthCredentialsError): OAuthCredentials.get_access_token(url=f"https://{str(uuid4())}/oauth/token", client_id=str(uuid4()), client_secret=str(uuid4()), audience=str(uuid4())) diff --git a/tests/unit/grpc_internals/zeebe_adapter_base_test.py b/tests/unit/grpc_internals/zeebe_adapter_base_test.py index e5c5b0ec..364dc43d 100644 --- a/tests/unit/grpc_internals/zeebe_adapter_base_test.py +++ b/tests/unit/grpc_internals/zeebe_adapter_base_test.py @@ -7,7 +7,7 @@ from pyzeebe.credentials.camunda_cloud_credentials import CamundaCloudCredentials from pyzeebe.credentials.oauth_credentials import OAuthCredentials -from pyzeebe.exceptions import ZeebeBackPressure, ZeebeGatewayUnavailable, ZeebeInternalError +from pyzeebe.exceptions import ZeebeBackPressureError, ZeebeGatewayUnavailableError, ZeebeInternalError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase from tests.unit.utils.grpc_utils import GRPCStatusCode from tests.unit.utils.random_utils import RANDOM_RANGE @@ -143,14 +143,14 @@ def test_common_zeebe_grpc_error_internal(zeebe_adapter): def test_common_zeebe_grpc_error_back_pressure(zeebe_adapter): error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.RESOURCE_EXHAUSTED) - with pytest.raises(ZeebeBackPressure): + with pytest.raises(ZeebeBackPressureError): zeebe_adapter._common_zeebe_grpc_errors(error) def test_common_zeebe_grpc_error_gateway_unavailable(zeebe_adapter): error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.UNAVAILABLE) - with pytest.raises(ZeebeGatewayUnavailable): + with pytest.raises(ZeebeGatewayUnavailableError): zeebe_adapter._common_zeebe_grpc_errors(error) @@ -166,7 +166,7 @@ def test_close_after_retried_unavailable(zeebe_adapter): error._state = GRPCStatusCode(grpc.StatusCode.UNAVAILABLE) zeebe_adapter._close = MagicMock() zeebe_adapter._max_connection_retries = 1 - with pytest.raises(ZeebeGatewayUnavailable): + with pytest.raises(ZeebeGatewayUnavailableError): zeebe_adapter._common_zeebe_grpc_errors(error) zeebe_adapter._close.assert_called_once() diff --git a/tests/unit/grpc_internals/zeebe_job_adapter_test.py b/tests/unit/grpc_internals/zeebe_job_adapter_test.py index fcd770c9..dda7f03c 100644 --- a/tests/unit/grpc_internals/zeebe_job_adapter_test.py +++ b/tests/unit/grpc_internals/zeebe_job_adapter_test.py @@ -6,7 +6,7 @@ import pytest from zeebe_grpc.gateway_pb2 import * -from pyzeebe.exceptions import ActivateJobsRequestInvalid, JobAlreadyDeactivated, JobNotFound +from pyzeebe.exceptions import ActivateJobsRequestInvalidError, JobAlreadyDeactivatedError, JobNotFoundError from pyzeebe.job.job import Job from pyzeebe.task.task import Task from tests.unit.utils.grpc_utils import GRPCStatusCode @@ -38,26 +38,26 @@ def test_activate_jobs(zeebe_adapter, grpc_servicer, task): def test_activate_jobs_invalid_worker(zeebe_adapter): - with pytest.raises(ActivateJobsRequestInvalid): + with pytest.raises(ActivateJobsRequestInvalidError): next(zeebe_adapter.activate_jobs(task_type=str(uuid4()), worker=None, timeout=randint(10, 100), request_timeout=100, max_jobs_to_activate=1, variables_to_fetch=[])) def test_activate_jobs_invalid_job_timeout(zeebe_adapter): - with pytest.raises(ActivateJobsRequestInvalid): + with pytest.raises(ActivateJobsRequestInvalidError): next(zeebe_adapter.activate_jobs(task_type=str(uuid4()), worker=str(uuid4()), timeout=0, request_timeout=100, max_jobs_to_activate=1, variables_to_fetch=[])) def test_activate_jobs_invalid_task_type(zeebe_adapter): - with pytest.raises(ActivateJobsRequestInvalid): + with pytest.raises(ActivateJobsRequestInvalidError): next(zeebe_adapter.activate_jobs(task_type=None, worker=str(uuid4()), timeout=randint(10, 100), request_timeout=100, max_jobs_to_activate=1, variables_to_fetch=[])) def test_activate_jobs_invalid_max_jobs(zeebe_adapter): - with pytest.raises(ActivateJobsRequestInvalid): + with pytest.raises(ActivateJobsRequestInvalidError): next(zeebe_adapter.activate_jobs(task_type=str(uuid4()), worker=str(uuid4()), timeout=randint(10, 100), request_timeout=100, max_jobs_to_activate=0, variables_to_fetch=[])) @@ -82,13 +82,13 @@ def test_complete_job(zeebe_adapter, first_active_job: Job): def test_complete_job_not_found(zeebe_adapter): - with pytest.raises(JobNotFound): + with pytest.raises(JobNotFoundError): zeebe_adapter.complete_job(job_key=randint(0, RANDOM_RANGE), variables={}) def test_complete_job_already_completed(zeebe_adapter, first_active_job: Job): zeebe_adapter.complete_job(job_key=first_active_job.key, variables={}) - with pytest.raises(JobAlreadyDeactivated): + with pytest.raises(JobAlreadyDeactivatedError): zeebe_adapter.complete_job(job_key=first_active_job.key, variables={}) @@ -110,13 +110,13 @@ def test_fail_job(zeebe_adapter, first_active_job: Job): def test_fail_job_not_found(zeebe_adapter): - with pytest.raises(JobNotFound): + with pytest.raises(JobNotFoundError): zeebe_adapter.fail_job(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) def test_fail_job_already_failed(zeebe_adapter, first_active_job: Job): zeebe_adapter.fail_job(job_key=first_active_job.key, message=str(uuid4())) - with pytest.raises(JobAlreadyDeactivated): + with pytest.raises(JobAlreadyDeactivatedError): zeebe_adapter.fail_job(job_key=first_active_job.key, message=str(uuid4())) @@ -138,13 +138,13 @@ def test_throw_error(zeebe_adapter, first_active_job: Job): def test_throw_error_job_not_found(zeebe_adapter): - with pytest.raises(JobNotFound): + with pytest.raises(JobNotFoundError): zeebe_adapter.throw_error(job_key=randint(0, RANDOM_RANGE), message=str(uuid4())) def test_throw_error_already_thrown(zeebe_adapter, first_active_job: Job): zeebe_adapter.throw_error(job_key=first_active_job.key, message=str(uuid4())) - with pytest.raises(JobAlreadyDeactivated): + with pytest.raises(JobAlreadyDeactivatedError): zeebe_adapter.throw_error(job_key=first_active_job.key, message=str(uuid4())) diff --git a/tests/unit/grpc_internals/zeebe_message_adapter_test.py b/tests/unit/grpc_internals/zeebe_message_adapter_test.py index 43ccb03e..b257f59a 100644 --- a/tests/unit/grpc_internals/zeebe_message_adapter_test.py +++ b/tests/unit/grpc_internals/zeebe_message_adapter_test.py @@ -6,7 +6,7 @@ import pytest from zeebe_grpc.gateway_pb2 import * -from pyzeebe.exceptions import MessageAlreadyExists +from pyzeebe.exceptions import MessageAlreadyExistsError from tests.unit.utils.grpc_utils import GRPCStatusCode from tests.unit.utils.random_utils import RANDOM_RANGE @@ -37,7 +37,7 @@ def test_punlish_message_invalid_time_to_live(zeebe_adapter): def test_publish_message_already_exists(zeebe_adapter): message_id = str(uuid4()) - with pytest.raises(MessageAlreadyExists): + with pytest.raises(MessageAlreadyExistsError): zeebe_adapter.publish_message(message_id=message_id, name=str(uuid4()), variables={}, correlation_key=str(uuid4()), time_to_live_in_milliseconds=randint(0, RANDOM_RANGE)) diff --git a/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py b/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py index 8c7ecbca..34ac188c 100644 --- a/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py +++ b/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py @@ -6,8 +6,8 @@ import grpc import pytest -from pyzeebe.exceptions import InvalidJSON, WorkflowNotFound, WorkflowInstanceNotFound, WorkflowHasNoStartEvent, \ - WorkflowInvalid +from pyzeebe.exceptions import InvalidJSONError, WorkflowNotFoundError, WorkflowInstanceNotFoundError, WorkflowHasNoStartEventError, \ + WorkflowInvalidError from tests.unit.utils.grpc_utils import GRPCStatusCode from tests.unit.utils.random_utils import RANDOM_RANGE @@ -75,7 +75,7 @@ def test_cancel_workflow_instance_already_cancelled(zeebe_adapter): zeebe_adapter._gateway_stub.CancelWorkflowInstance = MagicMock(side_effect=error) - with pytest.raises(WorkflowInstanceNotFound): + with pytest.raises(WorkflowInstanceNotFoundError): zeebe_adapter.cancel_workflow_instance(workflow_instance_key=randint(0, RANDOM_RANGE)) @@ -100,7 +100,7 @@ def test_deploy_workflow_workflow_invalid(zeebe_adapter): zeebe_adapter._gateway_stub.DeployWorkflow = MagicMock(side_effect=error) - with pytest.raises(WorkflowInvalid): + with pytest.raises(WorkflowInvalidError): zeebe_adapter.deploy_workflow() @@ -130,21 +130,21 @@ def test_get_workflow_request_object(zeebe_adapter): def test_create_workflow_errors_not_found(zeebe_adapter): error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.NOT_FOUND) - with pytest.raises(WorkflowNotFound): + with pytest.raises(WorkflowNotFoundError): zeebe_adapter._create_workflow_errors(error, str(uuid4()), randint(0, 10, ), {}) def test_create_workflow_errors_invalid_json(zeebe_adapter): error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.INVALID_ARGUMENT) - with pytest.raises(InvalidJSON): + with pytest.raises(InvalidJSONError): zeebe_adapter._create_workflow_errors(error, str(uuid4()), randint(0, 10, ), {}) def test_create_workflow_errors_workflow_has_no_start_event(zeebe_adapter): error = grpc.RpcError() error._state = GRPCStatusCode(grpc.StatusCode.FAILED_PRECONDITION) - with pytest.raises(WorkflowHasNoStartEvent): + with pytest.raises(WorkflowHasNoStartEventError): zeebe_adapter._create_workflow_errors(error, str(uuid4()), randint(0, 10, ), {}) diff --git a/tests/unit/job/job_test.py b/tests/unit/job/job_test.py index 49a5daed..09f0abbb 100644 --- a/tests/unit/job/job_test.py +++ b/tests/unit/job/job_test.py @@ -3,7 +3,7 @@ import pytest -from pyzeebe.exceptions import NoZeebeAdapter +from pyzeebe.exceptions import NoZeebeAdapterError def test_success(job_with_adapter): @@ -13,7 +13,7 @@ def test_success(job_with_adapter): def test_success_no_zeebe_adapter(job_without_adapter): - with pytest.raises(NoZeebeAdapter): + with pytest.raises(NoZeebeAdapterError): job_without_adapter.set_success_status() @@ -25,7 +25,7 @@ def test_error(job_with_adapter): def test_error_no_zeebe_adapter(job_without_adapter): - with pytest.raises(NoZeebeAdapter): + with pytest.raises(NoZeebeAdapterError): message = str(uuid4()) job_without_adapter.set_error_status(message) @@ -38,6 +38,6 @@ def test_failure(job_with_adapter): def test_failure_no_zeebe_adapter(job_without_adapter): - with pytest.raises(NoZeebeAdapter): + with pytest.raises(NoZeebeAdapterError): message = str(uuid4()) job_without_adapter.set_failure_status(message) diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 58e0da87..3b24c943 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -3,7 +3,7 @@ import pytest from pyzeebe import TaskDecorator -from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskType +from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskTypeError from pyzeebe.task.task import Task from pyzeebe.worker.task_router import ZeebeTaskRouter from tests.unit.utils.random_utils import randint @@ -67,7 +67,7 @@ def test_remove_fake_task(router: ZeebeTaskRouter): def test_check_is_task_duplicate_with_duplicate(router: ZeebeTaskRouter, task: Task): router.tasks.append(task) - with pytest.raises(DuplicateTaskType): + with pytest.raises(DuplicateTaskTypeError): router._is_task_duplicate(task.type) diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index ad8c347a..0c86d0c6 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -7,7 +7,7 @@ import pytest from pyzeebe import TaskDecorator, ZeebeTaskRouter -from pyzeebe.exceptions import DuplicateTaskType, MaxConsecutiveTaskThreadError +from pyzeebe.exceptions import DuplicateTaskTypeError, MaxConsecutiveTaskThreadError from pyzeebe.job.job import Job from pyzeebe.task.task import Task from pyzeebe.worker.worker import ZeebeWorker @@ -21,7 +21,7 @@ def test_add_task(self, zeebe_worker: ZeebeWorker, task: Task): def test_raises_on_duplicate(self, zeebe_worker: ZeebeWorker, task: Task): zeebe_worker._add_task(task) - with pytest.raises(DuplicateTaskType): + with pytest.raises(DuplicateTaskTypeError): zeebe_worker._add_task(task) def test_only_one_task_added(self, zeebe_worker: ZeebeWorker): From 5010b12a812d3aa5b46bb64609a47844c4f52372 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 11:59:56 +0200 Subject: [PATCH 29/47] Rename all exceptions files to errors --- docs/exceptions.rst | 40 +++++++++---------- pyzeebe/__init__.py | 2 +- .../credentials/camunda_cloud_credentials.py | 2 +- pyzeebe/credentials/oauth_credentials.py | 2 +- pyzeebe/errors/__init__.py | 6 +++ .../credentials_errors.py} | 6 +-- .../job_errors.py} | 8 ++-- pyzeebe/errors/message_errors.py | 5 +++ .../pyzeebe_errors.py} | 12 +++--- .../workflow_errors.py} | 12 +++--- pyzeebe/errors/zeebe_errors.py | 13 ++++++ pyzeebe/exceptions/__init__.py | 6 --- pyzeebe/exceptions/message_exceptions.py | 5 --- pyzeebe/exceptions/zeebe_exceptions.py | 13 ------ pyzeebe/grpc_internals/zeebe_adapter_base.py | 2 +- pyzeebe/grpc_internals/zeebe_job_adapter.py | 2 +- .../grpc_internals/zeebe_message_adapter.py | 2 +- .../grpc_internals/zeebe_workflow_adapter.py | 2 +- pyzeebe/job/job.py | 20 +++++----- pyzeebe/task/task_config.py | 2 +- pyzeebe/worker/task_router.py | 4 +- pyzeebe/worker/worker.py | 10 ++--- tests/integration/integration_test.py | 2 +- tests/unit/client/client_test.py | 2 +- .../camunda_cloud_credentials_test.py | 2 +- .../credentials/oauth_credentials_test.py | 2 +- .../grpc_internals/zeebe_adapter_base_test.py | 2 +- .../grpc_internals/zeebe_job_adapter_test.py | 2 +- .../zeebe_message_adapter_test.py | 2 +- .../zeebe_workflow_adapter_test.py | 2 +- tests/unit/job/job_test.py | 2 +- tests/unit/task/task_config_test.py | 2 +- tests/unit/worker/task_router_test.py | 2 +- tests/unit/worker/worker_test.py | 2 +- 34 files changed, 100 insertions(+), 100 deletions(-) create mode 100644 pyzeebe/errors/__init__.py rename pyzeebe/{exceptions/credentials_exceptions.py => errors/credentials_errors.py} (67%) rename pyzeebe/{exceptions/job_exceptions.py => errors/job_errors.py} (79%) create mode 100644 pyzeebe/errors/message_errors.py rename pyzeebe/{exceptions/pyzeebe_exceptions.py => errors/pyzeebe_errors.py} (56%) rename pyzeebe/{exceptions/workflow_exceptions.py => errors/workflow_errors.py} (68%) create mode 100644 pyzeebe/errors/zeebe_errors.py delete mode 100644 pyzeebe/exceptions/__init__.py delete mode 100644 pyzeebe/exceptions/message_exceptions.py delete mode 100644 pyzeebe/exceptions/zeebe_exceptions.py diff --git a/docs/exceptions.rst b/docs/exceptions.rst index 838c9176..3f74518d 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -1,43 +1,43 @@ ========== -Exceptions +errors ========== All ``pyzeebe`` exceptions inherit from :py:class:`PyZeebeError` -.. autoexception:: pyzeebe.exceptions.PyZeebeError +.. autoexception:: pyzeebe.errors.PyZeebeError -.. autoexception:: pyzeebe.exceptions.TaskNotFoundError +.. autoexception:: pyzeebe.errors.TaskNotFoundError -.. autoexception:: pyzeebe.exceptions.NoVariableNameGivenError +.. autoexception:: pyzeebe.errors.NoVariableNameGivenError -.. autoexception:: pyzeebe.exceptions.NoZeebeAdapterError +.. autoexception:: pyzeebe.errors.NoZeebeAdapterError -.. autoexception:: pyzeebe.exceptions.DuplicateTaskTypeError +.. autoexception:: pyzeebe.errors.DuplicateTaskTypeError -.. autoexception:: pyzeebe.exceptions.ActivateJobsRequestInvalidError +.. autoexception:: pyzeebe.errors.ActivateJobsRequestInvalidError -.. autoexception:: pyzeebe.exceptions.JobAlreadyDeactivatedError +.. autoexception:: pyzeebe.errors.JobAlreadyDeactivatedError -.. autoexception:: pyzeebe.exceptions.JobNotFoundError +.. autoexception:: pyzeebe.errors.JobNotFoundError -.. autoexception:: pyzeebe.exceptions.MessageAlreadyExistsError +.. autoexception:: pyzeebe.errors.MessageAlreadyExistsError -.. autoexception:: pyzeebe.exceptions.WorkflowNotFoundError +.. autoexception:: pyzeebe.errors.WorkflowNotFoundError -.. autoexception:: pyzeebe.exceptions.WorkflowInstanceNotFoundError +.. autoexception:: pyzeebe.errors.WorkflowInstanceNotFoundError -.. autoexception:: pyzeebe.exceptions.WorkflowHasNoStartEventError +.. autoexception:: pyzeebe.errors.WorkflowHasNoStartEventError -.. autoexception:: pyzeebe.exceptions.WorkflowInvalidError +.. autoexception:: pyzeebe.errors.WorkflowInvalidError -.. autoexception:: pyzeebe.exceptions.InvalidJSONError +.. autoexception:: pyzeebe.errors.InvalidJSONError -.. autoexception:: pyzeebe.exceptions.ZeebeBackPressureError +.. autoexception:: pyzeebe.errors.ZeebeBackPressureError -.. autoexception:: pyzeebe.exceptions.ZeebeGatewayUnavailableError +.. autoexception:: pyzeebe.errors.ZeebeGatewayUnavailableError -.. autoexception:: pyzeebe.exceptions.ZeebeInternalError +.. autoexception:: pyzeebe.errors.ZeebeInternalError -.. autoexception:: pyzeebe.exceptions.InvalidOAuthCredentialsError +.. autoexception:: pyzeebe.errors.InvalidOAuthCredentialsError -.. autoexception:: pyzeebe.exceptions.InvalidCamundaCloudCredentialsError +.. autoexception:: pyzeebe.errors.InvalidCamundaCloudCredentialsError diff --git a/pyzeebe/__init__.py b/pyzeebe/__init__.py index ed7580b1..36167fd3 100644 --- a/pyzeebe/__init__.py +++ b/pyzeebe/__init__.py @@ -1,6 +1,6 @@ __version__ = "2.2.3" -from pyzeebe import exceptions +from pyzeebe import errors from pyzeebe.client.client import ZeebeClient from pyzeebe.credentials.camunda_cloud_credentials import CamundaCloudCredentials from pyzeebe.credentials.oauth_credentials import OAuthCredentials diff --git a/pyzeebe/credentials/camunda_cloud_credentials.py b/pyzeebe/credentials/camunda_cloud_credentials.py index 3a16fdcc..40e6477f 100644 --- a/pyzeebe/credentials/camunda_cloud_credentials.py +++ b/pyzeebe/credentials/camunda_cloud_credentials.py @@ -1,5 +1,5 @@ from pyzeebe.credentials.oauth_credentials import OAuthCredentials -from pyzeebe.exceptions import InvalidOAuthCredentialsError, InvalidCamundaCloudCredentialsError +from pyzeebe.errors import InvalidOAuthCredentialsError, InvalidCamundaCloudCredentialsError class CamundaCloudCredentials(OAuthCredentials): diff --git a/pyzeebe/credentials/oauth_credentials.py b/pyzeebe/credentials/oauth_credentials.py index 68b6bd9e..0c0985a7 100644 --- a/pyzeebe/credentials/oauth_credentials.py +++ b/pyzeebe/credentials/oauth_credentials.py @@ -4,7 +4,7 @@ from requests_oauthlib import OAuth2Session from pyzeebe.credentials.base_credentials import BaseCredentials -from pyzeebe.exceptions import InvalidOAuthCredentialsError +from pyzeebe.errors import InvalidOAuthCredentialsError class OAuthCredentials(BaseCredentials): diff --git a/pyzeebe/errors/__init__.py b/pyzeebe/errors/__init__.py new file mode 100644 index 00000000..dd348601 --- /dev/null +++ b/pyzeebe/errors/__init__.py @@ -0,0 +1,6 @@ +from .credentials_errors import * +from .job_errors import * +from .message_errors import * +from .pyzeebe_errors import * +from .workflow_errors import * +from .zeebe_errors import * diff --git a/pyzeebe/exceptions/credentials_exceptions.py b/pyzeebe/errors/credentials_errors.py similarity index 67% rename from pyzeebe/exceptions/credentials_exceptions.py rename to pyzeebe/errors/credentials_errors.py index ba763432..9c41e3c4 100644 --- a/pyzeebe/exceptions/credentials_exceptions.py +++ b/pyzeebe/errors/credentials_errors.py @@ -1,12 +1,12 @@ -from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException +from pyzeebe.errors.pyzeebe_errors import PyZeebeError -class InvalidOAuthCredentialsError(PyZeebeException): +class InvalidOAuthCredentialsError(PyZeebeError): def __init__(self, url: str, client_id: str, audience: str): super().__init__( f"Invalid OAuth credentials supplied for {url} with audience {audience} and client id {client_id}") -class InvalidCamundaCloudCredentialsError(PyZeebeException): +class InvalidCamundaCloudCredentialsError(PyZeebeError): def __init__(self, client_id: str, cluster_id: str): super().__init__(f"Invalid credentials supplied for cluster {cluster_id} with client {client_id}") diff --git a/pyzeebe/exceptions/job_exceptions.py b/pyzeebe/errors/job_errors.py similarity index 79% rename from pyzeebe/exceptions/job_exceptions.py rename to pyzeebe/errors/job_errors.py index 4b9eda39..849b430a 100644 --- a/pyzeebe/exceptions/job_exceptions.py +++ b/pyzeebe/errors/job_errors.py @@ -1,7 +1,7 @@ -from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException +from pyzeebe.errors.pyzeebe_errors import PyZeebeError -class ActivateJobsRequestInvalidError(PyZeebeException): +class ActivateJobsRequestInvalidError(PyZeebeError): def __init__(self, task_type: str, worker: str, timeout: int, max_jobs_to_activate: int): msg = "Failed to activate jobs. Reasons:" if task_type == "" or task_type is None: @@ -16,13 +16,13 @@ def __init__(self, task_type: str, worker: str, timeout: int, max_jobs_to_activa super().__init__(msg) -class JobAlreadyDeactivatedError(PyZeebeException): +class JobAlreadyDeactivatedError(PyZeebeError): def __init__(self, job_key: int): super().__init__(f"Job {job_key} was already stopped (Completed/Failed/Error)") self.job_key = job_key -class JobNotFoundError(PyZeebeException): +class JobNotFoundError(PyZeebeError): def __init__(self, job_key: int): super().__init__(f"Job {job_key} not found") self.job_key = job_key diff --git a/pyzeebe/errors/message_errors.py b/pyzeebe/errors/message_errors.py new file mode 100644 index 00000000..fc22a962 --- /dev/null +++ b/pyzeebe/errors/message_errors.py @@ -0,0 +1,5 @@ +from pyzeebe.errors.pyzeebe_errors import PyZeebeError + + +class MessageAlreadyExistsError(PyZeebeError): + pass diff --git a/pyzeebe/exceptions/pyzeebe_exceptions.py b/pyzeebe/errors/pyzeebe_errors.py similarity index 56% rename from pyzeebe/exceptions/pyzeebe_exceptions.py rename to pyzeebe/errors/pyzeebe_errors.py index a43c105e..89118fb1 100644 --- a/pyzeebe/exceptions/pyzeebe_exceptions.py +++ b/pyzeebe/errors/pyzeebe_errors.py @@ -1,26 +1,26 @@ -class PyZeebeException(Exception): +class PyZeebeError(Exception): pass -class TaskNotFoundError(PyZeebeException): +class TaskNotFoundError(PyZeebeError): pass -class NoVariableNameGivenError(PyZeebeException): +class NoVariableNameGivenError(PyZeebeError): def __init__(self, task_type: str): super().__init__(f"No variable name given for single_value task {task_type}") self.task_type = task_type -class NoZeebeAdapterError(PyZeebeException): +class NoZeebeAdapterError(PyZeebeError): pass -class DuplicateTaskTypeError(PyZeebeException): +class DuplicateTaskTypeError(PyZeebeError): def __init__(self, task_type: str): super().__init__(f"Task with type {task_type} already exists") self.task_type = task_type -class MaxConsecutiveTaskThreadError(PyZeebeException): +class MaxConsecutiveTaskThreadError(PyZeebeError): pass diff --git a/pyzeebe/exceptions/workflow_exceptions.py b/pyzeebe/errors/workflow_errors.py similarity index 68% rename from pyzeebe/exceptions/workflow_exceptions.py rename to pyzeebe/errors/workflow_errors.py index 502717d1..f6bb55fd 100644 --- a/pyzeebe/exceptions/workflow_exceptions.py +++ b/pyzeebe/errors/workflow_errors.py @@ -1,7 +1,7 @@ -from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException +from pyzeebe.errors.pyzeebe_errors import PyZeebeError -class WorkflowNotFoundError(PyZeebeException): +class WorkflowNotFoundError(PyZeebeError): def __init__(self, bpmn_process_id: str, version: int): super().__init__( f"Workflow definition: {bpmn_process_id} with {version} was not found") @@ -9,21 +9,21 @@ def __init__(self, bpmn_process_id: str, version: int): self.version = version -class WorkflowInstanceNotFoundError(PyZeebeException): +class WorkflowInstanceNotFoundError(PyZeebeError): def __init__(self, workflow_instance_key: int): super().__init__(f"Workflow instance key: {workflow_instance_key} was not found") self.workflow_instance_key = workflow_instance_key -class WorkflowHasNoStartEventError(PyZeebeException): +class WorkflowHasNoStartEventError(PyZeebeError): def __init__(self, bpmn_process_id: str): super().__init__(f"Workflow {bpmn_process_id} has no start event that can be called manually") self.bpmn_process_id = bpmn_process_id -class WorkflowInvalidError(PyZeebeException): +class WorkflowInvalidError(PyZeebeError): pass -class InvalidJSONError(PyZeebeException): +class InvalidJSONError(PyZeebeError): pass diff --git a/pyzeebe/errors/zeebe_errors.py b/pyzeebe/errors/zeebe_errors.py new file mode 100644 index 00000000..e2bf3cc5 --- /dev/null +++ b/pyzeebe/errors/zeebe_errors.py @@ -0,0 +1,13 @@ +from pyzeebe.errors.pyzeebe_errors import PyZeebeError + + +class ZeebeBackPressureError(PyZeebeError): + pass + + +class ZeebeGatewayUnavailableError(PyZeebeError): + pass + + +class ZeebeInternalError(PyZeebeError): + pass diff --git a/pyzeebe/exceptions/__init__.py b/pyzeebe/exceptions/__init__.py deleted file mode 100644 index 7e2ff3a4..00000000 --- a/pyzeebe/exceptions/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .credentials_exceptions import * -from .job_exceptions import * -from .message_exceptions import * -from .pyzeebe_exceptions import * -from .workflow_exceptions import * -from .zeebe_exceptions import * diff --git a/pyzeebe/exceptions/message_exceptions.py b/pyzeebe/exceptions/message_exceptions.py deleted file mode 100644 index d204bd67..00000000 --- a/pyzeebe/exceptions/message_exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException - - -class MessageAlreadyExistsError(PyZeebeException): - pass diff --git a/pyzeebe/exceptions/zeebe_exceptions.py b/pyzeebe/exceptions/zeebe_exceptions.py deleted file mode 100644 index 3b9b39db..00000000 --- a/pyzeebe/exceptions/zeebe_exceptions.py +++ /dev/null @@ -1,13 +0,0 @@ -from pyzeebe.exceptions.pyzeebe_exceptions import PyZeebeException - - -class ZeebeBackPressureError(PyZeebeException): - pass - - -class ZeebeGatewayUnavailableError(PyZeebeException): - pass - - -class ZeebeInternalError(PyZeebeException): - pass diff --git a/pyzeebe/grpc_internals/zeebe_adapter_base.py b/pyzeebe/grpc_internals/zeebe_adapter_base.py index df5ac638..db79b3f9 100644 --- a/pyzeebe/grpc_internals/zeebe_adapter_base.py +++ b/pyzeebe/grpc_internals/zeebe_adapter_base.py @@ -5,7 +5,7 @@ from zeebe_grpc.gateway_pb2_grpc import GatewayStub from pyzeebe.credentials.base_credentials import BaseCredentials -from pyzeebe.exceptions import ZeebeBackPressureError, ZeebeGatewayUnavailableError, ZeebeInternalError +from pyzeebe.errors import ZeebeBackPressureError, ZeebeGatewayUnavailableError, ZeebeInternalError logger = logging.getLogger(__name__) diff --git a/pyzeebe/grpc_internals/zeebe_job_adapter.py b/pyzeebe/grpc_internals/zeebe_job_adapter.py index 6a30bdc4..86b29f8a 100644 --- a/pyzeebe/grpc_internals/zeebe_job_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_job_adapter.py @@ -6,7 +6,7 @@ from zeebe_grpc.gateway_pb2 import ActivateJobsRequest, CompleteJobRequest, CompleteJobResponse, FailJobRequest, \ FailJobResponse, ThrowErrorRequest, ThrowErrorResponse -from pyzeebe.exceptions import ActivateJobsRequestInvalidError, JobAlreadyDeactivatedError, JobNotFoundError +from pyzeebe.errors import ActivateJobsRequestInvalidError, JobAlreadyDeactivatedError, JobNotFoundError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase from pyzeebe.job.job import Job diff --git a/pyzeebe/grpc_internals/zeebe_message_adapter.py b/pyzeebe/grpc_internals/zeebe_message_adapter.py index 8c4cc72f..8bf16aee 100644 --- a/pyzeebe/grpc_internals/zeebe_message_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_message_adapter.py @@ -4,7 +4,7 @@ import grpc from zeebe_grpc.gateway_pb2 import PublishMessageRequest, PublishMessageResponse -from pyzeebe.exceptions import MessageAlreadyExistsError +from pyzeebe.errors import MessageAlreadyExistsError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase diff --git a/pyzeebe/grpc_internals/zeebe_workflow_adapter.py b/pyzeebe/grpc_internals/zeebe_workflow_adapter.py index 2dbc5d60..9eb0f996 100644 --- a/pyzeebe/grpc_internals/zeebe_workflow_adapter.py +++ b/pyzeebe/grpc_internals/zeebe_workflow_adapter.py @@ -6,7 +6,7 @@ from zeebe_grpc.gateway_pb2 import CreateWorkflowInstanceRequest, CreateWorkflowInstanceWithResultRequest, \ CancelWorkflowInstanceRequest, WorkflowRequestObject, DeployWorkflowRequest, DeployWorkflowResponse -from pyzeebe.exceptions import InvalidJSONError, WorkflowNotFoundError, WorkflowInstanceNotFoundError, WorkflowHasNoStartEventError, \ +from pyzeebe.errors import InvalidJSONError, WorkflowNotFoundError, WorkflowInstanceNotFoundError, WorkflowHasNoStartEventError, \ WorkflowInvalidError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase diff --git a/pyzeebe/job/job.py b/pyzeebe/job/job.py index 4fd293e7..c9490aab 100644 --- a/pyzeebe/job/job.py +++ b/pyzeebe/job/job.py @@ -1,6 +1,6 @@ from typing import Dict -from pyzeebe.exceptions import NoZeebeAdapterError +from pyzeebe.errors import NoZeebeAdapterError from pyzeebe.job.job_status import JobStatus @@ -30,9 +30,9 @@ def set_success_status(self) -> None: Success status means that the job has been completed as intended. Raises: - NoZeebeAdapter: If the job does not have a configured ZeebeAdapter - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + NoZeebeAdapterError: If the job does not have a configured ZeebeAdapter + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ @@ -50,9 +50,9 @@ def set_failure_status(self, message: str) -> None: message (str): The failure message that Zeebe will receive Raises: - NoZeebeAdapter: If the job does not have a configured ZeebeAdapter - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + NoZeebeAdapterError: If the job does not have a configured ZeebeAdapter + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ @@ -70,9 +70,9 @@ def set_error_status(self, message: str) -> None: message (str): The error message that Zeebe will receive Raises: - NoZeebeAdapter: If the job does not have a configured ZeebeAdapter - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + NoZeebeAdapterError: If the job does not have a configured ZeebeAdapter + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index 2b7196f3..efe06c07 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -1,7 +1,7 @@ import logging from typing import List, Optional -from pyzeebe.exceptions import NoVariableNameGivenError +from pyzeebe.errors import NoVariableNameGivenError from pyzeebe.job.job import Job from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.types import TaskDecorator diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 1f8fb02d..4a8d56e2 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -2,7 +2,7 @@ from typing import Tuple, List, Callable, Union from pyzeebe import TaskDecorator -from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskTypeError +from pyzeebe.errors import TaskNotFoundError, DuplicateTaskTypeError from pyzeebe.task import task_builder from pyzeebe.task.task import Task from pyzeebe.task.task_config import TaskConfig @@ -29,7 +29,7 @@ def task(self, task_config: Union[TaskConfig, str]): task_config (Union[str, TaskConfig]): Either the task type or a task configuration object Raises: - DuplicateTaskType: If a task from the router already exists in the worker + DuplicateTaskTypeError: If a task from the router already exists in the worker """ if isinstance(task_config, str): diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 9932f33f..928d4036 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -6,7 +6,7 @@ from pyzeebe import TaskDecorator from pyzeebe.credentials.base_credentials import BaseCredentials -from pyzeebe.exceptions.pyzeebe_exceptions import MaxConsecutiveTaskThreadError +from pyzeebe.errors.pyzeebe_errors import MaxConsecutiveTaskThreadError from pyzeebe.grpc_internals.zeebe_adapter import ZeebeAdapter from pyzeebe.job.job import Job from pyzeebe.task.task import Task @@ -51,9 +51,9 @@ def work(self, watch: bool = False) -> None: watch (bool): Start a watcher thread that restarts task threads on error Raises: - ActivateJobsRequestInvalid: If one of the worker's task has invalid types - ZeebeBackPressure: If Zeebe is currently in back pressure (too many requests) - ZeebeGatewayUnavailable: If the Zeebe gateway is unavailable + ActivateJobsRequestInvalidError: If one of the worker's task has invalid types + ZeebeBackPressureError: If Zeebe is currently in back pressure (too many requests) + ZeebeGatewayUnavailableError: If the Zeebe gateway is unavailable ZeebeInternalError: If Zeebe experiences an internal error """ @@ -185,7 +185,7 @@ def include_router(self, *routers: ZeebeTaskRouter) -> None: Adds all router's tasks to the worker. Raises: - DuplicateTaskType: If a task from the router already exists in the worker + DuplicateTaskTypeError: If a task from the router already exists in the worker """ for router in routers: diff --git a/tests/integration/integration_test.py b/tests/integration/integration_test.py index f869d9b3..bedf69dc 100644 --- a/tests/integration/integration_test.py +++ b/tests/integration/integration_test.py @@ -5,7 +5,7 @@ import pytest from pyzeebe import ZeebeClient, ZeebeWorker, Job, TaskConfig -from pyzeebe.exceptions import WorkflowNotFoundError +from pyzeebe.errors import WorkflowNotFoundError @pytest.fixture(scope="session") diff --git a/tests/unit/client/client_test.py b/tests/unit/client/client_test.py index c20e7873..f247da4a 100644 --- a/tests/unit/client/client_test.py +++ b/tests/unit/client/client_test.py @@ -4,7 +4,7 @@ import pytest -from pyzeebe.exceptions import WorkflowNotFoundError +from pyzeebe.errors import WorkflowNotFoundError def test_run_workflow(zeebe_client, grpc_servicer): diff --git a/tests/unit/credentials/camunda_cloud_credentials_test.py b/tests/unit/credentials/camunda_cloud_credentials_test.py index e79bb610..933d6b87 100644 --- a/tests/unit/credentials/camunda_cloud_credentials_test.py +++ b/tests/unit/credentials/camunda_cloud_credentials_test.py @@ -4,7 +4,7 @@ import pytest from pyzeebe.credentials.camunda_cloud_credentials import CamundaCloudCredentials -from pyzeebe.exceptions import InvalidOAuthCredentialsError, InvalidCamundaCloudCredentialsError +from pyzeebe.errors import InvalidOAuthCredentialsError, InvalidCamundaCloudCredentialsError def test_init(): diff --git a/tests/unit/credentials/oauth_credentials_test.py b/tests/unit/credentials/oauth_credentials_test.py index e940facb..c2ea1706 100644 --- a/tests/unit/credentials/oauth_credentials_test.py +++ b/tests/unit/credentials/oauth_credentials_test.py @@ -5,7 +5,7 @@ from requests import HTTPError from pyzeebe.credentials.oauth_credentials import OAuthCredentials -from pyzeebe.exceptions import InvalidOAuthCredentialsError +from pyzeebe.errors import InvalidOAuthCredentialsError def test_get_access_token(): diff --git a/tests/unit/grpc_internals/zeebe_adapter_base_test.py b/tests/unit/grpc_internals/zeebe_adapter_base_test.py index 364dc43d..bea811b5 100644 --- a/tests/unit/grpc_internals/zeebe_adapter_base_test.py +++ b/tests/unit/grpc_internals/zeebe_adapter_base_test.py @@ -7,7 +7,7 @@ from pyzeebe.credentials.camunda_cloud_credentials import CamundaCloudCredentials from pyzeebe.credentials.oauth_credentials import OAuthCredentials -from pyzeebe.exceptions import ZeebeBackPressureError, ZeebeGatewayUnavailableError, ZeebeInternalError +from pyzeebe.errors import ZeebeBackPressureError, ZeebeGatewayUnavailableError, ZeebeInternalError from pyzeebe.grpc_internals.zeebe_adapter_base import ZeebeAdapterBase from tests.unit.utils.grpc_utils import GRPCStatusCode from tests.unit.utils.random_utils import RANDOM_RANGE diff --git a/tests/unit/grpc_internals/zeebe_job_adapter_test.py b/tests/unit/grpc_internals/zeebe_job_adapter_test.py index dda7f03c..79418927 100644 --- a/tests/unit/grpc_internals/zeebe_job_adapter_test.py +++ b/tests/unit/grpc_internals/zeebe_job_adapter_test.py @@ -6,7 +6,7 @@ import pytest from zeebe_grpc.gateway_pb2 import * -from pyzeebe.exceptions import ActivateJobsRequestInvalidError, JobAlreadyDeactivatedError, JobNotFoundError +from pyzeebe.errors import ActivateJobsRequestInvalidError, JobAlreadyDeactivatedError, JobNotFoundError from pyzeebe.job.job import Job from pyzeebe.task.task import Task from tests.unit.utils.grpc_utils import GRPCStatusCode diff --git a/tests/unit/grpc_internals/zeebe_message_adapter_test.py b/tests/unit/grpc_internals/zeebe_message_adapter_test.py index b257f59a..8f4dab66 100644 --- a/tests/unit/grpc_internals/zeebe_message_adapter_test.py +++ b/tests/unit/grpc_internals/zeebe_message_adapter_test.py @@ -6,7 +6,7 @@ import pytest from zeebe_grpc.gateway_pb2 import * -from pyzeebe.exceptions import MessageAlreadyExistsError +from pyzeebe.errors import MessageAlreadyExistsError from tests.unit.utils.grpc_utils import GRPCStatusCode from tests.unit.utils.random_utils import RANDOM_RANGE diff --git a/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py b/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py index 34ac188c..d24493db 100644 --- a/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py +++ b/tests/unit/grpc_internals/zeebe_workflow_adapter_test.py @@ -6,7 +6,7 @@ import grpc import pytest -from pyzeebe.exceptions import InvalidJSONError, WorkflowNotFoundError, WorkflowInstanceNotFoundError, WorkflowHasNoStartEventError, \ +from pyzeebe.errors import InvalidJSONError, WorkflowNotFoundError, WorkflowInstanceNotFoundError, WorkflowHasNoStartEventError, \ WorkflowInvalidError from tests.unit.utils.grpc_utils import GRPCStatusCode from tests.unit.utils.random_utils import RANDOM_RANGE diff --git a/tests/unit/job/job_test.py b/tests/unit/job/job_test.py index 09f0abbb..6f190ee3 100644 --- a/tests/unit/job/job_test.py +++ b/tests/unit/job/job_test.py @@ -3,7 +3,7 @@ import pytest -from pyzeebe.exceptions import NoZeebeAdapterError +from pyzeebe.errors import NoZeebeAdapterError def test_success(job_with_adapter): diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py index bff43109..fd4bc1d7 100644 --- a/tests/unit/task/task_config_test.py +++ b/tests/unit/task/task_config_test.py @@ -4,7 +4,7 @@ import pytest from pyzeebe import Job -from pyzeebe.exceptions import NoVariableNameGivenError +from pyzeebe.errors import NoVariableNameGivenError from pyzeebe.task.task_config import TaskConfig, default_exception_handler diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 3b24c943..077417b7 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -3,7 +3,7 @@ import pytest from pyzeebe import TaskDecorator -from pyzeebe.exceptions import TaskNotFoundError, DuplicateTaskTypeError +from pyzeebe.errors import TaskNotFoundError, DuplicateTaskTypeError from pyzeebe.task.task import Task from pyzeebe.worker.task_router import ZeebeTaskRouter from tests.unit.utils.random_utils import randint diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index 0c86d0c6..008e4bfd 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -7,7 +7,7 @@ import pytest from pyzeebe import TaskDecorator, ZeebeTaskRouter -from pyzeebe.exceptions import DuplicateTaskTypeError, MaxConsecutiveTaskThreadError +from pyzeebe.errors import DuplicateTaskTypeError, MaxConsecutiveTaskThreadError from pyzeebe.job.job import Job from pyzeebe.task.task import Task from pyzeebe.worker.worker import ZeebeWorker From d861280524817a83f4d5cea96e44e3984df7d95b Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 12:57:56 +0200 Subject: [PATCH 30/47] Refactor build_job_handler tests to fit arrange, act, assert structure --- tests/unit/task/task_builder_test.py | 34 +++++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/unit/task/task_builder_test.py b/tests/unit/task/task_builder_test.py index b962e28d..2b9e991e 100644 --- a/tests/unit/task/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -41,35 +41,43 @@ def test_exception_handler_called(self, original_task_function: Callable, task_c mocked_job_with_adapter: Job): exception = Exception() original_task_function.side_effect = exception + job_handler = task_builder.build_job_handler(original_task_function, task_config) - task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job_handler(mocked_job_with_adapter) task_config.exception_handler.assert_called_with(exception, mocked_job_with_adapter) def test_parameters_are_provided_to_task(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): mocked_job_with_adapter.variables = {"x": 1} + job_handler = task_builder.build_job_handler(original_task_function, task_config) - task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job_handler(mocked_job_with_adapter) original_task_function.assert_called_with(x=1) def test_variables_are_added_to_result(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): original_task_function.return_value = {"x": 1} + job_handler = task_builder.build_job_handler(original_task_function, task_config) - job = task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job = job_handler(mocked_job_with_adapter) assert job.variables.pop("x") == 1 def test_complete_job_called(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): - task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job_handler = task_builder.build_job_handler(original_task_function, task_config) + + job_handler(mocked_job_with_adapter) + mocked_job_with_adapter.set_success_status.assert_called_once() def test_returned_task_runs_original_function(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): - task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job_handler = task_builder.build_job_handler(original_task_function, task_config) + + job_handler(mocked_job_with_adapter) original_task_function.assert_called_once() @@ -77,8 +85,9 @@ def test_before_decorator_called(self, original_task_function: Callable, decorat task_config: TaskConfig, mocked_job_with_adapter: Job): task_config.before.append(decorator) + job_handler = task_builder.build_job_handler(original_task_function, task_config) - task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job_handler(mocked_job_with_adapter) task_config.before.pop().assert_called_once() @@ -86,8 +95,9 @@ def test_after_decorator_called(self, original_task_function: Callable, decorato task_config: TaskConfig, mocked_job_with_adapter: Job): task_config.after.append(decorator) + job_handler = task_builder.build_job_handler(original_task_function, task_config) - task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job_handler(mocked_job_with_adapter) task_config.after.pop().assert_called_once() @@ -95,19 +105,21 @@ def test_failing_decorator_continues(self, original_task_function: Callable, dec task_config: TaskConfig, mocked_job_with_adapter: Job): decorator.side_effect = Exception() task_config.before.append(decorator) + job_handler = task_builder.build_job_handler(original_task_function, task_config) + + job_handler(mocked_job_with_adapter) - # Assert no exception is raised - task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) decorator.assert_called_once() task_config.exception_handler.assert_not_called() def test_decorator_variables_are_added(self, original_task_function: Callable, decorator: TaskDecorator, task_config: TaskConfig, mocked_job_with_adapter: Job): + mocked_job_with_adapter.variables = {"x": 2} decorator_return_value = mocked_job_with_adapter - decorator_return_value.variables = {"x": 2} decorator.return_value = decorator_return_value + job_handler = task_builder.build_job_handler(original_task_function, task_config) - job = task_builder.build_job_handler(original_task_function, task_config)(mocked_job_with_adapter) + job = job_handler(mocked_job_with_adapter) assert "x" in job.variables From 1da1d0e0bb33275acf7c5886997a4339820d9cce Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 18:31:01 +0200 Subject: [PATCH 31/47] Use pytest parameterize to simplify TestGetFunctionParameters --- tests/unit/task/task_builder_test.py | 92 +++++++--------------------- tests/unit/utils/dummy_functions.py | 42 +++++++++++++ 2 files changed, 65 insertions(+), 69 deletions(-) create mode 100644 tests/unit/utils/dummy_functions.py diff --git a/tests/unit/task/task_builder_test.py b/tests/unit/task/task_builder_test.py index 2b9e991e..82927359 100644 --- a/tests/unit/task/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -1,9 +1,12 @@ -from typing import Callable +from typing import Callable, List + +import pytest from pyzeebe import Job, TaskDecorator from pyzeebe.task import task_builder from pyzeebe.task.task import Task from pyzeebe.task.task_config import TaskConfig +from tests.unit.utils import dummy_functions class TestBuildTask: @@ -132,71 +135,22 @@ def test_converting_to_dict(self): class TestGetFunctionParameters: - def test_get_no_param(self): - def no_parameters(): - pass - - assert task_builder.get_parameters_from_function(no_parameters) == [] - - def test_get_single_param(self): - def dummy_function(x): - pass - - assert task_builder.get_parameters_from_function(dummy_function) == ["x"] - - def test_get_multiple_params(self): - def dummy_function(x, y, z): - pass - - assert task_builder.get_parameters_from_function(dummy_function) == ["x", "y", "z"] - - def test_get_param_from_lambda(self): - assert task_builder.get_parameters_from_function(lambda x: None) == ["x"] - - def test_get_one_positional_param(self): - def one_pos_func(x): - pass - - assert task_builder.get_parameters_from_function(one_pos_func) == ["x"] - - def test_get_multiple_positional_params(self): - def mul_pos_func(x, y, z): - pass - - assert task_builder.get_parameters_from_function(mul_pos_func) == ["x", "y", "z"] - - def test_get_one_keyword_param(self): - def one_key_func(x=0): - pass - - assert task_builder.get_parameters_from_function(one_key_func) == ["x"] - - def test_get_multiple_keyword_params(self): - def mul_key_func(x=0, y=0, z=0): - pass - - assert task_builder.get_parameters_from_function(mul_key_func) == ["x", "y", "z"] - - def test_get_positional_and_keyword_params(self): - def pos_and_key_func(x, y=0): - pass - - assert task_builder.get_parameters_from_function(pos_and_key_func) == ["x", "y"] - - def test_get_params_from_args(self): - def args_func(*args): - pass - - assert task_builder.get_parameters_from_function(args_func) == [] - - def test_get_params_from_kwargs(self): - def kwargs_func(**kwargs): - pass - - assert task_builder.get_parameters_from_function(kwargs_func) == [] - - def test_get_standard_named_params(self): - def func(args, kwargs): - pass - - assert task_builder.get_parameters_from_function(func) == ["args", "kwargs"] + @pytest.mark.parametrize("fn,expected", [ + (dummy_functions.no_param, []), + (dummy_functions.one_param, ["x"]), + (dummy_functions.multiple_params, ["x", "y", "z"]), + (dummy_functions.one_keyword_param, ["x"]), + (dummy_functions.multiple_keyword_param, ["x", "y", "z"]), + (dummy_functions.positional_and_keyword_params, ["x", "y"]), + (dummy_functions.args_param, []), + (dummy_functions.kwargs_param, []), + (dummy_functions.standard_named_params, ["args", "kwargs"]), + (dummy_functions.lambda_no_params, []), + (dummy_functions.lambda_one_param, ["x"]), + (dummy_functions.lambda_multiple_params, ["x", "y", "z"]), + (dummy_functions.lambda_one_keyword_param, ["x"]), + (dummy_functions.lambda_multiple_keyword_params, ["x", "y", "z"]), + (dummy_functions.lambda_positional_and_keyword_params, ["x", "y"]) + ]) + def test_get_params(self, fn: Callable, expected: List[str]): + assert task_builder.get_parameters_from_function(fn) == expected diff --git a/tests/unit/utils/dummy_functions.py b/tests/unit/utils/dummy_functions.py new file mode 100644 index 00000000..b0768eb6 --- /dev/null +++ b/tests/unit/utils/dummy_functions.py @@ -0,0 +1,42 @@ +def no_param(): + pass + + +def one_param(x): + pass + + +def multiple_params(x, y, z): + pass + + +def one_keyword_param(x=0): + pass + + +def multiple_keyword_param(x=0, y=0, z=0): + pass + + +def positional_and_keyword_params(x, y=0): + pass + + +def args_param(*args): + pass + + +def kwargs_param(**kwargs): + pass + + +def standard_named_params(args, kwargs): + pass + + +lambda_no_params = lambda: None +lambda_one_param = lambda x: None +lambda_multiple_params = lambda x, y, z: None +lambda_one_keyword_param = lambda x=0: None +lambda_multiple_keyword_params = lambda x=0, y=0, z=0: None +lambda_positional_and_keyword_params = lambda x, y=0: None From 2a2eaf717d9ddf4907c572102671187908735d65 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 18:37:22 +0200 Subject: [PATCH 32/47] Improve default_exception_handler test name --- tests/unit/task/task_config_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py index fd4bc1d7..0119f8e7 100644 --- a/tests/unit/task/task_config_test.py +++ b/tests/unit/task/task_config_test.py @@ -13,7 +13,7 @@ def test_add_non_dict_task_without_variable_name(): TaskConfig(str(uuid4()), single_value=True) -def test_default_exception_handler(mocked_job_with_adapter: Job): +def test_default_exception_handler_logs_a_warning(mocked_job_with_adapter: Job): with patch("pyzeebe.task.task_config.logger.warning") as logging_mock: default_exception_handler(Exception(), mocked_job_with_adapter) From f746baad0d2d185fad515a160115bf4d62a620b0 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sat, 6 Mar 2021 18:44:35 +0200 Subject: [PATCH 33/47] Add happy path to task_config tests --- tests/unit/task/task_config_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py index 0119f8e7..4899c559 100644 --- a/tests/unit/task/task_config_test.py +++ b/tests/unit/task/task_config_test.py @@ -8,6 +8,24 @@ from pyzeebe.task.task_config import TaskConfig, default_exception_handler +def test_add_dict_task_with_correct_parameters(task_type: str): + expected_config_as_dict = { + "type": task_type, + "exception_handler": default_exception_handler, + "timeout": 10000, + "max_jobs_to_activate": 32, + "variables_to_fetch": None, + "single_value": False, + "variable_name": None, + "before": [], + "after": [] + } + + config = TaskConfig(task_type) + + assert config.__dict__ == expected_config_as_dict + + def test_add_non_dict_task_without_variable_name(): with pytest.raises(NoVariableNameGivenError): TaskConfig(str(uuid4()), single_value=True) From 8d7547b29bffed26ccefa0b836d57acb110f8b62 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 14 Mar 2021 22:08:21 +0200 Subject: [PATCH 34/47] Add docstrings to before and after methods --- pyzeebe/worker/task_router.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 4a8d56e2..a1722903 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -62,9 +62,21 @@ def _is_task_duplicate(self, task_type: str) -> None: return def before(self, *decorators: TaskDecorator) -> None: + """ + Add decorators to be performed before a job is run + + Args: + decorators (Iterable[TaskDecorator]): The decorators to be performed before each job is run + """ self._before.extend(decorators) def after(self, *decorators: TaskDecorator) -> None: + """ + Add decorators to be performed after a job is run + + Args: + decorators (Iterable[TaskDecorator]): The decorators to be performed after each job is run + """ self._after.extend(decorators) def remove_task(self, task_type: str) -> Task: From f63f1d0e1e9f4eefbd8a0be176ac57f373848123 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 14 Mar 2021 22:13:11 +0200 Subject: [PATCH 35/47] Rename timeout field in TaskConfig to timeout_ms --- pyzeebe/task/task_config.py | 4 ++-- pyzeebe/worker/worker.py | 2 +- tests/unit/task/task_config_test.py | 2 +- tests/unit/worker/worker_test.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index efe06c07..af4b8778 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -16,7 +16,7 @@ def default_exception_handler(e: Exception, job: Job) -> None: class TaskConfig: def __init__(self, type: str, exception_handler: ExceptionHandler = default_exception_handler, - timeout: int = 10000, max_jobs_to_activate: int = 32, variables_to_fetch: Optional[List[str]] = None, + timeout_ms: int = 10000, max_jobs_to_activate: int = 32, variables_to_fetch: Optional[List[str]] = None, single_value: bool = False, variable_name: Optional[str] = None, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): if single_value and not variable_name: @@ -24,7 +24,7 @@ def __init__(self, type: str, exception_handler: ExceptionHandler = default_exce self.type = type self.exception_handler = exception_handler - self.timeout = timeout + self.timeout_ms = timeout_ms self.max_jobs_to_activate = max_jobs_to_activate self.variables_to_fetch = variables_to_fetch self.single_value = single_value diff --git a/pyzeebe/worker/worker.py b/pyzeebe/worker/worker.py index 928d4036..5e06d42e 100644 --- a/pyzeebe/worker/worker.py +++ b/pyzeebe/worker/worker.py @@ -175,7 +175,7 @@ def _handle_jobs(self, task: Task) -> None: def _get_jobs(self, task: Task) -> Generator[Job, None, None]: logger.debug(f"Activating jobs for task: {task}") - return self.zeebe_adapter.activate_jobs(task_type=task.type, worker=self.name, timeout=task.config.timeout, + return self.zeebe_adapter.activate_jobs(task_type=task.type, worker=self.name, timeout=task.config.timeout_ms, max_jobs_to_activate=task.config.max_jobs_to_activate, variables_to_fetch=task.config.variables_to_fetch, request_timeout=self.request_timeout) diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py index 4899c559..57ea1c0e 100644 --- a/tests/unit/task/task_config_test.py +++ b/tests/unit/task/task_config_test.py @@ -12,7 +12,7 @@ def test_add_dict_task_with_correct_parameters(task_type: str): expected_config_as_dict = { "type": task_type, "exception_handler": default_exception_handler, - "timeout": 10000, + "timeout_ms": 10000, "max_jobs_to_activate": 32, "variables_to_fetch": None, "single_value": False, diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index 008e4bfd..b0f178ca 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -171,7 +171,7 @@ def test_activate_jobs_called(self, zeebe_worker: ZeebeWorker, task: Task): zeebe_worker.zeebe_adapter.activate_jobs = MagicMock() zeebe_worker._get_jobs(task) zeebe_worker.zeebe_adapter.activate_jobs.assert_called_with(task_type=task.type, worker=zeebe_worker.name, - timeout=task.config.timeout, + timeout=task.config.timeout_ms, max_jobs_to_activate=task.config.max_jobs_to_activate, variables_to_fetch=task.config.variables_to_fetch, request_timeout=zeebe_worker.request_timeout) From 62c6002f821e9f59af261b6f1735434b5f4206cb Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 14 Mar 2021 22:19:21 +0200 Subject: [PATCH 36/47] Clean get_parameters_from_function function inner names --- pyzeebe/task/task_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index 1699a615..fb767aaf 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -68,9 +68,9 @@ def inner_fn(*args, **kwargs): return inner_fn -def get_parameters_from_function(fn: Callable) -> List[str]: - function_signature = inspect.signature(fn) - for parameter_name, parameter in function_signature.parameters.items(): +def get_parameters_from_function(task_function: Callable) -> List[str]: + function_signature = inspect.signature(task_function) + for _, parameter in function_signature.parameters.items(): if parameter.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): return [] return list(function_signature.parameters) From 2b8ddb2ff09b07c3dc01b19fa13e80bd7e82c566 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 14 Mar 2021 22:24:10 +0200 Subject: [PATCH 37/47] Add repr to Task and TaskConfig objects --- pyzeebe/task/task.py | 4 ++++ pyzeebe/task/task_config.py | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pyzeebe/task/task.py b/pyzeebe/task/task.py index 913ac3a1..c4bdb7e3 100644 --- a/pyzeebe/task/task.py +++ b/pyzeebe/task/task.py @@ -13,3 +13,7 @@ def __init__(self, original_function: Callable, job_handler: JobHandler, config: @property def type(self): return self.config.type + + def __repr__(self): + return f"Task(config= {self.config}, original_function={self.original_function}, " \ + f"job_handler={self.job_handler})" diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index af4b8778..930c4633 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -16,7 +16,8 @@ def default_exception_handler(e: Exception, job: Job) -> None: class TaskConfig: def __init__(self, type: str, exception_handler: ExceptionHandler = default_exception_handler, - timeout_ms: int = 10000, max_jobs_to_activate: int = 32, variables_to_fetch: Optional[List[str]] = None, + timeout_ms: int = 10000, max_jobs_to_activate: int = 32, + variables_to_fetch: Optional[List[str]] = None, single_value: bool = False, variable_name: Optional[str] = None, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): if single_value and not variable_name: @@ -31,3 +32,9 @@ def __init__(self, type: str, exception_handler: ExceptionHandler = default_exce self.variable_name = variable_name self.before = before or [] self.after = after or [] + + def __repr__(self): + return f"TaskConfig(type={self.type}, exception_handler={self.exception_handler}, " \ + f"timeout_ms={self.timeout_ms}, max_jobs_to_activate={self.max_jobs_to_activate}, " \ + f"variables_to_fetch={self.variables_to_fetch}, single_value={self.single_value}, " \ + f"variable_name={self.variable_name}, before={self.before}, after={self.after})" From bd568e27c9210f175926d63ef047e15f5ef1c709 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 14 Mar 2021 22:24:48 +0200 Subject: [PATCH 38/47] Rename wrapper to task_wrapper --- pyzeebe/worker/task_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index a1722903..eadcaa51 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -36,12 +36,12 @@ def task(self, task_config: Union[TaskConfig, str]): task_config = TaskConfig(task_config) config_with_decorators = self._add_decorators_to_config(task_config) - def wrapper(fn: Callable): + def task_wrapper(fn: Callable): task = task_builder.build_task(fn, config_with_decorators) self._add_task(task) return fn - return wrapper + return task_wrapper def _add_task(self, task: Task): self._is_task_duplicate(task.type) From 3483cf16cfaa10a73701373ee8829f353b202aef Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Sun, 14 Mar 2021 22:28:05 +0200 Subject: [PATCH 39/47] Rename fixture from task_handler_mock to job_handler_spy --- tests/unit/worker/worker_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index b0f178ca..a36d3fb8 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -90,7 +90,7 @@ def get_jobs_mock(self, zeebe_worker: ZeebeWorker): return zeebe_worker._get_jobs @pytest.fixture(autouse=True) - def task_handler_mock(self, task: Task): + def job_handler_spy(self, task: Task): task.job_handler = MagicMock(wraps=task.job_handler) def test_handle_no_job(self, zeebe_worker: ZeebeWorker, task: Task, get_jobs_mock: MagicMock): From 83c4ce61be5e4ad9913c32eb40eac5ef37c2cca7 Mon Sep 17 00:00:00 2001 From: JonatanMartens Date: Mon, 15 Mar 2021 22:13:50 +0200 Subject: [PATCH 40/47] Call route._is_task_duplicate in test_no_duplicate_task_type_error_is_raised --- tests/unit/worker/task_router_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 077417b7..479f5905 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -71,8 +71,8 @@ def test_check_is_task_duplicate_with_duplicate(router: ZeebeTaskRouter, task: T router._is_task_duplicate(task.type) -def test_check_is_task_duplicate_no_duplicate(router: ZeebeTaskRouter, task: Task): - router.tasks.append(task) +def test_no_duplicate_task_type_error_is_raised(router: ZeebeTaskRouter, task: Task): + router._is_task_duplicate(task.type) def test_add_before_decorator(router: ZeebeTaskRouter, decorator: TaskDecorator): From 3c5723799c01804bc0f58f747614c967eaf8641f Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sat, 20 Mar 2021 16:25:21 +0200 Subject: [PATCH 41/47] Receive regular params instead of task config --- pyzeebe/__init__.py | 4 +- pyzeebe/task/task_config.py | 29 ++++------- pyzeebe/worker/task_router.py | 69 ++++++++++++++++++++++----- tests/conftest.py | 12 ++++- tests/unit/task/task_config_test.py | 39 --------------- tests/unit/utils/random_utils.py | 2 +- tests/unit/worker/task_router_test.py | 16 +++++-- tests/unit/worker/worker_test.py | 18 +------ 8 files changed, 93 insertions(+), 96 deletions(-) delete mode 100644 tests/unit/task/task_config_test.py diff --git a/pyzeebe/__init__.py b/pyzeebe/__init__.py index 36167fd3..ad6e7a0f 100644 --- a/pyzeebe/__init__.py +++ b/pyzeebe/__init__.py @@ -7,7 +7,7 @@ from pyzeebe.job.job import Job from pyzeebe.job.job_status import JobStatus from pyzeebe.task.exception_handler import ExceptionHandler -from pyzeebe.task.task_config import TaskConfig, default_exception_handler +from pyzeebe.task.task_config import TaskConfig from pyzeebe.task.types import TaskDecorator -from pyzeebe.worker.task_router import ZeebeTaskRouter +from pyzeebe.worker.task_router import ZeebeTaskRouter, default_exception_handler from pyzeebe.worker.worker import ZeebeWorker diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index 930c4633..bb25638e 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -1,28 +1,15 @@ -import logging -from typing import List, Optional +from typing import List -from pyzeebe.errors import NoVariableNameGivenError -from pyzeebe.job.job import Job from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.types import TaskDecorator -logger = logging.getLogger(__name__) - - -def default_exception_handler(e: Exception, job: Job) -> None: - logger.warning(f"Task type: {job.type} - failed job {job}. Error: {e}.") - job.set_failure_status(f"Failed job. Error: {e}") - class TaskConfig: - def __init__(self, type: str, exception_handler: ExceptionHandler = default_exception_handler, - timeout_ms: int = 10000, max_jobs_to_activate: int = 32, - variables_to_fetch: Optional[List[str]] = None, - single_value: bool = False, variable_name: Optional[str] = None, before: List[TaskDecorator] = None, - after: List[TaskDecorator] = None): - if single_value and not variable_name: - raise NoVariableNameGivenError(type) - + def __init__(self, type: str, exception_handler: ExceptionHandler, + timeout_ms: int, max_jobs_to_activate: int, + variables_to_fetch: List[str], + single_value: bool, variable_name: str, before: List[TaskDecorator], + after: List[TaskDecorator]): self.type = type self.exception_handler = exception_handler self.timeout_ms = timeout_ms @@ -30,8 +17,8 @@ def __init__(self, type: str, exception_handler: ExceptionHandler = default_exce self.variables_to_fetch = variables_to_fetch self.single_value = single_value self.variable_name = variable_name - self.before = before or [] - self.after = after or [] + self.before = before + self.after = after def __repr__(self): return f"TaskConfig(type={self.type}, exception_handler={self.exception_handler}, " \ diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index eadcaa51..5b767383 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -1,15 +1,23 @@ import logging -from typing import Tuple, List, Callable, Union +from typing import Callable, List, Tuple, Optional from pyzeebe import TaskDecorator -from pyzeebe.errors import TaskNotFoundError, DuplicateTaskTypeError +from pyzeebe.errors import (DuplicateTaskTypeError, NoVariableNameGivenError, + TaskNotFoundError) +from pyzeebe.job.job import Job from pyzeebe.task import task_builder +from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.task import Task from pyzeebe.task.task_config import TaskConfig logger = logging.getLogger(__name__) +def default_exception_handler(e: Exception, job: Job) -> None: + logger.warning(f"Task type: {job.type} - failed job {job}. Error: {e}.") + job.set_failure_status(f"Failed job. Error: {e}") + + class ZeebeTaskRouter: def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator] = None): """ @@ -21,25 +29,60 @@ def __init__(self, before: List[TaskDecorator] = None, after: List[TaskDecorator self._after: List[TaskDecorator] = after or [] self.tasks: List[Task] = [] - def task(self, task_config: Union[TaskConfig, str]): + def task(self, task_type: str, exception_handler: ExceptionHandler = default_exception_handler, + variables_to_fetch: Optional[List[str]] = None, timeout_ms: int = 10000, max_jobs_to_activate: int = 32, + before: List[TaskDecorator] = None, after: List[TaskDecorator] = None, single_value: bool = False, + variable_name: str = None): """ Decorator to create a task - Args: - task_config (Union[str, TaskConfig]): Either the task type or a task configuration object + task_type (str): The task type + + exception_handler (ExceptionHandler): Handler that will be called when a job fails. + + variables_to_fetch (Optional[List[str]]): The variables to request from Zeebe when activating jobs. + + timeout_ms (int): Maximum duration of the task in milliseconds. If the timeout is surpassed Zeebe will give up + on the worker and retry it. Default: 10000 (10 seconds). + max_jobs_to_activate (int): Maximum jobs the worker will execute in parallel (of this task). Default: 32 + + before (List[TaskDecorator]): All decorators which should be performed before the task. + + after (List[TaskDecorator]): All decorators which should be performed after the task. + + single_value (bool): If the function returns a single value (int, string, list) and not a dictionary set + this to True. Default: False + + variable_name (str): If single_value then this will be the variable name given to zeebe: + { : } Raises: DuplicateTaskTypeError: If a task from the router already exists in the worker - + NoVariableNameGivenError: When single_value is set, but no variable_name is given """ - if isinstance(task_config, str): - task_config = TaskConfig(task_config) - config_with_decorators = self._add_decorators_to_config(task_config) - - def task_wrapper(fn: Callable): - task = task_builder.build_task(fn, config_with_decorators) + if single_value and not variable_name: + raise NoVariableNameGivenError(task_type) + + def task_wrapper(task_function: Callable): + config = TaskConfig( + task_type, + exception_handler, + timeout_ms, + max_jobs_to_activate, + variables_to_fetch or task_builder.get_parameters_from_function( + task_function), + single_value, + variable_name or "", + before or [], + after or [] + ) + config_with_decorators = self._add_decorators_to_config(config) + + task = task_builder.build_task( + task_function, config_with_decorators + ) self._add_task(task) - return fn + return task_function return task_wrapper diff --git a/tests/conftest.py b/tests/conftest.py index 5c64086e..865c7a7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,7 +69,17 @@ def first_active_job(task, job_from_task, grpc_servicer) -> str: @pytest.fixture def task_config(task_type): - return TaskConfig(task_type, MagicMock()) + return TaskConfig( + type=task_type, + exception_handler=MagicMock(), + timeout_ms=10000, + max_jobs_to_activate=32, + variables_to_fetch=[], + single_value=False, + variable_name="", + before=[], + after=[] + ) @pytest.fixture diff --git a/tests/unit/task/task_config_test.py b/tests/unit/task/task_config_test.py deleted file mode 100644 index 57ea1c0e..00000000 --- a/tests/unit/task/task_config_test.py +++ /dev/null @@ -1,39 +0,0 @@ -from unittest.mock import patch -from uuid import uuid4 - -import pytest - -from pyzeebe import Job -from pyzeebe.errors import NoVariableNameGivenError -from pyzeebe.task.task_config import TaskConfig, default_exception_handler - - -def test_add_dict_task_with_correct_parameters(task_type: str): - expected_config_as_dict = { - "type": task_type, - "exception_handler": default_exception_handler, - "timeout_ms": 10000, - "max_jobs_to_activate": 32, - "variables_to_fetch": None, - "single_value": False, - "variable_name": None, - "before": [], - "after": [] - } - - config = TaskConfig(task_type) - - assert config.__dict__ == expected_config_as_dict - - -def test_add_non_dict_task_without_variable_name(): - with pytest.raises(NoVariableNameGivenError): - TaskConfig(str(uuid4()), single_value=True) - - -def test_default_exception_handler_logs_a_warning(mocked_job_with_adapter: Job): - with patch("pyzeebe.task.task_config.logger.warning") as logging_mock: - default_exception_handler(Exception(), mocked_job_with_adapter) - - mocked_job_with_adapter.set_failure_status.assert_called() - logging_mock.assert_called() diff --git a/tests/unit/utils/random_utils.py b/tests/unit/utils/random_utils.py index 0a5988fb..4f95a155 100644 --- a/tests/unit/utils/random_utils.py +++ b/tests/unit/utils/random_utils.py @@ -10,7 +10,7 @@ RANDOM_RANGE = 1000000000 -def random_job(task: Task = task_builder.build_task(lambda x: {"x": x}, TaskConfig("test")), +def random_job(task: Task = task_builder.build_task(lambda x: {"x": x}, TaskConfig("test", lambda: None, 10000, 32, [], False, "", [], [])), zeebe_adapter: ZeebeAdapter = None) -> Job: return Job(_type=task.type, key=randint(0, RANDOM_RANGE), worker=str(uuid4()), retries=randint(0, 10), workflow_instance_key=randint(0, RANDOM_RANGE), diff --git a/tests/unit/worker/task_router_test.py b/tests/unit/worker/task_router_test.py index 479f5905..92d42971 100644 --- a/tests/unit/worker/task_router_test.py +++ b/tests/unit/worker/task_router_test.py @@ -1,11 +1,13 @@ +from unittest.mock import patch from uuid import uuid4 import pytest - from pyzeebe import TaskDecorator -from pyzeebe.errors import TaskNotFoundError, DuplicateTaskTypeError +from pyzeebe.errors import DuplicateTaskTypeError, TaskNotFoundError +from pyzeebe.job.job import Job from pyzeebe.task.task import Task -from pyzeebe.worker.task_router import ZeebeTaskRouter +from pyzeebe.worker.task_router import (ZeebeTaskRouter, + default_exception_handler) from tests.unit.utils.random_utils import randint @@ -97,3 +99,11 @@ def test_add_after_decorator_through_constructor(decorator: TaskDecorator): router = ZeebeTaskRouter(after=[decorator]) assert len(router._after) == 1 + + +def test_default_exception_handler_logs_a_warning(mocked_job_with_adapter: Job): + with patch("pyzeebe.worker.task_router.logger.warning") as logging_mock: + default_exception_handler(Exception(), mocked_job_with_adapter) + + mocked_job_with_adapter.set_failure_status.assert_called() + logging_mock.assert_called() diff --git a/tests/unit/worker/worker_test.py b/tests/unit/worker/worker_test.py index a36d3fb8..4af0cd69 100644 --- a/tests/unit/worker/worker_test.py +++ b/tests/unit/worker/worker_test.py @@ -43,7 +43,8 @@ def test_variables_to_fetch_match_function_parameters(self, zeebe_worker: ZeebeW def dummy_function(x): pass - assert zeebe_worker.get_task(task_type).config.variables_to_fetch == expected_variables_to_fetch + assert zeebe_worker.get_task( + task_type).config.variables_to_fetch == expected_variables_to_fetch class TestDecorator: @@ -67,21 +68,6 @@ def test_add_constructor_after_decorator(self, decorator: TaskDecorator): assert len(zeebe_worker._after) == 1 assert decorator in zeebe_worker._after - def test_decorator_failed(self, zeebe_worker: ZeebeWorker, task: Task, decorator: TaskDecorator, - job_from_task: Job): - decorator.side_effect = Exception() - zeebe_worker.before(decorator) - zeebe_worker.after(decorator) - - @zeebe_worker.task(task.config) - def dummy_function(): - pass - - task = zeebe_worker.get_task(task.type) - task.job_handler(job_from_task) - - assert decorator.call_count == 2 - class TestHandleJobs: @pytest.fixture(autouse=True) From 876d81915f25a7f2ff8592c6f1d2c0502f6738f1 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sat, 20 Mar 2021 16:54:11 +0200 Subject: [PATCH 42/47] Add test case for no variables added to result --- pyzeebe/task/task_builder.py | 9 ++-- tests/unit/task/task_builder_test.py | 62 +++++++++++++++++----------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/pyzeebe/task/task_builder.py b/pyzeebe/task/task_builder.py index fb767aaf..2d4d1fdb 100644 --- a/pyzeebe/task/task_builder.py +++ b/pyzeebe/task/task_builder.py @@ -11,11 +11,9 @@ def build_task(task_function: Callable, task_config: TaskConfig) -> Task: - if not task_config.variables_to_fetch: - task_config.variables_to_fetch = get_parameters_from_function(task_function) - if task_config.single_value: - task_function = convert_to_dict_function(task_function, task_config.variable_name) + task_function = convert_to_dict_function( + task_function, task_config.variable_name) return Task(task_function, build_job_handler(task_function, task_config), task_config) @@ -26,7 +24,8 @@ def build_job_handler(task_function: Callable, task_config: TaskConfig) -> JobHa def job_handler(job: Job) -> Job: job = before_decorator_runner(job) - job.variables, succeeded = run_original_task_function(task_function, task_config, job) + job.variables, succeeded = run_original_task_function( + task_function, task_config, job) job = after_decorator_runner(job) if succeeded: job.set_success_status() diff --git a/tests/unit/task/task_builder_test.py b/tests/unit/task/task_builder_test.py index 82927359..2e398fd3 100644 --- a/tests/unit/task/task_builder_test.py +++ b/tests/unit/task/task_builder_test.py @@ -10,50 +10,57 @@ class TestBuildTask: + @pytest.fixture + def single_value_task_config(self, task_config: TaskConfig): + task_config.single_value = True + task_config.variable_name = "y" + + return task_config + def test_returns_task(self, original_task_function: Callable, task_config: TaskConfig): task = task_builder.build_task(original_task_function, task_config) assert isinstance(task, Task) - def test_single_value_func(self, task_config: TaskConfig, mocked_job_with_adapter: Job): - task_config.single_value = True - task_config.variable_name = "y" - mocked_job_with_adapter.variables = {"x": 1} - - job = task_builder.build_task(lambda x: x, task_config).job_handler(mocked_job_with_adapter) + def test_single_value_func(self, single_value_task_config: TaskConfig, mocked_job_with_adapter: Job): + task = task_builder.build_task(lambda: 1, single_value_task_config) + job = task.job_handler(mocked_job_with_adapter) assert job.variables.pop("y") == 1 - def test_variables_to_fetch_added_to_task_config(self, task_config: TaskConfig): - expected_variables_to_fetch = ["x", "y"] - - def dummy_fn(x, y): - pass + def test_no_additional_variables_are_added_to_result(self, single_value_task_config: TaskConfig, mocked_job_with_adapter: Job): + mocked_job_with_adapter.variables = {"x": 1} - task = task_builder.build_task(dummy_fn, task_config) + task = task_builder.build_task(lambda x: x, single_value_task_config) + job = task.job_handler(mocked_job_with_adapter) - assert task.config.variables_to_fetch == expected_variables_to_fetch + assert len(job.variables.keys()) == 1 + assert set(job.variables.keys()) == {"y"} class TestBuildJobHandler: def test_returned_task_is_callable(self, original_task_function: Callable, task_config: TaskConfig): - task = task_builder.build_job_handler(original_task_function, task_config) + task = task_builder.build_job_handler( + original_task_function, task_config) assert callable(task) def test_exception_handler_called(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): exception = Exception() original_task_function.side_effect = exception - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job_handler(mocked_job_with_adapter) - task_config.exception_handler.assert_called_with(exception, mocked_job_with_adapter) + task_config.exception_handler.assert_called_with( + exception, mocked_job_with_adapter) def test_parameters_are_provided_to_task(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): mocked_job_with_adapter.variables = {"x": 1} - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job_handler(mocked_job_with_adapter) @@ -62,7 +69,8 @@ def test_parameters_are_provided_to_task(self, original_task_function: Callable, def test_variables_are_added_to_result(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): original_task_function.return_value = {"x": 1} - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job = job_handler(mocked_job_with_adapter) @@ -70,7 +78,8 @@ def test_variables_are_added_to_result(self, original_task_function: Callable, t def test_complete_job_called(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job_handler(mocked_job_with_adapter) @@ -78,7 +87,8 @@ def test_complete_job_called(self, original_task_function: Callable, task_config def test_returned_task_runs_original_function(self, original_task_function: Callable, task_config: TaskConfig, mocked_job_with_adapter: Job): - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job_handler(mocked_job_with_adapter) @@ -88,7 +98,8 @@ def test_before_decorator_called(self, original_task_function: Callable, decorat task_config: TaskConfig, mocked_job_with_adapter: Job): task_config.before.append(decorator) - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job_handler(mocked_job_with_adapter) @@ -98,7 +109,8 @@ def test_after_decorator_called(self, original_task_function: Callable, decorato task_config: TaskConfig, mocked_job_with_adapter: Job): task_config.after.append(decorator) - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job_handler(mocked_job_with_adapter) @@ -108,7 +120,8 @@ def test_failing_decorator_continues(self, original_task_function: Callable, dec task_config: TaskConfig, mocked_job_with_adapter: Job): decorator.side_effect = Exception() task_config.before.append(decorator) - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job_handler(mocked_job_with_adapter) @@ -120,7 +133,8 @@ def test_decorator_variables_are_added(self, original_task_function: Callable, d mocked_job_with_adapter.variables = {"x": 2} decorator_return_value = mocked_job_with_adapter decorator.return_value = decorator_return_value - job_handler = task_builder.build_job_handler(original_task_function, task_config) + job_handler = task_builder.build_job_handler( + original_task_function, task_config) job = job_handler(mocked_job_with_adapter) From 6d9e85edc9cde8acfbdc73e32e4338b89848da62 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sat, 20 Mar 2021 20:31:39 +0200 Subject: [PATCH 43/47] Run tests on every push --- .github/workflows/test-python-package.yml | 3 +-- .github/workflows/test-zeebe-integration.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-python-package.yml b/.github/workflows/test-python-package.yml index 4773b327..64c3def3 100644 --- a/.github/workflows/test-python-package.yml +++ b/.github/workflows/test-python-package.yml @@ -2,9 +2,8 @@ name: Test pyzeebe on: push: - branches: [ master, development, feature/*, bugfix/*, maintenance/* ] pull_request: - branches: [ master, development, feature/*, bugfix/*, maintenance/* ] + branches: [ master, development ] jobs: build: diff --git a/.github/workflows/test-zeebe-integration.yml b/.github/workflows/test-zeebe-integration.yml index 6b5254fa..6fd8623a 100644 --- a/.github/workflows/test-zeebe-integration.yml +++ b/.github/workflows/test-zeebe-integration.yml @@ -2,9 +2,8 @@ name: Integration test pyzeebe on: push: - branches: [ master, development, feature/*, bugfix/*, maintenance/* ] pull_request: - branches: [ master, development, feature/*, bugfix/*, maintenance/* ] + branches: [ master, development ] jobs: test: From 34e7f268f595442bfb6eb35a682c8d427e5614f2 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sat, 20 Mar 2021 20:42:52 +0200 Subject: [PATCH 44/47] Fix integration tests to work without task config --- tests/integration/integration_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/integration_test.py b/tests/integration/integration_test.py index 15b1f5cc..576a5245 100644 --- a/tests/integration/integration_test.py +++ b/tests/integration/integration_test.py @@ -4,7 +4,7 @@ import pytest -from pyzeebe import ZeebeClient, ZeebeWorker, Job, TaskConfig +from pyzeebe import ZeebeClient, ZeebeWorker, Job from pyzeebe.errors import WorkflowNotFoundError @@ -20,7 +20,7 @@ def zeebe_worker(): def exception_handler(exc: Exception, job: Job) -> None: job.set_error_status(f"Failed to run task {job.type}. Reason: {exc}") - @worker.task(TaskConfig("test", exception_handler)) + @worker.task("test", exception_handler) def task_handler(should_throw: bool, input: str) -> Dict: if should_throw: raise Exception("Error thrown") From 4a8ba92b284ebed3951413c04e06e3b51850e964 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sat, 20 Mar 2021 21:22:55 +0200 Subject: [PATCH 45/47] Update ZeebeWorker documentation --- docs/worker_reference.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/worker_reference.rst b/docs/worker_reference.rst index 8ac5e46a..4cb774f8 100644 --- a/docs/worker_reference.rst +++ b/docs/worker_reference.rst @@ -2,9 +2,10 @@ Worker Reference ================ -The :py:class:`ZeebeTaskHandler` class from which both :py:class:`ZeebeWorker` and :py:class:`ZeebeTaskRouter` inherit. +The :py:class:`ZeebeWorker` class inherits from :py:class:`ZeebeTaskRouter` class. +This means that all methods that :py:class:`ZeebeTaskRouter` has will also appear in :py:class:`ZeebeWorker`. -.. autoclass:: pyzeebe.worker.task_handler.ZeebeTaskHandler +.. autoclass:: pyzeebe.ZeebeTaskRouter :members: :undoc-members: @@ -13,9 +14,6 @@ The :py:class:`ZeebeTaskHandler` class from which both :py:class:`ZeebeWorker` a :members: :undoc-members: -.. autoclass:: pyzeebe.ZeebeTaskRouter - :members: - :undoc-members: .. autoclass:: pyzeebe.Job :members: From 22042f4be4624f341aa463d1da260d55a9d38f73 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sat, 27 Mar 2021 15:30:38 +0300 Subject: [PATCH 46/47] Remove docstring whitespace in ZeebeTaskRouter --- pyzeebe/worker/task_router.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 5b767383..2d878e90 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -37,23 +37,15 @@ def task(self, task_type: str, exception_handler: ExceptionHandler = default_exc Decorator to create a task Args: task_type (str): The task type - exception_handler (ExceptionHandler): Handler that will be called when a job fails. - variables_to_fetch (Optional[List[str]]): The variables to request from Zeebe when activating jobs. - timeout_ms (int): Maximum duration of the task in milliseconds. If the timeout is surpassed Zeebe will give up on the worker and retry it. Default: 10000 (10 seconds). - max_jobs_to_activate (int): Maximum jobs the worker will execute in parallel (of this task). Default: 32 - before (List[TaskDecorator]): All decorators which should be performed before the task. - after (List[TaskDecorator]): All decorators which should be performed after the task. - single_value (bool): If the function returns a single value (int, string, list) and not a dictionary set this to True. Default: False - variable_name (str): If single_value then this will be the variable name given to zeebe: { : } Raises: From 16213695c3c64facf9fe0d51aa46e25364bb9e40 Mon Sep 17 00:00:00 2001 From: Jonatan Martens Date: Sat, 27 Mar 2021 15:40:56 +0300 Subject: [PATCH 47/47] Move variable name check back to TaskConfig --- pyzeebe/task/task_config.py | 4 ++++ pyzeebe/worker/task_router.py | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyzeebe/task/task_config.py b/pyzeebe/task/task_config.py index bb25638e..0ba9f529 100644 --- a/pyzeebe/task/task_config.py +++ b/pyzeebe/task/task_config.py @@ -1,5 +1,6 @@ from typing import List +from pyzeebe.errors import NoVariableNameGivenError from pyzeebe.task.exception_handler import ExceptionHandler from pyzeebe.task.types import TaskDecorator @@ -10,6 +11,9 @@ def __init__(self, type: str, exception_handler: ExceptionHandler, variables_to_fetch: List[str], single_value: bool, variable_name: str, before: List[TaskDecorator], after: List[TaskDecorator]): + if single_value and not variable_name: + raise NoVariableNameGivenError(type) + self.type = type self.exception_handler = exception_handler self.timeout_ms = timeout_ms diff --git a/pyzeebe/worker/task_router.py b/pyzeebe/worker/task_router.py index 2d878e90..17a35918 100644 --- a/pyzeebe/worker/task_router.py +++ b/pyzeebe/worker/task_router.py @@ -52,9 +52,6 @@ def task(self, task_type: str, exception_handler: ExceptionHandler = default_exc DuplicateTaskTypeError: If a task from the router already exists in the worker NoVariableNameGivenError: When single_value is set, but no variable_name is given """ - if single_value and not variable_name: - raise NoVariableNameGivenError(task_type) - def task_wrapper(task_function: Callable): config = TaskConfig( task_type,