From 86365f16c35e43976ee771b9f3c3729f6df3de28 Mon Sep 17 00:00:00 2001 From: Pocoder Date: Tue, 6 Sep 2022 10:20:02 +0200 Subject: [PATCH 1/2] format docs --- toloka_provider/hooks/toloka.py | 6 +- toloka_provider/sensors/toloka.py | 4 +- toloka_provider/tasks/toloka.py | 91 ++++++++++++++++--------------- toloka_provider/utils/utils.py | 5 ++ 4 files changed, 58 insertions(+), 48 deletions(-) diff --git a/toloka_provider/hooks/toloka.py b/toloka_provider/hooks/toloka.py index b13d728..0c9017d 100644 --- a/toloka_provider/hooks/toloka.py +++ b/toloka_provider/hooks/toloka.py @@ -14,9 +14,9 @@ class TolokaHook(BaseHook): """ Hook to interact with Toloka. - Performs a connection to Toloka and retrieves client. + Performs a connection to Toloka and retrieves client. - :param toloka_conn_id: Airflow Connection with OAuth token for Toloka. + :param toloka_conn_id: Airflow Connection with OAuth token for Toloka. You can learn more about how to get it [here](https://toloka.ai/docs/api/concepts/access.html#access__token). """ @@ -62,6 +62,7 @@ def get_conn(self) -> TolokaClient: @staticmethod def get_connection_form_widgets() -> Dict[str, Any]: + """Adds fields for connection in UI""" from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget from flask_babel import lazy_gettext from wtforms import PasswordField, StringField, validators, ValidationError @@ -86,6 +87,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: @staticmethod def get_ui_field_behaviour() -> Dict[str, Any]: + """Hides unused fields in UI""" return { 'hidden_fields': ['port', 'host', 'login', 'schema', 'extra', 'password'], 'relabeling': {}, diff --git a/toloka_provider/sensors/toloka.py b/toloka_provider/sensors/toloka.py index 200a860..b9a8cfb 100644 --- a/toloka_provider/sensors/toloka.py +++ b/toloka_provider/sensors/toloka.py @@ -15,7 +15,7 @@ class WaitPoolSensor(BaseSensorOperator): """ Wait given pool until close. - :param pool: Either a `Pool` object or it's config or a pool_id value. + :param toloka_pool: Either a `Pool` object or it's config or a pool_id value. :param toloka_conn_id: Airflow connection with toloka credentials. :param success_on_reasons: Container of Pool.CloseReason. `WaitPoolSensor` will wait until both `pool` becomes closed and `success_on_reasons` @@ -39,6 +39,7 @@ def __init__( reason) for reason in success_on_reasons] if success_on_reasons is not None else list(Pool.CloseReason) def poke(self, context: 'Context') -> bool: + """Checks if pool is closed""" pool_id = extract_id(self.toloka_pool, Pool) toloka_hook = TolokaHook(toloka_conn_id=self.toloka_conn_id) toloka_client = toloka_hook.get_conn() @@ -81,6 +82,7 @@ def __init__( self.success_on_statuses = [AppBatch.Status(status) for status in success_on_statuses] def poke(self, context: 'Context') -> bool: + """Checks if batch status in success_on_statuses""" app_project_id = extract_id(self.app_project, AppProject) app_batch_id = extract_id(self.batch, AppBatch) toloka_hook = TolokaHook(toloka_conn_id=self.toloka_conn_id) diff --git a/toloka_provider/tasks/toloka.py b/toloka_provider/tasks/toloka.py index 7e4fb85..cace0a3 100644 --- a/toloka_provider/tasks/toloka.py +++ b/toloka_provider/tasks/toloka.py @@ -25,11 +25,11 @@ def create_project( toloka_conn_id: str = 'toloka_default', ) -> Union[Project, str]: """Create a Project object from given config. - Args: - obj: Either a `Project` object itself or a config to make a `Project`. - toloka_conn_id: Airflow connection with toloka credentials. - Returns: - Project object if custom XCom backend is configured or its JSON serialized version otherwise. + + :param obj: Either a `Project` object itself or a config to make a `Project`. + :param toloka_conn_id: Airflow connection with toloka credentials. + :returns: Project object if custom XCom backend is configured or its JSON serialized version otherwise. + """ toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id) toloka_client = toloka_hook.get_conn() @@ -48,12 +48,12 @@ def create_exam_pool( toloka_conn_id: str = 'toloka_default', ) -> Union[Training, str]: """Create a Training pool object from given config. - Args: - obj: Either a `Training` object itself or a config to make a `Training`. - project: Project to assign a training pool to. May pass either an object, config or project_id. - toloka_conn_id: Airflow connection with toloka credentials. - Returns: - Training object if custom XCom backend is configured or its JSON serialized version otherwise. + + :param obj: Either a `Training` object itself or a config to make a `Training`. + :param project: Project to assign a training pool to. May pass either an object, config or project_id. + :param toloka_conn_id: Airflow connection with toloka credentials. + :returns: Training object if custom XCom backend is configured or its JSON serialized version otherwise. + """ toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id) toloka_client = toloka_hook.get_conn() @@ -76,17 +76,17 @@ def create_pool( toloka_conn_id: str = 'toloka_default', ) -> Union[Pool, str]: """Create a Pool object from given config. - Args: - obj: Either a `Pool` object itself or a config to make a `Pool`. - project: Project to assign a pool to. May pass either an object, config or project_id. - exam_pool: Related training pool. May pass either an object, config or pool_id. - expiration: Expiration setting. May pass any of: - * `None` if this setting is already present; - * `datetime` object to set exact datetime; - * `timedelta` to set expiration related to the current time. - toloka_conn_id: Airflow connection with toloka credentials. - Returns: - Pool object if custom XCom backend is configured or its JSON serialized version otherwise. + + :param obj: Either a `Pool` object itself or a config to make a `Pool`. + :param project: Project to assign a pool to. May pass either an object, config or project_id. + :param exam_pool: Related training pool. May pass either an object, config or pool_id. + :param expiration: Expiration setting. May pass any of: + * `None` if this setting is already present; + * `datetime` object to set exact datetime; + * `timedelta` to set expiration related to the current time. + :param toloka_conn_id: Airflow connection with toloka credentials. + :returns: Pool object if custom XCom backend is configured or its JSON serialized version otherwise. + """ toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id) toloka_client = toloka_hook.get_conn() @@ -114,12 +114,13 @@ def create_tasks( additional_args: Optional[Dict] = None, ) -> None: """Create a list of tasks for a given pool. - Args: - tasks: List of either a `Task` objects or a task conofigurations. - pool: Allow to set tasks pool if it's not present in the tasks themselves. - May be either a `Pool` or `Training` object or config or a pool_id value. - toloka_conn_id: Airflow connection with toloka credentials. - additional_args: Any other args presented in `toloka.client.task.CreateTasksParameters`. + + :param tasks: List of either a `Task` objects or a task conofigurations. + :param pool: Allow to set tasks pool if it's not present in the tasks themselves. + May be either a `Pool` or `Training` object or config or a pool_id value. + :param toloka_conn_id: Airflow connection with toloka credentials. + :param additional_args: Any other args presented in `toloka.client.task.CreateTasksParameters`. + """ toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id) toloka_client = toloka_hook.get_conn() @@ -147,11 +148,11 @@ def open_pool( toloka_conn_id: str = 'toloka_default', ) -> Union[Pool, str]: """Open given pool. - Args: - obj: Pool id or `Pool` object of it's config. - toloka_conn_id: Airflow connection with toloka credentials. - Returns: - Pool object if custom XCom backend is configured or its JSON serialized version otherwise. + + :param obj: Pool id or `Pool` object of it's config. + :param toloka_conn_id: Airflow connection with toloka credentials. + :returns: Pool object if custom XCom backend is configured or its JSON serialized version otherwise. + """ toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id) toloka_client = toloka_hook.get_conn() @@ -170,11 +171,11 @@ def open_exam_pool( toloka_conn_id: str = 'toloka_default', ) -> Union[Pool, str]: """Open given training pool. - Args: - obj: Training pool_id or `Training` object of it's config. - toloka_conn_id: Airflow connection with toloka credentials. - Returns: - Training object if custom XCom backend is configured or its JSON serialized version otherwise. + + :param obj: Training pool_id or `Training` object of it's config. + :param toloka_conn_id: Airflow connection with toloka credentials. + :returns: Training object if custom XCom backend is configured or its JSON serialized version otherwise. + """ toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id) toloka_client = toloka_hook.get_conn() @@ -195,13 +196,13 @@ def get_assignments( additional_args: Optional[Dict] = None, ) -> List[Union[Assignment, str]]: """Get all assignments of selected status from pool. - Args: - pool: Either a `Pool` object or it's config or a pool_id. - status: A status or a list of statuses to get. All statuses (None) by default. - toloka_conn_id: Airflow connection with toloka credentials. - additional_args: Any other args presented in `toloka.client.search_requests.AssignmentSearchRequest`. - Returns: - List of `Assignment` objects if custom XCom backend is configured or of its JSON serialized versions otherwise. + + :param pool: Either a `Pool` object or it's config or a pool_id. + :param status: A status or a list of statuses to get. All statuses (None) by default. + :param toloka_conn_id: Airflow connection with toloka credentials. + :param additional_args: Any other args presented in `toloka.client.search_requests.AssignmentSearchRequest`. + :returns: List of `Assignment` objects if custom XCom backend is configured or of its JSON serialized versions otherwise. + """ toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id) toloka_client = toloka_hook.get_conn() diff --git a/toloka_provider/utils/utils.py b/toloka_provider/utils/utils.py index fd3c26e..26b3cf6 100644 --- a/toloka_provider/utils/utils.py +++ b/toloka_provider/utils/utils.py @@ -12,6 +12,7 @@ def serialize_if_default_xcom_backend(func): + """Call to_json method to serialize toloka objects to pass through base xcom""" @wraps(func) def wrapper(*args, **kwargs): @@ -30,6 +31,8 @@ def wrapper(*args, **kwargs): def structure_from_conf(obj: Any, cl: Type[T]) -> T: + """Create toloka object from bytes or json config""" + if isinstance(obj, cl): return obj if isinstance(obj, bytes): @@ -44,6 +47,8 @@ def structure_from_conf(obj: Any, cl: Type[T]) -> T: def extract_id(obj: Any, cl: Type[T]) -> str: + """Extracts id attr from toloka objects or its config""" + if isinstance(obj, str): try: obj = json.loads(obj, parse_float=Decimal) From 0c5086a1c4a1999c83ec68ca9c210714355f786c Mon Sep 17 00:00:00 2001 From: Pocoder Date: Tue, 6 Sep 2022 10:20:15 +0200 Subject: [PATCH 2/2] update version to 0.0.8 --- CHANGELOG.md | 5 +++++ setup.py | 3 ++- toloka_provider/__init__.py | 2 +- toloka_provider/hooks/toloka.py | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da68ae0..bc70de2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +0.0.8 +------------------- +* Add `success_on_reasons` param to `WaitAppBatchSensor` init +* Format all docs according to sphinx standard + 0.0.7 ------------------- * Add `WaitAppBatchSensor` for waiting app batches diff --git a/setup.py b/setup.py index 87c196e..d879e42 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='airflow-provider-toloka', packages=['toloka_provider', *(f'toloka_provider.{package}' for package in find_packages('toloka_provider'))], - version='0.0.7', + version='0.0.8', description='A Toloka provider for Apache Airflow', long_description=readme, long_description_content_type='text/markdown', @@ -33,6 +33,7 @@ classifiers=[ 'Development Status :: 4 - Beta', 'Framework :: Apache Airflow', + 'Framework :: Apache Airflow :: Provider', 'Intended Audience :: Developers', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: Apache Software License', diff --git a/toloka_provider/__init__.py b/toloka_provider/__init__.py index 2c2bb0b..02bf0a0 100644 --- a/toloka_provider/__init__.py +++ b/toloka_provider/__init__.py @@ -9,5 +9,5 @@ def get_provider_info(): 'hook-class-name': 'toloka_provider.hooks.toloka.TolokaHook', }, ], - 'version': ['0.0.7'], + 'version': ['0.0.8'], } diff --git a/toloka_provider/hooks/toloka.py b/toloka_provider/hooks/toloka.py index 0c9017d..e37ba6a 100644 --- a/toloka_provider/hooks/toloka.py +++ b/toloka_provider/hooks/toloka.py @@ -14,9 +14,9 @@ class TolokaHook(BaseHook): """ Hook to interact with Toloka. - Performs a connection to Toloka and retrieves client. + Performs a connection to Toloka and retrieves client. - :param toloka_conn_id: Airflow Connection with OAuth token for Toloka. + :param toloka_conn_id: Airflow Connection with OAuth token for Toloka. You can learn more about how to get it [here](https://toloka.ai/docs/api/concepts/access.html#access__token). """