Merge pull request #11 from Toloka/docs-to-sphinx

Format doc strings according to sphinx

CHANGELOG.md

@@ -1,3 +1,8 @@
+0.0.8
+-------------------
+* Add `success_on_reasons` param to `WaitAppBatchSensor` init
+* Format all docs according to sphinx standard
+
 0.0.7
 -------------------
 * Add `WaitAppBatchSensor` for waiting app batches

setup.py

@@ -9,7 +9,7 @@
 setup(
     name='airflow-provider-toloka',
     packages=['toloka_provider', *(f'toloka_provider.{package}' for package in find_packages('toloka_provider'))],
-    version='0.0.7',
+    version='0.0.8',
     description='A Toloka provider for Apache Airflow',
     long_description=readme,
     long_description_content_type='text/markdown',
@@ -33,6 +33,7 @@
     classifiers=[
         'Development Status :: 4 - Beta',
         'Framework :: Apache Airflow',
+        'Framework :: Apache Airflow :: Provider',
         'Intended Audience :: Developers',
         'Intended Audience :: Science/Research',
         'License :: OSI Approved :: Apache Software License',

toloka_provider/__init__.py

@@ -9,5 +9,5 @@ def get_provider_info():
                 'hook-class-name': 'toloka_provider.hooks.toloka.TolokaHook',
             },
         ],
-        'version': ['0.0.7'],
+        'version': ['0.0.8'],
     }

toloka_provider/hooks/toloka.py

@@ -62,6 +62,7 @@ def get_conn(self) -> TolokaClient:
 
     @staticmethod
     def get_connection_form_widgets() -> Dict[str, Any]:
+        """Adds fields for connection in UI"""
         from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
         from flask_babel import lazy_gettext
         from wtforms import PasswordField, StringField, validators, ValidationError
@@ -86,6 +87,7 @@ def get_connection_form_widgets() -> Dict[str, Any]:
 
     @staticmethod
     def get_ui_field_behaviour() -> Dict[str, Any]:
+        """Hides unused fields in UI"""
         return {
             'hidden_fields': ['port', 'host', 'login', 'schema', 'extra', 'password'],
             'relabeling': {},

toloka_provider/sensors/toloka.py

@@ -15,7 +15,7 @@ class WaitPoolSensor(BaseSensorOperator):
     """
     Wait given pool until close.
 
-    :param pool: Either a `Pool` object or it's config or a pool_id value.
+    :param toloka_pool: Either a `Pool` object or it's config or a pool_id value.
     :param toloka_conn_id: Airflow connection with toloka credentials.
     :param success_on_reasons: Container of Pool.CloseReason. `WaitPoolSensor`
         will wait until both `pool` becomes closed and `success_on_reasons`
@@ -39,6 +39,7 @@ def __init__(
             reason) for reason in success_on_reasons] if success_on_reasons is not None else list(Pool.CloseReason)
 
     def poke(self, context: 'Context') -> bool:
+        """Checks if pool is closed"""
         pool_id = extract_id(self.toloka_pool, Pool)
         toloka_hook = TolokaHook(toloka_conn_id=self.toloka_conn_id)
         toloka_client = toloka_hook.get_conn()
@@ -81,6 +82,7 @@ def __init__(
         self.success_on_statuses = [AppBatch.Status(status) for status in success_on_statuses]
 
     def poke(self, context: 'Context') -> bool:
+        """Checks if batch status in success_on_statuses"""
         app_project_id = extract_id(self.app_project, AppProject)
         app_batch_id = extract_id(self.batch, AppBatch)
         toloka_hook = TolokaHook(toloka_conn_id=self.toloka_conn_id)

toloka_provider/tasks/toloka.py

@@ -25,11 +25,11 @@ def create_project(
     toloka_conn_id: str = 'toloka_default',
 ) -> Union[Project, str]:
     """Create a Project object from given config.
-    Args:
-        obj: Either a `Project` object itself or a config to make a `Project`.
-        toloka_conn_id: Airflow connection with toloka credentials.
-    Returns:
-        Project object if custom XCom backend is configured or its JSON serialized version otherwise.
+
+    :param obj: Either a `Project` object itself or a config to make a `Project`.
+    :param toloka_conn_id: Airflow connection with toloka credentials.
+    :returns: Project object if custom XCom backend is configured or its JSON serialized version otherwise.
+
     """
     toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id)
     toloka_client = toloka_hook.get_conn()
@@ -48,12 +48,12 @@ def create_exam_pool(
     toloka_conn_id: str = 'toloka_default',
 ) -> Union[Training, str]:
     """Create a Training pool object from given config.
-    Args:
-        obj: Either a `Training` object itself or a config to make a `Training`.
-        project: Project to assign a training pool to. May pass either an object, config or project_id.
-        toloka_conn_id: Airflow connection with toloka credentials.
-    Returns:
-        Training object if custom XCom backend is configured or its JSON serialized version otherwise.
+
+    :param obj: Either a `Training` object itself or a config to make a `Training`.
+    :param project: Project to assign a training pool to. May pass either an object, config or project_id.
+    :param toloka_conn_id: Airflow connection with toloka credentials.
+    :returns: Training object if custom XCom backend is configured or its JSON serialized version otherwise.
+
     """
     toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id)
     toloka_client = toloka_hook.get_conn()
@@ -76,17 +76,17 @@ def create_pool(
     toloka_conn_id: str = 'toloka_default',
 ) -> Union[Pool, str]:
     """Create a Pool object from given config.
-    Args:
-        obj: Either a `Pool` object itself or a config to make a `Pool`.
-        project: Project to assign a pool to. May pass either an object, config or project_id.
-        exam_pool: Related training pool. May pass either an object, config or pool_id.
-        expiration: Expiration setting. May pass any of:
-            * `None` if this setting is already present;
-            * `datetime` object to set exact datetime;
-            * `timedelta` to set expiration related to the current time.
-        toloka_conn_id: Airflow connection with toloka credentials.
-    Returns:
-        Pool object if custom XCom backend is configured or its JSON serialized version otherwise.
+
+    :param obj: Either a `Pool` object itself or a config to make a `Pool`.
+    :param project: Project to assign a pool to. May pass either an object, config or project_id.
+    :param exam_pool: Related training pool. May pass either an object, config or pool_id.
+    :param expiration: Expiration setting. May pass any of:
+        * `None` if this setting is already present;
+        * `datetime` object to set exact datetime;
+        * `timedelta` to set expiration related to the current time.
+    :param toloka_conn_id: Airflow connection with toloka credentials.
+    :returns: Pool object if custom XCom backend is configured or its JSON serialized version otherwise.
+
     """
     toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id)
     toloka_client = toloka_hook.get_conn()
@@ -114,12 +114,13 @@ def create_tasks(
     additional_args: Optional[Dict] = None,
 ) -> None:
     """Create a list of tasks for a given pool.
-    Args:
-        tasks: List of either a `Task` objects or a task conofigurations.
-        pool: Allow to set tasks pool if it's not present in the tasks themselves.
-            May be either a `Pool` or `Training` object or config or a pool_id value.
-        toloka_conn_id: Airflow connection with toloka credentials.
-        additional_args: Any other args presented in `toloka.client.task.CreateTasksParameters`.
+
+    :param tasks: List of either a `Task` objects or a task conofigurations.
+    :param pool: Allow to set tasks pool if it's not present in the tasks themselves.
+        May be either a `Pool` or `Training` object or config or a pool_id value.
+    :param toloka_conn_id: Airflow connection with toloka credentials.
+    :param additional_args: Any other args presented in `toloka.client.task.CreateTasksParameters`.
+    
     """
     toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id)
     toloka_client = toloka_hook.get_conn()
@@ -147,11 +148,11 @@ def open_pool(
     toloka_conn_id: str = 'toloka_default',
 ) -> Union[Pool, str]:
     """Open given pool.
-    Args:
-        obj: Pool id or `Pool` object of it's config.
-        toloka_conn_id: Airflow connection with toloka credentials.
-    Returns:
-        Pool object if custom XCom backend is configured or its JSON serialized version otherwise.
+
+    :param obj: Pool id or `Pool` object of it's config.
+    :param toloka_conn_id: Airflow connection with toloka credentials.
+    :returns: Pool object if custom XCom backend is configured or its JSON serialized version otherwise.
+
     """
     toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id)
     toloka_client = toloka_hook.get_conn()
@@ -170,11 +171,11 @@ def open_exam_pool(
     toloka_conn_id: str = 'toloka_default',
 ) -> Union[Pool, str]:
     """Open given training pool.
-    Args:
-        obj: Training pool_id or `Training` object of it's config.
-        toloka_conn_id: Airflow connection with toloka credentials.
-    Returns:
-        Training object if custom XCom backend is configured or its JSON serialized version otherwise.
+
+    :param obj: Training pool_id or `Training` object of it's config.
+    :param toloka_conn_id: Airflow connection with toloka credentials.
+    :returns: Training object if custom XCom backend is configured or its JSON serialized version otherwise.
+
     """
     toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id)
     toloka_client = toloka_hook.get_conn()
@@ -195,13 +196,13 @@ def get_assignments(
     additional_args: Optional[Dict] = None,
 ) -> List[Union[Assignment, str]]:
     """Get all assignments of selected status from pool.
-    Args:
-        pool: Either a `Pool` object or it's config or a pool_id.
-        status: A status or a list of statuses to get. All statuses (None) by default.
-        toloka_conn_id: Airflow connection with toloka credentials.
-        additional_args: Any other args presented in `toloka.client.search_requests.AssignmentSearchRequest`.
-    Returns:
-        List of `Assignment` objects if custom XCom backend is configured or of its JSON serialized versions otherwise.
+
+    :param pool: Either a `Pool` object or it's config or a pool_id.
+    :param status: A status or a list of statuses to get. All statuses (None) by default.
+    :param toloka_conn_id: Airflow connection with toloka credentials.
+    :param additional_args: Any other args presented in `toloka.client.search_requests.AssignmentSearchRequest`.
+    :returns: List of `Assignment` objects if custom XCom backend is configured or of its JSON serialized versions otherwise.
+
     """
     toloka_hook = TolokaHook(toloka_conn_id=toloka_conn_id)
     toloka_client = toloka_hook.get_conn()

toloka_provider/utils/utils.py

@@ -12,6 +12,7 @@
 
 
 def serialize_if_default_xcom_backend(func):
+    """Call to_json method to serialize toloka objects to pass through base xcom"""
 
     @wraps(func)
     def wrapper(*args, **kwargs):
@@ -30,6 +31,8 @@ def wrapper(*args, **kwargs):
 
 
 def structure_from_conf(obj: Any, cl: Type[T]) -> T:
+    """Create toloka object from bytes or json config"""
+
     if isinstance(obj, cl):
         return obj
     if isinstance(obj, bytes):
@@ -44,6 +47,8 @@ def structure_from_conf(obj: Any, cl: Type[T]) -> T:
 
 
 def extract_id(obj: Any, cl: Type[T]) -> str:
+    """Extracts id attr from toloka objects or its config"""
+
     if isinstance(obj, str):
         try:
             obj = json.loads(obj, parse_float=Decimal)