diff --git a/.devexp.json b/.devexp.json new file mode 100644 index 00000000..cf1b3919 --- /dev/null +++ b/.devexp.json @@ -0,0 +1,12 @@ +{ + "strict": true, + "notification": "email", + "static_members": [ + "losev", + "sinosov", + "tulinev", + "vlad-mois" + ], + "approve_count": 1, + "total_reviewers": 1 +} diff --git a/CHANGELOG.rst b/CHANGELOG.md similarity index 90% rename from CHANGELOG.rst rename to CHANGELOG.md index 1740a0d0..fce9d8da 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +0.1.8 +------------------- +* Added `get_aggregated_solutions` method +* Supported webhooks related methods +* Tracebacks in expanded methods do not show confusing TypeError as an original exception + + 0.1.7 ------------------- * Fixed error on ARRAY_JSON typed fields specs structuring diff --git a/examples/SQUAD2.0/SQUAD2.0_processing.ipynb b/examples/SQUAD2.0/SQUAD2.0_processing.ipynb index 6102cd6a..a58a385f 100644 --- a/examples/SQUAD2.0/SQUAD2.0_processing.ipynb +++ b/examples/SQUAD2.0/SQUAD2.0_processing.ipynb @@ -962,25 +962,17 @@ " return task_to_assignment\n", "\n", "\n", - "def get_aggregation_results():\n", + "def get_aggregation_results(pool_id):\n", " print('Start aggregation in the verification pool')\n", " aggregation_operation = toloka_client.aggregate_solutions_by_pool(\n", " type='DAWID_SKENE',\n", - " pool_id=verification_pool.id,\n", + " pool_id=pool_id,\n", " fields=[toloka.aggregation.PoolAggregatedSolutionRequest.Field(name='is_correct')]\n", " )\n", " aggregation_operation = toloka_client.wait_operation(aggregation_operation)\n", " print('Results aggregated')\n", "\n", - " aggregation_result = toloka_client.find_aggregated_solutions(aggregation_operation.id)\n", - " verification_results = aggregation_result.items\n", - " while aggregation_result.has_more:\n", - " aggregation_result = toloka_client.find_aggregated_solutions(\n", - " aggregation_operation.id,\n", - " task_id_gt=aggregation_result.items[len(aggregation_result.items) - 1].task_id,\n", - " )\n", - " verification_results = verification_results + aggregation_result.items\n", - " return verification_results\n", + " return list(toloka_client.get_aggregated_solutions(aggregation_operation.id))\n", "\n", "\n", "def set_answers_status(verification_results):\n", @@ -1053,7 +1045,7 @@ " print(f'Verification pool {verification_pool.id} is finally closed!')\n", "\n", " # Aggregation operation\n", - " verification_results = get_aggregation_results()\n", + " verification_results = get_aggregation_results(verification_pool.id)\n", " # Reject or accept tasks in the segmentation pool\n", " set_answers_status(verification_results)\n", "\n", @@ -1172,4 +1164,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/examples/image_segmentation/image_segmentation.ipynb b/examples/image_segmentation/image_segmentation.ipynb index 6eac5d35..71877435 100644 --- a/examples/image_segmentation/image_segmentation.ipynb +++ b/examples/image_segmentation/image_segmentation.ipynb @@ -843,12 +843,12 @@ "cell_type": "code", "metadata": {}, "source": [ - "def get_aggregation_results():\n", + "def get_aggregation_results(pool_id, skill_id):\n", " print('Start aggregation in the verification pool')\n", " aggregation_operation = toloka_client.aggregate_solutions_by_pool(\n", " type=toloka.aggregation.AggregatedSolutionType.WEIGHTED_DYNAMIC_OVERLAP,\n", - " pool_id=verification_pool.id, # Aggregate in this pool\n", - " answer_weight_skill_id=verification_skill.id, # Aggregate by this skill\n", + " pool_id=pool_id, # Aggregate in this pool\n", + " answer_weight_skill_id=skill_id, # Aggregate by this skill\n", " fields=[toloka.aggregation.PoolAggregatedSolutionRequest.Field(name='result')] # Aggregate this field\n", " )\n", "\n", @@ -857,19 +857,7 @@ " print('Results aggregated')\n", "\n", " # Get aggregated results\n", - " # Set a limit to show how to iterate over aggregation results\n", - " aggregation_result = toloka_client.find_aggregated_solutions(aggregation_operation.id, limit=5)\n", - " verification_results = aggregation_result.items\n", - " # If we have more results, let's get them\n", - " while aggregation_result.has_more:\n", - " aggregation_result = toloka_client.find_aggregated_solutions(\n", - " aggregation_operation.id,\n", - " # We have to establish which id we want to get results from (or else we'll loop back)\n", - " # This is usually the last item id in the previous request\n", - " task_id_gt=aggregation_result.items[len(aggregation_result.items) - 1].task_id,\n", - " )\n", - " verification_results = verification_results + aggregation_result.items\n", - " return verification_results\n", + " return list(toloka_client.get_aggregated_solutions(aggregation_operation.id))\n", "\n", "\n", "def set_segmentation_status(verification_results):\n", @@ -890,7 +878,7 @@ "\n", "\n", "# Aggregation operation\n", - "verification_results = get_aggregation_results()\n", + "verification_results = get_aggregation_results(verification_pool.id, verification_skill.id)\n", "# Reject or accept tasks in the segmentation pool\n", "set_segmentation_status(verification_results)" ], @@ -931,7 +919,7 @@ " print(f'Verification pool {verification_pool.id} is finally closed!')\n", "\n", " # Aggregation operation\n", - " verification_results = get_aggregation_results()\n", + " verification_results = get_aggregation_results(verification_pool.id, verification_skill.id)\n", " # Reject or accept tasks in the segmentation pool\n", " set_segmentation_status(verification_results)\n", "\n", @@ -1026,4 +1014,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/src/client/__init__.py b/src/client/__init__.py index 156585fc..3eb41614 100644 --- a/src/client/__init__.py +++ b/src/client/__init__.py @@ -15,21 +15,22 @@ 'Pool', 'Project', ] -import datetime -from decimal import Decimal -import time -from enum import Enum, unique -from typing import List, Optional, Union, BinaryIO, Tuple, Generator -import pandas as pd import attr - +import datetime +import functools import io import logging +import pandas as pd import requests +import time +import uuid + +from decimal import Decimal +from enum import Enum, unique from requests.adapters import HTTPAdapter +from typing import BinaryIO, Generator, List, Optional, Tuple, Union from urllib3.util.retry import Retry -import uuid from . import actions # noqa: F401 from . import aggregation @@ -312,7 +313,8 @@ def aggregate_solutions_by_task(self, request: aggregation.WeightedDynamicOverla def find_aggregated_solutions(self, operation_id: str, request: search_requests.AggregatedSolutionSearchRequest, sort: Union[List[str], search_requests.AggregatedSolutionSortItems, None] = None, limit: Optional[int] = None) -> search_results.AggregatedSolutionSearchResult: - """Gets aggregated responses after the AggregatedSolutionOperation completes + """Gets aggregated responses after the AggregatedSolutionOperation completes. + It is better to use the "get_aggregated_solutions" method, that allows to iterate through all results. Note: In all aggregation purposes we are strongly recommending using our crowd-kit library, that have more aggregation methods and can perform on your computers: https://github.com/Toloka/crowd-kit @@ -347,6 +349,31 @@ def find_aggregated_solutions(self, operation_id: str, request: search_requests. response = self._search_request('get', f'/v1/aggregated-solutions/{operation_id}', request, sort, limit) return structure(response, search_results.AggregatedSolutionSearchResult) + @expand('request') + def get_aggregated_solutions(self, operation_id: str, request: search_requests.AggregatedSolutionSearchRequest) -> Generator[AggregatedSolution, None, None]: + """Finds all aggregated responses after the AggregatedSolutionOperation completes + + Note: In all aggregation purposes we are strongly recommending using our crowd-kit library, that have more aggregation + methods and can perform on your computers: https://github.com/Toloka/crowd-kit + + Args: + operation_id: From what aggregation operation you want to get results. + request: How to filter search results. + + Yields: + AggregatedSolution: The next object corresponding to the request parameters. + + Example: + How to get all aggregated solutions from pool. + + >>> # run toloka_client.aggregate_solutions_by_pool and wait operation for closing. + >>> aggregation_results = list(toloka_client.get_aggregated_solutions(aggregation_operation.id)) + >>> print(len(aggregation_results)) + ... + """ + find_function = functools.partial(self.find_aggregated_solutions, operation_id) + return self._find_all(find_function, request, sort_field='task_id') + # Assignments section def accept_assignment(self, assignment_id: str, public_comment: str) -> Assignment: @@ -2176,7 +2203,7 @@ def delete_user_skill(self, user_skill_id: str) -> None: """ self._raw_request('delete', f'/v1/user-skills/{user_skill_id}') - def create_webhook_subscriptions(self, subscriptions: List[WebhookSubscription]) -> batch_create_results.WebhookSubscriptionBatchCreateResult: + def upsert_webhook_subscriptions(self, subscriptions: List[WebhookSubscription]) -> batch_create_results.WebhookSubscriptionBatchCreateResult: """Creates (upsert) many webhook-subscriptions. Args: @@ -2192,7 +2219,7 @@ def create_webhook_subscriptions(self, subscriptions: List[WebhookSubscription]) Example: How to create several subscriptions. - >>> created_result = toloka_client.create_webhook_subscriptions([ + >>> created_result = toloka_client.upsert_webhook_subscriptions([ >>> { >>> 'webhook_url': 'https://awesome-requester.com/toloka-webhook', >>> 'event_type': toloka.webhook_subscription.WebhookSubscription.EventType.ASSIGNMENT_CREATED, diff --git a/src/client/__init__.pyi b/src/client/__init__.pyi index e06573dd..42c242d9 100644 --- a/src/client/__init__.pyi +++ b/src/client/__init__.pyi @@ -1120,7 +1120,7 @@ class TolokaClient: """ ... - def create_webhook_subscriptions(self, subscriptions: List[WebhookSubscription]) -> WebhookSubscriptionBatchCreateResult: + def upsert_webhook_subscriptions(self, subscriptions: List[WebhookSubscription]) -> WebhookSubscriptionBatchCreateResult: """Creates (upsert) many webhook-subscriptions. Args: @@ -1136,7 +1136,7 @@ class TolokaClient: Example: How to create several subscriptions. - >>> created_result = toloka_client.create_webhook_subscriptions([ + >>> created_result = toloka_client.upsert_webhook_subscriptions([ >>> { >>> 'webhook_url': 'https://awesome-requester.com/toloka-webhook', >>> 'event_type': toloka.webhook_subscription.WebhookSubscription.EventType.ASSIGNMENT_CREATED, @@ -1197,7 +1197,8 @@ class TolokaClient: @overload def find_aggregated_solutions(self, operation_id: str, task_id_lt: Optional[str] = None, task_id_lte: Optional[str] = None, task_id_gt: Optional[str] = None, task_id_gte: Optional[str] = None, sort: Union[List[str], AggregatedSolutionSortItems, None] = None, limit: Optional[int] = None) -> AggregatedSolutionSearchResult: - """Gets aggregated responses after the AggregatedSolutionOperation completes + """Gets aggregated responses after the AggregatedSolutionOperation completes. + It is better to use the "get_aggregated_solutions" method, that allows to iterate through all results. Note: In all aggregation purposes we are strongly recommending using our crowd-kit library, that have more aggregation methods and can perform on your computers: https://github.com/Toloka/crowd-kit @@ -1232,7 +1233,8 @@ class TolokaClient: @overload def find_aggregated_solutions(self, operation_id: str, request: AggregatedSolutionSearchRequest, sort: Union[List[str], AggregatedSolutionSortItems, None] = None, limit: Optional[int] = None) -> AggregatedSolutionSearchResult: - """Gets aggregated responses after the AggregatedSolutionOperation completes + """Gets aggregated responses after the AggregatedSolutionOperation completes. + It is better to use the "get_aggregated_solutions" method, that allows to iterate through all results. Note: In all aggregation purposes we are strongly recommending using our crowd-kit library, that have more aggregation methods and can perform on your computers: https://github.com/Toloka/crowd-kit @@ -1265,6 +1267,54 @@ class TolokaClient: """ ... + @overload + def get_aggregated_solutions(self, operation_id: str, task_id_lt: Optional[str] = None, task_id_lte: Optional[str] = None, task_id_gt: Optional[str] = None, task_id_gte: Optional[str] = None) -> Generator[AggregatedSolution, None, None]: + """Finds all aggregated responses after the AggregatedSolutionOperation completes + + Note: In all aggregation purposes we are strongly recommending using our crowd-kit library, that have more aggregation + methods and can perform on your computers: https://github.com/Toloka/crowd-kit + + Args: + operation_id: From what aggregation operation you want to get results. + request: How to filter search results. + + Yields: + AggregatedSolution: The next object corresponding to the request parameters. + + Example: + How to get all aggregated solutions from pool. + + >>> # run toloka_client.aggregate_solutions_by_pool and wait operation for closing. + >>> aggregation_results = list(toloka_client.get_aggregated_solutions(aggregation_operation.id)) + >>> print(len(aggregation_results)) + ... + """ + ... + + @overload + def get_aggregated_solutions(self, operation_id: str, request: AggregatedSolutionSearchRequest) -> Generator[AggregatedSolution, None, None]: + """Finds all aggregated responses after the AggregatedSolutionOperation completes + + Note: In all aggregation purposes we are strongly recommending using our crowd-kit library, that have more aggregation + methods and can perform on your computers: https://github.com/Toloka/crowd-kit + + Args: + operation_id: From what aggregation operation you want to get results. + request: How to filter search results. + + Yields: + AggregatedSolution: The next object corresponding to the request parameters. + + Example: + How to get all aggregated solutions from pool. + + >>> # run toloka_client.aggregate_solutions_by_pool and wait operation for closing. + >>> aggregation_results = list(toloka_client.get_aggregated_solutions(aggregation_operation.id)) + >>> print(len(aggregation_results)) + ... + """ + ... + @overload def find_assignments(self, status: Union[str, Assignment.Status, List[Union[str, Assignment.Status]]] = None, task_id: Optional[str] = None, task_suite_id: Optional[str] = None, pool_id: Optional[str] = None, user_id: Optional[str] = None, id_lt: Optional[str] = None, id_lte: Optional[str] = None, id_gt: Optional[str] = None, id_gte: Optional[str] = None, created_lt: Optional[datetime] = None, created_lte: Optional[datetime] = None, created_gt: Optional[datetime] = None, created_gte: Optional[datetime] = None, submitted_lt: Optional[datetime] = None, submitted_lte: Optional[datetime] = None, submitted_gt: Optional[datetime] = None, submitted_gte: Optional[datetime] = None, sort: Union[List[str], AssignmentSortItems, None] = None, limit: Optional[int] = None) -> AssignmentSearchResult: """Finds all assignments that match certain rules diff --git a/src/client/__version__.py b/src/client/__version__.py index 93aa807e..410e5a4d 100644 --- a/src/client/__version__.py +++ b/src/client/__version__.py @@ -1,3 +1,3 @@ __title__ = 'toloka-kit' -__version__ = '0.1.7' +__version__ = '0.1.8' __license__ = 'Apache 2.0' diff --git a/src/client/project/template_builder/plugins.py b/src/client/project/template_builder/plugins.py index 54eae499..2d70a24e 100644 --- a/src/client/project/template_builder/plugins.py +++ b/src/client/project/template_builder/plugins.py @@ -26,6 +26,16 @@ class HotkeysPluginV1(BasePluginV1, spec_value=ComponentType.PLUGIN_HOTKEYS): Attributes: key_ + [a-z|0-9|up|down]: An action that is triggered when you press the specified keyboard key. The keyboard shortcut is set in the key, and the action is specified in the value + + Example: + How to create hotkeys for classification buttons. + + >>> hot_keys_plugin = tb.HotkeysPluginV1( + >>> key_1=tb.SetActionV1(data=tb.OutputData(path='result'), payload='cat'), + >>> key_2=tb.SetActionV1(data=tb.OutputData(path='result'), payload='dog'), + >>> key_3=tb.SetActionV1(data=tb.OutputData(path='result'), payload='other'), + >>> ) + ... """ key_a: base_component_or(Any) = attribute(default=None, origin='a') diff --git a/src/client/project/template_builder/plugins.pyi b/src/client/project/template_builder/plugins.pyi index 5ccbbf91..6be2ab89 100644 --- a/src/client/project/template_builder/plugins.pyi +++ b/src/client/project/template_builder/plugins.pyi @@ -31,6 +31,16 @@ class HotkeysPluginV1(BasePluginV1): Attributes: key_ + [a-z|0-9|up|down]: An action that is triggered when you press the specified keyboard key. The keyboard shortcut is set in the key, and the action is specified in the value + + Example: + How to create hotkeys for classification buttons. + + >>> hot_keys_plugin = tb.HotkeysPluginV1( + >>> key_1=tb.SetActionV1(data=tb.OutputData(path='result'), payload='cat'), + >>> key_2=tb.SetActionV1(data=tb.OutputData(path='result'), payload='dog'), + >>> key_3=tb.SetActionV1(data=tb.OutputData(path='result'), payload='other'), + >>> ) + ... """ def __init__(self, *, version: Optional[str] = '1.0.0', key_a: Optional[Any] = None, key_b: Optional[Any] = None, key_c: Optional[Any] = None, key_d: Optional[Any] = None, key_e: Optional[Any] = None, key_f: Optional[Any] = None, key_g: Optional[Any] = None, key_h: Optional[Any] = None, key_i: Optional[Any] = None, key_j: Optional[Any] = None, key_k: Optional[Any] = None, key_l: Optional[Any] = None, key_m: Optional[Any] = None, key_n: Optional[Any] = None, key_o: Optional[Any] = None, key_p: Optional[Any] = None, key_q: Optional[Any] = None, key_r: Optional[Any] = None, key_s: Optional[Any] = None, key_t: Optional[Any] = None, key_u: Optional[Any] = None, key_v: Optional[Any] = None, key_w: Optional[Any] = None, key_x: Optional[Any] = None, key_y: Optional[Any] = None, key_z: Optional[Any] = None, key_0: Optional[Any] = None, key_1: Optional[Any] = None, key_2: Optional[Any] = None, key_3: Optional[Any] = None, key_4: Optional[Any] = None, key_5: Optional[Any] = None, key_6: Optional[Any] = None, key_7: Optional[Any] = None, key_8: Optional[Any] = None, key_9: Optional[Any] = None, key_up: Optional[Any] = None, key_down: Optional[Any] = None) -> None: diff --git a/src/client/webhook_subscription.py b/src/client/webhook_subscription.py index 736a251a..2b31a886 100644 --- a/src/client/webhook_subscription.py +++ b/src/client/webhook_subscription.py @@ -45,6 +45,7 @@ class EventType(Enum): webhook_url: str event_type: EventType pool_id: str + secret_key: str # Readonly id: str diff --git a/src/client/webhook_subscription.pyi b/src/client/webhook_subscription.pyi index 9db0d7b2..47892ca0 100644 --- a/src/client/webhook_subscription.pyi +++ b/src/client/webhook_subscription.pyi @@ -41,7 +41,7 @@ class WebhookSubscription(BaseTolokaObject): ASSIGNMENT_APPROVED = 'ASSIGNMENT_APPROVED' ASSIGNMENT_REJECTED = 'ASSIGNMENT_REJECTED' - def __init__(self, *, webhook_url: Optional[str] = None, event_type: Optional[EventType] = None, pool_id: Optional[str] = None, id: Optional[str] = None, created: Optional[datetime] = None) -> None: + def __init__(self, *, webhook_url: Optional[str] = None, event_type: Optional[EventType] = None, pool_id: Optional[str] = None, secret_key: Optional[str] = None, id: Optional[str] = None, created: Optional[datetime] = None) -> None: """Method generated by attrs for class WebhookSubscription. """ ... @@ -50,5 +50,6 @@ class WebhookSubscription(BaseTolokaObject): webhook_url: Optional[str] event_type: Optional[EventType] pool_id: Optional[str] + secret_key: Optional[str] id: Optional[str] created: Optional[datetime] diff --git a/tests/test_aggregated_solution.py b/tests/test_aggregated_solution.py index bac46646..8b532e5f 100644 --- a/tests/test_aggregated_solution.py +++ b/tests/test_aggregated_solution.py @@ -148,3 +148,55 @@ def aggregated_solutions(request, context): limit=42, ) assert raw_result == client.unstructure(result) + + +def test_get_aggregated_solutions(requests_mock, toloka_client, toloka_url): + backend_solutions = [ + { + 'pool_id': '11', + 'task_id': '111', + 'output_values': {'out1': True}, + 'confidence': 0.111 + }, + { + 'pool_id': '11', + 'task_id': '112', + 'output_values': {'out1': True}, + 'confidence': 0.112 + }, + { + 'pool_id': '11', + 'task_id': '113', + 'output_values': {'out1': True}, + 'confidence': 0.113 + }, + { + 'pool_id': '11', + 'task_id': '114', + 'output_values': {'out1': True}, + 'confidence': 0.114 + }, + { + 'pool_id': '11', + 'task_id': '115', + 'output_values': {'out1': True}, + 'confidence': 0.115 + } + ] + + def find_aggregated_solutions_mock(request, _): + params = parse_qs(urlparse(request.url).query) + task_id_gt = params.pop('task_id_gt', None) + assert {'sort': ['task_id']} == params, params + solutions_greater = [ + item + for item in backend_solutions + if task_id_gt is None or item['task_id'] > task_id_gt[0] + ][:2] # For test purposes return 2 items at a time. + has_more = (solutions_greater[-1]['task_id'] != backend_solutions[-1]['task_id']) + return {'items': solutions_greater, 'has_more': has_more} + + requests_mock.get(f'{toloka_url}/aggregated-solutions/some_op_id', + json=find_aggregated_solutions_mock, + status_code=200) + assert backend_solutions == client.unstructure(list(toloka_client.get_aggregated_solutions('some_op_id'))) diff --git a/tests/test_webhook_subscription.py b/tests/test_webhook_subscription.py index f4a5a7c9..bb9fea42 100644 --- a/tests/test_webhook_subscription.py +++ b/tests/test_webhook_subscription.py @@ -1,5 +1,4 @@ import datetime -from operator import itemgetter from urllib.parse import urlparse, parse_qs from uuid import uuid4 @@ -28,7 +27,7 @@ def webhook_subscriptions_map(): @pytest.fixture -def create_webhook_subscriptions_result_map(): +def upsert_webhook_subscriptions_result_map(): return { 'items': { '0': { @@ -57,20 +56,20 @@ def create_webhook_subscriptions_result_map(): } -def test_create_webhook_subscriptions( +def test_upsert_webhook_subscriptions( requests_mock, toloka_client, toloka_url, - webhook_subscriptions_map, create_webhook_subscriptions_result_map + webhook_subscriptions_map, upsert_webhook_subscriptions_result_map ): - def create_webhook_subscriptions_mock(request, _): + def upsert_webhook_subscriptions_mock(request, _): assert webhook_subscriptions_map == request.json() - return create_webhook_subscriptions_result_map + return upsert_webhook_subscriptions_result_map - # create_webhook_subscriptions -> operation - requests_mock.put(f'{toloka_url}/webhook-subscriptions', json=create_webhook_subscriptions_mock, status_code=201) + # upsert_webhook_subscriptions -> operation + requests_mock.put(f'{toloka_url}/webhook-subscriptions', json=upsert_webhook_subscriptions_mock, status_code=201) - result = toloka_client.create_webhook_subscriptions(webhook_subscriptions_map) - assert create_webhook_subscriptions_result_map == client.unstructure(result) + result = toloka_client.upsert_webhook_subscriptions(webhook_subscriptions_map) + assert upsert_webhook_subscriptions_result_map == client.unstructure(result) @pytest.fixture