From 791d1f7bd4ddc799abbf9aa5d4bd50767fe96b18 Mon Sep 17 00:00:00 2001 From: mr-fedulow Date: Fri, 17 Dec 2021 17:46:37 +0300 Subject: [PATCH 01/18] Add support of python3.10 Add support of python 3.10 ref:22f74dc1eaa336c7c777733bcfe48ffe7ccf9aeb --- .github/workflows/tests.yml | 2 +- setup.py | 1 + src/client/__init__.py | 2 +- src/streaming/storage.py | 2 +- tox.ini | 3 ++- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 22008c31..a7d21a7d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index f07fa123..4fec7ae3 100755 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Scientific/Engineering', 'Topic :: Software Development', diff --git a/src/client/__init__.py b/src/client/__init__.py index d5f6f64b..44f6698d 100644 --- a/src/client/__init__.py +++ b/src/client/__init__.py @@ -249,7 +249,7 @@ def _default_retryer_factory( retry_quotas=retry_quotas, total=retries, status_forcelist=list(status_list), - method_whitelist=['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST', 'PATCH'], + allowed_methods=['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST', 'PATCH'], backoff_factor=2, # summary retry time more than 10 seconds ) diff --git a/src/streaming/storage.py b/src/streaming/storage.py index 54157de4..a5e3d8b5 100644 --- a/src/streaming/storage.py +++ b/src/streaming/storage.py @@ -175,7 +175,7 @@ class S3Storage(BaseExternalLockerStorage): @classmethod def _is_not_found_error(cls, exc: Exception) -> bool: response = getattr(exc, 'response', None) - if isinstance(response, collections.Mapping): + if isinstance(response, collections.abc.Mapping): error_info = response.get('Error') if error_info and 'Code' in error_info and 'Message' in error_info: return error_info['Code'] == '404' and error_info['Message'] == 'Not Found' diff --git a/tox.ini b/tox.ini index 61eef083..67931e3f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 3.3.0 -envlist = py3{6,7,8,9}-attrs{20,21} +envlist = py3{6,7,8,9,10}-attrs{20,21} isolated_build = True requires = setuptools >= 36.2.0 @@ -10,6 +10,7 @@ python = 3.7: py37-attrs{20,21} 3.8: py38-attrs{20,21} 3.9: py39-attrs{20,21} + 3.10: py310-attrs{20,21} [testenv] deps = From 151aad301fd9f4c4313c36b26e34eabe32ac5999 Mon Sep 17 00:00:00 2001 From: Vladimir Losev Date: Fri, 17 Dec 2021 19:02:55 +0300 Subject: [PATCH 02/18] Ditamap fix ref:c6c9f067f3b210dadb554d4a80bd1aa329ade784 --- docs/reference/_reference.ditamap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/_reference.ditamap b/docs/reference/_reference.ditamap index a79bd25e..08cd667d 100644 --- a/docs/reference/_reference.ditamap +++ b/docs/reference/_reference.ditamap @@ -999,7 +999,7 @@ - + From ab44b5dc2f1be7a16ad3b70615d822e7abc1f768 Mon Sep 17 00:00:00 2001 From: mr-fedulow Date: Wed, 22 Dec 2021 13:31:19 +0300 Subject: [PATCH 03/18] Add map_provider to AssignmentsIssuingViewConfig Add map_provider to AssignmentsIssuingViewConfig ref:55514a08ab113d27009471761836acd10a4efa36 --- ...ct.AssignmentsIssuingViewConfig.MapProvider.md | 11 +++++++++++ ...roject.Project.AssignmentsIssuingViewConfig.md | 3 ++- src/client/project/__init__.py | 13 +++++++++++++ src/client/project/__init__.pyi | 15 ++++++++++++++- tests/test_project.py | 8 ++++++-- 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.MapProvider.md diff --git a/docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.MapProvider.md b/docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.MapProvider.md new file mode 100644 index 00000000..5f5003ea --- /dev/null +++ b/docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.MapProvider.md @@ -0,0 +1,11 @@ +# MapProvider +`toloka.client.project.Project.AssignmentsIssuingViewConfig.MapProvider` + +Map provider for assignments_issuing_view_config: + +## Attributes Description + +| Name | Value | Description | +| :------| :-----------| :----------| +`YANDEX`|'YANDEX'|

Use Yandex.Maps as a map provider

+`GOOGLE`|'GOOGLE'|

Use Google Maps as a map provider

diff --git a/docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.md b/docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.md index 9da5fcd3..fafb1dc0 100644 --- a/docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.md +++ b/docs/reference/toloka.client.project.Project.AssignmentsIssuingViewConfig.md @@ -6,7 +6,8 @@ AssignmentsIssuingViewConfig( self, *, title_template: Optional[str] = None, - description_template: Optional[str] = None + description_template: Optional[str] = None, + map_provider: Optional[MapProvider] = None ) ``` diff --git a/src/client/project/__init__.py b/src/client/project/__init__.py index b44910ed..541ab547 100644 --- a/src/client/project/__init__.py +++ b/src/client/project/__init__.py @@ -149,8 +149,21 @@ class AssignmentsIssuingViewConfig(BaseTolokaObject): description_template: Brief description of the task. Users will see it in the task preview mode. """ + @unique + class MapProvider(ExtendableStrEnum): + """Map provider for assignments_issuing_view_config: + + Attributes: + YANDEX: Use Yandex.Maps as a map provider + GOOGLE: Use Google Maps as a map provider + """ + + YANDEX = 'YANDEX' + GOOGLE = 'GOOGLE' + title_template: str description_template: str + map_provider: Optional[MapProvider] = None QualityControl = QualityControl diff --git a/src/client/project/__init__.pyi b/src/client/project/__init__.pyi index ecbe366f..30c2ace6 100644 --- a/src/client/project/__init__.pyi +++ b/src/client/project/__init__.pyi @@ -151,11 +151,23 @@ class Project(toloka.client.primitives.base.BaseTolokaObject): description_template: Brief description of the task. Users will see it in the task preview mode. """ + class MapProvider(toloka.util._extendable_enum.ExtendableStrEnum): + """Map provider for assignments_issuing_view_config: + + Attributes: + YANDEX: Use Yandex.Maps as a map provider + GOOGLE: Use Google Maps as a map provider + """ + + YANDEX = 'YANDEX' + GOOGLE = 'GOOGLE' + def __init__( self, *, title_template: typing.Optional[str] = None, - description_template: typing.Optional[str] = None + description_template: typing.Optional[str] = None, + map_provider: typing.Optional[MapProvider] = None ) -> None: """Method generated by attrs for class Project.AssignmentsIssuingViewConfig. """ @@ -164,6 +176,7 @@ class Project(toloka.client.primitives.base.BaseTolokaObject): _unexpected: typing.Optional[typing.Dict[str, typing.Any]] title_template: typing.Optional[str] description_template: typing.Optional[str] + map_provider: typing.Optional[MapProvider] def set_default_language(self, language: str): """Sets the source language used in the fields public_name, public_description, and public_instructions. diff --git a/tests/test_project.py b/tests/test_project.py index cde2e117..bd60613d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -247,6 +247,7 @@ def simple_poject_map(): 'assignments_issuing_view_config': { 'title_template': 'Company: {{inputParams[\'company\']}}', 'description_template': 'Check if company {{inputParams[\'company\']}} exists', + 'map_provider': 'YANDEX', }, 'metadata': {'projectMetadataKey': ['projectMetadataValue']}, 'quality_control': { @@ -450,7 +451,8 @@ def test_project_update(requests_mock, toloka_client, toloka_url): 'max_active_assignments_count': 5, 'assignments_issuing_view_config': { 'title_template': 'Company: {{inputParams[\'company\']}}', - 'description_template': 'Check if company {{inputParams[\'company\']}} exists' + 'description_template': 'Check if company {{inputParams[\'company\']}} exists', + 'map_provider': 'GOOGLE' }, } @@ -488,6 +490,7 @@ def update_project(request, context): assignments_issuing_view_config=client.project.Project.AssignmentsIssuingViewConfig( title_template='Company: {{inputParams[\'company\']}}', description_template='Check if company {{inputParams[\'company\']}} exists', + map_provider=client.project.Project.AssignmentsIssuingViewConfig.MapProvider.GOOGLE ), max_active_assignments_count=5 ) @@ -507,7 +510,8 @@ def update_project(request, context): public_instructions='Check if company exists', assignments_issuing_view_config=client.project.Project.AssignmentsIssuingViewConfig( title_template='Company: {{inputParams[\'company\']}}', - description_template='Check if company {{inputParams[\'company\']}} exists' + description_template='Check if company {{inputParams[\'company\']}} exists', + map_provider=client.project.Project.AssignmentsIssuingViewConfig.MapProvider.GOOGLE ), max_active_assignments_count=5, id='10', From 3865a880a1acc1f8674b92bcf5c610ecedc018f6 Mon Sep 17 00:00:00 2001 From: Sukhorosov Aleksey Date: Thu, 23 Dec 2021 12:36:28 +0300 Subject: [PATCH 04/18] create socket correctly when using ipv4 ref:49aa49cc4179fb130ce233b04f2ecea20af9f2a2 --- examples/metrics/graphite.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/metrics/graphite.ipynb b/examples/metrics/graphite.ipynb index 4998b48b..070e3cbd 100644 --- a/examples/metrics/graphite.ipynb +++ b/examples/metrics/graphite.ipynb @@ -186,7 +186,7 @@ " s.connect((self.carbon_address, self.carbon_port, 0, 0))\n", " else:\n", " s = socket.socket()\n", - " s.connect(self.carbon_address, self.carbon_port)\n", + " s.connect((self.carbon_address, self.carbon_port))\n", "\n", " for metric in metric_dict:\n", " for timestamp, value in metric_dict[metric]:\n", @@ -354,4 +354,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file From 727db46b8a7e404fd8e8659d9d082c0f90868641 Mon Sep 17 00:00:00 2001 From: pocoder Date: Fri, 24 Dec 2021 15:02:16 +0300 Subject: [PATCH 05/18] Add headers in metrics apply add_headers to metrics methods ref:03aa1791897f6551fef53446957159f31aa805a5 --- ...til._managing_headers.async_add_headers.md | 19 +++++++++ src/metrics/collector.py | 4 +- src/metrics/jupyter_dashboard.py | 5 +++ src/metrics/metrics.py | 26 ++++++------ src/metrics/pool_metrics.py | 40 ++++++++++--------- src/util/_managing_headers.py | 31 ++++++++++++++ src/util/_managing_headers.pyi | 13 ++++++ 7 files changed, 106 insertions(+), 32 deletions(-) create mode 100644 docs/reference/toloka.util._managing_headers.async_add_headers.md diff --git a/docs/reference/toloka.util._managing_headers.async_add_headers.md b/docs/reference/toloka.util._managing_headers.async_add_headers.md new file mode 100644 index 00000000..17f047b7 --- /dev/null +++ b/docs/reference/toloka.util._managing_headers.async_add_headers.md @@ -0,0 +1,19 @@ +# async_add_headers +`toloka.util._managing_headers.async_add_headers` + +``` +async_add_headers(client: str) +``` + +This decorator add 3 headers into resulting http request called by async function: + + +1) X-Caller-Context: high-level abstraction like client, metrics, streaming +2) X-Top-Level-Method: first function, that was called and then called other functions which provoked request +3) X-Low-Level-Method: last function before calling TolokaClient _method (_raw_request for example) + +## Parameters Description + +| Parameters | Type | Description | +| :----------| :----| :-----------| +`client`|**str**|

name of high-level abstraction for X-Caller-Context

diff --git a/src/metrics/collector.py b/src/metrics/collector.py index f6695aa9..90b2ba62 100644 --- a/src/metrics/collector.py +++ b/src/metrics/collector.py @@ -9,6 +9,7 @@ from .metrics import BaseMetric from ..util.async_utils import ComplexException, get_task_traceback +from ..util._managing_headers import async_add_headers logger = logging.getLogger(__name__) @@ -47,7 +48,7 @@ class MetricCollector: >>> asyncio.run(collector.run()) """ - metrics : List[BaseMetric] + metrics: List[BaseMetric] _callback: Callable[[NamedMetrics], None] def __init__(self, metrics: List[BaseMetric], callback: Callable[[NamedMetrics], None]): @@ -69,6 +70,7 @@ def create_async_tasks(coro): task.new_coro = coro return task + @async_add_headers('metrics') async def run(self): """Starts collecting metrics. And never stops.""" tasks = [MetricCollector.create_async_tasks(m.get_lines) for m in self.metrics] diff --git a/src/metrics/jupyter_dashboard.py b/src/metrics/jupyter_dashboard.py index d7f5231c..7472c40d 100644 --- a/src/metrics/jupyter_dashboard.py +++ b/src/metrics/jupyter_dashboard.py @@ -31,6 +31,8 @@ from .metrics import BaseMetric from ..util.async_utils import get_task_traceback +from ..util._managing_headers import add_headers + logger = logging.getLogger(__name__) @@ -94,6 +96,7 @@ def create_async_tasks(metric: BaseMetric, loop: asyncio.AbstractEventLoop): task.metric_inst = metric return task + @add_headers('metrics') def update_metrics(self): """Gathers all metrics, and stores them in lines. """ @@ -316,6 +319,7 @@ def update_charts(self, n_intervals: int) -> Union[go.Figure, List[go.Figure]]: figures.append(new_figure) return figures if len(figures) > 1 else figures[0] + @add_headers('metrics') def run_dash(self, mode: str = 'inline', height: int = None, host: str = '127.0.0.1', port: str = '8050'): """Starts dashboard. Starts server for online updating charts. @@ -336,6 +340,7 @@ def run_dash(self, mode: str = 'inline', height: int = None, host: str = '127.0. self._port = port self._dashboard.run_server(mode=mode, height=height, host=host, port=port) + @add_headers('metrics') def stop_dash(self): """Stops server. And stops updating dashboard. """ diff --git a/src/metrics/metrics.py b/src/metrics/metrics.py index 413cbf89..1a533945 100644 --- a/src/metrics/metrics.py +++ b/src/metrics/metrics.py @@ -27,6 +27,7 @@ Requester, TolokaClient, ) +from ..util._managing_headers import async_add_headers from ..util.async_utils import Cooldown from ..streaming import cursor @@ -76,6 +77,7 @@ def atoloka_client(self): assert self.toloka_client is not None return AsyncTolokaClient.from_sync_client(self.toloka_client) + @async_add_headers('metrics') async def get_lines(self) -> Dict[str, List[Tuple[Any, Any]]]: """Gather and return metrics @@ -144,7 +146,7 @@ class Balance(BaseMetric): } """ - balance_name : Optional[str] = None + balance_name: Optional[str] = None def __attrs_post_init__(self): if self.balance_name is None: @@ -182,10 +184,10 @@ class NewUserBonuses(BaseMetric): 'bonus_money': [(datetime.datetime(2021, 11, 18, 8, 29, 9, 734377), Decimal('0'))] } """ - _count_name : Optional[str] = None - _money_name : Optional[str] = None + _count_name: Optional[str] = None + _money_name: Optional[str] = None - _join_events : bool = False + _join_events: bool = False def __attrs_post_init__(self): metric_names = self.get_line_names() @@ -266,11 +268,11 @@ class NewUserSkills(BaseMetric): 'values': [(datetime.datetime(2021, 11, 18, 8, 31, 54, 11000), Decimal('50.000000000000'))], } """ - _skill_id : str = attr.ib(kw_only=False) - _count_name : Optional[str] = None - _value_name : Optional[str] = None + _skill_id: str = attr.ib(kw_only=False) + _count_name: Optional[str] = None + _value_name: Optional[str] = None - _join_events : bool = False + _join_events: bool = False def __attrs_post_init__(self): metric_names = self.get_line_names() @@ -360,10 +362,10 @@ class NewMessageThreads(BaseMetric): 'checking_proj': [(datetime.datetime(2021, 11, 19, 12, 42, 50, 554830), 1)], } """ - _count_name : Optional[str] = None - _projects_name : Dict[str, str] = {} # {project_id: line_name} - _pools_name : Dict[str, str] = {} # {pool_id: line_name} - _join_events : bool = False + _count_name: Optional[str] = None + _projects_name: Dict[str, str] = {} # {project_id: line_name} + _pools_name: Dict[str, str] = {} # {pool_id: line_name} + _join_events: bool = False def __attrs_post_init__(self): metric_names = self.get_line_names() diff --git a/src/metrics/pool_metrics.py b/src/metrics/pool_metrics.py index a0250575..ef4d7984 100644 --- a/src/metrics/pool_metrics.py +++ b/src/metrics/pool_metrics.py @@ -31,10 +31,12 @@ from ..client.operations import Operation from ..client._converter import structure from ..streaming import cursor +from ..util._managing_headers import add_headers from .metrics import BaseMetric @lru_cache(maxsize=128) +@add_headers('metrics') def get_pool(pool_id: str, toloka_client: TolokaClient) -> Pool: return toloka_client.get_pool(pool_id) @@ -42,7 +44,7 @@ def get_pool(pool_id: str, toloka_client: TolokaClient) -> Pool: @attr.s(auto_attribs=True) class BasePoolMetric(BaseMetric): """Base class for all pool metrics""" - pool_id : str = attr.ib(kw_only=False) + pool_id: str = attr.ib(kw_only=False) @cached_property def beautiful_name(self) -> str: @@ -85,14 +87,14 @@ class AssignmentEventsInPool(BasePoolMetric): 'rejected_events_in_pool': [], } """ - _created_name : Optional[str] = None - _submitted_name : Optional[str] = None - _accepted_name : Optional[str] = None - _rejected_name : Optional[str] = None - _skipped_name : Optional[str] = None - _expired_name : Optional[str] = None + _created_name: Optional[str] = None + _submitted_name: Optional[str] = None + _accepted_name: Optional[str] = None + _rejected_name: Optional[str] = None + _skipped_name: Optional[str] = None + _expired_name: Optional[str] = None - _join_events : bool = False + _join_events: bool = False _status_dict = { '_created_name': 'CREATED', @@ -169,7 +171,7 @@ class PoolCompletedPercentage(BasePoolMetric): 'completion_percentage': [(datetime.datetime(2021, 8, 11, 15, 13, 4, 31000), 55)], } """ - _percents_name : Optional[str] = None + _percents_name: Optional[str] = None def __attrs_post_init__(self): if self._percents_name is None: @@ -221,10 +223,10 @@ class AssignmentsInPool(BasePoolMetric): 'accepted_assignments_in_pool': [(datetime.datetime(2021, 8, 12, 10, 4, 45, 951156), 75)], } """ - _submitted_name : Optional[str] = None - _accepted_name : Optional[str] = None - _rejected_name : Optional[str] = None - _skipped_name : Optional[str] = None + _submitted_name: Optional[str] = None + _accepted_name: Optional[str] = None + _rejected_name: Optional[str] = None + _skipped_name: Optional[str] = None _analytics_dict = { '_submitted_name': analytics_request.SubmitedAssignmentsCountPoolAnalytics, @@ -295,7 +297,7 @@ class TasksInPool(BasePoolMetric): 'tasks_count': [(datetime.datetime(2021, 11, 18, 9, 36, 34, 163000), 40)], } """ - _tasks_name : Optional[str] = None + _tasks_name: Optional[str] = None def __attrs_post_init__(self): if self._tasks_name is None: @@ -338,7 +340,7 @@ class SpentBudgetOnPool(BasePoolMetric): 'spent_money': [(datetime.datetime(2021, 11, 18, 9, 36, 34, 163000), Decimal('0.3'))], } """ - _money_name : Optional[str] = None + _money_name: Optional[str] = None def __attrs_post_init__(self): if self._money_name is None: @@ -382,7 +384,7 @@ class WorkersByFilterOnPool(BasePoolMetric): 'workers_count': [(datetime.datetime(2021, 11, 18, 9, 36, 34, 163000), 2697)], } """ - _workers_name : Optional[str] = None + _workers_name: Optional[str] = None _interval_hours: int = attr.ib(default=1) def __attrs_post_init__(self): @@ -454,10 +456,10 @@ class BansInPool(BasePoolMetric): 'fast': [(datetime.datetime(2021, 11, 18, 13, 32, 50, 453000), 1)], } """ - _count_name : Optional[str] = None - _filter_by_comment : Optional[Dict[str, str]] = None # {'comment': 'line_name'} + _count_name: Optional[str] = None + _filter_by_comment: Optional[Dict[str, str]] = None # {'comment': 'line_name'} - _join_events : bool = False + _join_events: bool = False def __attrs_post_init__(self): metric_names = self.get_line_names() diff --git a/src/util/_managing_headers.py b/src/util/_managing_headers.py index 26220192..3b1c0c5c 100644 --- a/src/util/_managing_headers.py +++ b/src/util/_managing_headers.py @@ -1,5 +1,6 @@ __all__ = [ 'add_headers', + 'async_add_headers', 'caller_context_var', 'top_level_method_var', 'low_level_method_var', @@ -56,3 +57,33 @@ def wrapped(*args, **kwargs): return wrapped return wrapper + + +def async_add_headers(client: str): + """ + This decorator add 3 headers into resulting http request called by async function: + 1) X-Caller-Context: high-level abstraction like client, metrics, streaming + 2) X-Top-Level-Method: first function, that was called and then called other functions which provoked request + 3) X-Low-Level-Method: last function before calling TolokaClient _method (_raw_request for example) + + Args: + client: name of high-level abstraction for X-Caller-Context + """ + + def wrapper(func): + + @functools.wraps(func) + async def wrapped(*args, **kwargs): + ctx = copy_context() + with ExitStack() as stack: + stack.enter_context(SetVariable(low_level_method_var, func.__name__)) + if caller_context_var not in ctx: + stack.enter_context(SetVariable(caller_context_var, client)) + if top_level_method_var not in ctx: + stack.enter_context(SetVariable(top_level_method_var, func.__name__)) + + return await func(*args, **kwargs) + + return wrapped + + return wrapper diff --git a/src/util/_managing_headers.pyi b/src/util/_managing_headers.pyi index b39222ff..13ae55c6 100644 --- a/src/util/_managing_headers.pyi +++ b/src/util/_managing_headers.pyi @@ -1,5 +1,6 @@ __all__ = [ 'add_headers', + 'async_add_headers', 'caller_context_var', 'top_level_method_var', 'low_level_method_var', @@ -23,3 +24,15 @@ def add_headers(client: str): client: name of high-level abstraction for X-Caller-Context """ ... + + +def async_add_headers(client: str): + """This decorator add 3 headers into resulting http request called by async function: + 1) X-Caller-Context: high-level abstraction like client, metrics, streaming + 2) X-Top-Level-Method: first function, that was called and then called other functions which provoked request + 3) X-Low-Level-Method: last function before calling TolokaClient _method (_raw_request for example) + + Args: + client: name of high-level abstraction for X-Caller-Context + """ + ... From d2b3f5260c57f236a4ec2d89c52aad8d3acf43d1 Mon Sep 17 00:00:00 2001 From: mr-fedulow Date: Fri, 24 Dec 2021 15:17:30 +0300 Subject: [PATCH 06/18] Stop support of python 3.6 Remove python-3.6 from setup.py and tox.ini, up python version where needed. Remove redundant checks for python version less than 3.7. Remove redundant libraries from required ref:851336f3e7fbf197145229dd31e243a9b09ce3a3 --- .github/workflows/tests.yml | 2 +- README.md | 2 +- ....client.TolokaClient.add_message_thread_to_folders.md | 4 ++-- ...nt.TolokaClient.remove_message_thread_from_folders.md | 4 ++-- setup.py | 9 ++------- src/client/__init__.pyi | 4 ++-- src/client/_converter.py | 5 ----- src/client/primitives/base.py | 4 +--- tests/slow_tests/conftest.py | 7 +++---- tests/slow_tests/test_retries.py | 8 -------- tox.ini | 3 +-- 11 files changed, 15 insertions(+), 37 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a7d21a7d..22ff26de 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/README.md b/README.md index 52721648..ef394563 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Main advantages of Toloka: Requirements -------------- -- Python 3.6+ +- Python 3.7+ - Register in [Toloka.ai](https://toloka.ai/?utm_source=github&utm_medium=site&utm_campaign=tolokakit) as requester. Registration process described [here.](https://toloka.ai/docs/guide/concepts/access.html?utm_source=github&utm_medium=site&utm_campaign=tolokakit) - [Topping up your account.](https://toloka.ai/docs/guide/concepts/refill.html?utm_source=github&utm_medium=site&utm_campaign=tolokakit) - Getting an OAuth. Learn more in [help](https://toloka.ai/docs/api/concepts/access.html?utm_source=github&utm_medium=site&utm_campaign=tolokakit) and in the image below. diff --git a/docs/reference/toloka.client.TolokaClient.add_message_thread_to_folders.md b/docs/reference/toloka.client.TolokaClient.add_message_thread_to_folders.md index 2ba3df74..040fb88c 100644 --- a/docs/reference/toloka.client.TolokaClient.add_message_thread_to_folders.md +++ b/docs/reference/toloka.client.TolokaClient.add_message_thread_to_folders.md @@ -5,7 +5,7 @@ add_message_thread_to_folders( self, message_thread_id: str, - folders: Union[List[Union[str, Folder]], MessageThreadFolders] + folders: Union[List[Union[Folder, str]], MessageThreadFolders] ) ``` @@ -16,7 +16,7 @@ Adds a message chain to one or more folders ("unread", "important" etc.) | Parameters | Type | Description | | :----------| :----| :-----------| `message_thread_id`|**str**|

ID of message chain.

-`folders`|**Union\[List\[Union\[str, [Folder](toloka.client.message_thread.Folder.md)\]\], [MessageThreadFolders](toloka.client.message_thread.MessageThreadFolders.md)\]**|

List of folders, where to move chain.

+`folders`|**Union\[List\[Union\[[Folder](toloka.client.message_thread.Folder.md), str\]\], [MessageThreadFolders](toloka.client.message_thread.MessageThreadFolders.md)\]**|

List of folders, where to move chain.

* **Returns:** diff --git a/docs/reference/toloka.client.TolokaClient.remove_message_thread_from_folders.md b/docs/reference/toloka.client.TolokaClient.remove_message_thread_from_folders.md index a4d3d403..7235d748 100644 --- a/docs/reference/toloka.client.TolokaClient.remove_message_thread_from_folders.md +++ b/docs/reference/toloka.client.TolokaClient.remove_message_thread_from_folders.md @@ -5,7 +5,7 @@ remove_message_thread_from_folders( self, message_thread_id: str, - folders: Union[List[Union[str, Folder]], MessageThreadFolders] + folders: Union[List[Union[Folder, str]], MessageThreadFolders] ) ``` @@ -16,7 +16,7 @@ Deletes a message chain from one or more folders ("unread", "important" etc.) | Parameters | Type | Description | | :----------| :----| :-----------| `message_thread_id`|**str**|

ID of message chain.

-`folders`|**Union\[List\[Union\[str, [Folder](toloka.client.message_thread.Folder.md)\]\], [MessageThreadFolders](toloka.client.message_thread.MessageThreadFolders.md)\]**|

List of folders, where from to remove chain.

+`folders`|**Union\[List\[Union\[[Folder](toloka.client.message_thread.Folder.md), str\]\], [MessageThreadFolders](toloka.client.message_thread.MessageThreadFolders.md)\]**|

List of folders, where from to remove chain.

* **Returns:** diff --git a/setup.py b/setup.py index 4fec7ae3..cfb28f16 100755 --- a/setup.py +++ b/setup.py @@ -29,16 +29,12 @@ license=about['__license__'], author='Vladimir Losev', author_email='losev@yandex-team.ru', - python_requires='>=3.6.0', + python_requires='>=3.7.0', install_requires=[ 'attrs >= 20.3.0', 'boto3 >= 1.4.7', - 'cattrs == 1.0.0 ; python_version < "3.7.0"', - 'cattrs >= 1.1.1; python_version >= "3.7.0"', + 'cattrs >= 1.1.1', 'cached-property; python_version < "3.8.0"', - 'contextvars; python_version<"3.7.0"', - 'immutables <= 0.15; python_version<"3.7.0"', # dependecy for contextvars - 'backports-datetime-fromisoformat; python_version < "3.7.0"', 'filelock >= 3.2.0', 'kazoo >= 2.6.1', 'requests', @@ -65,7 +61,6 @@ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', diff --git a/src/client/__init__.pyi b/src/client/__init__.pyi index 0bfb7efb..d09a494c 100644 --- a/src/client/__init__.pyi +++ b/src/client/__init__.pyi @@ -963,7 +963,7 @@ class TolokaClient: def add_message_thread_to_folders( self, message_thread_id: str, - folders: typing.Union[typing.List[typing.Union[str, toloka.client.message_thread.Folder]], toloka.client.message_thread.MessageThreadFolders] + folders: typing.Union[typing.List[typing.Union[toloka.client.message_thread.Folder, str]], toloka.client.message_thread.MessageThreadFolders] ) -> toloka.client.message_thread.MessageThread: """Adds a message chain to one or more folders ("unread", "important" etc.) @@ -1194,7 +1194,7 @@ class TolokaClient: def remove_message_thread_from_folders( self, message_thread_id: str, - folders: typing.Union[typing.List[typing.Union[str, toloka.client.message_thread.Folder]], toloka.client.message_thread.MessageThreadFolders] + folders: typing.Union[typing.List[typing.Union[toloka.client.message_thread.Folder, str]], toloka.client.message_thread.MessageThreadFolders] ) -> toloka.client.message_thread.MessageThread: """Deletes a message chain from one or more folders ("unread", "important" etc.) diff --git a/src/client/_converter.py b/src/client/_converter.py index be9efcc3..c97d6d18 100644 --- a/src/client/_converter.py +++ b/src/client/_converter.py @@ -3,17 +3,12 @@ from decimal import Decimal import typing import re -import sys import uuid from typing import List, Union import cattr from ..util._extendable_enum import ExtendableStrEnum -if sys.version_info[:2] < (3, 7): - from backports.datetime_fromisoformat import MonkeyPatch - MonkeyPatch.patch_fromisoformat() - converter = cattr.Converter() diff --git a/src/client/primitives/base.py b/src/client/primitives/base.py index d3e749e2..484d33ff 100644 --- a/src/client/primitives/base.py +++ b/src/client/primitives/base.py @@ -73,9 +73,7 @@ def convert_type_recursively(cur_type, type_converter): return type_converter(cur_type) new_args = tuple(convert_type_recursively(arg, type_converter=type_converter) for arg in cur_type.__args__) - # Python3.6 distinguish between generic types initialized with singleton tuple of type and type itself in some - # cases (i.e. List[Union[int, str]] != List[tuple(Union[int, str]])). - return origin[new_args] if len(new_args) > 1 else origin[new_args[0]] + return origin[new_args] return type_converter(cur_type) diff --git a/tests/slow_tests/conftest.py b/tests/slow_tests/conftest.py index 975a56ed..b55eacc5 100644 --- a/tests/slow_tests/conftest.py +++ b/tests/slow_tests/conftest.py @@ -1,6 +1,5 @@ import asyncio import json -import sys import time from copy import copy from decimal import Decimal @@ -10,9 +9,9 @@ import requests from toloka.client.requester import Requester -if sys.version_info >= (3, 7): - from aiohttp import web - from aiohttp.test_utils import unused_port + +from aiohttp import web +from aiohttp.test_utils import unused_port # local test requester server diff --git a/tests/slow_tests/test_retries.py b/tests/slow_tests/test_retries.py index fa922390..2942b89c 100644 --- a/tests/slow_tests/test_retries.py +++ b/tests/slow_tests/test_retries.py @@ -1,13 +1,10 @@ import pytest import requests -import sys from urllib3.util import Retry from toloka.client import TolokaClient -@pytest.mark.skipif(sys.version_info < (3, 7), reason='aiohttp requires python3.7 or higher') def test_socket_timeout_is_retried(timeout_server_url, fake_requester, retries_before_response): - toloka_client = TolokaClient( 'fake-token', url=timeout_server_url, @@ -19,9 +16,7 @@ def test_socket_timeout_is_retried(timeout_server_url, fake_requester, retries_b assert toloka_client.get_requester() == fake_requester -@pytest.mark.skipif(sys.version_info < (3, 7), reason='aiohttp requires python3.7 or higher') def test_socket_connection_error_when_not_retried_enough(timeout_server_url, retries_before_response): - toloka_client = TolokaClient( 'fake-token', url=timeout_server_url, @@ -37,7 +32,6 @@ def test_socket_connection_error_when_not_retried_enough(timeout_server_url, ret assert retries_left == 0 -@pytest.mark.skipif(sys.version_info < (3, 7), reason='aiohttp requires python3.7 or higher') def test_retries_off(connection_error_server_url, retries_before_response): toloka_client = TolokaClient( 'fake-token', @@ -53,7 +47,6 @@ def test_retries_off(connection_error_server_url, retries_before_response): assert retries_left == retries_before_response - 1 -@pytest.mark.skipif(sys.version_info < (3, 7), reason='aiohttp requires python3.7 or higher') def test_retries_from_int(connection_error_server_url, retries_before_response): toloka_client = TolokaClient( 'fake-token', @@ -69,7 +62,6 @@ def test_retries_from_int(connection_error_server_url, retries_before_response): assert retries_left == retries_before_response - 3 -@pytest.mark.skipif(sys.version_info < (3, 7), reason='aiohttp requires python3.7 or higher') def test_retries_from_class(connection_error_server_url, retries_before_response): toloka_client = TolokaClient( 'fake-token', diff --git a/tox.ini b/tox.ini index 67931e3f..b7c4c707 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,11 @@ [tox] minversion = 3.3.0 -envlist = py3{6,7,8,9,10}-attrs{20,21} +envlist = py3{7,8,9,10}-attrs{20,21} isolated_build = True requires = setuptools >= 36.2.0 [gh-actions] python = - 3.6: py36-attrs{20,21} 3.7: py37-attrs{20,21} 3.8: py38-attrs{20,21} 3.9: py39-attrs{20,21} From 69abbebb863a5e7e707b45c2c1900701f855de77 Mon Sep 17 00:00:00 2001 From: mr-fedulow Date: Mon, 27 Dec 2021 16:46:03 +0300 Subject: [PATCH 07/18] Add tests for toloka-kit ditamaps ref:d8f7b3f9a0fd2cfd031417cef30d954e54e626c5 --- docs/reference/_reference.ditamap | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/_reference.ditamap b/docs/reference/_reference.ditamap index 08cd667d..9d84eb86 100644 --- a/docs/reference/_reference.ditamap +++ b/docs/reference/_reference.ditamap @@ -507,6 +507,7 @@ + @@ -1023,13 +1024,13 @@ + - From 39ac772c6ce4ee2bb00819e7cacd96a1aa7febeb Mon Sep 17 00:00:00 2001 From: Sukhorosov Aleksey Date: Wed, 29 Dec 2021 15:55:28 +0300 Subject: [PATCH 08/18] changed default statuses ref:123c3550df7f8dcbabe513e1c5504d63beef22dd --- src/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/__init__.py b/src/client/__init__.py index 44f6698d..ce531248 100644 --- a/src/client/__init__.py +++ b/src/client/__init__.py @@ -243,7 +243,7 @@ def __init__( def _default_retryer_factory( retries: int, retry_quotas: Union[List[str], str, None], - status_list: Tuple[int] = tuple(code for code in requests.status_codes._codes if code >= 411 or code == 408), + status_list: Tuple[int] = (408, 409, 429, 500, 503), ) -> Retry: return TolokaRetry( retry_quotas=retry_quotas, From 1a0617ce5177b376b1dd1f03c7ef5d319f8e4d49 Mon Sep 17 00:00:00 2001 From: Sukhorosov Aleksey Date: Wed, 29 Dec 2021 17:11:52 +0300 Subject: [PATCH 09/18] stopping criterions ref:c0316f529fbdcb047104571e7d81bd66438ccb2b --- docs/reference/toloka.client.TolokaClient.md | 2 +- src/client/__init__.pyi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/toloka.client.TolokaClient.md b/docs/reference/toloka.client.TolokaClient.md index 0706bdd5..532dab75 100644 --- a/docs/reference/toloka.client.TolokaClient.md +++ b/docs/reference/toloka.client.TolokaClient.md @@ -7,7 +7,7 @@ TolokaClient( token: str, environment: Union[Environment, str, None] = None, retries: Union[int, Retry] = 3, - timeout: Union[float, Tuple[float, float]] = ..., + timeout: Union[float, Tuple[float, float]] = 10.0, url: Optional[str] = None, retry_quotas: Union[List[str], str, None] = 'MIN', retryer_factory: Optional[Callable[[], Retry]] = None diff --git a/src/client/__init__.pyi b/src/client/__init__.pyi index d09a494c..cbcc9947 100644 --- a/src/client/__init__.pyi +++ b/src/client/__init__.pyi @@ -212,7 +212,7 @@ class TolokaClient: token: str, environment: typing.Union[Environment, str, None] = None, retries: typing.Union[int, requests.packages.urllib3.util.retry.Retry] = 3, - timeout: typing.Union[float, typing.Tuple[float, float]] = ..., + timeout: typing.Union[float, typing.Tuple[float, float]] = 10.0, url: typing.Optional[str] = None, retry_quotas: typing.Union[typing.List[str], str, None] = 'MIN', retryer_factory: typing.Optional[typing.Callable[[], requests.packages.urllib3.util.retry.Retry]] = None From c2905b5010599aed93a88f5110487c1b002a5e8a Mon Sep 17 00:00:00 2001 From: Evgeny Tulin Date: Wed, 12 Jan 2022 12:58:34 +0300 Subject: [PATCH 10/18] PoolCompletedPercentage metric raises an error for closed pools ref:032a46d0fb2315442da8da17abf4906a2340da7e --- ....metrics.pool_metrics.PoolCompletedPercentage.md | 3 +++ src/metrics/collector.py | 13 +++++++------ src/metrics/pool_metrics.py | 3 +++ src/metrics/pool_metrics.pyi | 2 ++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/reference/toloka.metrics.pool_metrics.PoolCompletedPercentage.md b/docs/reference/toloka.metrics.pool_metrics.PoolCompletedPercentage.md index 8c094c7f..27d5cb3b 100644 --- a/docs/reference/toloka.metrics.pool_metrics.PoolCompletedPercentage.md +++ b/docs/reference/toloka.metrics.pool_metrics.PoolCompletedPercentage.md @@ -14,6 +14,9 @@ PoolCompletedPercentage( Track pool completion in percentage + +You can't gather this metric from a pool with infinite task suites. For example, if you have infinite overlap on a pool. + ## Parameters Description | Parameters | Type | Description | diff --git a/src/metrics/collector.py b/src/metrics/collector.py index 90b2ba62..1aa7cb2f 100644 --- a/src/metrics/collector.py +++ b/src/metrics/collector.py @@ -3,12 +3,13 @@ ] import asyncio +import datetime import logging from typing import Any, Callable, Dict, List, Tuple from .metrics import BaseMetric -from ..util.async_utils import ComplexException, get_task_traceback +from ..util.async_utils import get_task_traceback from ..util._managing_headers import async_add_headers logger = logging.getLogger(__name__) @@ -79,10 +80,8 @@ async def run(self): done, pending = await asyncio.wait(tasks, timeout=1) # check errors errored = [task for task in done if task.exception() is not None] - if errored: - for task in errored: - logger.error('Got error in metric:\n%s', get_task_traceback(task)) - raise ComplexException([task.exception() for task in errored]) + for task in errored: + logger.error('Got error in metric:\n%s', get_task_traceback(task)) # rerun completed tasks tasks = pending | {MetricCollector.create_async_tasks(done_task.new_coro) for done_task in done} @@ -90,10 +89,12 @@ async def run(self): # join metrics metrics_points = {} for done_task in done: + if done_task.exception() is not None: + continue for name, points in done_task.result().items(): if name in metrics_points: logger.error(f'Duplicated metrics name detected: "{name}". Only one metric was returned.') - metrics_points[name] = points + metrics_points[name] = [(dt.replace(tzinfo=datetime.timezone.utc), val) for dt, val in points] if metrics_points: self._callback(metrics_points) diff --git a/src/metrics/pool_metrics.py b/src/metrics/pool_metrics.py index ef4d7984..bf8e5108 100644 --- a/src/metrics/pool_metrics.py +++ b/src/metrics/pool_metrics.py @@ -16,6 +16,7 @@ from operator import attrgetter import sys + if sys.version_info[:2] >= (3, 8): from functools import cached_property else: @@ -155,6 +156,8 @@ async def _get_lines_impl(self) -> Dict[str, List[Tuple[Any, Any]]]: class PoolCompletedPercentage(BasePoolMetric): """Track pool completion in percentage + You can't gather this metric from a pool with infinite task suites. For example, if you have infinite overlap on a pool. + Args: pool_id: From which pool track metrics. percents_name: Metric name for pool completion percentage. Default 'completion_percentage'. diff --git a/src/metrics/pool_metrics.pyi b/src/metrics/pool_metrics.pyi index 3ded3d43..9ee3076d 100644 --- a/src/metrics/pool_metrics.pyi +++ b/src/metrics/pool_metrics.pyi @@ -106,6 +106,8 @@ class AssignmentEventsInPool(BasePoolMetric): class PoolCompletedPercentage(BasePoolMetric): """Track pool completion in percentage + You can't gather this metric from a pool with infinite task suites. For example, if you have infinite overlap on a pool. + Args: pool_id: From which pool track metrics. percents_name: Metric name for pool completion percentage. Default 'completion_percentage'. From 1fe66a1f56b79b627e341d6c6cce4b48447382c6 Mon Sep 17 00:00:00 2001 From: Vladislav Moiseev Date: Tue, 18 Jan 2022 13:24:48 +0300 Subject: [PATCH 11/18] Display results of the pipeline example. ref:acfa9090aed11a9b84a4a4f439805eea157c9a90 --- .../streaming_pipelines.ipynb | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/examples/6.streaming_pipelines/streaming_pipelines.ipynb b/examples/6.streaming_pipelines/streaming_pipelines.ipynb index 1d38a996..57825877 100644 --- a/examples/6.streaming_pipelines/streaming_pipelines.ipynb +++ b/examples/6.streaming_pipelines/streaming_pipelines.ipynb @@ -36,7 +36,7 @@ "outputs": [], "source": [ "%%capture\n", - "!pip install requests toloka-kit==0.1.15 crowd-kit==0.0.7" + "!pip install requests toloka-kit==0.1.22 crowd-kit==0.0.9" ] }, { @@ -131,7 +131,7 @@ " conditions=[AssessmentEvent == AssessmentEvent.REJECT],\n", " action=ChangeOverlap(delta=1, open_pool=True),\n", ")\n", - "client.update_pool(find_items_pool.id, find_items_pool)" + "client.update_pool(find_items_pool.id, find_items_pool);" ] }, { @@ -339,7 +339,7 @@ "found_items_observer.on_submitted(handle_found_items)\n", "found_items_observer.on_accepted(AcceptedItemsToComparison(client))\n", "verification_observer.on_accepted(VerificationDoneHandler(client))\n", - "sbs_observer.on_accepted(HandleSbS(client))" + "sbs_handler = sbs_observer.on_accepted(HandleSbS(client))" ] }, { @@ -366,7 +366,7 @@ " Task(pool_id=find_items_pool.id, overlap=OVERLAP_FIND_ITEMS, input_values={'image': image})\n", " for image in images\n", "]\n", - "client.create_tasks(tasks, open_pool=True)" + "client.create_tasks(tasks, open_pool=True);" ] }, { @@ -391,6 +391,26 @@ "else:\n", " await pipeline.run()" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Display results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Image, display\n", + "\n", + "for image, scores in sbs_handler.scores_by_image.items():\n", + " display(Image(url=image, height=200))\n", + " print(f'{scores.nlargest(1)}\\n')" + ] } ], "metadata": { From 6766a89e92f940e0de58c64ecd9e0a7a093b567e Mon Sep 17 00:00:00 2001 From: pocoder Date: Tue, 18 Jan 2022 16:43:33 +0300 Subject: [PATCH 12/18] Add tracing headers to streaming add streaming headers ref:caa1e12743a3e62dcd524442c09d3814a9b927a9 --- src/streaming/observer.py | 6 ++++++ src/streaming/pipeline.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/streaming/observer.py b/src/streaming/observer.py index 530a2a94..c23b9ccd 100644 --- a/src/streaming/observer.py +++ b/src/streaming/observer.py @@ -16,6 +16,7 @@ from ..client.assignment import Assignment from ..client.pool import Pool from ..util.async_utils import AsyncInterfaceWrapper, ComplexException, ensure_async, get_task_traceback +from ..util._managing_headers import async_add_headers from .cursor import AssignmentCursor, TolokaClientSyncOrAsyncType from .event import AssignmentEvent @@ -38,6 +39,7 @@ async def __call__(self) -> None: async def should_resume(self) -> bool: return False + @async_add_headers('streaming') async def run(self, period: datetime.timedelta = datetime.timedelta(seconds=60)) -> None: """For standalone usage (out of a Pipeline).""" while True: @@ -114,6 +116,7 @@ class BasePoolObserver(BaseObserver): def _get_unique_key(self) -> Tuple: return super()._get_unique_key() + (self.pool_id,) + @async_add_headers('streaming') async def should_resume(self) -> bool: logger.info('Check resume by pool status: %s', self.pool_id) pool = await self.toloka_client.get_pool(self.pool_id) @@ -222,6 +225,7 @@ def on_status_change(self, callback: CallbackForPoolType) -> CallbackForPoolType self.register_callback(callback, status) return callback + @async_add_headers('streaming') async def __call__(self) -> None: if not self._callbacks: return @@ -268,6 +272,7 @@ def inject(self, injection: '_CallbacksCursorConsumer') -> None: def add_callback(self, callback: CallbackForAssignmentEventsType) -> None: self.callbacks.append(ensure_async(callback)) + @async_add_headers('streaming') async def __call__(self, pool_id: str) -> None: async with self.cursor.try_fetch_all() as fetched: if not fetched: @@ -382,6 +387,7 @@ def on_expired(self, callback: CallbackForAssignmentEventsType) -> CallbackForAs # Run section. + @async_add_headers('streaming') async def __call__(self) -> None: if not self._callbacks: return diff --git a/src/streaming/pipeline.py b/src/streaming/pipeline.py index e946b3a4..0fbab85f 100644 --- a/src/streaming/pipeline.py +++ b/src/streaming/pipeline.py @@ -14,6 +14,7 @@ from .observer import BaseObserver from .storage import BaseStorage from ..util.async_utils import ComplexException +from ..util._managing_headers import async_add_headers logger = logging.getLogger(__name__) @@ -32,6 +33,7 @@ class _Worker: observer: BaseObserver = attr.ib() should_resume: bool = attr.ib(default=False) + @async_add_headers('streaming') async def __call__(self) -> None: await self.observer() self.should_resume = await self.observer.should_resume() @@ -202,6 +204,7 @@ def _process_done_tasks( logger.error('Got error in: %s', task) raise ComplexException([task.exception() for task in errored]) + @async_add_headers('streaming') async def run(self) -> None: if not self._observers: raise ValueError('No observers registered') From e97a244d35e5f886cde551451f9070f36d015652 Mon Sep 17 00:00:00 2001 From: Sukhorosov Aleksey Date: Wed, 19 Jan 2022 13:02:33 +0300 Subject: [PATCH 13/18] verified languages support ref:a60a1c37abb1c25b6ddfbdb05549883395bcb62f --- .../toloka.client.filter.Languages.md | 4 +- functional_tests/test_filter.py | 18 ++++++ src/client/filter.py | 64 ++++++++++++++++++- src/client/filter.pyi | 5 +- src/client/primitives/operators.py | 8 +-- tests/test_filter.py | 24 +++++++ 6 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 functional_tests/test_filter.py create mode 100644 tests/test_filter.py diff --git a/docs/reference/toloka.client.filter.Languages.md b/docs/reference/toloka.client.filter.Languages.md index f4ede009..51768658 100644 --- a/docs/reference/toloka.client.filter.Languages.md +++ b/docs/reference/toloka.client.filter.Languages.md @@ -5,7 +5,8 @@ Languages( self, operator: InclusionOperator, - value: Union[str, List[str]] + value: Union[str, List[str]], + verified: bool = False ) ``` @@ -17,3 +18,4 @@ Use to select users by languages specified by the user in the profile. | :----------| :----| :-----------| `operator`|**[InclusionOperator](toloka.client.primitives.operators.InclusionOperator.md)**|

Comparison operator in the condition. For example, for a condition "The user must be 18 years old or older» used date of birth and operator GTE («Greater than or equal»). Possible key values operator depends on the data type in the field value

`value`|**Union\[str, List\[str\]\]**|

Languages specified by the user in the profile (two-letter ISO code of the standard ISO 639-1 in upper case).

+`verified`|**-**|

If set to True only users who have passed language test will be selected. Currently only following ISO codes are supported: DE, EN, FR, JA, PT, SV, RU, AR, ES.

diff --git a/functional_tests/test_filter.py b/functional_tests/test_filter.py new file mode 100644 index 00000000..e970fbb8 --- /dev/null +++ b/functional_tests/test_filter.py @@ -0,0 +1,18 @@ +import json +import requests +from toloka.client.filter import Languages + + +def test_all_verified_languages_skills_exist(client): + skills = Languages.verified_languages_to_skills.values() + for skill in skills: + assert client.get_skill(skill).id == skill + + +def test_verified_languages_to_skills_mapping_is_up_to_date(): + local_mapping = Languages.verified_languages_to_skills + remote_mapping = json.loads(requests.get('https://toloka.yandex.com/api/env').content)['config']['public_verifiedLanguages'] + remote_keys = remote_mapping[::2] # every first element in list + remote_values = remote_mapping[1::2] # every second element in list + remote_mapping = dict(zip(remote_keys, remote_values)) + assert local_mapping == remote_mapping diff --git a/src/client/filter.py b/src/client/filter.py index e065dba2..266c30ed 100644 --- a/src/client/filter.py +++ b/src/client/filter.py @@ -31,8 +31,9 @@ 'UserAgentVersionBugfix', 'Rating' ] +import inspect from enum import unique -from typing import Any, List, Optional, Union +from typing import Any, List, Optional, Union, ClassVar, Dict from .primitives.base import BaseTolokaObject from .primitives.operators import ( @@ -41,6 +42,7 @@ IdentityConditionMixin, ComparableConditionMixin, InclusionConditionMixin, + InclusionOperator, ) from ..util._codegen import attribute from ..util._docstrings import inherit_docstrings @@ -328,10 +330,70 @@ class Languages(Profile, InclusionConditionMixin, spec_value=Profile.Key.LANGUAG Attributes: value: Languages specified by the user in the profile (two-letter ISO code of the standard ISO 639-1 in upper case). + verified: If set to True only users who have passed language test will be selected. Currently only following + ISO codes are supported: DE, EN, FR, JA, PT, SV, RU, AR, ES. """ + verified_languages_to_skills: ClassVar[Dict[str, str]] = { + 'DE': '26377', + 'EN': '26366', + 'FR': '26711', + 'JA': '26513', + 'PT': '26714', + 'SV': '29789', + 'RU': '26296', + 'AR': '30724', + 'ES': '32346', + } + + VERIFIED_LANGUAGE_SKILL_VALUE: ClassVar[int] = 100 + value: Union[str, List[str]] = attribute(required=True) + def __new__(cls, *args, **kwargs): + """Handle "verified" parameter + + If class is instantiated with `verified=True` parameter then return + `FilterAnd([verified_skill_for_language_1, ..., verified_skill_for_language_n, language_1, ..., language_n])` + condition for API compatibility reasons. + """ + bound_args = inspect.signature(cls.__init__).bind(None, *args, **kwargs).arguments + value = bound_args['value'] + operator = bound_args['operator'] + + if bound_args.get('verified', False) and operator == InclusionOperator.IN: + skills_mapping = cls.verified_languages_to_skills + try: + conditions = [Languages(operator=operator, value=value)] + if isinstance(value, list): + conditions.extend([Skill(skills_mapping[value]).eq(cls.VERIFIED_LANGUAGE_SKILL_VALUE) for value in value]) + else: + conditions.append(Skill(skills_mapping[value]).eq(cls.VERIFIED_LANGUAGE_SKILL_VALUE)) + + return FilterAnd(conditions) + except KeyError: + if not isinstance(value, str): + unsupported_languages = set(value) - skills_mapping.keys() + else: + unsupported_languages = [value] + raise ValueError( + 'Following languages are not supported as verified languages:\n' + '\n'.join(unsupported_languages) + ) + else: + if kwargs.pop('verified', False): + raise ValueError('"Language not in" filter does not support verified=True argument') + return super().__new__(cls, *args, **kwargs) + + +# add fake parameter "verified: bool = False" to Languages.__init__ signature. This parameter will be consumed in +# Languages.__new__ while the actual __init__ is managed by attrs. +languages_init_signature = inspect.signature(Languages.__init__) +languages_init_signature_parameters = dict(languages_init_signature.parameters) +languages_init_signature_parameters['verified'] = inspect.Parameter( + name='verified', kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, default=False, annotation=bool, +) +Languages.__init__.__signature__ = languages_init_signature.replace(parameters=languages_init_signature_parameters.values()) + @inherit_docstrings class RegionByPhone(Computed, InclusionConditionMixin, spec_value=Computed.Key.REGION_BY_PHONE): diff --git a/src/client/filter.pyi b/src/client/filter.pyi index 261d0ec7..2fcba510 100644 --- a/src/client/filter.pyi +++ b/src/client/filter.pyi @@ -442,12 +442,15 @@ class Languages(Profile, toloka.client.primitives.operators.InclusionConditionMi For example, for a condition "The user must be 18 years old or older» used date of birth and operator GTE («Greater than or equal»). Possible key values operator depends on the data type in the field value value: Languages specified by the user in the profile (two-letter ISO code of the standard ISO 639-1 in upper case). + verified: If set to True only users who have passed language test will be selected. Currently only following + ISO codes are supported: DE, EN, FR, JA, PT, SV, RU, AR, ES. """ def __init__( self, operator: toloka.client.primitives.operators.InclusionOperator, - value: typing.Union[str, typing.List[str]] + value: typing.Union[str, typing.List[str]], + verified: bool = False ) -> None: """Method generated by attrs for class Languages. """ diff --git a/src/client/primitives/operators.py b/src/client/primitives/operators.py index c1b0d43b..61631ccf 100644 --- a/src/client/primitives/operators.py +++ b/src/client/primitives/operators.py @@ -67,11 +67,11 @@ def _eq_compatible_with_help(cls, value): class _InclusionConditionMetaclass(BaseTolokaObjectMetaclass): - def include(cls, value): - return cls(operator=InclusionOperator.IN, value=value) + def include(cls, *args, **kwargs): + return cls(InclusionOperator.IN, *args, **kwargs) - def exclude(cls, value): - return cls(operator=InclusionOperator.NOT_IN, value=value) + def exclude(cls, *args, **kwargs): + return cls(InclusionOperator.NOT_IN, *args, **kwargs) in_ = include not_in = exclude diff --git a/tests/test_filter.py b/tests/test_filter.py new file mode 100644 index 00000000..571dd78f --- /dev/null +++ b/tests/test_filter.py @@ -0,0 +1,24 @@ +import pytest +from toloka.client.filter import Languages, FilterAnd, Skill + + +def test_simple_language(): + assert Languages.in_('EN').unstructure() == {'operator': 'IN', 'value': 'EN', 'key': 'languages', 'category': 'profile'} + + +def test_verified_language_not_in_is_incorrect(): + with pytest.raises(ValueError): + Languages.not_in('EN', verified=True) + + +def test_unknown_verified_language_is_incorrect(): + with pytest.raises(ValueError): + Languages.in_(['fake language 1', 'EN'], verified=True) + + +def test_language_multiple(): + assert Languages.in_('EN', verified=True) == FilterAnd([Languages.in_('EN'), Skill('26366').eq(100)]) + + +def test_verified_language_multiple(): + assert Languages.in_(['EN', 'RU'], verified=True) == FilterAnd([Languages.in_(['EN', 'RU']), Skill('26366').eq(100), Skill('26296').eq(100)]) From 8e9c7655a28ebe95baefadfb2fca5be45a7e14fd Mon Sep 17 00:00:00 2001 From: pocoder Date: Fri, 21 Jan 2022 15:56:22 +0300 Subject: [PATCH 14/18] Fix bonuses remove outdated functionality ref:3778dcd5369fa0c397f7875e3cf658624ad0592a --- ...a.client.TolokaClient.create_user_bonus.md | 10 +- ...client.TolokaClient.create_user_bonuses.md | 20 +++- ....TolokaClient.create_user_bonuses_async.md | 20 +++- .../toloka.client.user_bonus.UserBonus.md | 24 +++-- src/client/__init__.py | 50 +++++++-- src/client/__init__.pyi | 100 ++++++++++++++---- src/client/user_bonus.py | 29 ++--- src/client/user_bonus.pyi | 31 +++--- 8 files changed, 209 insertions(+), 75 deletions(-) diff --git a/docs/reference/toloka.client.TolokaClient.create_user_bonus.md b/docs/reference/toloka.client.TolokaClient.create_user_bonus.md index 2ec79984..bef37e41 100644 --- a/docs/reference/toloka.client.TolokaClient.create_user_bonus.md +++ b/docs/reference/toloka.client.TolokaClient.create_user_bonus.md @@ -32,8 +32,14 @@ new_bonus = toloka_client.create_user_bonus( UserBonus( user_id='1', amount=decimal.Decimal('0.50'), - public_title='Perfect job!', - public_message='You are the best performer!', + public_title={ + 'EN': 'Perfect job!', + 'RU': 'Прекрасная работа!', + }, + public_message={ + 'EN': 'You are the best performer!', + 'RU': 'Молодец!', + }, assignment_id='012345' ) ) diff --git a/docs/reference/toloka.client.TolokaClient.create_user_bonuses.md b/docs/reference/toloka.client.TolokaClient.create_user_bonuses.md index 4b7ef027..b6b2738c 100644 --- a/docs/reference/toloka.client.TolokaClient.create_user_bonuses.md +++ b/docs/reference/toloka.client.TolokaClient.create_user_bonuses.md @@ -32,14 +32,26 @@ new_bonuses=[ UserBonus( user_id='1', amount=decimal.Decimal('0.50'), - public_title='Perfect job!', - public_message='You are the best performer!', + public_title={ + 'EN': 'Perfect job!', + 'RU': 'Прекрасная работа!', + }, + public_message={ + 'EN': 'You are the best performer!', + 'RU': 'Молодец!', + }, assignment_id='1'), UserBonus( user_id='2', amount=decimal.Decimal('1.0'), - public_title='Excellent work!', - public_message='You completed all the tasks!', + public_title={ + 'EN': 'Excellent work!', + 'RU': 'Отличная работа!', + }, + public_message={ + 'EN': 'You have completed all tasks!', + 'RU': 'Сделаны все задания!', + }, assignment_id='2') ] toloka_client.create_user_bonuses(new_bonuses) diff --git a/docs/reference/toloka.client.TolokaClient.create_user_bonuses_async.md b/docs/reference/toloka.client.TolokaClient.create_user_bonuses_async.md index c5b8c289..b499ceee 100644 --- a/docs/reference/toloka.client.TolokaClient.create_user_bonuses_async.md +++ b/docs/reference/toloka.client.TolokaClient.create_user_bonuses_async.md @@ -30,14 +30,26 @@ new_bonuses=[ UserBonus( user_id='1', amount=decimal.Decimal('0.50'), - public_title='Perfect job!', - public_message='You are the best performer!', + public_title={ + 'EN': 'Perfect job!', + 'RU': 'Прекрасная работа!', + }, + public_message={ + 'EN': 'You are the best performer!', + 'RU': 'Молодец!', + }, assignment_id='1'), UserBonus( user_id='2', amount=decimal.Decimal('1.0'), - public_title='Excellent work!', - public_message='You completed all the tasks!', + public_title={ + 'EN': 'Excellent work!', + 'RU': 'Превосходная работа!', + }, + public_message={ + 'EN': 'You have completed all tasks!', + 'RU': 'Сделаны все задания!', + }, assignment_id='2') ] create_bonuses = toloka_client.create_user_bonuses_async(new_bonuses) diff --git a/docs/reference/toloka.client.user_bonus.UserBonus.md b/docs/reference/toloka.client.user_bonus.UserBonus.md index 7f4a807d..83d4c182 100644 --- a/docs/reference/toloka.client.user_bonus.UserBonus.md +++ b/docs/reference/toloka.client.user_bonus.UserBonus.md @@ -8,8 +8,8 @@ UserBonus( user_id: Optional[str] = None, amount: Optional[Decimal] = None, private_comment: Optional[str] = None, - public_title: Optional[Any] = None, - public_message: Optional[Any] = None, + public_title: Optional[Dict[str, str]] = None, + public_message: Optional[Dict[str, str]] = None, without_message: Optional[bool] = None, assignment_id: Optional[str] = None, id: Optional[str] = None, @@ -29,8 +29,8 @@ It's addition to payment for completed tasks. `user_id`|**Optional\[str\]**|

Performer ID to whom the bonus will be issued.

`amount`|**Optional\[Decimal\]**|

The bonus amount in dollars. Can be from 0.01 to 100 dollars per user per time.

`private_comment`|**Optional\[str\]**|

Comments that are only visible to the requester.

-`public_title`|**Optional\[Any\]**|

Message header for the user. You can provide a title in several languages (the message will come in the user's language).

-`public_message`|**Optional\[Any\]**|

Message text for the user. You can provide text in several languages (the message will come in the user's language).

+`public_title`|**Optional\[Dict\[str, str\]\]**|

Message header for the user. You can provide a title in several languages (the message will come in the user's language). Format {'language': 'title', ... }. The language can be RU/EN/TR/ID/FR.

+`public_message`|**Optional\[Dict\[str, str\]\]**|

Message text for the user. You can provide text in several languages (the message will come in the user's language). Format {'language': 'message', ... }. The language can be RU/EN/TR/ID/FR.

`without_message`|**Optional\[bool\]**|

Do not send a bonus message to the user. To award a bonus without a message, specify null for public_title and public_message and True for without_message.

`assignment_id`|**Optional\[str\]**|

The answer to the task for which this bonus was issued.

`id`|**Optional\[str\]**|

Internal ID of the issued bonus. Read only.

@@ -45,28 +45,32 @@ new_bonus = toloka_client.create_user_bonus( UserBonus( user_id='1', amount='0.50', - public_title='Perfect job!', - public_message='You are the best performer EVER!' + public_title={ + 'EN': 'Perfect job!', + }, + public_message={ + 'EN': 'You are the best performer EVER', + }, assignment_id='012345' ) ) ``` -Hoiw to create bonus with message in several languages. +How to create bonus with message in several languages. ```python new_bonus = toloka_client.create_user_bonus( UserBonus( user_id='1', amount='0.10', - public_title= { + public_title={ 'EN': 'Good Job!', 'RU': 'Молодец!', }, - public_message: { + public_message={ 'EN': 'Ten tasks completed', 'RU': 'Выполнено 10 заданий', - }, + } ) ) ``` diff --git a/src/client/__init__.py b/src/client/__init__.py index ce531248..8f45ea5a 100644 --- a/src/client/__init__.py +++ b/src/client/__init__.py @@ -2640,8 +2640,14 @@ def create_user_bonus(self, user_bonus: UserBonus, parameters: Optional[UserBonu >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='012345' >>> ) >>> ) @@ -2675,14 +2681,26 @@ def create_user_bonuses(self, user_bonuses: List[UserBonus], parameters: Optiona >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='1'), >>> UserBonus( >>> user_id='2', >>> amount=decimal.Decimal('1.0'), - >>> public_title='Excellent work!', - >>> public_message='You completed all the tasks!', + >>> public_title={ + >>> 'EN': 'Excellent work!', + >>> 'RU': 'Отличная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You have completed all tasks!', + >>> 'RU': 'Сделаны все задания!', + >>> }, >>> assignment_id='2') >>> ] >>> toloka_client.create_user_bonuses(new_bonuses) @@ -2714,14 +2732,26 @@ def create_user_bonuses_async(self, user_bonuses: List[UserBonus], parameters: O >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='1'), >>> UserBonus( >>> user_id='2', >>> amount=decimal.Decimal('1.0'), - >>> public_title='Excellent work!', - >>> public_message='You completed all the tasks!', + >>> public_title={ + >>> 'EN': 'Excellent work!', + >>> 'RU': 'Превосходная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You have completed all tasks!', + >>> 'RU': 'Сделаны все задания!', + >>> }, >>> assignment_id='2') >>> ] >>> create_bonuses = toloka_client.create_user_bonuses_async(new_bonuses) diff --git a/src/client/__init__.pyi b/src/client/__init__.pyi index cbcc9947..76373163 100644 --- a/src/client/__init__.pyi +++ b/src/client/__init__.pyi @@ -3665,8 +3665,14 @@ class TolokaClient: >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='012345' >>> ) >>> ) @@ -3701,8 +3707,14 @@ class TolokaClient: >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='012345' >>> ) >>> ) @@ -3735,14 +3747,26 @@ class TolokaClient: >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='1'), >>> UserBonus( >>> user_id='2', >>> amount=decimal.Decimal('1.0'), - >>> public_title='Excellent work!', - >>> public_message='You completed all the tasks!', + >>> public_title={ + >>> 'EN': 'Excellent work!', + >>> 'RU': 'Отличная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You have completed all tasks!', + >>> 'RU': 'Сделаны все задания!', + >>> }, >>> assignment_id='2') >>> ] >>> toloka_client.create_user_bonuses(new_bonuses) @@ -3777,14 +3801,26 @@ class TolokaClient: >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='1'), >>> UserBonus( >>> user_id='2', >>> amount=decimal.Decimal('1.0'), - >>> public_title='Excellent work!', - >>> public_message='You completed all the tasks!', + >>> public_title={ + >>> 'EN': 'Excellent work!', + >>> 'RU': 'Отличная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You have completed all tasks!', + >>> 'RU': 'Сделаны все задания!', + >>> }, >>> assignment_id='2') >>> ] >>> toloka_client.create_user_bonuses(new_bonuses) @@ -3815,14 +3851,26 @@ class TolokaClient: >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='1'), >>> UserBonus( >>> user_id='2', >>> amount=decimal.Decimal('1.0'), - >>> public_title='Excellent work!', - >>> public_message='You completed all the tasks!', + >>> public_title={ + >>> 'EN': 'Excellent work!', + >>> 'RU': 'Превосходная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You have completed all tasks!', + >>> 'RU': 'Сделаны все задания!', + >>> }, >>> assignment_id='2') >>> ] >>> create_bonuses = toloka_client.create_user_bonuses_async(new_bonuses) @@ -3856,14 +3904,26 @@ class TolokaClient: >>> UserBonus( >>> user_id='1', >>> amount=decimal.Decimal('0.50'), - >>> public_title='Perfect job!', - >>> public_message='You are the best performer!', + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> 'RU': 'Прекрасная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer!', + >>> 'RU': 'Молодец!', + >>> }, >>> assignment_id='1'), >>> UserBonus( >>> user_id='2', >>> amount=decimal.Decimal('1.0'), - >>> public_title='Excellent work!', - >>> public_message='You completed all the tasks!', + >>> public_title={ + >>> 'EN': 'Excellent work!', + >>> 'RU': 'Превосходная работа!', + >>> }, + >>> public_message={ + >>> 'EN': 'You have completed all tasks!', + >>> 'RU': 'Сделаны все задания!', + >>> }, >>> assignment_id='2') >>> ] >>> create_bonuses = toloka_client.create_user_bonuses_async(new_bonuses) diff --git a/src/client/user_bonus.py b/src/client/user_bonus.py index f716d075..729e150d 100644 --- a/src/client/user_bonus.py +++ b/src/client/user_bonus.py @@ -5,7 +5,7 @@ from attr.validators import optional, instance_of import datetime from decimal import Decimal -from typing import Any +from typing import Dict from .primitives.base import BaseTolokaObject from .primitives.parameter import Parameters @@ -22,9 +22,11 @@ class UserBonus(BaseTolokaObject): amount: The bonus amount in dollars. Can be from 0.01 to 100 dollars per user per time. private_comment: Comments that are only visible to the requester. public_title: Message header for the user. You can provide a title in several languages - (the message will come in the user's language). + (the message will come in the user's language). Format {'language': 'title', ... }. + The language can be RU/EN/TR/ID/FR. public_message: Message text for the user. You can provide text in several languages - (the message will come in the user's language). + (the message will come in the user's language). Format {'language': 'message', ... }. + The language can be RU/EN/TR/ID/FR. without_message: Do not send a bonus message to the user. To award a bonus without a message, specify null for public_title and public_message and True for without_message. assignment_id: The answer to the task for which this bonus was issued. @@ -38,27 +40,30 @@ class UserBonus(BaseTolokaObject): >>> UserBonus( >>> user_id='1', >>> amount='0.50', - >>> public_title='Perfect job!', - >>> public_message='You are the best performer EVER!' + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer EVER', + >>> }, >>> assignment_id='012345' >>> ) >>> ) - ... - Hoiw to create bonus with message in several languages. + How to create bonus with message in several languages. >>> new_bonus = toloka_client.create_user_bonus( >>> UserBonus( >>> user_id='1', >>> amount='0.10', - >>> public_title= { + >>> public_title={ >>> 'EN': 'Good Job!', >>> 'RU': 'Молодец!', >>> }, - >>> public_message: { + >>> public_message={ >>> 'EN': 'Ten tasks completed', >>> 'RU': 'Выполнено 10 заданий', - >>> }, + >>> } >>> ) >>> ) ... @@ -68,8 +73,8 @@ class UserBonus(BaseTolokaObject): amount: Decimal = attribute(validator=optional(instance_of(Decimal))) private_comment: str - public_title: Any - public_message: Any + public_title: Dict[str, str] + public_message: Dict[str, str] without_message: bool assignment_id: str diff --git a/src/client/user_bonus.pyi b/src/client/user_bonus.pyi index b76064e3..273e2f8c 100644 --- a/src/client/user_bonus.pyi +++ b/src/client/user_bonus.pyi @@ -19,9 +19,11 @@ class UserBonus(toloka.client.primitives.base.BaseTolokaObject): amount: The bonus amount in dollars. Can be from 0.01 to 100 dollars per user per time. private_comment: Comments that are only visible to the requester. public_title: Message header for the user. You can provide a title in several languages - (the message will come in the user's language). + (the message will come in the user's language). Format {'language': 'title', ... }. + The language can be RU/EN/TR/ID/FR. public_message: Message text for the user. You can provide text in several languages - (the message will come in the user's language). + (the message will come in the user's language). Format {'language': 'message', ... }. + The language can be RU/EN/TR/ID/FR. without_message: Do not send a bonus message to the user. To award a bonus without a message, specify null for public_title and public_message and True for without_message. assignment_id: The answer to the task for which this bonus was issued. @@ -35,27 +37,30 @@ class UserBonus(toloka.client.primitives.base.BaseTolokaObject): >>> UserBonus( >>> user_id='1', >>> amount='0.50', - >>> public_title='Perfect job!', - >>> public_message='You are the best performer EVER!' + >>> public_title={ + >>> 'EN': 'Perfect job!', + >>> }, + >>> public_message={ + >>> 'EN': 'You are the best performer EVER', + >>> }, >>> assignment_id='012345' >>> ) >>> ) - ... - Hoiw to create bonus with message in several languages. + How to create bonus with message in several languages. >>> new_bonus = toloka_client.create_user_bonus( >>> UserBonus( >>> user_id='1', >>> amount='0.10', - >>> public_title= { + >>> public_title={ >>> 'EN': 'Good Job!', >>> 'RU': 'Молодец!', >>> }, - >>> public_message: { + >>> public_message={ >>> 'EN': 'Ten tasks completed', >>> 'RU': 'Выполнено 10 заданий', - >>> }, + >>> } >>> ) >>> ) ... @@ -67,8 +72,8 @@ class UserBonus(toloka.client.primitives.base.BaseTolokaObject): user_id: typing.Optional[str] = None, amount: typing.Optional[decimal.Decimal] = None, private_comment: typing.Optional[str] = None, - public_title: typing.Optional[typing.Any] = None, - public_message: typing.Optional[typing.Any] = None, + public_title: typing.Optional[typing.Dict[str, str]] = None, + public_message: typing.Optional[typing.Dict[str, str]] = None, without_message: typing.Optional[bool] = None, assignment_id: typing.Optional[str] = None, id: typing.Optional[str] = None, @@ -82,8 +87,8 @@ class UserBonus(toloka.client.primitives.base.BaseTolokaObject): user_id: typing.Optional[str] amount: typing.Optional[decimal.Decimal] private_comment: typing.Optional[str] - public_title: typing.Optional[typing.Any] - public_message: typing.Optional[typing.Any] + public_title: typing.Optional[typing.Dict[str, str]] + public_message: typing.Optional[typing.Dict[str, str]] without_message: typing.Optional[bool] assignment_id: typing.Optional[str] id: typing.Optional[str] From 37c74b79d25d51897a12c1500d75c28d52f86118 Mon Sep 17 00:00:00 2001 From: Sukhorosov Aleksey Date: Tue, 25 Jan 2022 12:39:00 +0300 Subject: [PATCH 15/18] TB components tests ref:68989fd9e35da07e583e8a6e8c37a8738ef9f23f --- ...a.async_client.client.AsyncTolokaClient.md | 2 +- docs/reference/toloka.client.TolokaClient.md | 4 +- ...t.template_builder.fields.SelectFieldV1.md | 4 +- ...emplate_builder.helpers.ReplaceHelperV1.md | 2 +- ...builder.helpers.YandexDiskProxyHelperV1.md | 4 +- ...oject.template_builder.view.VideoViewV1.md | 3 +- ...lient.project.view_spec.ClassicViewSpec.md | 2 +- functional_tests/client/__init__.py | 0 .../{ => client}/test_assignment.py | 0 functional_tests/{ => client}/test_client.py | 0 functional_tests/{ => client}/test_filter.py | 0 functional_tests/{ => client}/test_project.py | 0 functional_tests/{ => client}/test_tasks.py | 4 +- functional_tests/conftest.py | 6 +- functional_tests/template_builder/__init__.py | 22 ++ functional_tests/template_builder/conftest.py | 45 ++++ .../template_builder/test_actions.py | 77 ++++++ .../template_builder/test_conditions.py | 139 +++++++++++ .../template_builder/test_data.py | 70 ++++++ .../template_builder/test_fields.py | 223 ++++++++++++++++++ .../template_builder/test_helpers.py | 138 +++++++++++ .../template_builder/test_layouts.py | 96 ++++++++ .../template_builder/test_plugins.py | 73 ++++++ .../template_builder/test_views.py | 218 +++++++++++++++++ src/async_client/client.py | 2 +- src/async_client/client.pyi | 2 +- src/client/__init__.py | 4 +- src/client/__init__.pyi | 4 +- src/client/collectors.py | 2 +- src/client/project/template_builder/fields.py | 2 +- .../project/template_builder/fields.pyi | 4 +- .../project/template_builder/helpers.py | 6 +- .../project/template_builder/helpers.pyi | 6 +- src/client/project/template_builder/view.py | 1 + src/client/project/template_builder/view.pyi | 6 +- src/client/project/view_spec.py | 2 +- src/client/project/view_spec.pyi | 2 +- tests/template_builder/test_completeness.py | 60 +++++ tox.ini | 2 + 39 files changed, 1202 insertions(+), 35 deletions(-) create mode 100644 functional_tests/client/__init__.py rename functional_tests/{ => client}/test_assignment.py (100%) rename functional_tests/{ => client}/test_client.py (100%) rename functional_tests/{ => client}/test_filter.py (100%) rename functional_tests/{ => client}/test_project.py (100%) rename functional_tests/{ => client}/test_tasks.py (96%) create mode 100644 functional_tests/template_builder/__init__.py create mode 100644 functional_tests/template_builder/conftest.py create mode 100644 functional_tests/template_builder/test_actions.py create mode 100644 functional_tests/template_builder/test_conditions.py create mode 100644 functional_tests/template_builder/test_data.py create mode 100644 functional_tests/template_builder/test_fields.py create mode 100644 functional_tests/template_builder/test_helpers.py create mode 100644 functional_tests/template_builder/test_layouts.py create mode 100644 functional_tests/template_builder/test_plugins.py create mode 100644 functional_tests/template_builder/test_views.py create mode 100644 tests/template_builder/test_completeness.py diff --git a/docs/reference/toloka.async_client.client.AsyncTolokaClient.md b/docs/reference/toloka.async_client.client.AsyncTolokaClient.md index 194c1189..c6806c3b 100644 --- a/docs/reference/toloka.async_client.client.AsyncTolokaClient.md +++ b/docs/reference/toloka.async_client.client.AsyncTolokaClient.md @@ -9,7 +9,7 @@ AsyncTolokaClient( ) ``` -Class that implements interaction with [Toloka API], in an asynchronous way. +Class that implements interaction with [Toloka API](https://toloka.ai/docs/api/concepts/about.html), in an asynchronous way. All methods are wrapped as async. So all methods calls must be awaited. diff --git a/docs/reference/toloka.client.TolokaClient.md b/docs/reference/toloka.client.TolokaClient.md index 532dab75..15182e6e 100644 --- a/docs/reference/toloka.client.TolokaClient.md +++ b/docs/reference/toloka.client.TolokaClient.md @@ -14,7 +14,7 @@ TolokaClient( ) ``` -Class that implements interaction with [Toloka API](https://yandex.com/dev/toloka/doc/concepts/about.html). +Class that implements interaction with [Toloka API](https://toloka.ai/docs/api/concepts/about.html). Objects of other classes are created and modified only in memory of your computer. @@ -30,7 +30,7 @@ Call `TolokaClient.update_project` and pass the `Project` to apply your changes. | Parameters | Type | Description | | :----------| :----| :-----------| -`token`|**str**|

Your OAuth token for Toloka. You can learn more about how to get it [here](https://yandex.com/dev/toloka/doc/concepts/access.html#access__token)

+`token`|**str**|

Your OAuth token for Toloka. You can learn more about how to get it [here](https://toloka.ai/docs/api/concepts/access.html#access__token)

`environment`|**Union\[[Environment](toloka.client.TolokaClient.Environment.md), str, None\]**|

There are two environments in Toloka:

  • `SANDBOX` – [Testing environment](https://sandbox.toloka.yandex.com) for Toloka requesters. You can test complex projects before starting them on real performers. Nobody will see your tasks, and it's free.
  • `PRODUCTION` – [Production environment](https://toloka.yandex.com) for Toloka requesters. You spend money there and get the results. You need to register in each environment separately. OAuth tokens are generated in each environment separately too.

Default value: `None`.

`retries`|**Union\[int, Retry\]**|

Retry policy for failed API requests. Possible values:

  • `int` – The number of retries for all requests. In this case, the retry policy is created automatically.
  • `Retry` object – Deprecated type. Use `retryer_factory` parameter instead.

Default value: `3`.

`timeout`|**Union\[float, Tuple\[float, float\]\]**|

Number of seconds that [Requests library](https://docs.python-requests.org/en/master) will wait for your client to establish connection to a remote machine. Possible values:

  • `float` – Single value for both connect and read timeouts.
  • `Tuple[float, float]` – Tuple sets the values for connect and read timeouts separately.
  • `None` – Set the timeout to `None` only if you are willing to wait the [Response](https://docs.python-requests.org/en/master/api/#requests.Response) for unlimited number of seconds.

Default value: `10.0`.

diff --git a/docs/reference/toloka.client.project.template_builder.fields.SelectFieldV1.md b/docs/reference/toloka.client.project.template_builder.fields.SelectFieldV1.md index 52b87995..8a97243b 100644 --- a/docs/reference/toloka.client.project.template_builder.fields.SelectFieldV1.md +++ b/docs/reference/toloka.client.project.template_builder.fields.SelectFieldV1.md @@ -5,7 +5,7 @@ SelectFieldV1( self, data: Optional[BaseComponent] = None, - options: Optional[Union[BaseComponent, Option]] = None, + options: Optional[Union[BaseComponent, List[Union[BaseComponent, Option]]]] = None, *, placeholder: Optional[Any] = None, hint: Optional[Any] = None, @@ -30,7 +30,7 @@ To allow selecting multiple options, use the field.checkbox-group component. | Parameters | Type | Description | | :----------| :----| :-----------| `data`|**Optional\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md)\]**|

Data with values that will be processed or changed.

-`options`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), [Option](toloka.client.project.template_builder.fields.SelectFieldV1.Option.md)\]\]**|

Options to choose from.

+`options`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), List\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), [Option](toloka.client.project.template_builder.fields.SelectFieldV1.Option.md)\]\]\]\]**|

Options to choose from.

`placeholder`|**Optional\[Any\]**|

The text that will be displayed if none of the options is selected.

`hint`|**Optional\[Any\]**|

Hint text.

`label`|**Optional\[Any\]**|

Label above the component.

diff --git a/docs/reference/toloka.client.project.template_builder.helpers.ReplaceHelperV1.md b/docs/reference/toloka.client.project.template_builder.helpers.ReplaceHelperV1.md index 47a35320..d2813cb8 100644 --- a/docs/reference/toloka.client.project.template_builder.helpers.ReplaceHelperV1.md +++ b/docs/reference/toloka.client.project.template_builder.helpers.ReplaceHelperV1.md @@ -15,7 +15,7 @@ ReplaceHelperV1( Allows you to replace some parts of the string with others. -This helper function returns a string in which all occurrences of 'findindataare replaced withreplace`. +This helper function returns a string in which all occurrences of `find` in `data` are replaced with `replace`. Because the helper.replace helper returns a string, it can be used in properties that accept string values. ## Parameters Description diff --git a/docs/reference/toloka.client.project.template_builder.helpers.YandexDiskProxyHelperV1.md b/docs/reference/toloka.client.project.template_builder.helpers.YandexDiskProxyHelperV1.md index 93081865..b422e94b 100644 --- a/docs/reference/toloka.client.project.template_builder.helpers.YandexDiskProxyHelperV1.md +++ b/docs/reference/toloka.client.project.template_builder.helpers.YandexDiskProxyHelperV1.md @@ -14,7 +14,7 @@ You can use this component to download files from Yandex.Disk. To use YandexDiskProxyHelper, connect Yandex.Disk to your Toloka account and add the proxy by following -the [instructions](https://yandex.com/support/toloka-requester/concepts/prepare-data.html?lang=en) +the [instructions](https://toloka.ai/docs/guide/concepts/prepare-data.html?lang=en) Select the component that you want to add, such as view.image for an image or view.audio for an audio file. In the url property of this component, use YandexDiskProxyHelper. @@ -22,4 +22,4 @@ In the url property of this component, use YandexDiskProxyHelper. | Parameters | Type | Description | | :----------| :----| :-----------| -`path`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), str\]\]**|

Path to the file in the /&lt;Proxy name&gt;/&lt;File name&gt;.&lt;type&gt; format

+`path`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), str\]\]**|

Path to the file in the <Proxy name>/<File name>.<type> format

diff --git a/docs/reference/toloka.client.project.template_builder.view.VideoViewV1.md b/docs/reference/toloka.client.project.template_builder.view.VideoViewV1.md index 75bc8578..25a5d85b 100644 --- a/docs/reference/toloka.client.project.template_builder.view.VideoViewV1.md +++ b/docs/reference/toloka.client.project.template_builder.view.VideoViewV1.md @@ -9,6 +9,7 @@ VideoViewV1( full_height: Optional[Union[BaseComponent, bool]] = None, max_width: Optional[Union[BaseComponent, float]] = None, min_width: Optional[Union[BaseComponent, float]] = None, + ratio: Optional[Union[BaseComponent, List[Union[BaseComponent, float]]]] = None, hint: Optional[Any] = None, label: Optional[Any] = None, validation: Optional[BaseComponent] = None, @@ -33,7 +34,7 @@ cropped. `full_height`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), bool\]\]**|

If true, the element takes up all the vertical free space. The element is set to a minimum height of 400 pixels.

`max_width`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), float\]\]**|

Maximum width of the element in pixels, must be greater than min_width.

`min_width`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), float\]\]**|

Minimum width of the element in pixels. Takes priority over max_width.

+`ratio`|**Optional\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), List\[Union\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md), float\]\]\]\]**|

The aspect ratio of the video block. An array of two numbers: the first sets the width of the block and the second sets the height.

`hint`|**Optional\[Any\]**|

Hint text.

`label`|**Optional\[Any\]**|

Label above the component.

`validation`|**Optional\[[BaseComponent](toloka.client.project.template_builder.base.BaseComponent.md)\]**|

Validation based on condition.

-`ratio`|**-**|

The aspect ratio of the video block. An array of two numbers: the first sets the width of the block and the second sets the height.

diff --git a/docs/reference/toloka.client.project.view_spec.ClassicViewSpec.md b/docs/reference/toloka.client.project.view_spec.ClassicViewSpec.md index 07a82740..b3ef93ba 100644 --- a/docs/reference/toloka.client.project.view_spec.ClassicViewSpec.md +++ b/docs/reference/toloka.client.project.view_spec.ClassicViewSpec.md @@ -16,7 +16,7 @@ ClassicViewSpec( A classic view specification defined with HTML, CSS and JS. -For more information, see [Toloka Requester's guide](https://yandex.ru/support/toloka-requester/?lang=en) +For more information, see [Toloka Requester's guide](https://toloka.ai/ru/docs/guide/?lang=en) ## Parameters Description diff --git a/functional_tests/client/__init__.py b/functional_tests/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/functional_tests/test_assignment.py b/functional_tests/client/test_assignment.py similarity index 100% rename from functional_tests/test_assignment.py rename to functional_tests/client/test_assignment.py diff --git a/functional_tests/test_client.py b/functional_tests/client/test_client.py similarity index 100% rename from functional_tests/test_client.py rename to functional_tests/client/test_client.py diff --git a/functional_tests/test_filter.py b/functional_tests/client/test_filter.py similarity index 100% rename from functional_tests/test_filter.py rename to functional_tests/client/test_filter.py diff --git a/functional_tests/test_project.py b/functional_tests/client/test_project.py similarity index 100% rename from functional_tests/test_project.py rename to functional_tests/client/test_project.py diff --git a/functional_tests/test_tasks.py b/functional_tests/client/test_tasks.py similarity index 96% rename from functional_tests/test_tasks.py rename to functional_tests/client/test_tasks.py index 7ecd75d0..a45741e5 100644 --- a/functional_tests/test_tasks.py +++ b/functional_tests/client/test_tasks.py @@ -4,7 +4,7 @@ from toloka.client.task import Task -@pytest.fixture(scope='session') +@pytest.fixture(scope='package') def pool(client, empty_project): new_pool = Pool( project_id=empty_project.id, @@ -20,7 +20,7 @@ def pool(client, empty_project): assert client.get_pool(new_pool.id).status == Pool.Status.ARCHIVED -@pytest.fixture(scope='session') +@pytest.fixture(scope='package') def task(client, pool): new_task = Task(input_values={'url': 'https://toloka.yandex.com'}, pool_id=pool.id, overlap=1) new_task = client.create_task(new_task) diff --git a/functional_tests/conftest.py b/functional_tests/conftest.py index d1634af7..4329f991 100644 --- a/functional_tests/conftest.py +++ b/functional_tests/conftest.py @@ -1,10 +1,10 @@ import os -import pytest +import pytest from toloka.client import TolokaClient from toloka.client.project import Project -from toloka.client.project.task_spec import TaskSpec from toloka.client.project.field_spec import UrlSpec, StringSpec +from toloka.client.project.task_spec import TaskSpec from toloka.client.project.view_spec import ClassicViewSpec @@ -29,7 +29,7 @@ def client(token): return client -@pytest.fixture(scope='session') +@pytest.fixture(scope='package') def empty_project(client, dummy_task_spec): test_project = Project( public_name='Test project', diff --git a/functional_tests/template_builder/__init__.py b/functional_tests/template_builder/__init__.py new file mode 100644 index 00000000..5c24eabb --- /dev/null +++ b/functional_tests/template_builder/__init__.py @@ -0,0 +1,22 @@ +import json + + +def assert_view_spec_uploads_to_project(client, project, view_spec): + def compare_recursively(left, right): + if not isinstance(left, dict): + assert left == right + else: + keys = left.keys() | right.keys() + keys.discard('_unexpected') + keys.discard('localizationConfig') + for key in keys: + # config is serialized to string so to compare two dicts serialized to strings we need to deserialize + # them first + if key == 'config': + compare_recursively(json.loads(left[key]), json.loads(right[key])) + else: + compare_recursively(left[key], right[key]) + + project.task_spec.view_spec = view_spec + created_project = client.update_project(project.id, project) + compare_recursively(created_project.task_spec.view_spec.unstructure(), view_spec.unstructure()) diff --git a/functional_tests/template_builder/conftest.py b/functional_tests/template_builder/conftest.py new file mode 100644 index 00000000..cc51e355 --- /dev/null +++ b/functional_tests/template_builder/conftest.py @@ -0,0 +1,45 @@ +import pytest +import functools + + +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def add_label_hint_validation(empty_condition): + def func(cls): + return functools.partial( + cls, + hint='hint', + label='label', + validation=empty_condition, + ) + + return func + + +@pytest.fixture +def empty_condition(): + return tb.EmptyConditionV1( + tb.InputData('url'), + hint='hint' + ) + + +@pytest.fixture +def text_view(add_label_hint_validation): + return add_label_hint_validation(tb.TextViewV1)( + 'Hello world', + ) + + +@pytest.fixture +def notify_action(): + return tb.NotifyActionV1( + tb.NotifyActionV1.Payload( + content='Hello World!', + theme='info', + delay=100, + duration=100, + ), + ) diff --git a/functional_tests/template_builder/test_actions.py b/functional_tests/template_builder/test_actions.py new file mode 100644 index 00000000..cc042a58 --- /dev/null +++ b/functional_tests/template_builder/test_actions.py @@ -0,0 +1,77 @@ +import pytest + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def bulk_action(notify_action): + return tb.BulkActionV1(notify_action) + + +@pytest.fixture +def open_close_action(): + return tb.OpenCloseActionV1( + tb.base.RefComponent('fake_ref') + ) + + +@pytest.fixture +def open_link_action(): + return tb.OpenLinkActionV1( + 'http://fake_url' + ) + + +@pytest.fixture +def play_pause_action(): + return tb.PlayPauseActionV1( + tb.base.RefComponent('fake_ref') + ) + + +@pytest.fixture(params=[e.value for e in tb.RotateActionV1.Payload]) +def rotate_action(request): + return tb.RotateActionV1( + tb.base.RefComponent('fake_ref'), + request.param # payload + ) + + +@pytest.fixture +def set_action(): + return tb.SetActionV1( + tb.OutputData('label'), True + ) + + +@pytest.fixture +def toggle_action(): + return tb.ToggleActionV1( + tb.OutputData('label') + ) + + +@pytest.mark.parametrize( + 'action_component', [ + lazy_fixture('bulk_action'), + lazy_fixture('notify_action'), + lazy_fixture('open_close_action'), + lazy_fixture('open_link_action'), + lazy_fixture('play_pause_action'), + lazy_fixture('rotate_action'), + lazy_fixture('set_action'), + lazy_fixture('toggle_action'), + ] +) +def test_action_component(action_component, client, empty_project): + view_spec = TemplateBuilderViewSpec( + view=tb.ActionButtonViewV1( + action=action_component + ) + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/functional_tests/template_builder/test_conditions.py b/functional_tests/template_builder/test_conditions.py new file mode 100644 index 00000000..678dea6e --- /dev/null +++ b/functional_tests/template_builder/test_conditions.py @@ -0,0 +1,139 @@ +import pytest + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def all_condition(empty_condition): + return tb.AllConditionV1( + [empty_condition], + hint='hint' + ) + + +@pytest.fixture +def any_condition(empty_condition): + return tb.AnyConditionV1( + [empty_condition], + hint='hint' + ) + + +@pytest.fixture +def distance_condition(): + return tb.DistanceConditionV1( + 'coord0', 'coord1', 100., + hint='hint' + ) + + +@pytest.fixture +def equals_condition(): + return tb.EqualsConditionV1( + 'value', tb.InputData('url'), + hint='hint' + ) + + +@pytest.fixture +def link_opened_condition(): + return tb.LinkOpenedConditionV1( + 'http://fake-url', + hint='hint' + ) + + +@pytest.fixture +def not_condition(empty_condition): + return tb.NotConditionV1( + empty_condition, + hint='hint' + ) + + +@pytest.fixture +def played_condition(): + return tb.PlayedConditionV1( + hint='hint' + ) + + +@pytest.fixture +def played_fully_condition(): + return tb.PlayedFullyConditionV1( + hint='hint' + ) + + +@pytest.fixture +def required_condition(): + return tb.RequiredConditionV1( + tb.InputData('url'), + hint='hint' + ) + + +@pytest.fixture +def same_domain_condition(): + return tb.SameDomainConditionV1( + tb.InputData('url'), + 'http://fake-url', + hint='hint' + ) + + +@pytest.fixture +def schema_condition(): + return tb.SchemaConditionV1( + tb.InputData('url'), + # example schema from https://json-schema.org/learn/getting-started-step-by-step.html + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Product", + "description": "A product in the catalog", + "type": "object" + }, + hint='hint' + ) + + +@pytest.fixture +def sub_array_condition(): + return tb.SubArrayConditionV1( + ['http://fake-url'], + tb.InputData('url'), + hint='hint' + ) + + +@pytest.mark.parametrize( + 'condition_component', [ + lazy_fixture('any_condition'), + lazy_fixture('all_condition'), + lazy_fixture('distance_condition'), + lazy_fixture('empty_condition'), + lazy_fixture('equals_condition'), + lazy_fixture('link_opened_condition'), + lazy_fixture('not_condition'), + lazy_fixture('played_condition'), + lazy_fixture('played_fully_condition'), + lazy_fixture('required_condition'), + lazy_fixture('same_domain_condition'), + lazy_fixture('schema_condition'), + lazy_fixture('sub_array_condition'), + ] +) +def test_condition_component(condition_component, client, empty_project): + view_spec = TemplateBuilderViewSpec( + view=tb.TextViewV1( + content='Hello world', + validation=condition_component, + ) + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/functional_tests/template_builder/test_data.py b/functional_tests/template_builder/test_data.py new file mode 100644 index 00000000..66d061c9 --- /dev/null +++ b/functional_tests/template_builder/test_data.py @@ -0,0 +1,70 @@ +import pytest + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def input_data(): + return tb.InputData( + 'test_data', 10, + ) + + +@pytest.fixture +def internal_data(): + return tb.InternalData( + 'test_data', 10, + ) + + +@pytest.fixture +def local_data(): + return tb.LocalData( + 'test_data', 10, + ) + + +@pytest.fixture +def location_data(): + return tb.LocationData() + + +@pytest.fixture +def output_data(): + return tb.OutputData( + 'test_data', + ) + + +@pytest.fixture +def relative_data(): + return tb.RelativeData( + 'test_data', 10, + ) + + +@pytest.mark.parametrize( + 'data_component', [ + lazy_fixture('input_data'), + lazy_fixture('internal_data'), + lazy_fixture('local_data'), + lazy_fixture('location_data'), + lazy_fixture('output_data'), + lazy_fixture('relative_data'), + ] +) +def test_data_component(data_component, client, empty_project): + view_spec = TemplateBuilderViewSpec( + view=tb.TextViewV1( + content='Hello world', + validation=tb.EmptyConditionV1( + data=data_component, + ) + ) + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/functional_tests/template_builder/test_fields.py b/functional_tests/template_builder/test_fields.py new file mode 100644 index 00000000..7a4019ec --- /dev/null +++ b/functional_tests/template_builder/test_fields.py @@ -0,0 +1,223 @@ +import pytest + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def audio_field(add_label_hint_validation): + return add_label_hint_validation(tb.AudioFieldV1)( + tb.OutputData('label'), + multiple=True, + ) + + +@pytest.fixture +def button_radio_field(add_label_hint_validation): + return add_label_hint_validation(tb.ButtonRadioFieldV1)( + tb.OutputData('label'), 10 + ) + + +@pytest.fixture +def button_radio_group_field(add_label_hint_validation): + return add_label_hint_validation(tb.ButtonRadioGroupFieldV1)( + tb.OutputData('label'), + [tb.GroupFieldOption('yes', 'yes')], + ) + + +@pytest.fixture +def checkbox_field(add_label_hint_validation): + return add_label_hint_validation(tb.CheckboxFieldV1)( + tb.OutputData('label'), + disabled=True, + preserve_false=True, + ) + + +@pytest.fixture +def checkbox_group_field(add_label_hint_validation): + return add_label_hint_validation(tb.CheckboxGroupFieldV1)( + tb.OutputData('label'), + [tb.GroupFieldOption('yes', 'yes')], + disabled=True, + preserve_false=True, + ) + + +@pytest.fixture +def date_field(add_label_hint_validation): + return add_label_hint_validation(tb.DateFieldV1)( + tb.OutputData('label'), 'date', + block_list=['2000-01-01'], + max=['2050-01-01'], + min=['1000-01-01'], + placeholder='this is placeholder', + ) + + +@pytest.fixture +def email_field(add_label_hint_validation): + return add_label_hint_validation(tb.EmailFieldV1)( + tb.OutputData('label'), + placeholder='this is placeholder', + ) + + +@pytest.fixture +def file_field(add_label_hint_validation): + return add_label_hint_validation(tb.FileFieldV1)( + tb.OutputData('label'), + ['image/png'], + multiple=True, + ) + + +@pytest.fixture +def image_annotation_field(add_label_hint_validation): + return add_label_hint_validation(tb.ImageAnnotationFieldV1)( + tb.OutputData('label'), + 'fake_image_path', + disabled=True, + full_height=True, + labels=[tb.ImageAnnotationFieldV1.Label('yes', 'yes')], + min_width=300., + ratio=[1., 2.], + shapes={ + 'point': True, + 'polygon': True, + 'rectangle': True, + }, + ) + + +@pytest.fixture +def list_field(add_label_hint_validation): + return add_label_hint_validation(tb.ListFieldV1)( + tb.OutputData('label'), + tb.TextFieldV1( + data=tb.RelativeData() + ), + button_label='button label', + direction='vertical', + editable=True, + max_length=3., + size='s', + ) + + +@pytest.fixture +def media_file_field(add_label_hint_validation): + return add_label_hint_validation(tb.MediaFileFieldV1)( + tb.OutputData('label'), + tb.MediaFileFieldV1.Accept( + file_system=True, + gallery=True, + photo=True, + video=True, + ), + multiple=True, + ) + + +@pytest.fixture +def number_field(add_label_hint_validation): + return add_label_hint_validation(tb.NumberFieldV1)( + tb.OutputData('label'), + maximum=100, + minimum=-100, + placeholder='placeholder', + ) + + +@pytest.fixture +def phone_number_field(add_label_hint_validation): + return add_label_hint_validation(tb.PhoneNumberFieldV1)( + tb.OutputData('label'), + placeholder='placeholder', + ) + + +@pytest.fixture +def radio_group_field(add_label_hint_validation): + return add_label_hint_validation(tb.RadioGroupFieldV1)( + tb.OutputData('label'), + [tb.GroupFieldOption('yes', 'yes')], + disabled=True, + ) + + +@pytest.fixture +def select_field(add_label_hint_validation): + return add_label_hint_validation(tb.SelectFieldV1)( + tb.OutputData('label'), + [tb.SelectFieldV1.Option(label='yes', value='yes')], + placeholder='placeholder', + ) + + +@pytest.fixture +def text_field(add_label_hint_validation): + return add_label_hint_validation(tb.TextFieldV1)( + tb.OutputData('label'), + disabled=True, + placeholder='placeholder', + ) + + +@pytest.fixture +def text_annotation_field(add_label_hint_validation): + return add_label_hint_validation(tb.TextAnnotationFieldV1)( + tb.OutputData('label'), + adjust='words', + content='this is content', + disabled=True, + labels=[tb.TextAnnotationFieldV1.Label('yes', 'yes')], + ) + + +@pytest.fixture +def textarea_field(add_label_hint_validation): + return add_label_hint_validation(tb.TextareaFieldV1)( + tb.OutputData('label'), + disabled=True, + placeholder='placeholder', + resizable=False, + rows=3., + ) + + +@pytest.mark.parametrize( + 'field_component', [ + lazy_fixture('audio_field'), + lazy_fixture('button_radio_field'), + lazy_fixture('button_radio_group_field'), + lazy_fixture('checkbox_field'), + lazy_fixture('checkbox_group_field'), + lazy_fixture('date_field'), + lazy_fixture('email_field'), + lazy_fixture('file_field'), + lazy_fixture('image_annotation_field'), + lazy_fixture('list_field'), + lazy_fixture('media_file_field'), + lazy_fixture('number_field'), + lazy_fixture('phone_number_field'), + lazy_fixture('radio_group_field'), + lazy_fixture('select_field'), + lazy_fixture('text_field'), + lazy_fixture('text_annotation_field'), + lazy_fixture('textarea_field'), + ] +) +def test_field_component(field_component, client, empty_project): + view_spec = TemplateBuilderViewSpec( + view=tb.ListViewV1( + items=[field_component] + ) + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/functional_tests/template_builder/test_helpers.py b/functional_tests/template_builder/test_helpers.py new file mode 100644 index 00000000..bc7d0df9 --- /dev/null +++ b/functional_tests/template_builder/test_helpers.py @@ -0,0 +1,138 @@ +import pytest + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def concat_arrays_helper(): + return tb.ConcatArraysHelperV1([[1, 2, 3], [4, 5, 6]]) + + +@pytest.fixture +def entries_2_object_helper(): + return tb.Entries2ObjectHelperV1( + [tb.Entries2ObjectHelperV1.Entry(key='key', value='value')] + ) + + +@pytest.fixture +def if_helper(empty_condition): + return tb.IfHelperV1( + empty_condition, + tb.TextViewV1('then'), + else_=tb.TextViewV1('else'), + ) + + +@pytest.fixture +def join_helper(): + return tb.JoinHelperV1( + ['Hello', 'world', '!'], + ' ', + ) + + +@pytest.fixture +def object_2_entries_helper(): + return tb.Object2EntriesHelperV1( + { + 'key1': 'value1', + 'key2': 'value2', + } + ) + + +@pytest.fixture +def replace_helper(): + return tb.ReplaceHelperV1( + 'ababbaba', + 'ab', + 'c' + ) + + +@pytest.fixture +def search_query_helper(): + return tb.SearchQueryHelperV1( + 'Toloka', + 'wikipedia', + ) + + +@pytest.fixture +def switch_helper(empty_condition): + return tb.SwitchHelperV1( + [ + tb.SwitchHelperV1.Case( + condition=empty_condition, + result=tb.TextViewV1('first') + ), + tb.SwitchHelperV1.Case( + condition=tb.NotConditionV1(condition=empty_condition), + result=tb.TextViewV1('second') + ) + ], + tb.TextViewV1('default') + ) + + +@pytest.fixture(params=[e.value for e in tb.TextTransformHelperV1.Transformation]) +def text_transform_helper(request): + return tb.TextTransformHelperV1( + 'sample', + request.param # transformation + ) + + +@pytest.fixture +def transform_helper(): + return tb.TransformHelperV1( + tb.ImageViewV1( + url=tb.RelativeData('item') + ), + tb.InputData('url'), + ) + + +@pytest.fixture +def translate_helper(): + return tb.TranslateHelperV1( + 'test-key' + ) + + +@pytest.fixture +def yandex_disk_proxy_helper(): + return tb.YandexDiskProxyHelperV1( + 'fake-proxy/my.file' + ) + + +@pytest.mark.parametrize( + 'helper_component', [ + lazy_fixture('concat_arrays_helper'), + lazy_fixture('entries_2_object_helper'), + lazy_fixture('if_helper'), + lazy_fixture('join_helper'), + lazy_fixture('object_2_entries_helper'), + lazy_fixture('replace_helper'), + lazy_fixture('search_query_helper'), + lazy_fixture('switch_helper'), + lazy_fixture('text_transform_helper'), + lazy_fixture('transform_helper'), + lazy_fixture('translate_helper'), + lazy_fixture('yandex_disk_proxy_helper'), + ] +) +def test_helper_component(helper_component, client, empty_project): + view_spec = TemplateBuilderViewSpec( + view=tb.TextViewV1( + content=helper_component + ) + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/functional_tests/template_builder/test_layouts.py b/functional_tests/template_builder/test_layouts.py new file mode 100644 index 00000000..6fea5f66 --- /dev/null +++ b/functional_tests/template_builder/test_layouts.py @@ -0,0 +1,96 @@ +import pytest + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def bars_layout(empty_condition): + return tb.BarsLayoutV1( + tb.TextViewV1('text_view'), + bar_before=tb.TextViewV1('i am before bar'), + bar_after=tb.TextViewV1('i am after bar'), + validation=empty_condition + ) + + +@pytest.fixture +def columns_layout(empty_condition): + return tb.ColumnsLayoutV1( + [ + tb.TextViewV1('column 1'), + tb.TextViewV1('column 2'), + ], + full_height=True, + min_width=100., + ratio=[1., 2.], + vertical_align='top', + validation=empty_condition, + ) + + +@pytest.fixture +def compare_layout(empty_condition): + return tb.CompareLayoutV1( + tb.TextViewV1('this is controls'), + [ + tb.CompareLayoutItem( + content=tb.TextViewV1('left'), + controls=tb.TextViewV1('left_controls'), + ), + tb.CompareLayoutItem( + content=tb.TextViewV1('right'), + controls=tb.TextViewV1('right_controls'), + ), + ], + min_width=300., + wide_common_controls=True, + validation=empty_condition, + ) + + +@pytest.fixture +def side_by_side_layout(empty_condition): + return tb.SideBySideLayoutV1( + tb.CheckboxFieldV1( + data=tb.InputData('url') + ), + [ + tb.TextViewV1('left'), + tb.TextViewV1('right'), + ], + min_item_width=300., + validation=empty_condition + ) + + +@pytest.fixture +def sidebar_layout(empty_condition): + return tb.SidebarLayoutV1( + tb.TextViewV1('this is content'), + tb.TextViewV1('this is controls'), + controls_width=300., + extra_controls=tb.TextViewV1('this is extra controls'), + min_width=300., + validation=empty_condition, + ) + + +@pytest.mark.parametrize( + 'layout_component', [ + lazy_fixture('bars_layout'), + lazy_fixture('columns_layout'), + lazy_fixture('compare_layout'), + lazy_fixture('side_by_side_layout'), + lazy_fixture('sidebar_layout'), + ] +) +def test_layout_component(layout_component, client, empty_project): + view_spec = TemplateBuilderViewSpec( + view=layout_component + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/functional_tests/template_builder/test_plugins.py b/functional_tests/template_builder/test_plugins.py new file mode 100644 index 00000000..cb16f77f --- /dev/null +++ b/functional_tests/template_builder/test_plugins.py @@ -0,0 +1,73 @@ +import pytest + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def image_annotation_hotkeys_plugin(empty_condition): + return tb.ImageAnnotationHotkeysPluginV1( + cancel='a', + confirm='b', + labels=["Moscow", "Tokyo", "New York"], + point='c', + polygon='d', + rectangle='e', + select='f', + ) + + +@pytest.fixture +def text_annotation_hotkeys_plugin(): + return tb.TextAnnotationHotkeysPluginV1( + ['1', '2', '3'], 'c' + ) + + +@pytest.fixture +def hotkeys_plugin(): + keys = list('abcdefghijklmnopqrstuvwxyz0123456789') + ['up', 'down'] + return tb.HotkeysPluginV1(**{f'key_{key}': tb.SetActionV1(tb.OutputData('label'), 'cat') for key in keys}) + + +@pytest.fixture +def trigger_plugin(empty_condition): + return tb.TriggerPluginV1( + action=tb.SetActionV1(tb.OutputData('label'), 'cat'), + condition=empty_condition, + fire_immediately=False, + on_change_of=tb.InputData('url'), + ) + + +@pytest.fixture(params=[e.value for e in tb.TolokaPluginV1.TolokaPluginLayout.Kind]) +def toloka_plugin(request): + return tb.TolokaPluginV1( + request.param, + task_width=100., # kind + notifications=[ + tb.TextViewV1('notification 1'), + tb.TextViewV1('notification 2'), + ] + ) + + +@pytest.mark.parametrize( + 'plugin_component', [ + lazy_fixture('image_annotation_hotkeys_plugin'), + lazy_fixture('text_annotation_hotkeys_plugin'), + lazy_fixture('hotkeys_plugin'), + lazy_fixture('trigger_plugin'), + lazy_fixture('toloka_plugin'), + ] +) +def test_plugin_component(plugin_component, client, empty_project, text_view): + view_spec = TemplateBuilderViewSpec( + view=text_view, + plugins=[plugin_component] + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/functional_tests/template_builder/test_views.py b/functional_tests/template_builder/test_views.py new file mode 100644 index 00000000..85028a19 --- /dev/null +++ b/functional_tests/template_builder/test_views.py @@ -0,0 +1,218 @@ +import pytest +import functools + +from . import assert_view_spec_uploads_to_project +from pytest_lazyfixture import lazy_fixture +from toloka.client.project import TemplateBuilderViewSpec +from toloka.client.project import template_builder as tb + + +@pytest.fixture +def action_button_view(notify_action, add_label_hint_validation): + return add_label_hint_validation(tb.ActionButtonViewV1)( + notify_action, + ) + + +@pytest.fixture(params=[e.value for e in tb.AlertViewV1.Theme]) +def alert_view(request, text_view, add_label_hint_validation): + return add_label_hint_validation(tb.AlertViewV1)( + text_view, + theme=request.param, + ) + + +@pytest.fixture +def audio_view(add_label_hint_validation): + return add_label_hint_validation(tb.AudioViewV1)( + 'http://fake-url', + loop=True, + ) + + +@pytest.fixture +def collapse_view(text_view, add_label_hint_validation): + return add_label_hint_validation(tb.CollapseViewV1)( + text_view, + default_opened=True, + ) + + +@pytest.fixture +def partial_device_frame_view(text_view, add_label_hint_validation): + return functools.partial( + add_label_hint_validation(tb.DeviceFrameViewV1), + text_view, + max_width=500., + min_width=300. + ) + + +@pytest.fixture +def device_frame_view(partial_device_frame_view): + return partial_device_frame_view(ratio=[1., 2.]) + + +@pytest.fixture +def device_frame_view_full_height(partial_device_frame_view): + return partial_device_frame_view(full_height=True) + + +@pytest.fixture +def divider_view(add_label_hint_validation): + return add_label_hint_validation(tb.DividerViewV1)() + + +@pytest.fixture +def group_view(text_view, add_label_hint_validation): + return add_label_hint_validation(tb.GroupViewV1)(text_view) + + +@pytest.fixture +def partial_iframe_view(add_label_hint_validation): + return functools.partial( + add_label_hint_validation(tb.IframeViewV1), + 'http://fake-url', + max_width=500., + min_width=300., + ) + + +@pytest.fixture +def iframe_view(partial_iframe_view): + return partial_iframe_view(ratio=[1., 2.]) + + +@pytest.fixture +def iframe_view_full_height(partial_iframe_view): + return partial_iframe_view(full_height=True) + + +@pytest.fixture +def partial_image_view(add_label_hint_validation): + return functools.partial( + add_label_hint_validation(tb.ImageViewV1), + 'http://fake-url', + max_width=500., + min_width=300., + no_border=False, + no_lazy_load=True, + popup=False, + rotatable=True, + scrollable=True, + ) + + +@pytest.fixture +def image_view(partial_image_view): + return partial_image_view(ratio=[1., 2.]) + + +@pytest.fixture +def image_view_full_height(partial_image_view): + return partial_image_view(full_height=True) + + +@pytest.fixture +def labeled_list_view(text_view, add_label_hint_validation): + return add_label_hint_validation(tb.LabeledListViewV1)( + [ + tb.LabeledListViewV1.Item( + text_view, + 'label', + center_label=True, + hint='hint', + ), + tb.LabeledListViewV1.Item( + text_view, + 'label', + center_label=True, + hint='hint', + ) + ], + min_width=300., + ) + + +@pytest.fixture +def link_view(add_label_hint_validation): + return add_label_hint_validation(tb.LinkViewV1)( + 'http://fake-url', + content='LINK', + ) + + +@pytest.fixture +def link_group_view(add_label_hint_validation): + return add_label_hint_validation(tb.LinkGroupViewV1)( + [ + tb.LinkGroupViewV1.Link( + 'LINK_1', + 'http://fake-url', + theme='primary' + ), + tb.LinkGroupViewV1.Link( + 'http://fake-url' + 'LINK_2', + ) + ], + ) + + +@pytest.fixture +def list_view(text_view, add_label_hint_validation): + return add_label_hint_validation(tb.ListViewV1)( + [text_view, text_view], + direction='horizontal', + size='s', + ) + + +@pytest.fixture +def markdown_view(add_label_hint_validation): + return add_label_hint_validation(tb.MarkdownViewV1)( + '## i am markdown header', + ) + + +@pytest.fixture +def video_view(add_label_hint_validation): + return add_label_hint_validation(tb.VideoViewV1)( + 'http://fake-url', + ratio=[1., 2.], + min_width=300., + max_width=500., + full_height=True, + ) + + +@pytest.mark.parametrize( + 'view_component', [ + lazy_fixture('action_button_view'), + lazy_fixture('alert_view'), + lazy_fixture('audio_view'), + lazy_fixture('collapse_view'), + lazy_fixture('device_frame_view'), + lazy_fixture('device_frame_view_full_height'), + lazy_fixture('divider_view'), + lazy_fixture('group_view'), + lazy_fixture('iframe_view'), + lazy_fixture('iframe_view_full_height'), + lazy_fixture('image_view'), + lazy_fixture('image_view_full_height'), + lazy_fixture('labeled_list_view'), + lazy_fixture('link_view'), + lazy_fixture('link_group_view'), + lazy_fixture('list_view'), + lazy_fixture('markdown_view'), + lazy_fixture('text_view'), + lazy_fixture('video_view'), + ] +) +def test_view_component(view_component, client, empty_project): + view_spec = TemplateBuilderViewSpec( + view=view_component + ) + assert_view_spec_uploads_to_project( + client, empty_project, view_spec + ) diff --git a/src/async_client/client.py b/src/async_client/client.py index fa168ef2..1566a55f 100644 --- a/src/async_client/client.py +++ b/src/async_client/client.py @@ -13,7 +13,7 @@ class AsyncTolokaClient: - """Class that implements interaction with [Toloka API], in an asynchronous way. + """Class that implements interaction with [Toloka API](https://toloka.ai/docs/api/concepts/about.html), in an asynchronous way. All methods are wrapped as async. So all methods calls must be awaited. All arguments, same as in TolokaClient. diff --git a/src/async_client/client.pyi b/src/async_client/client.pyi index bd781a83..0b2f4c8e 100644 --- a/src/async_client/client.pyi +++ b/src/async_client/client.pyi @@ -7,7 +7,7 @@ import toloka.client.operations class AsyncTolokaClient: - """Class that implements interaction with [Toloka API], in an asynchronous way. + """Class that implements interaction with [Toloka API](https://toloka.ai/docs/api/concepts/about.html), in an asynchronous way. All methods are wrapped as async. So all methods calls must be awaited. All arguments, same as in TolokaClient. diff --git a/src/client/__init__.py b/src/client/__init__.py index 8f45ea5a..eda9c391 100644 --- a/src/client/__init__.py +++ b/src/client/__init__.py @@ -137,7 +137,7 @@ class TolokaClient: - """Class that implements interaction with [Toloka API](https://yandex.com/dev/toloka/doc/concepts/about.html). + """Class that implements interaction with [Toloka API](https://toloka.ai/docs/api/concepts/about.html). Objects of other classes are created and modified only in memory of your computer. You can transfer information about these objects to Toloka only by calling one of the `TolokaClient` methods. @@ -149,7 +149,7 @@ class TolokaClient: Call `TolokaClient.update_project` and pass the `Project` to apply your changes. Args: - token: Your OAuth token for Toloka. You can learn more about how to get it [here](https://yandex.com/dev/toloka/doc/concepts/access.html#access__token) + token: Your OAuth token for Toloka. You can learn more about how to get it [here](https://toloka.ai/docs/api/concepts/access.html#access__token) environment: There are two environments in Toloka: * `SANDBOX` – [Testing environment](https://sandbox.toloka.yandex.com) for Toloka requesters. You can test complex projects before starting them on real performers. Nobody will see your tasks, and it's free. diff --git a/src/client/__init__.pyi b/src/client/__init__.pyi index 76373163..71d7c9a7 100644 --- a/src/client/__init__.pyi +++ b/src/client/__init__.pyi @@ -144,7 +144,7 @@ from toloka.client.user_bonus import UserBonus class TolokaClient: - """Class that implements interaction with [Toloka API](https://yandex.com/dev/toloka/doc/concepts/about.html). + """Class that implements interaction with [Toloka API](https://toloka.ai/docs/api/concepts/about.html). Objects of other classes are created and modified only in memory of your computer. You can transfer information about these objects to Toloka only by calling one of the `TolokaClient` methods. @@ -156,7 +156,7 @@ class TolokaClient: Call `TolokaClient.update_project` and pass the `Project` to apply your changes. Args: - token: Your OAuth token for Toloka. You can learn more about how to get it [here](https://yandex.com/dev/toloka/doc/concepts/access.html#access__token) + token: Your OAuth token for Toloka. You can learn more about how to get it [here](https://toloka.ai/docs/api/concepts/access.html#access__token) environment: There are two environments in Toloka: * `SANDBOX` – [Testing environment](https://sandbox.toloka.yandex.com) for Toloka requesters. You can test complex projects before starting them on real performers. Nobody will see your tasks, and it's free. diff --git a/src/client/collectors.py b/src/client/collectors.py index 81bcc3ec..199c2419 100644 --- a/src/client/collectors.py +++ b/src/client/collectors.py @@ -13,7 +13,7 @@ 'UsersAssessment' ] """ -https://yandex.ru/dev/toloka/doc/concepts/quality_control-docpage/ +https://toloka.ai/ru/docs/guide/concepts/control.html?lang=en """ from enum import unique diff --git a/src/client/project/template_builder/fields.py b/src/client/project/template_builder/fields.py index 5160ce84..4715b211 100644 --- a/src/client/project/template_builder/fields.py +++ b/src/client/project/template_builder/fields.py @@ -475,7 +475,7 @@ class Option(BaseTemplate): label: base_component_or(Any) value: base_component_or(Any) - options: base_component_or(Option) + options: base_component_or(List[base_component_or(Option)], 'ListBaseComponentOrOption') placeholder: base_component_or(Any) = attribute(kw_only=True) diff --git a/src/client/project/template_builder/fields.pyi b/src/client/project/template_builder/fields.pyi index f9ec04be..8ac5a402 100644 --- a/src/client/project/template_builder/fields.pyi +++ b/src/client/project/template_builder/fields.pyi @@ -863,7 +863,7 @@ class SelectFieldV1(BaseFieldV1): def __init__( self, data: typing.Optional[toloka.client.project.template_builder.base.BaseComponent] = None, - options: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, Option]] = None, + options: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, typing.List[typing.Union[toloka.client.project.template_builder.base.BaseComponent, Option]]]] = None, *, placeholder: typing.Optional[typing.Any] = None, hint: typing.Optional[typing.Any] = None, @@ -881,7 +881,7 @@ class SelectFieldV1(BaseFieldV1): label: typing.Optional[typing.Any] validation: typing.Optional[toloka.client.project.template_builder.base.BaseComponent] version: typing.Optional[str] - options: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, Option]] + options: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, typing.List[typing.Union[toloka.client.project.template_builder.base.BaseComponent, Option]]]] placeholder: typing.Optional[typing.Any] diff --git a/src/client/project/template_builder/helpers.py b/src/client/project/template_builder/helpers.py index 4e3c88eb..ac7565a7 100644 --- a/src/client/project/template_builder/helpers.py +++ b/src/client/project/template_builder/helpers.py @@ -144,7 +144,7 @@ class Object2EntriesHelperV1(BaseHelperV1, spec_value=ComponentType.HELPER_OBJEC class ReplaceHelperV1(BaseHelperV1, spec_value=ComponentType.HELPER_REPLACE): """Allows you to replace some parts of the string with others. - This helper function returns a string in which all occurrences of 'findindataare replaced withreplace`. + This helper function returns a string in which all occurrences of `find` in `data` are replaced with `replace`. Because the helper.replace helper returns a string, it can be used in properties that accept string values. Attributes: data: Data to perform the replacement on. @@ -290,11 +290,11 @@ class YandexDiskProxyHelperV1(BaseHelperV1, spec_value=ComponentType.HELPER_YAND """You can use this component to download files from Yandex.Disk. To use YandexDiskProxyHelper, connect Yandex.Disk to your Toloka account and add the proxy by following - the [instructions](https://yandex.com/support/toloka-requester/concepts/prepare-data.html?lang=en) + the [instructions](https://toloka.ai/docs/guide/concepts/prepare-data.html?lang=en) Select the component that you want to add, such as view.image for an image or view.audio for an audio file. In the url property of this component, use YandexDiskProxyHelper. Attributes: - path: Path to the file in the /<Proxy name>/<File name>.<type> format + path: Path to the file in the /. format """ path: base_component_or(str) diff --git a/src/client/project/template_builder/helpers.pyi b/src/client/project/template_builder/helpers.pyi index 5226c24a..98f8a482 100644 --- a/src/client/project/template_builder/helpers.pyi +++ b/src/client/project/template_builder/helpers.pyi @@ -219,7 +219,7 @@ class Object2EntriesHelperV1(BaseHelperV1): class ReplaceHelperV1(BaseHelperV1): """Allows you to replace some parts of the string with others. - This helper function returns a string in which all occurrences of 'findindataare replaced withreplace`. + This helper function returns a string in which all occurrences of `find` in `data` are replaced with `replace`. Because the helper.replace helper returns a string, it can be used in properties that accept string values. Attributes: data: Data to perform the replacement on. @@ -456,11 +456,11 @@ class YandexDiskProxyHelperV1(BaseHelperV1): """You can use this component to download files from Yandex.Disk. To use YandexDiskProxyHelper, connect Yandex.Disk to your Toloka account and add the proxy by following - the [instructions](https://yandex.com/support/toloka-requester/concepts/prepare-data.html?lang=en) + the [instructions](https://toloka.ai/docs/guide/concepts/prepare-data.html?lang=en) Select the component that you want to add, such as view.image for an image or view.audio for an audio file. In the url property of this component, use YandexDiskProxyHelper. Attributes: - path: Path to the file in the /<Proxy name>/<File name>.<type> format + path: Path to the file in the /. format """ def __init__( diff --git a/src/client/project/template_builder/view.py b/src/client/project/template_builder/view.py index 5fe914b6..c7104549 100644 --- a/src/client/project/template_builder/view.py +++ b/src/client/project/template_builder/view.py @@ -423,3 +423,4 @@ class VideoViewV1(BaseViewV1, spec_value=ComponentType.VIEW_VIDEO): full_height: base_component_or(bool) = attribute(origin='fullHeight', kw_only=True) max_width: base_component_or(float) = attribute(origin='maxWidth', kw_only=True) min_width: base_component_or(float) = attribute(origin='minWidth', kw_only=True) + ratio: base_component_or(List[base_component_or(float)], 'ListBaseComponentOrFloat') = attribute(kw_only=True) diff --git a/src/client/project/template_builder/view.pyi b/src/client/project/template_builder/view.pyi index e64f2c23..e4be732d 100644 --- a/src/client/project/template_builder/view.pyi +++ b/src/client/project/template_builder/view.pyi @@ -757,11 +757,11 @@ class VideoViewV1(BaseViewV1): of 400 pixels. max_width: Maximum width of the element in pixels, must be greater than min_width. min_width: Minimum width of the element in pixels. Takes priority over max_width. + ratio: The aspect ratio of the video block. An array of two numbers: the first sets the width of the block and + the second sets the height. hint: Hint text. label: Label above the component. validation: Validation based on condition. - ratio: The aspect ratio of the video block. An array of two numbers: the first sets the width of the block and - the second sets the height. """ def __init__( @@ -771,6 +771,7 @@ class VideoViewV1(BaseViewV1): full_height: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, bool]] = None, max_width: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, float]] = None, min_width: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, float]] = None, + ratio: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, typing.List[typing.Union[toloka.client.project.template_builder.base.BaseComponent, float]]]] = None, hint: typing.Optional[typing.Any] = None, label: typing.Optional[typing.Any] = None, validation: typing.Optional[toloka.client.project.template_builder.base.BaseComponent] = None, @@ -789,3 +790,4 @@ class VideoViewV1(BaseViewV1): full_height: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, bool]] max_width: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, float]] min_width: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, float]] + ratio: typing.Optional[typing.Union[toloka.client.project.template_builder.base.BaseComponent, typing.List[typing.Union[toloka.client.project.template_builder.base.BaseComponent, float]]]] diff --git a/src/client/project/view_spec.py b/src/client/project/view_spec.py index 5d6d659a..06925aee 100644 --- a/src/client/project/view_spec.py +++ b/src/client/project/view_spec.py @@ -64,7 +64,7 @@ class Settings(BaseTolokaObject): class ClassicViewSpec(ViewSpec, spec_value=ViewSpec.CLASSIC): """A classic view specification defined with HTML, CSS and JS. - For more information, see [Toloka Requester's guide](https://yandex.ru/support/toloka-requester/?lang=en) + For more information, see [Toloka Requester's guide](https://toloka.ai/ru/docs/guide/?lang=en) Attributes: script: JavaScript interface for the task. diff --git a/src/client/project/view_spec.pyi b/src/client/project/view_spec.pyi index 7ba53187..398467d4 100644 --- a/src/client/project/view_spec.pyi +++ b/src/client/project/view_spec.pyi @@ -79,7 +79,7 @@ class ViewSpec(toloka.client.primitives.base.BaseTolokaObject): class ClassicViewSpec(ViewSpec): """A classic view specification defined with HTML, CSS and JS. - For more information, see [Toloka Requester's guide](https://yandex.ru/support/toloka-requester/?lang=en) + For more information, see [Toloka Requester's guide](https://toloka.ai/ru/docs/guide/?lang=en) Attributes: script: JavaScript interface for the task. diff --git a/tests/template_builder/test_completeness.py b/tests/template_builder/test_completeness.py new file mode 100644 index 00000000..b90f4bb2 --- /dev/null +++ b/tests/template_builder/test_completeness.py @@ -0,0 +1,60 @@ +import pytest +import requests + +from toloka.client.project.template_builder.base import ComponentType + + +UNDOCUMENTED_COMPONENTS = { + 'action.list-push', + 'action.list-remove', + 'action.set-current-time', + 'action.set-duration', + 'action.set-media-status', + 'action.set-volume', + 'action.web-phone-call', + 'action.web-phone-call-next', + 'action.zoom-in-out', + 'condition.contains', + 'condition.less', + 'condition.more', + 'core', + 'field.bounding-box', + 'field.multi-select', + 'helper.length', + 'helper.sum', + 'view.debug', + 'view.hint', + 'view.json', + 'view.table', + 'view.web-phone', + 'view.web-phone-next', + 'view.with-label', + 'field.button-checkbox', +} + + +def is_component_blacklisted(component): + return ( + component['type'].startswith('@') and not component['type'].startswith('@yandex-toloka') or + component['type'].startswith('lib.') or + component['type'] in UNDOCUMENTED_COMPONENTS + ) + + +@pytest.fixture +def tb_components_list(): + components = list( + component for component in requests.get('https://tb.yandex.net/registry2/latest').json()['found'] + if not is_component_blacklisted(component) + ) + return components + + +@pytest.fixture +def implemented_tb_classes(): + return set(ct.value for ct in ComponentType) + + +def test_component_is_implemented(tb_components_list, implemented_tb_classes): + for tb_component in tb_components_list: + assert tb_component['type'] in implemented_tb_classes diff --git a/tox.ini b/tox.ini index b7c4c707..78105673 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,8 @@ commands = # Functional tests on real production version of Toloka. [testenv:py38-functional-tests] +deps = + pytest-lazy-fixture passenv = TOLOKA_TOKEN commands = From c0c0d2b27c5277e3c7fe3d85d24279f8f124c47c Mon Sep 17 00:00:00 2001 From: Vladimir Losev Date: Thu, 27 Jan 2022 02:35:36 +0300 Subject: [PATCH 16/18] Added support for native Python generation ref:8ce2534988b80c6e1893b3dce05cfc286c1eb08c --- src/client/__init__.pyi | 8 ++++---- src/client/primitives/retry.pyi | 10 +++++----- tox.ini | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/client/__init__.pyi b/src/client/__init__.pyi index 71d7c9a7..373314a7 100644 --- a/src/client/__init__.pyi +++ b/src/client/__init__.pyi @@ -56,7 +56,6 @@ import datetime import decimal import enum import pandas -import requests.packages.urllib3.util.retry import toloka.client.aggregation import toloka.client.analytics_request import toloka.client.app @@ -82,6 +81,7 @@ import toloka.client.user_restriction import toloka.client.user_skill import toloka.client.webhook_subscription import typing +import urllib3.util.retry import uuid from toloka.client import ( @@ -211,11 +211,11 @@ class TolokaClient: self, token: str, environment: typing.Union[Environment, str, None] = None, - retries: typing.Union[int, requests.packages.urllib3.util.retry.Retry] = 3, + retries: typing.Union[int, urllib3.util.retry.Retry] = 3, timeout: typing.Union[float, typing.Tuple[float, float]] = 10.0, url: typing.Optional[str] = None, retry_quotas: typing.Union[typing.List[str], str, None] = 'MIN', - retryer_factory: typing.Optional[typing.Callable[[], requests.packages.urllib3.util.retry.Retry]] = None + retryer_factory: typing.Optional[typing.Callable[[], urllib3.util.retry.Retry]] = None ): ... @typing.overload @@ -5334,4 +5334,4 @@ class TolokaClient: token: str default_timeout: typing.Union[float, typing.Tuple[float, float]] url: typing.Optional[str] - retryer_factory: typing.Optional[typing.Callable[[], requests.packages.urllib3.util.retry.Retry]] + retryer_factory: typing.Optional[typing.Callable[[], urllib3.util.retry.Retry]] diff --git a/src/client/primitives/retry.pyi b/src/client/primitives/retry.pyi index f00ece18..f219056a 100644 --- a/src/client/primitives/retry.pyi +++ b/src/client/primitives/retry.pyi @@ -3,12 +3,12 @@ __all__ = [ 'PreloadingHTTPAdapter', ] import requests.adapters -import requests.packages.urllib3.response -import requests.packages.urllib3.util.retry import typing +import urllib3.response +import urllib3.util.retry -class TolokaRetry(requests.packages.urllib3.util.retry.Retry): +class TolokaRetry(urllib3.util.retry.Retry): """Retry toloka quotas. By default only minutes quotas. Args: @@ -32,13 +32,13 @@ class TolokaRetry(requests.packages.urllib3.util.retry.Retry): def new(self, **kwargs): ... - def get_retry_after(self, response: requests.packages.urllib3.response.HTTPResponse) -> typing.Optional[float]: ... + def get_retry_after(self, response: urllib3.response.HTTPResponse) -> typing.Optional[float]: ... def increment( self, *args, **kwargs - ) -> requests.packages.urllib3.util.retry.Retry: ... + ) -> urllib3.util.retry.Retry: ... _retry_quotas: typing.Union[typing.List[str], str, None] diff --git a/tox.ini b/tox.ini index 78105673..7cff5c3c 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ deps = pytest requests_mock types-requests + types-urllib3 attrs20: attrs==20.3.0 attrs21: attrs>=21.2.0 py36: contextvars From 7a7202f4673ad8cab59bb66a6877cf9c9a0dfe07 Mon Sep 17 00:00:00 2001 From: Vladimir Losev Date: Thu, 27 Jan 2022 03:52:20 +0300 Subject: [PATCH 17/18] Python 3.10 added or fixed ref:aca0da75de37c469933dc9d3e4d7d739c5194abc --- .github/workflows/tests.yml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 22ff26de..55ab1858 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/tox.ini b/tox.ini index 7cff5c3c..17589908 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] minversion = 3.3.0 +# attrs{20,21} appear due to the issue https://github.com/Toloka/toloka-kit/issues/37 envlist = py3{7,8,9,10}-attrs{20,21} isolated_build = True requires = setuptools >= 36.2.0 @@ -25,8 +26,7 @@ deps = types-urllib3 attrs20: attrs==20.3.0 attrs21: attrs>=21.2.0 - py36: contextvars - py36-attrs21: types-contextvars + commands = pytest tests attrs21: mypy src From b2eb9152825c3758b1bdf052653daef5abfc663f Mon Sep 17 00:00:00 2001 From: Vladimir Losev Date: Tue, 1 Feb 2022 15:18:17 +0300 Subject: [PATCH 18/18] toloka-kit==0.1.23 release ref:dd53cd6c32fdddb3befdb4005b113c22b044dd55 --- CHANGELOG.md | 15 +++++++++++++++ src/__version__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be8fe95b..f50866b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +0.1.23 +------------------- +Python versions support: +* Python 3.10 support added +* Python 3.6 support dropped + +Improvements: +* Added native support for verfied language skills +* Added native support for map provider's selection for pedestrian tasks in `AssignmentsIssuingViewConfig` +* Improved default status code dependent retry policies +* Requests originated from `toloka.streaming` or `toloka.metrics` are now marked with additional headers. This makes it easier for us to collect these features' usage statistics +* Annotations now use `urllib3` instead of weird `requests.packages.urllib3` +* Actualized outdated docstrings in `toloka.client.user_bonus` + + 0.1.22 ------------------- Fixes: diff --git a/src/__version__.py b/src/__version__.py index 4f99040c..cded50a7 100644 --- a/src/__version__.py +++ b/src/__version__.py @@ -1,3 +1,3 @@ __title__ = 'toloka-kit' -__version__ = '0.1.22' +__version__ = '0.1.23' __license__ = 'Apache 2.0'