From 4fc74712301e6bcc7c2a25869587a4f9ca681917 Mon Sep 17 00:00:00 2001 From: Sergey Ryabov Date: Thu, 12 Dec 2024 19:47:44 +0000 Subject: [PATCH] feat: Add AgentQL integration --- .../provider/builtin/agentql/_assets/icon.svg | 44 +++++++++++ .../tools/provider/builtin/agentql/agentql.py | 20 +++++ .../provider/builtin/agentql/agentql.yaml | 27 +++++++ .../builtin/agentql/tools/query_data.py | 63 +++++++++++++++ .../builtin/agentql/tools/query_data.yaml | 78 +++++++++++++++++++ 5 files changed, 232 insertions(+) create mode 100644 api/core/tools/provider/builtin/agentql/_assets/icon.svg create mode 100644 api/core/tools/provider/builtin/agentql/agentql.py create mode 100644 api/core/tools/provider/builtin/agentql/agentql.yaml create mode 100644 api/core/tools/provider/builtin/agentql/tools/query_data.py create mode 100644 api/core/tools/provider/builtin/agentql/tools/query_data.yaml diff --git a/api/core/tools/provider/builtin/agentql/_assets/icon.svg b/api/core/tools/provider/builtin/agentql/_assets/icon.svg new file mode 100644 index 00000000000000..8ea199f9041f32 --- /dev/null +++ b/api/core/tools/provider/builtin/agentql/_assets/icon.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + diff --git a/api/core/tools/provider/builtin/agentql/agentql.py b/api/core/tools/provider/builtin/agentql/agentql.py new file mode 100644 index 00000000000000..16b903c97161f9 --- /dev/null +++ b/api/core/tools/provider/builtin/agentql/agentql.py @@ -0,0 +1,20 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.agentql.tools.query_data import QueryDataTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class AgentQlProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + QueryDataTool().fork_tool_runtime( + runtime={ + "credentials": credentials, + } + ).invoke( + user_id="", + tool_parameters={"url": "https://scrapeme.live/shop", "query": "{ title }"}, + ) + except Exception as e: + raise ToolProviderCredentialValidationError() from e diff --git a/api/core/tools/provider/builtin/agentql/agentql.yaml b/api/core/tools/provider/builtin/agentql/agentql.yaml new file mode 100644 index 00000000000000..73337ecb3ccd95 --- /dev/null +++ b/api/core/tools/provider/builtin/agentql/agentql.yaml @@ -0,0 +1,27 @@ +identity: + author: TinyFish + name: agentql + label: + en_US: AgentQL + zh_Hans: AgentQL + description: + en_US: AgentQL + zh_Hans: AgentQL + icon: icon.svg + tags: + - productivity + - utilities +credentials_for_provider: + api_key: + type: secret-input + required: true + label: + en_US: AgentQL API key + zh_Hans: AgentQL API key + placeholder: + en_US: Please enter your AgentQL API key + zh_Hans: 请输入你的 AgentQL API 密钥 + help: + en_US: Get your AgentQL API key from our DevPortal + zh_Hans: 从 DevPortal 获取您的 AgentQL API key + url: https://dev.agentql.com/ diff --git a/api/core/tools/provider/builtin/agentql/tools/query_data.py b/api/core/tools/provider/builtin/agentql/tools/query_data.py new file mode 100644 index 00000000000000..0aaf8e2fb32c2c --- /dev/null +++ b/api/core/tools/provider/builtin/agentql/tools/query_data.py @@ -0,0 +1,63 @@ +import logging +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + +logger = logging.getLogger(__name__) + + +class QueryDataTool(BuiltinTool): + def _invoke( + self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + url = tool_parameters["url"] + query = tool_parameters["query"] + timeout = tool_parameters["timeout"] + + params = {} + params["mode"] = tool_parameters.get("mode", "fast") + params["wait_for"] = tool_parameters.get("wait_for", 0) + params["is_scroll_to_bottom_enabled"] = tool_parameters.get("is_scroll_to_bottom_enabled", False) + params["is_screenshot_enabled"] = tool_parameters.get("is_screenshot_enabled", False) + + endpoint = "https://api.agentql.com/v1/query-data" + headers = { + "X-API-Key": self.runtime.credentials["api_key"], + "Content-Type": "application/json", + } + + payload = { + "url": url, + "query": query, + "params": params, + } + + try: + response = httpx.post(endpoint, headers=headers, json=payload, timeout=timeout) + response.raise_for_status() + + json = response.json() + return self.create_json_message(json) + + except httpx.HTTPStatusError as e: + response = e.response + if response.status_code in [401, 403]: + msg = "Please, provide a valid API Key. You can create one at https://dev.agentql.com." + else: + try: + error_json = response.json() + logger.error( # noqa: TRY400 + f"Failure response: '{response.status_code} {response.reason_phrase}' with body: {error_json}" + ) + msg = error_json["error_info"] if "error_info" in error_json else error_json["detail"] + except (ValueError, TypeError): + msg = f"HTTP {e}." + raise ValueError(msg) from e diff --git a/api/core/tools/provider/builtin/agentql/tools/query_data.yaml b/api/core/tools/provider/builtin/agentql/tools/query_data.yaml new file mode 100644 index 00000000000000..9786a92ad34a1e --- /dev/null +++ b/api/core/tools/provider/builtin/agentql/tools/query_data.yaml @@ -0,0 +1,78 @@ +identity: + name: query_data + author: Tiny Fish + label: + en_US: AgentQL Query Data +description: + human: + en_US: Extract structured data from a given URL. + llm: A tool for extracting structured data matching a given AgentQL query from a given URL. # Introduction passed to LLM, in order to make LLM better understand this tool, we suggest to write as detailed information about this tool as possible here, so that LLM can understand and use this tool +parameters: + - name: url + type: string + required: true + label: + en_US: URL + human_description: + en_US: The public URL of the webpage to extract data from. + form: form + - name: query + type: string + required: true + label: + en_US: AgentQL Query + human_description: + en_US: The AgentQL query to execute. Read more at https://docs.agentql.com/agentql-query. + form: form +############## Advanced Options ####################### + - name: timeout + type: number + min: 0 + default: 900 + label: + en_US: Timeout + human_description: + en_US: Timeout in seconds for the request. Increase if data extraction takes too long. + form: form + - name: mode + type: select + required: true + options: + - value: fast + label: + en_US: Fast + - value: standard + label: + en_US: Standard + default: fast + label: + en_US: Request Mode + human_description: + en_US: Specifies the extraction mode — 'standard' for complex or high-volume data, or 'fast' for typical use cases. You can read more about the mode options in Guide. + form: form + - name: wait_for + type: number + min: 0 + max: 10 + default: 0 + label: + en_US: Wait For + human_description: + en_US: Wait time in seconds for the page to load before data extraction. + form: form + - name: is_scroll_to_bottom_enabled + type: boolean + default: false + label: + en_US: Is Scroll To Bottom + human_description: + en_US: Toggle scrolling to bottom of the page before data extraction. + form: form + - name: is_screenshot_enabled + type: boolean + default: false + label: + en_US: Is Screenshot Enabled + human_description: + en_US: Toggle screenshot capture. + form: form