From 5be033d2f87de551d739a2f85ce05f7cfd6f6311 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Apr 2024 16:47:04 -0700 Subject: [PATCH 001/273] Nick: init --- .../rag/extractor/entity/datasource_type.py | 1 + .../rag/extractor/entity/extract_setting.py | 19 +++ api/core/rag/extractor/extract_processor.py | 8 ++ .../rag/extractor/firecrawl/firecrawl_app.py | 136 ++++++++++++++++++ .../rag/extractor/firecrawl/web_extractor.py | 54 +++++++ api/libs/bearer_data_source.py | 53 +++++++ 6 files changed, 271 insertions(+) create mode 100644 api/core/rag/extractor/firecrawl/firecrawl_app.py create mode 100644 api/core/rag/extractor/firecrawl/web_extractor.py create mode 100644 api/libs/bearer_data_source.py diff --git a/api/core/rag/extractor/entity/datasource_type.py b/api/core/rag/extractor/entity/datasource_type.py index 2c79e7b97b18d8..902835571d94f4 100644 --- a/api/core/rag/extractor/entity/datasource_type.py +++ b/api/core/rag/extractor/entity/datasource_type.py @@ -4,3 +4,4 @@ class DatasourceType(Enum): FILE = "upload_file" NOTION = "notion_import" + URL = "url" diff --git a/api/core/rag/extractor/entity/extract_setting.py b/api/core/rag/extractor/entity/extract_setting.py index 49cd4d0c033143..688bf7250c0927 100644 --- a/api/core/rag/extractor/entity/extract_setting.py +++ b/api/core/rag/extractor/entity/extract_setting.py @@ -20,6 +20,24 @@ class Config: def __init__(self, **data) -> None: super().__init__(**data) +class FirecrawlInfo(BaseModel): + """ + Firecrawl import info. + """ + url: str + mode: str + # [Review] Not sure if api key and base url belong here. + firecrawl_api_key: str # should this even be here? + firecrawl_base_url: str = 'https://api.firecrawl.dev' # should this even be here? + document: Document = None + tenant_id: str + + class Config: + arbitrary_types_allowed = True + + def __init__(self, **data) -> None: + super().__init__(**data) + class ExtractSetting(BaseModel): """ @@ -28,6 +46,7 @@ class ExtractSetting(BaseModel): datasource_type: str upload_file: UploadFile = None notion_info: NotionInfo = None + firecrawl_info: FirecrawlInfo = None document_model: str = None class Config: diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index 1136e11f765246..b11232f727eb6e 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -9,6 +9,7 @@ from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting from core.rag.extractor.excel_extractor import ExcelExtractor +from core.rag.extractor.firecrawl.web_extractor import FirecrawlWebExtractor from core.rag.extractor.html_extractor import HtmlExtractor from core.rag.extractor.markdown_extractor import MarkdownExtractor from core.rag.extractor.notion_extractor import NotionExtractor @@ -141,5 +142,12 @@ def extract(cls, extract_setting: ExtractSetting, is_automatic: bool = False, tenant_id=extract_setting.notion_info.tenant_id, ) return extractor.extract() + elif extract_setting.datasource_type == DatasourceType.URL.value: + # [Review] Not sure if api key and base url belong here. + extractor = FirecrawlWebExtractor( + api_key=extract_setting.firecrawl_info.firecrawl_api_key, + base_url=extract_setting.firecrawl_info.firecrawl_base_url, + url=extract_setting.firecrawl_info.url, mode=extract_setting.firecrawl_info.mode) + return extractor.extract() else: raise ValueError(f"Unsupported datasource type: {extract_setting.datasource_type}") diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py new file mode 100644 index 00000000000000..1af8778353764d --- /dev/null +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -0,0 +1,136 @@ +import os +import requests +import time + +class FirecrawlApp: + def __init__(self, api_key=None, base_url='https://api.firecrawl.dev'): + self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') + self.base_url = base_url + if self.api_key is None: + raise ValueError('No API key provided') + + def scrape_url(self, url, params=None): + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.api_key}' + } + json_data = {'url': url} + if params: + json_data.update(params) + response = requests.post( + f'{self.base_url}/v0/scrape', + headers=headers, + json=json_data + ) + if response.status_code == 200: + response = response.json() + if response['success'] == True: + return response['data'] + else: + raise Exception(f'Failed to scrape URL. Error: {response["error"]}') + + elif response.status_code in [402, 409, 500]: + error_message = response.json().get('error', 'Unknown error occurred') + raise Exception(f'Failed to scrape URL. Status code: {response.status_code}. Error: {error_message}') + else: + raise Exception(f'Failed to scrape URL. Status code: {response.status_code}') + + def search(self, query, params=None): + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.api_key}' + } + json_data = {'query': query} + if params: + json_data.update(params) + response = requests.post( + f'{self.base_url}/v0/search', + headers=headers, + json=json_data + ) + if response.status_code == 200: + response = response.json() + if response['success'] == True: + return response['data'] + else: + raise Exception(f'Failed to search. Error: {response["error"]}') + + elif response.status_code in [402, 409, 500]: + error_message = response.json().get('error', 'Unknown error occurred') + raise Exception(f'Failed to search. Status code: {response.status_code}. Error: {error_message}') + else: + raise Exception(f'Failed to search. Status code: {response.status_code}') + + def crawl_url(self, url, params=None, wait_until_done=True, timeout=2): + headers = self._prepare_headers() + json_data = {'url': url} + if params: + json_data.update(params) + response = self._post_request(f'{self.base_url}/v0/crawl', json_data, headers) + if response.status_code == 200: + job_id = response.json().get('jobId') + if wait_until_done: + return self._monitor_job_status(job_id, headers, timeout) + else: + return {'jobId': job_id} + else: + self._handle_error(response, 'start crawl job') + + def check_crawl_status(self, job_id): + headers = self._prepare_headers() + response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) + if response.status_code == 200: + return response.json() + else: + self._handle_error(response, 'check crawl status') + + def _prepare_headers(self): + return { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.api_key}' + } + + def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5): + for attempt in range(retries): + response = requests.post(url, headers=headers, json=data) + if response.status_code == 502: + time.sleep(backoff_factor * (2 ** attempt)) + else: + return response + return response + + def _get_request(self, url, headers, retries=3, backoff_factor=0.5): + for attempt in range(retries): + response = requests.get(url, headers=headers) + if response.status_code == 502: + time.sleep(backoff_factor * (2 ** attempt)) + else: + return response + return response + + def _monitor_job_status(self, job_id, headers, timeout): + import time + while True: + status_response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) + if status_response.status_code == 200: + status_data = status_response.json() + if status_data['status'] == 'completed': + if 'data' in status_data: + return status_data['data'] + else: + raise Exception('Crawl job completed but no data was returned') + elif status_data['status'] in ['active', 'paused', 'pending', 'queued']: + if timeout < 2: + timeout = 2 + time.sleep(timeout) # Wait for the specified timeout before checking again + else: + raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}') + else: + self._handle_error(status_response, 'check crawl status') + + def _handle_error(self, response, action): + if response.status_code in [402, 409, 500]: + error_message = response.json().get('error', 'Unknown error occurred') + raise Exception(f'Failed to {action}. Status code: {response.status_code}. Error: {error_message}') + else: + raise Exception(f'Unexpected error occurred while trying to {action}. Status code: {response.status_code}') diff --git a/api/core/rag/extractor/firecrawl/web_extractor.py b/api/core/rag/extractor/firecrawl/web_extractor.py new file mode 100644 index 00000000000000..5b822c0d337745 --- /dev/null +++ b/api/core/rag/extractor/firecrawl/web_extractor.py @@ -0,0 +1,54 @@ +"""Abstract interface for document loader implementations.""" +from core.rag.extractor.extractor_base import BaseExtractor +from core.rag.models.document import Document +from api.core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp + + +class FirecrawlWebExtractor(BaseExtractor): + + """ + Load html files. + + + Args: + url: The URL to scrape. + api_key: The API key for Firecrawl. + base_url: The base URL for the Firecrawl API. Defaults to 'https://api.firecrawl.dev'. + mode: The mode of operation. Defaults to 'scrape'. Options are 'crawl', 'scrape' and 'crawl_return_urls'. + """ + + + def __init__( + self, + url: str, + api_key: str, + base_url: str = 'https://api.firecrawl.dev', + mode: str = 'scrape', + ): + """Initialize with url, api_key, base_url and mode.""" + self._url = url + self._api_key = api_key + self._base_url = base_url + self._mode = mode + self._firecrawl_app = FirecrawlApp(api_key=self._api_key, base_url=self._base_url) + + def extract(self) -> list[Document]: + if self._mode == 'scrape': + content = self._scrape_url() + return [Document(page_content=content.get('markdown', ''))] + elif self._mode == 'crawl': + pages = self._crawl_url() + return [Document(page_content=page.get('markdown', '')) for page in pages] + elif self._mode == 'crawl_return_urls': + urls = self._crawl_url(return_only_urls=True) + return [Document(page_content=url) for url in urls] + + def _scrape_url(self): + return self._firecrawl_app.scrape_url(self._url) + + def _crawl_url(self, return_only_urls=False): + return self._firecrawl_app.crawl_url(self._url, { + "crawlerOptions": { + "returnOnlyUrls": return_only_urls + } + }) diff --git a/api/libs/bearer_data_source.py b/api/libs/bearer_data_source.py new file mode 100644 index 00000000000000..22da5bf10ab833 --- /dev/null +++ b/api/libs/bearer_data_source.py @@ -0,0 +1,53 @@ +import requests +import time +from typing import Iterator, Literal, Optional + +class BearerDataSource: + def __init__(self, api_key: str): + self.api_key = api_key + + def access_token(self) -> str: + raise NotImplementedError() + + +class FireCrawlDataSource(BearerDataSource): + _FIRECRAWL_API_URL = 'https://api.firecrawl.dev/v0' + + def __init__(self, api_key: str, mode: Literal["crawl", "scrape"] = "crawl"): + super().__init__(api_key) + self.mode = mode + + def access_token(self) -> str: + return self.api_key + + def get_page_content(self, url: str): + if self.mode not in ("crawl", "scrape"): + raise ValueError( + f"Unrecognized mode '{self.mode}'. Expected one of 'crawl', 'scrape'." + ) + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.api_key}' + } + if self.mode == "scrape": + response = requests.post(f'{self._FIRECRAWL_API_URL}/scrape', headers=headers, json={"url": url}) + data = response.json().get('data', {}) + return { + 'page_content': data.get('markdown', ''), + 'metadata': data.get('metadata', {}) + } + elif self.mode == "crawl": + response = requests.post(f'{self._FIRECRAWL_API_URL}/crawl', headers=headers, json={"url": url}) + job_id = response.json().get('jobId') + while True: + response = requests.get(f'{self._FIRECRAWL_API_URL}/crawl/status/{job_id}', headers=headers) + data = response.json() + if data.get('status') == 'completed': + break + time.sleep(5) # wait for 5 seconds before checking the status again + return [ + { + 'page_content': doc.get('markdown', ''), + 'metadata': doc.get('metadata', {}) + } for doc in data.get('data', []) + ] From bd2fbbe87dcf6f8b84fea8f134f101fbb5c28e41 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Apr 2024 16:48:09 -0700 Subject: [PATCH 002/273] Nick: more readable --- api/core/rag/extractor/extract_processor.py | 2 +- .../firecrawl/{web_extractor.py => firecrawl_web_extractor.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename api/core/rag/extractor/firecrawl/{web_extractor.py => firecrawl_web_extractor.py} (100%) diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index b11232f727eb6e..85bb3bef853ba2 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -9,7 +9,7 @@ from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting from core.rag.extractor.excel_extractor import ExcelExtractor -from core.rag.extractor.firecrawl.web_extractor import FirecrawlWebExtractor +from api.core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor from core.rag.extractor.html_extractor import HtmlExtractor from core.rag.extractor.markdown_extractor import MarkdownExtractor from core.rag.extractor.notion_extractor import NotionExtractor diff --git a/api/core/rag/extractor/firecrawl/web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py similarity index 100% rename from api/core/rag/extractor/firecrawl/web_extractor.py rename to api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py From b318b5e171b7f3ba0da59eed047c68d64d3bd051 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Apr 2024 17:19:55 -0700 Subject: [PATCH 003/273] Nick: added tests and envs --- api/.env.example | 6 ++- .../rag/extractor/entity/extract_setting.py | 9 ++-- .../firecrawl/firecrawl_web_extractor.py | 25 +++++++---- api/libs/bearer_data_source.py | 45 ++----------------- api/pyproject.toml | 4 +- .../unit_tests/core/rag/extractor/__init__.py | 0 .../core/rag/extractor/firecrawl/__init__.py | 0 .../rag/extractor/firecrawl/test_firecrawl.py | 35 +++++++++++++++ 8 files changed, 69 insertions(+), 55 deletions(-) create mode 100644 api/tests/unit_tests/core/rag/extractor/__init__.py create mode 100644 api/tests/unit_tests/core/rag/extractor/firecrawl/__init__.py create mode 100644 api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py diff --git a/api/.env.example b/api/.env.example index c61cb60d3ebff9..6e3cb581c53385 100644 --- a/api/.env.example +++ b/api/.env.example @@ -145,4 +145,8 @@ API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 API_TOOL_DEFAULT_READ_TIMEOUT=60 # Log file path -LOG_FILE= \ No newline at end of file +LOG_FILE= + +# Firecrawl Web Extractor +FIRECRAWL_API_KEY= + diff --git a/api/core/rag/extractor/entity/extract_setting.py b/api/core/rag/extractor/entity/extract_setting.py index 688bf7250c0927..02ae80b27aeffc 100644 --- a/api/core/rag/extractor/entity/extract_setting.py +++ b/api/core/rag/extractor/entity/extract_setting.py @@ -20,15 +20,18 @@ class Config: def __init__(self, **data) -> None: super().__init__(**data) +import os + class FirecrawlInfo(BaseModel): """ Firecrawl import info. """ url: str mode: str - # [Review] Not sure if api key and base url belong here. - firecrawl_api_key: str # should this even be here? - firecrawl_base_url: str = 'https://api.firecrawl.dev' # should this even be here? + ## [Review] Not sure if these belong here + firecrawl_api_key: str + firecrawl_base_url: str + ## --- document: Document = None tenant_id: str diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index 5b822c0d337745..05088cb48db942 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -1,7 +1,8 @@ """Abstract interface for document loader implementations.""" +import os from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document -from api.core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp +from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp class FirecrawlWebExtractor(BaseExtractor): @@ -33,15 +34,23 @@ def __init__( self._firecrawl_app = FirecrawlApp(api_key=self._api_key, base_url=self._base_url) def extract(self) -> list[Document]: + documents = [] if self._mode == 'scrape': content = self._scrape_url() - return [Document(page_content=content.get('markdown', ''))] - elif self._mode == 'crawl': - pages = self._crawl_url() - return [Document(page_content=page.get('markdown', '')) for page in pages] - elif self._mode == 'crawl_return_urls': - urls = self._crawl_url(return_only_urls=True) - return [Document(page_content=url) for url in urls] + if content: + documents.append(self._create_document(content)) + elif self._mode in ['crawl', 'crawl_return_urls']: + items = self._crawl_url(return_only_urls=(self._mode == 'crawl_return_urls')) + for item in items: + if item: + documents.append(self._create_document(item, is_url=(self._mode == 'crawl_return_urls'))) + return documents + + def _create_document(self, content, is_url=False): + if is_url: + return Document(page_content=content.get('url', '')) + else: + return Document(page_content=content.get('markdown', '')) def _scrape_url(self): return self._firecrawl_app.scrape_url(self._url) diff --git a/api/libs/bearer_data_source.py b/api/libs/bearer_data_source.py index 22da5bf10ab833..002f2de914edd5 100644 --- a/api/libs/bearer_data_source.py +++ b/api/libs/bearer_data_source.py @@ -1,7 +1,6 @@ -import requests -import time from typing import Iterator, Literal, Optional +# [REVIEW] Implement if Needed? Do we need a new type of data source class BearerDataSource: def __init__(self, api_key: str): self.api_key = api_key @@ -11,43 +10,5 @@ def access_token(self) -> str: class FireCrawlDataSource(BearerDataSource): - _FIRECRAWL_API_URL = 'https://api.firecrawl.dev/v0' - - def __init__(self, api_key: str, mode: Literal["crawl", "scrape"] = "crawl"): - super().__init__(api_key) - self.mode = mode - - def access_token(self) -> str: - return self.api_key - - def get_page_content(self, url: str): - if self.mode not in ("crawl", "scrape"): - raise ValueError( - f"Unrecognized mode '{self.mode}'. Expected one of 'crawl', 'scrape'." - ) - headers = { - 'Content-Type': 'application/json', - 'Authorization': f'Bearer {self.api_key}' - } - if self.mode == "scrape": - response = requests.post(f'{self._FIRECRAWL_API_URL}/scrape', headers=headers, json={"url": url}) - data = response.json().get('data', {}) - return { - 'page_content': data.get('markdown', ''), - 'metadata': data.get('metadata', {}) - } - elif self.mode == "crawl": - response = requests.post(f'{self._FIRECRAWL_API_URL}/crawl', headers=headers, json={"url": url}) - job_id = response.json().get('jobId') - while True: - response = requests.get(f'{self._FIRECRAWL_API_URL}/crawl/status/{job_id}', headers=headers) - data = response.json() - if data.get('status') == 'completed': - break - time.sleep(5) # wait for 5 seconds before checking the status again - return [ - { - 'page_content': doc.get('markdown', ''), - 'metadata': doc.get('metadata', {}) - } for doc in data.get('data', []) - ] + # [REVIEW] Implement if Needed? Do we need a new type of data source + pass \ No newline at end of file diff --git a/api/pyproject.toml b/api/pyproject.toml index 0002c6143647a8..5ec91aee38ef08 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -55,4 +55,6 @@ HUGGINGFACE_TEXT_GEN_ENDPOINT_URL = "a" HUGGINGFACE_TEXT2TEXT_GEN_ENDPOINT_URL = "b" HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL = "c" MOCK_SWITCH = "true" -CODE_MAX_STRING_LENGTH = "80000" \ No newline at end of file +CODE_MAX_STRING_LENGTH = "80000" +FIRECRAWL_API_KEY = "fc-" + diff --git a/api/tests/unit_tests/core/rag/extractor/__init__.py b/api/tests/unit_tests/core/rag/extractor/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/__init__.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py new file mode 100644 index 00000000000000..5500104a58b84b --- /dev/null +++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py @@ -0,0 +1,35 @@ +import os +from core.rag.models.document import Document +from core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor + +def test_firecrawl_web_extractor_scrape_mode(): + url = "https://dify.ai" + api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-' + base_url = 'https://api.firecrawl.dev' + mode = 'scrape' + firecrawl_web_extractor = FirecrawlWebExtractor(url, api_key, base_url, mode) + documents = firecrawl_web_extractor.extract() + print(documents) + assert isinstance(documents, list) + assert all(isinstance(doc, Document) for doc in documents) + +def test_firecrawl_web_extractor_crawl_mode(): + url = "https://firecrawl.dev" + api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-' + base_url = 'https://api.firecrawl.dev' + mode = 'crawl' + firecrawl_web_extractor = FirecrawlWebExtractor(url, api_key, base_url, mode) + documents = firecrawl_web_extractor.extract() + print(documents) + assert isinstance(documents, list) + assert all(isinstance(doc, Document) for doc in documents) + +def test_firecrawl_web_extractor_crawl_return_urls_mode(): + url = "https://mendable.ai" + api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-' + base_url = 'https://api.firecrawl.dev' + mode = 'crawl_return_urls' + firecrawl_web_extractor = FirecrawlWebExtractor(url, api_key, base_url, mode) + documents = firecrawl_web_extractor.extract() + assert isinstance(documents, list) + assert all(isinstance(doc, Document) for doc in documents) From 21cfd5e801b814e6e26c559be98cb03569ddb681 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Apr 2024 17:25:34 -0700 Subject: [PATCH 004/273] Nick: --- api/.env.example | 1 + api/config.py | 4 ++++ api/core/rag/extractor/firecrawl/firecrawl_app.py | 2 +- api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py | 4 +--- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/.env.example b/api/.env.example index 6e3cb581c53385..ab36d704425b7d 100644 --- a/api/.env.example +++ b/api/.env.example @@ -149,4 +149,5 @@ LOG_FILE= # Firecrawl Web Extractor FIRECRAWL_API_KEY= +FIRECRAWL_BASE_URL= diff --git a/api/config.py b/api/config.py index 8fb6b83cb3cc60..9c53477f5a1fac 100644 --- a/api/config.py +++ b/api/config.py @@ -297,6 +297,10 @@ def __init__(self): self.NOTION_INTERNAL_SECRET = get_env('NOTION_INTERNAL_SECRET') self.NOTION_INTEGRATION_TOKEN = get_env('NOTION_INTEGRATION_TOKEN') + # Firecrawl integration setting + self.FIRECRAWL_API_KEY = get_env('FIRECRAWL_API_KEY') + self.FIRECRAWL_BASE_URL = get_env('FIRECRAWL_BASE_URL') + # ------------------------ # Platform Configurations. # ------------------------ diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 1af8778353764d..eeac19a01acd9c 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -5,7 +5,7 @@ class FirecrawlApp: def __init__(self, api_key=None, base_url='https://api.firecrawl.dev'): self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') - self.base_url = base_url + self.base_url = base_url or os.getenv('FIRECRAWL_BASE_URL') if self.api_key is None: raise ValueError('No API key provided') diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index 05088cb48db942..c410eebd1f112c 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -1,5 +1,3 @@ -"""Abstract interface for document loader implementations.""" -import os from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp @@ -8,7 +6,7 @@ class FirecrawlWebExtractor(BaseExtractor): """ - Load html files. + Crawl and scrape websites and return content in clean llm-ready markdown. Args: From 0cb6bed148014b9475f51dd174fc0a989c20a3e7 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 29 Apr 2024 17:36:36 -0700 Subject: [PATCH 005/273] Update extract_setting.py --- api/core/rag/extractor/entity/extract_setting.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/core/rag/extractor/entity/extract_setting.py b/api/core/rag/extractor/entity/extract_setting.py index 02ae80b27aeffc..69b17039fbf200 100644 --- a/api/core/rag/extractor/entity/extract_setting.py +++ b/api/core/rag/extractor/entity/extract_setting.py @@ -20,8 +20,6 @@ class Config: def __init__(self, **data) -> None: super().__init__(**data) -import os - class FirecrawlInfo(BaseModel): """ Firecrawl import info. From 01b0baccd54ff6a0b8a50572dabca5d61ace47e2 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 1 May 2024 12:38:01 -0700 Subject: [PATCH 006/273] Update firecrawl_app.py --- .../rag/extractor/firecrawl/firecrawl_app.py | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index eeac19a01acd9c..e10ef60834fea8 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -61,7 +61,8 @@ def search(self, query, params=None): else: raise Exception(f'Failed to search. Status code: {response.status_code}') - def crawl_url(self, url, params=None, wait_until_done=True, timeout=2): + def crawl_url(self, url, params=None, wait_until_done=True, polling_interval=2, timeout=500): + start_time = time.time() headers = self._prepare_headers() json_data = {'url': url} if params: @@ -70,12 +71,19 @@ def crawl_url(self, url, params=None, wait_until_done=True, timeout=2): if response.status_code == 200: job_id = response.json().get('jobId') if wait_until_done: - return self._monitor_job_status(job_id, headers, timeout) + while True: + elapsed_time = time.time() - start_time + if elapsed_time > timeout: + raise Exception('Firecrawl: Crawl job timed out.') + result = self._monitor_job_status(job_id, headers, polling_interval) + if result is not None: + return result else: return {'jobId': job_id} else: self._handle_error(response, 'start crawl job') + def check_crawl_status(self, job_id): headers = self._prepare_headers() response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) @@ -107,26 +115,22 @@ def _get_request(self, url, headers, retries=3, backoff_factor=0.5): else: return response return response - - def _monitor_job_status(self, job_id, headers, timeout): - import time - while True: - status_response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) - if status_response.status_code == 200: - status_data = status_response.json() - if status_data['status'] == 'completed': - if 'data' in status_data: - return status_data['data'] - else: - raise Exception('Crawl job completed but no data was returned') - elif status_data['status'] in ['active', 'paused', 'pending', 'queued']: - if timeout < 2: - timeout = 2 - time.sleep(timeout) # Wait for the specified timeout before checking again + + def _monitor_job_status(self, job_id, headers, polling_interval): + status_response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) + if status_response.status_code == 200: + status_data = status_response.json() + if status_data['status'] == 'completed': + if 'data' in status_data: + return status_data['data'] else: - raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}') + raise Exception('Crawl job completed but no data was returned') + elif status_data['status'] in ['active', 'paused', 'pending', 'queued']: + time.sleep(max(polling_interval, 2)) # Wait for the specified polling_interval before checking again else: - self._handle_error(status_response, 'check crawl status') + raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}') + else: + self._handle_error(status_response, 'check crawl status') def _handle_error(self, response, action): if response.status_code in [402, 409, 500]: From 59d1ae9a905dd5518cf3027b503d59719c26b408 Mon Sep 17 00:00:00 2001 From: chenhe Date: Tue, 7 May 2024 15:30:18 +0800 Subject: [PATCH 007/273] init --- api/core/rag/extractor/extract_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index 85bb3bef853ba2..dc416d5d376b3e 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -9,7 +9,7 @@ from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting from core.rag.extractor.excel_extractor import ExcelExtractor -from api.core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor +from core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor from core.rag.extractor.html_extractor import HtmlExtractor from core.rag.extractor.markdown_extractor import MarkdownExtractor from core.rag.extractor.notion_extractor import NotionExtractor From c0ddb786b1072be75564d701ae169e30e23a56c1 Mon Sep 17 00:00:00 2001 From: chenhe Date: Tue, 7 May 2024 15:35:40 +0800 Subject: [PATCH 008/273] lint fixes --- .../rag/extractor/firecrawl/firecrawl_app.py | 4 +++- .../firecrawl/firecrawl_web_extractor.py | 2 +- api/libs/bearer_data_source.py | 1 - .../rag/extractor/firecrawl/test_firecrawl.py | 4 +++- web/bun.lockb | Bin 0 -> 394413 bytes 5 files changed, 7 insertions(+), 4 deletions(-) create mode 100755 web/bun.lockb diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index e10ef60834fea8..e9cce18ffa7606 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -1,7 +1,9 @@ import os -import requests import time +import requests + + class FirecrawlApp: def __init__(self, api_key=None, base_url='https://api.firecrawl.dev'): self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index c410eebd1f112c..4d70ff74d4d20b 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -1,6 +1,6 @@ from core.rag.extractor.extractor_base import BaseExtractor -from core.rag.models.document import Document from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp +from core.rag.models.document import Document class FirecrawlWebExtractor(BaseExtractor): diff --git a/api/libs/bearer_data_source.py b/api/libs/bearer_data_source.py index 002f2de914edd5..e94f291246c16c 100644 --- a/api/libs/bearer_data_source.py +++ b/api/libs/bearer_data_source.py @@ -1,4 +1,3 @@ -from typing import Iterator, Literal, Optional # [REVIEW] Implement if Needed? Do we need a new type of data source class BearerDataSource: diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py index 5500104a58b84b..19b372649911b1 100644 --- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py +++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py @@ -1,6 +1,8 @@ import os -from core.rag.models.document import Document + from core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor +from core.rag.models.document import Document + def test_firecrawl_web_extractor_scrape_mode(): url = "https://dify.ai" diff --git a/web/bun.lockb b/web/bun.lockb new file mode 100755 index 0000000000000000000000000000000000000000..b30ced5c52268ea6570fec3c05751af617b795a5 GIT binary patch literal 394413 zcmbrH1zZ%}*T)yc1i=7N6cMl$RKzYs1Vsf=5G-VYC6tm-z{2kCZpH4zN3jbVTf`Q- zyWZbn=E&;%mz`Pe=UET;&iURGH|EYP`dC}mii?S^f!5rK;QItqn>LXi{@6BC;d zrBa;BuTT_6JzAvj{-#%3M0tan6OZP=%u5*H0sWf z6(ILQnn6~Cw1Qj(N&OUpEC)FSvJ~V~7%LB%DAv=&dXT8U8FkX14rvY<3uyt_3zGb4 z14;45sKWbE0-Q?<@g*q3!a`$W70qxRT_puXhDB1uP2mLPZ5pH@s`ywH zhE?IO3=IRWa6~)hE3RK0RTVMHKcCn2n3UK*G&TVK-A26_>isbY)K70n zit7;CDerrc>#(w@?$l@k4#Y>f+TxC;2*k58U{&zAGQ(NH$hT-tsyBdpKXPF2CDqy zf|1|w82o|Kq?VB1Oi0T2H0V)3)4`Gbfshn`Pe_`#1Cc!P{{tlT9}o~37ZIyayw?-* ze$Gy~o`pJ%+m^b*daGbB#A^Ua+!(ae{G1L+>t{|qA?{I-G){r_1wXn$(m2>R5b|CD zlK4W96xUl^#}FpPfGY_ZrHqbI;b&DuY-nt#DkiI;U^k_a&>zpIO!!CfUJ<2L6T$9( zjfHs@Sl1>oGBRA@(o~3R4z5!?XparWR@DgYlqb})mnNi28KY9zqMhPGJuq4stOy8G z#>50DW55s)U3evs6UhuEIwAk-V8 zPV=Xl*j@&b)KxTRVWm~ zeOaMMfWH*STphuW?VSbxIL>e}FULFuf3xqSHlZ=)y@Y)D;yUI32kJC`UO-ZRZ(xJw zV@emnZU^X5oGl;?A&+DBt8$+~0(alfr>nJ3}xf_z~hbUtr`$yoq zQm98kQoq5Fl($XLBe_bHul$Ah0|Nv*8K_fUf+M2>6(}9iPJV8Jf0T!XkaXQfCG?XW zEc9Zq4UoPAHed3i8YI~X2#M?y8mNp8ic$5$o)v>U1_p*kN6iTluA4!U|DiE9!C{g9 z3a?Nh{{~`vCh9bvI;c||{?W<+6)dgqE$F?4B;Kiy(4R6Q0q4Mgkj+6tJhLGw?}H%C zAXQ?!8zkl3UTiN5NqPk#Dc(1TkH-HoB+ZAY$e381^%PAag?fZCB9cyUw{V^OIuA+t z@sEq~k4#V~21N<=U1GalU!lGcb;`?hK{uoc~3@MjZpLtLFif%8Lf)PG5_-w$Y~>rDm-<6c;_gMKiTq5~oQqvK+zUu8TJ z)R5!Vy)Pdy=Nyj&qEjz(g;DgX!e*cgm6j+r(k;RoG6#!nUcNfKrMkWe~yD?(#b@iY!4rV0K(ge(euS|m1o zR0%w9KaCdrsxU*?Kk`G;eRT{Z<@@z?A^*{;n8>hxD#fjt!hPc+B*l3IlJc@)l%U^e zwou1jqlyVoD8{3Iy3YGgJnAJ;Pf&)3!99f|>XhdQNb0ZQT)|I2k-LLBwMWq)qNJft z`m}n%PtV|7Nm@c?F+qVRa^{py%g=_m;XX> zpM#|JW+>_hhG7#xsU_M|Mh9otEl{U%;P{s!4>aBrA<54{kaYdAIDcF zI^CaIK+^o_woZ7S>H&T7#~kfekj9WS9?F2o=m_{d9rnntEgOWm7DLkdS8fy}pYIN! zPVvh6I|{*FhT{9gx2<32}}-?j?=C!$VqVwyyUMg%KdP^W#U86@Rt zIr2&TW=I+j-5tX9SV+q2XffY>PWrJ^usd~^Fs`pqr??M6QoQM6T?tA3-V^0!@%lnY z@^b^2qyy^C6)g6&e^Dtx_rI&mIFC!yV^qi;&4-F3uunCH|2|b#}1&;~-wMU)u+6t25Xdqt4t{R|(y}rkV@%XzR z722sE-esJ-;lVNKD5zgB7VxZ3h%$@`Mtv#LBEwK zpF>jq!c_etpsu(lwx1K_bx5)os8U7YF+^dHI*tF|_ciPDLcd11UJ2KG$Jk`29Q#}L zqq+{-X`X&e6Y?J#91%%3_!p>C|8yT!hQ%m2Z$Y6Efi`hbifX3)dqf0$^34=#TS66cQ8~5u%C?jg1M7 z2+DqLs{uV)ceu6nBfaxignCezGBy_P)l@O`SP}!-H%=9uph&qY*i*$s(R5Orf*-WL z_Cp##9=;~T$L~=BBLiq7pkq+>IHg_}@`C3T8`ua|_Rcn|h>H%jq4QN#Ttq@ZWT0yL z4Pk#sfTaC#HTpM&91BVNVgw}Zmpm@pZwc!y5bZQSJRvDRw=x9(&O;iYKJK<42SFA^ zy>zDFZxiU#erFCz{ckxf)Z-v2zM}Vp`YT9U@2>a7`w8l$P+uyxw|^kyFAa6-|365Y zKU*Ow&k2tO{xT%B5_QUBby1cUum8k#%JU;gx?fy?r2XX(B>9Qwv}jeZ zDt-&!LLW6OoycYjzLl$_K59uKMLbG z8-j^7XE`M4w}d4B>OxXH%6JuC*hQ-uK?i$hWz)gUWEDhmiauV>oF{1fng z0q+?@;2+){WWOkwT2S!69Hc(DH+njX3XuEJKjm!|B=yVt;1wNVog9Ruai~>9C);mb zNQ%!$Uq_g?edz6kLZOUFhzNkapSVurWNM(3J#P#k$sV77`l3$r-UNCyZoI$kfE~(X zM?;?kxSEjE-pNQOdpvQ-SBBxch1RH3KXo8!oYsOT{jrdg#~?^+$j*@DuRSE? z&r+-xha|o4@R|05H;}~Nfh7OWL(=s*B?UXFVtqR#`LjZ-PlqJ^5s&!)6YA-(PvdByoKMxEj*R#WJ&AnG)3-{2qF3kb#g%AiQa6*1p?Fix~z97mn( z+uI6yd~T?SI{BS_bBT&l{C(X&E;KAQG$Lj>`k}b_dmsLOkH7!n@4dFe4$T)@#(i*J zT828!qk(q9_y$IXXCG)Tp-$s}6q5E&le&T&34O{>6WAke3+5fguMp>-dp)7OIV9zw z9wc4&kBkiS$BUJ7o3<{zZ?dNGiEG<>lqz90!_1^o@uAk8(;9t@HPBC(KJ~u|@12cb zB^;ab_}c9KiAU4t zAFIW_xstg$ZPKs#U)uH8arJn2t60>k=DUXtc((Y?L6e^&u7x}ZUB1k4cZm|Mofl`U zU1pl{xm?*V;~&;Fx?-tkv?SAW%e_}V88Z!CcbNKlb}IecGsSJ_Io&FqhwBcyyftLX z{`pUue;wb_dhn&WaXSw#+giWG!<7S``@c3AzGhjQh!svh`{-r795d!tA(y_xjxYOp zzWbwxE8O+UnuqHjQnuRpE#tzz#W9CI*ZjQc#EWHXQv9AS9~RWlr0v5lMW6}Ql09w5>}Z{)DtWZ7@#-4mo;Vfy=zG*~z?vG5 z+L&}6|7nERqs~xf*HpQ~Zxr3D}-~Dd%qx!w1x%b-Nn^`}tUaeP z*xhPqwH?ijJGHMra>J4vPYhxo?M{z0iKyiKsPxz`*N&gQQ)ocj*9#*qU+r-C>z0jm z>Non&^42czX2T}*+|%w zoi)YgV#5hGnVjl!%Iem~1xIF-PqQrJV16vT&GuQHXCB#^R;$#PVa65?hWe|k zmF>CZa@XQEsmV82n$`*39uS;SpnPoH$74I)`&j=dT6rHZHfp)GWF6k%n#E%IiF;czSXE-MtJa9l7in@H%n8{D|;A ziEsLC^FCeQ<-6yB(uG$K*q_O_s!!{Bo5zhE`TfZ3(2mt! z_t1a&;)%{t6US1=Z99*)U4N-#q4XY4J$iaRs=wm+=`8m>i?*9x_6)7DCg9=6xaG_H z_aE3bZsNy+lh1ygT;;W`d!<9`R#vc#3)2tz9BRMz{IV;}I+k zpWZbqX{6V$T(_vGkaZ@3`zmbeaHZwM%&zTcXYA1rJm)d0<-);*7EXCFrHoF7vd}2_2`&MOzPnRm++-uwB zf!oPf#m`t}R=nHw#)wsQ`fkax8Emn=Laj50&J6jKvZSixwVTGnEvwrBS+%9CxICJz}5imeSm8*|;%5 z75mKcemM2Rty8X7ji03K`_#&z+P*5}HSf9E7VpVXqx;t{+G*<7;1uut zo4*$RReX|dpTXVd>BS8EvU_i8<;Om@_DzQkI=+6ms>XN!Nl)(jX1+|-AG1Rh?QlGE z#$wwBQ|h%_;~A(5{P5$$uJ~3vJrrJzEAOu#=~jMv$iV#00S}kPEgr9myRO)9(z4Xz zgS$80Tj0FMiay)6>ZOh+p~gqDSUV{i}v=+q9V*U2SXM0Sn&dA96ok|9HjM<)iEi^nBQ2OzQKr zzD_oyPw5oD;l3{GE5~`{!IH6o56%s@**L7IWs^e|lTCUm_w1c#-D&KVjpcn@izIrr zvb?kUbd%3z=WjEea3)`u*^ig^>eh7Q)LPvpR^HL9QQ^p@$DVr3OF7V^O^3Rt@5NQ$ z=+ifA)zdG9m-LJHwP4um`~~|?NolrLxpZZ8V#+G>V!{5!{L>yMro3!X?__-b9m?;< z-)dGF_3c?1qnW-2y~ovxO(}lB>vYD?*KfVM%vm<|A^Wp*@GUEs!Ft7#mZj93mhtAz z$C`D@nJuw#8M1J9zPc?VHoCiAZ{gRxtznz2bDIwiSYDt~T7%4p1OsP-g#R+46V4Cp ze7pPo?ioq?WomoOyQsUh>D>hfj)bpIE?;7H!#9_@*?+I>ReJU$Z=W+!(W%AW%*rfo znO6C~7f1BVy{Y%6xZ}GGM@-{~2Zb+uccy5m&)r^6xHG7Ag5$iDgRP4`-_&fW@eZ%5 zZCZ>kJl6Hj`T28Cne`3dtzU4#)HLhikCvI2`!Qtd+Q%m=b+BA=q{Xe0Eqveh``T*p z`q#V0OqpBgVZ&vHbN0kdOmQf^c);9=$)0vUT6mn?9Ws0F<#MUFI&VL+Vq=-6Q9mQb z+*oga=G)epZprH`&&4<27@`__^8NJ|oqQK|dh9r^_|%a28kd(xU+A;7M_sq94L+4? zHV;l6*RyPkT6=d-8D={2<(S)67PHHkTXh=m<@fy8*D1=60}J$=-)zFli2fsj9@)Cb z)hn{_?u`)z@7HxQ+Ppe;;`wob6Lxk9y|w7&qERWWl7ASkN;SS#VDI6_7R}zJZaGz~ zUWlHLo#&YXYmZj1)#m%>?#6FF$9);$^=#UX>)$>dEArz0$l!@_mnKXNNos$6O@d1& z-SSgjC#O00OS8T@tkjmmmul*@8KrQox!Py?rXfSJN)&!IXK0QHW!O&xa)4TmQQh(Z+s?iZG z4(gRM_bzqj{ll3F^^2uU8@a@*xk(fIVqZ?Lxf@pL+Dg}iQ#R)9u6>y~uyozbBHOu>3O{&Vt^W;|io?EJ zJGl1AU0)r`N@`_P^j7Or9oF@De4}a4*G-pyuV9&~Tvx5K zs#l9{CpR_U8|c}z!k3~}J4_swyykPzq=8W-cgI-mJ-B@3ptv>@Kc*Hvn>=W9*!{wv z&t-K_EmNcH#fA^h7h3!D<5>$UAM=iv_io+i8Tw1LaZ4Gmt|bf9Y4vv;f5+Ld+s#U< z0#`O2K3;#T-z+!paW!rHGAF<9 zo;|a!c)WXWdn};Z!%~YX)gRenTBnQ7Rg6wNb8-FZpXB(p>n!D#o{bz{z3JIMy;{d^ zRY#R?t{;-tc1LDmivo={`djTC^z2KP-kqx>u5WYTb9tE)QwwK|+`Mk})e~EqM=w_1 z@7eIvzL|PEQ}6X_wy=>~`t;!qst4DyIb1tz;^vBb=RG?e_{DN+>-%%Vny%j%``q~L z^)<&zp7|V_YJI9rP{@@kb&Gy7+f*@r_&WA0@z5wQ)8hkLeYmLzd106_$o2Edu>7x< zd+t0Mr9Z!eN%T{XfB_xHR~ zIC162_`uGF$BHK2v?!JqV3QE!J@KVkSGU$9i}eT#8=7I54_C3;P#8E-X7^4dcHk*H`T0Cfs|f;cajr+M%G3ySfiKkZ`Ofu&*K2T^-ktg2fEtX99dNJm}&&!>f7Up?7-(=TQvyuDyB)r*i?PiO8 zgTEA4_4#RkX0Bh`0DY^mUN`RkJhQQ?ebG5p>#RIiHY@Ja_tXYoPOCfuhi_J$tKP_c z&+^{=;u3y#H$Sqmzhem@0nYCR92&GX7W|M|5e&&{1`I;dVt{T+(+ zX@@4%ZTINo&f(Pxt{l6>y1Lu4=HsG!|JT#?$+2hQN9OxH_BhS`j9qZWTd#{jD(|;- zTCAuvaCmXE_p3@Q(47`Evu2Mo1|8D7?RZsf zbirziIxOG1x`{;_-^6LI?ZyYr*s|}&$;<=80=KQ3*Uo!O(ee+*PF!PSaMbFm$7~O) za&=2ItaQHOLXJP;OQX*ndUqR)44-OiK! zOg9(0l0VblWq8PydDl*Uy8G&m`wpF<1Dn-PZarvv^A+_&515TG?p`@@L)m*DSMH8E z^~LmXlHu}G$v1}w-D_gfHuACE!olTt4SYJ-@KS6W-Qa=uPrn}@q<6Yk?LD3H{V>_N z^LE0s0X>#~SbD8QbGwN>Ql4CFJgl>A-2(qbO?&WsuUTq^9~VkSoIam5z0&HH4<75) z3GljA;fl_lIq$BHem1p!J-zYOeWpELIAy~ei`G5Atb13o!R^`$w{%-nZGro*k_(17 z1@5Z7y1B3JzVugxGN&F)^6oIr)_8GBdf7Ws?Y7t%e7o;yer590B*Um*tE0PYOm(zv zP$9Zwe(Tu=uFuLhoHN7WNcwJ{($_yt4&1xs!JD>buWfGiE0wb5OonMb#UO=MNxi9e zM~)c!Xwrnhd5(#Ce7@_}N2j8l-7r77=fnB!Ppq4`K^fo3%&qr@;#D(GKK+>acEZy1 zLqpu|MrNk?PK%g*{maAemV37Dc9=HasCV~=U8X$svYa`1^yhOc-)WCIoH&PZ zM|V71KFaf01zkQTjcD|^-Me!0Q?2Hk4{p}?U6lht>pxzzxjFmSx9a{+X7)Ya-F5o( zdRYqx`yB|)2pS$_Rw+EQ^jYi0ruiCvw%zDH_iRdZ*YzDIt?Xdub-m@5u?M!jZV}~L zW|{w#3l86WpLsOBV)QlUoofGZ-%fSBVh>+RaJaBwlj(GuSv3s;dRQNSesIXMipJ0U zJbkVNjURDkSc49mEC)x{aWFcQ_8`rA#DDP_iB-ykm7Q?m;&}IWshfSK^f&#S`tU^U zYvY@Y^1QGj_1v(dPTu7z51C`srqGx5{k)%if5PYZG9!IjndkRS=X3stfn^>PIkZ4g z_DW)H?*#jriz_A7tT9sM|GDE@{eH$ySr_~3hjpxUw}9J+qdso8&Q%%uzN~q>{kJ~9 z>Af_ybAE>rA5Fea>-J;7&yUk8C7j#vVIS`kz6Sf_&tzmZJLNdO=ke|%20xDK5N6oi zWl5^vllgZl>;IhW8@=e0i?UJ6UM*bo*3Ey#`>kTw{^S$R(Z`;&U3qtpcaj_Lm)?!n z^_o~*XZEI^PS*Of7d&kov^HjX72n&hsl%YIUCsU8PcIkcbPZxSp+12e-+7&nF)z2p1 zwO=`F(ewNj{90ChbNf|{>FZs*51+d~w08SaJ+7I)3~18i<<3>FnuhhOw)|nh9?zr! zn~EiW_PTzzMhS~{i52>}ZjLr~O^x64p+Ww}ZP(Q;RH;MT_T9%doMkh^@^Fch+fELd z<`}%bQmWzOH7?r%-(0soQ)xt-=ME;twxlFZj+pcJeeLALfrh5Km*cybwYgyNX4}b0 z*T)-uS-xIZ*XZTtE{AR(Gai5Ao>$t@m9>lt`CClg_Vj6|V<)evR`ov8wf3hX6LwCv z-M`z`(XQ%6&YR(*P(!mF9x*NhOD^^Iy}P=L$7wfZvF*pNS~o2?GGX+n4?R0h_z`oe zdac#XQy-1=A7mG{?cJ$!A;HJ1^-Ai0H^^ws=ftoNliKy-`}5J$Pj?LN)xPhE&5m>D z#4C#&S#bSqm4W@f6`%Liv~*^Au*u4rho0R#pPv8m%=!^?HX8iQvU>k1)?{vYwetRJ zXZX8cSdy8(ai_)F`097c4op1~ai#8utfJehs`@mWF{baOTgO+Kw(ebO-x#~c8xy8_ z-|PEn{@jZDJ4|2wUU|s!sLPx(ANW1WIIn8XHs=q&T~nd z2lp9<4!2xeRwKE`9&mtJ85OXcrs?q)vxq-yY!{?T3Uu6EyWq5q4k zW$NADxy~q{)5;~!Kl^9&Dp|;Wou$vRuOoj|d}dIgjQ+4nc6Z0zjNRgPs!+^6uXP)B zjox&LeVNepm&xIKuPbL%ZxDHN*};1Wl`l=Iw^{#2c>Ktnm!f~J`+DWxgi6*!oU6Ti zUctD{k!GdbqTRf$jrT8HeD%Wf8lCnm?pQgot!>l8s;qIHyNo|ESpVvo+uwZF_jqd= z)+xPnfZ>%I3w(Dete%VzeMcrCG(xp8|9I(PHZ)Be}) zo4Ic)UHP%)?J4uDqZ|gVOC8e0F!^o#XrH;3y9O+GcU5jo3)$toe8GvD^<3j^YCIcQ z-7Kr*?Lh}F1vDyAzkOm2+r_VJuba-cEs+%Tp!CQSbNBnKxqr*M;rdB7doqf(PwUb& zy6V{QdHXDA<;NlbgF;TD8=(W8K*&fA*a0{p_Oi)g`Bbi{W)ouYG=52S)4OF1)NP6kNrAJ^OiK>DSnhL@{`Ldw zhhFHv{FjMgg+tr^iy7N{URV1zDV5Z}e z{!`{=%-HEM>QU{+(f+>8O0VmmdS}?FlL@}zO)DqYDdsq3ss46%^Rjcp;{JZGs+v%; z$+Xkk{EIYgHguEDlr^tUZnQDBDsa)nux+=1kgJW$*L7BACC;=8dA0pjC;QlHC%)M0 z&#<|4BOx)}&3#74Lgz@p_(g z&2NZ##IN)h3kw@pQ%yV@bL8mF(=%U%Y`m57+%l%7+1UsGMa`M?Av2=-aAKP#otAo6-N*aeN1M$_9Uo=%A7p21aj0ZIE0Z{b({a-e4dNHw5$KzR*Q{E1G>3cS_T-8-sYkJHX1@AH1tuLnHa=^&#fY989BzcX?3{1>^&O4NEv)-+ z`1Ws2AEi62KhdsV(-#XCgt-)S8_++h+Dk{?|M~vF<2ql}#Ol_UF8lep_wVuN@mthy zpH1k+{Lv?#*sqV}d0R z4h32^x;yYh+S#XO9er(ietq8crcal)A(m5j9X=dtRcFk$2@8r&KepEX{DSJmZjH*? z5On+S*sWXF+JzaU#BFMG{?z`UqXl~GK5}zGx5>88s&6wqd)%)1@tcmVo^^6<{_H`g zXI|cE9;)Yq+wgt*kmHM|$M5f+%k_|Yy;jw#`C?|1am#MmS$;W~@#N8%TedUj@b_1) zJAU1{-Q$kefxcC4J|E(0%=6^$`yk&h`8k!p@8a*j9(7F2*THM#*=*O!=+FDt zXU=_Vw`#~L2R=76(kroiPm!mu0*oKc)w_CX`QO(or#0(vx6j6+`CKQ}nEbut1peNK zzu)8UfB1W^8U>a|U#*eoJIXU+tj*x>SLz0SN>7+;7!$K=($MC*md)KBJ}5QSYDdPe zE44c7X3cHh);Bn7_;S5dqfh#Lx3+HjZfGONRu_xX`chk&etmGyJxlTH8)B)DE#ouK z;5T9Ldw3xu`jD)xP+V!AAAc{P{WX^xr3xsPi0#?R_(NCxp$&BE<9ld5)bZo*cJcB* z1@kijH3pCGycPNI$?@x-X!Zr>?}+x9B`KvakI)pYVIRLEpeTqBewRQ}VZJ-~x_R(_ z9(V`v6gTBiYGM2Dz*GL@uw;XI8w5@M<97+N3w~EcQeoZ?d>8QKKetKgfOYSH$6Whu zU+(x9D}=u@CfcVQNX-M=4+Gx>Jo${@Gm;zw^Q%Q(+xdGBye;fgfApa>B&D$ZMi@Bt z{BygeK3H!u_~v5##K_J68Su@(6Gv-^`=!2eE38)pzYIh3k8~~Zk-L5?W5Kr110N0E zEf0Juc;`IuCio>L%0K0gKa>kih4a@Lyd8L|V=IuI|B>K5^Wgt=@ByO#1_!>TqW|POfp4Kn>LwS%=SpWH7yxj9^W&EWH+JC4&?o*EaZr~ljQ{JSGJ@wDF=85(> zZ`@8gxfRws3%<5m{PyTbeRDN$6;rW5|Dfd$MVYZV6p8O}wbOz7+FZCgJ{=|bP|A}LtsDDj`b(6vK{3C|vfmHq$_>N-!$bYGY z?bpO#QlR@cvD)y1!PER9miv_A1M6M^&(DwA*sqDd3`6-No5k>vyMKg&Zw#L1p48lv zE%tx8$Wt5Hm%IOFfT!m-?vEw)ms?>y7yP9SPci>e=Pvcj{7dla^H1*nZ&hBQXaoDa z|8c*$`L1q%GI*LlY z=5?gN&|zNJN;tm~FO?+|%sYc`ng{!jttXHq9Lg6d=&uyAmbv_wI`#<%^A5FIDI@>Rd$=tSD|o#{Q{zrT>FQ>yNdp6vwrF!d7Z%X zx|N&1(co$PC>~kH3M#B zY5Zya%2~tAzXVVBFB*G_U2go&m^_|p`I9^U=7Fc@Keo$$ah(5>toH*vrf_!tsSj@d z7su_aSErFM|7reclfO{#w0>#)wXuH^Jk39{&pBWpbCc|UQ9OLp^Ec@l<_7$Ajd>T5 zr`Tx@arSsR7&1Q#yn`Aq*ZvjoSOUNOmuue;$)owl<0hBy1fK0nWhq9Ee;j!A z{7dbF`Bd=D^Wgu_zwFD+zdejo{JeLt)YK3A9|j&S{LX(NO`v$4`TgMAsPS^|-vwGI z6z$Y_+J~eRw(kcX|7H76W5@GIQy^S8j`2$sG7lJ9cuuWG4Kv{j2= zZvJ0^=lIEQ_DfR?tY_;eiLt)?*z~O)A;c@SgQXY)a(;4wdjh9cC8eO z&T9FW>;Fjb7}!@jCO1!Snp3OY}Q1QVR38|KjE5 z&)hjDe`HrK?*m?a{jy)0Vqm?Q;1R;_`6GA!-4J=^fJK{&fDOoXF*0fT#N(pWWs1 z^#POrWLq2i!@&EA@uLgb>$ezQzO)qiLg3||ebVux%K+aMyb+Y8jve*Oyl01;_xDoQzB<1EJYKq+c@?d=9n^G93$J$Ni3t@#dEJkELGr-Szu zd5S@9{BOW_%LCsIFYhn~v*V|n%C$cmJmp``+@m4p@w)_`-rrH|^pUy-n6KiI^ZkJw zADHhC9bVux3j>cM zNcQ~IX8j!k?3D}cdgo_)&AZ@$L5uHgCpNx7#Oq!i{yfp>uaT$j3rSpP0~ zx_@%)a_!gf67FBc;54264fDG`F`}}zvd`r>38F;z-S0QiV{SoKgREqP^VgEaWcY=Lh zH?)UHDa>C2Pvb`n<)1!M3iCCwdE*F>M2t4`ClP!r@U;JN4yZ3ph3%gMk0VqPjh)oJ zllo>p0Eb8NpW|jJ)&3UnxCQ@S|59UM`#Ly$(Eh`|%jKJa#}<-3|B2;(rTRY=d`s|@ ze=2hMi{NSflmJUH$n{^pd*1Tb9efb%)4nU!HdKGdc@aFFUny=L1G=uMaGuKb5aut> zKdC-2KN5Tc*e?Zxw0Fqm_kpMR$G*$u%k<3o{!1!LxncjigU1v|B9=aK_rD$BYm52k zKB#|9h3!8Ck0WFfuUonYN-4~@z{3N!@Zb59!jca2Tfx)&2lAcfAdds-!eQ@h;rk8VKD@2^mw+%NT&TVcI{ z;CcO!KDXz_s_UhL=k=?JQNPZ1Dg@^I{X4qO6Lw(SDY$=8Pudfp3A2Hk}haUvK4(!u&1N$I{e~QBg@f178QF38l zI|To^e@N|s`N`lL!#<50tWlACMb=FYt8!r~b%yx&F@wPyRE{{%Mlz|7-AhIsZ29 zo%8(*`Of`HwI2_j?th#&x%^J>lt140<>pV(N5~(q8@YT3@HGF}zNWbOI>$L0Jg+~p z&3RzGe@W&KfX6E|t@p1IVG0Fa;b_eVgU2fbt@-`n^WuN;aN+!~?f&f#J}>_70iT!r z8Ab^2f3)?#D|mc`p>_U~!PEMo`A_pt?)=w_6rNuSfu(zp1wK*=&)?SI$-Y#(+&A-+ z!Snt>Ye(ujWd1Pt7O+pgQ=2{;&>)%DjS_x;hy6c)@+AM5YY)t~1yBAHq|NVy4?Mkp zDFT+}fYieAkBH8hzfx_J4dzq9^ZA9_r1-$PU%}J-A>YY9w`=Ny^=xAl3P12%XDOAR z1fKFw_KV|Vq6rkQvz=4m$$zSAGk<@Aw*ybz(jLV9Qa!iAdbMMP{8R48KDXz_(lyrW z4W9fbmi7+0`^O^iwEytFMe|TfVfz{2Yl?hfDWIsseBU_X{6piXP5yU)uMhj2KdCOT z{b%6${6lTzhm^v6&3-xShdA;>N?|?-yt;iUEa@=62z*}RPY2&V5BaOmU+|yv#(rsv zgY{y;)BdCF_-$0<$)?mf#P+km)B7hLH@SS}cwzt1miGtmpcX&-B{hG`z|;EY{K@q{ z6MRFp_%+$(>+Gj#!k@fc`vbtU|J1i!eh+xse~6JDm@apqduKz^_{>f|do3C^J9KrMc%h#kx=HtQh`PTrT-1)mr^q=QH-^Vri z!1kYs{&U`>-ou%<93*`IN_Fy|`;ltD8+e-k%(0Zp&j9ZXp6s(vnpnEdy7$3T{weR~ zYVD+xTVcNZ;GFwEx8=t1HP#IPPwSWKa^qhL-V?mG=cf;%|7=(8{HrxY@Sk~(gX8>{ zWW8|k?cl$*``2#p4ZxEfnt$96)pILs{}p(;|B(OG#&P7v(lzF7l7#(>IhIoSQ1CSW zIq$p=OXZh?cNP67M(+B50p2eU@pl-SbN`THx@kk&+&6R>Eu>e&uo~m|H^rH zK>ae`F^~Kx@bvsg{c#Mr`OVkZ{%-KJe-TS_M`|C;{{&C_2hTfhXFLCrtk-n-pU6z_)>Y z&cD<(!2E9T^}v%|TDx+e-^-88S%2K89RIt4Z!g*>UTPn#w+cMvpIOd>CLfr;1HLnO z(x?5K+o_&gVZFMegz+QY{P+~j4N|+t{9W+0eklKX_{bgq)X~EIhd67s{x$u`_Un!j zzCR#+%AH*QM}X(`M}5ehe|y2#6vt2My@Twq|KGv${v*}4I`2F-=lo1LlskV0f@k|u z$4=e;UhsVXq?l+9NGTk@@i<}sECPt^(;Svkn2!P90{N%7sSj?~)CcROfv5eG#!hPP zsBh-=#tY|Pw$1HQ?Y96=>%XuF$j#pv@Xp}buGAP<{}y;BF@NfDN%gcge4X5V(`zgfA*_kW{mlAr!8?MN8aMaJy#4f?_kY@qUl@3P{?LYB z1>P0@lV21Ats^Og{ckiQ=kI?LUq%WH9p)E}rm zrZE1@l9W=Ip9r4b{}5LSAF2B|^WVS+iM$lMti!x=mazY_-@Fc4?_ZMnh2UGmKGo^$ z&;9(1<96onf%gW_`*$%`&>)$2nVq-u(@gO6{!H8TcT%*^{>zQO=p14EY20c4R@M+V zU*`Coz`MbJVySSuQ2Y0vtTzpOZSYj5KDhngq;Q4x4uPlrgV>_`idv`#<|Hmv22+ z*gx4Hx%_DGG=FK|kjhd9*xpm{jescH!3iE~eL;cCgA=_7huOsH)L{rde*O{LMp5o{A%k65a|MN%IOBem8 z{LzQo|0kr@$a=O5a^Bz58jzX;=KaC*^Eb80y?<;6UsuduaV_jW247#~DLh`s^mA^7 z{Wo8jbN`^&<<6f7@D8GV?lU*Le2sPg1MjTHOI-)d=UXJ?pX+k{cLZ-M+Nb#C+K&R? zRLy_(gX8>{WWDvGefFE%|HW}T>)im~684GbHrB~avToVMf8PJ;J&u&Zd^C6m*r$0% zYnaw(knQE&Qi8Iejp{8$69ao!k0SClU+u zI!p7$H_Ib`9DEn}ukHS6zD&4(XdAybc!xaXuh4Sg{-LdXXYe%s+VYdZ)ANhA`~&cO z|B$-&Xo~Utv0fqUU-{6cBYx<*rowzT@QuasCr0l2n*rV{5Ai<%-!2cl{mQ)k{&fs^ zTECPJ%AwT4`P(b<#L*nklh^_CAHmc8TU+~YR^>c@()e?~n%rPL@6~yXAAC3XPqwMB z6l(welkHDU{`3EzlKI^~b#PdW9lP&40P)_pRV*|D?ES?s30lBe%kOMjM3lKiQ@4f8@?zNAON! z{A5pR4%q%t@O1tnj%>^2O*RVO|5Ja&@;t3JmsJLmf8pNXGEUsa{1z$bM|lYOD^95yes_I*8Xhp&UxT7z|;Pr zt^Foja_&FctiPGy>HaTg?yHY~26zXNS8u1Eb1NLL`PM(be~>zlnePSO4fbi=`IDPn zzQ+6pk*7M@mTSM%ww&?P#(pe#nt#NyPwZoEl66ml=l379hswSG{Q}b;Da@Y&PwS5u1AL^$z_t)CwFBp7Y>@NpT^Hxc_R)j{;BcFR6ce2GgS- zBowxP3%rBKmjNf2x7#n=Kj^;2eM&I@TemlO%0DqQesb@BYrwmJXWMe0pI?J-0-kNl zeSWWTK)8PpPd<>8QrO=Z@MNFHzc@Z}^S@WLPaMTA*Z+^;8^J!%Y~O3s4-yLd@1H8H zU-F-PkbC|)2)>PIpZh>J8VlPmd@$$y%447rh63~5!PEXjb*W=V{WCukyq{YBxSe!z zE38-WkTC!0+)VLHEzEm>$CzdNPrTF^m>&whxmy0YU6UWIcLqE?e~`Y^wM)9p=Q}KX z|4b}>y7|V z?{9eiXyW)f^B2L>{3*Z}BueHBoe<`aoUtPl%sYU0fPI?3a$;aUDUbGdi}txc@=Z!% z{mYsVLlfwL^{K=U^%*TSK{foy>?)={jp5{M&cOa**{b%6q@-Ti?P6>P= z(YVw+vi+{$JHWmncxsc&XPp+l|0Vl4$4M@1-}{V^e*(EpN++y4U5zgcQ0h8j{yX?K zh@a~`4>b9}`c7ws`#N;loHs^E3kL*$(QVR2-;G;$R z#L4BKfT#P9oViDK*uKSuob!t|_PxP7!9MRBEUB;D3frF#p3najyVNM(sV&$VSWgBx_=NxXD4o_dTxdF_JZg0Yi{~~Ut_+& zB_aQEVyC{CcM^H2`zHIr{4ns8KdJl9-~3+i6hF;7sbf!lu>EhM|J0@p-zZ(Ue{lR# z`(XRy!PEXv{&O4oqN%X%aq#vcPYkz9<-dWa=T|;I%U!?DmxcH_ZkAH*PXbT&iIX!2 zInMpyDSo!g?NaT(1<%iK#7Oml?Nq-a@NA#wfmHkc;A#J3UK7XH+5Qsnlt1#D){tEP zZ;JML+@<=#_RCz&S$}fYAoK1b&vo{Ln4bjRPV}F0&+V-DFUfjm!8?KH{QXP+?{?-(+z|Yiy6(6i=3T+p zf_<7h+I)Y!8a(ZPWS>5A=ig1yfAWXMKrUbMrZE0MwXz=oz6bm#`%-O_EzaKo@b2K{ zjJ-N|H-oz!)}{=Ndw`ycUg^WW^YV4vca8aq^f&!1T$FV$~isj&Z7 zz&j#-_M4@k_3uBKx6I5Lf2pjx{qEq~!an(5Wtmy-kLtKU&5V!{7? z?=6n@GHAD{+C5}_V#^7CJHY%k=+XM6{nrW~sX1WY`JO`2UG$%9$bEiS-2e0UFS%dp zE4RXWzTi8;KKV^;Qs*G^8^L>uJmrwvbF)p?SWoxCpTEB+H-FyX`TR+&Tz)2aN6~-s zi^fq(VgIj!cUI%2u%yGh`NN#wpD;zc-29Jwob&%rrPz(Aa|wm*KLD@pKeub@g7vCr z3FimWC*S4z?+>2GkNT9G|0Upg{*e!IdBZ1v{{02&LrUTJdxOtQ{?3D^{fp-<_p8YV z*7JIr^ZthzJx!o^o%zXs@pAKb5_}sq|K;+X#&OTY-cxkpFHrzUgoSn>a(y+`~Kk7*T3BTFBv>NKT`bkkvo6y zfOl2nIWL;RV7=$w{gUec6!5(N zQ0#K|zenJC{g7?B`D^@Ec>lqi-1#>cJni2cH;;qV_)mdXzkh4u_&VDw_wG-g<6ymi zN#-NKtKWa+@;kx%s?A@y@f*MYbN`iVKMcG*?9;p<+j99O;G3z9ALlDV6(4Jh7Xw$% z>&fE_eMY!K=?dx%a;f;GIPO$-Z3vjp#q+ zo<4HdkIToL=SSi>FZeOn!g_1LyNUi2!0oxhYI>i+Q~W%4!ALX7t+^(q`*82tCQ!ReE`={#jpY|yaQVQGu30{5w;C4-YuwJ(>!u^Nqa_!Ih zi|2l%+P?^%_7CRd+Ar`mZ+v6$oz=##h$hFy>+J7D@O=NIOWdxh8`irIp3Z+<*TnI4 z*0=d4oIj~f^H*wNUIm`d-^9z!|9{~5{IAXWDf2xie`K5e(i8{lMS^GlsZH+wcN{#e zUs|_P_a5?%?JIxe+&`#Y?(;_q_^z-|EX7}nevnYuzVXjL_g|?oFz*kZ_g`w0I)|8F z1->Et*8?l{-2sKi{B6-bwNa7lztyk2eShi;-UI%VKj!$zjlU9z*5GOEjPQ|KIQ}8v z8;iVDmh_n41D^8FYfoUrFy9kA%|Fh&-2G=McshSj ze-u0Sqsa%h{|Y?C&wG#5In2DJuFmiO{{h*S>wgFEzOYaJ5-WHASO?x;%pdtAcmFGz zPe;*PjGy*Ex%rz8-T^$Xf09xP=kGrFHlqJhIM!jlhMrFL_fHgeF}^54G9L@RKI~JS zK62w<3*H4h%^U7hg8ARNVfl5k&mY?I2f)+*qb;vjKu6It4}3Uy2k>&{KIMmVwE;Zs ze?0HyuAj%?>w(utpHjz8-F}sVIqRR=r55(TD|p&}*>9;nFh37G#V>W;uz$>_gKrL= z>QZe}AIz61r1SgVZ{s$$k(*@QVDS9>L}zDi&yA&PthWt3oxcsmHqN6|zCdAN{Kz)3 zB&8I#9|xZDPwR(@T>G2AlmEm@efLkgY`=XG9YsJM_)Xy5^T3zU*U5hW!gHqrIVYj8 z|9;@<`Hc|TgXKQ|&HzvMU+Rxosq2949|P|Up2m$>Gkl~J=BpV9_aANf!Qg5BXv^;g z?+Bj#VW}ww)-x)qll}gk=5J|y}fN9lfl#YQSNCD@jRmbax3irQ}B-9 zY28vEa^K(dG8D$2Y>{p5m+H9{)>{dl_D`}+W61kJZY*76{yzAYB2VA{OD)WY7zz6? z=S}J}1oOwh^ZiGg_A}+J~hS)^Ax%=lA<_%HeyyprI31n-0DK?DPF!?(@qj z@Q&bl?~&`jL2(^LFnG>C`=P=8-{T(x-bLiece(!W1+N58<4!S1U4xvzvL$lt)BKgY z|N4Wc@h<_E3Z0{*6t=$-JnjEvmvSd}{$2;)NVG4NCI8rdiIRf<)J8r~4x|+3y+oeo zZgG4Jq`=T&ekJ&(@Sk~}N1D1}{x0~I;Ms4v<7Z_o+<(X(?OhlPNrmnA08jSGe`@1) zO`WjbLhz0vFEw`RoB2%elt1!aYV7KKSreV?_oviWk^Yca*nY24!u~~=HvBX29pS&W zdm;PWzm^~(zLPh0yF zz&C;aw0@}%x%a>0;K_gDsYtyCaQ=&y%X$8!Hg3=EESEo6uPb;r*r&Q)F8Y68VtzIF zcHn8;rJg;gFXr=`3-+mv@+Y-0-x<80=)Y8!^q4;mp1!}*cKqGS3(rs5^0UCx{z>`M zCjXB_UR(S1Epqm6ZT6p0;OY6_Mod2ERjmF~u%4=duz$+2O`S5o2|V3DdEa6APs8E` z=AVf?)rpnM*SFO9pU&&kN&-*uBm+AxaAHdW4=dqVNeswDg`6rI{52^bo+m8d! z`zOyqibGRjehc`ZJdD3-72*6x{YlLo`NsBFf%g;hN9{C+q!i|>R280oIB#nU zHT(1oEv2yiyWr{kr)~UIs^#22xlcLv`+}$U7nFP21EluBdMm-R|Fr+p9F$U+e+It0 z7(b1j-0^d+E_{DUoYb)+|JeRO@U;Hq%sq8}FL>HNw6*_M^q>6UIY|9$D(rvN8aewX zw`pSOI_q`=Pw(%gKDTh+%+C<*v+vx_djFEFcMQBM>=RFIQpb>ai<-jxBc6TumreC{ z=2hSw#qm?WuIXpySA%x|FV%1Em-)xw>HBx`UFzDQKA1PR5%w=S`;ZS({a}6wczS<8 z_NB5={cS%3ybE}GZsjv1UDs6DzJqPf{-udgzs`ITcz%A>#{aY6+rWQq*Kes>IpfDZ zsr&h_pINs(cn8?0x;Fk#08jA~tIhns0iO0R+W)k%U!t~f|7P1#`{4M4!27^H%xAwl zk-85vzaPB6m_O=+{h*(7E6i7@BfP(%I=%a+c_5`Q-yJ-i|F|xNB^}mZ2i{%eX${HE z-&gRoe^8&a2XMbs&#katZ9C!p5v^URwy9s{M}w#PC*MD~JvZCxd|U#wTQz7RjvY5qx_ zgUow@r}--ygjW*Y&txuk$>g=j%T2YoGUB zpnf#80LH-OfWUtS`+&fYg?&Ka$H6`z@SnpzAn@a19}xHnu>UuL@9PlHhaifEI9U{s1k&06}~RT0r{|XaVg!LYJq%37sZ? z>p+LA-~L62;Pa4y2=bJJ)33Tp2age_?M39F) zOeeTLlAxX|6bJ3xA%byu0N3}0?xZd9giZ^{apMrfr zP)-Yz4vqtY&y5Kp*e+S&I3S2~!#Y6F;H4|<1A>?CFg+lG?c^aupyLk_ zj8`Z`V1VFrc?uEeM?eJiV&HN>P#zDLC&2OlL{R?)R1fr%0k_M9nFTW&B6$C~u%8Dr zA0jY7YKV;xLA_=;jwBf8Za5AI^6!CtBtg6vjspU}5B33p-w*Qx%mJ8#5P<=Lm&36C z|AIhj6k37zHxBFmCxSdD;Cg`IMc!Ku|6U$1lS1zY(+(gUkO$P%aLa z1A_fj8Y0k_g$T;!;PSr_)RTwH0YQBQ*arl@BJ2Z#?;>A^{r^NTZdy>ilMrnng5!-7 z+zv?)a)nl)A2*opa6Ke}o);7c`Fg|UNP_l0aQlaFIUuO-2m65F_!A5fgu5WI|o{l5|95f7IGf|p6q3g{%maX{dw!2aI|ie5r1 zpqCEU`x`+~CR~mr(9eR~zlO_^1iHCU926BmE3iEk!S#>?^@`y*AgEUk5$IRI@xKwY ztAfjs1nsJ!IQX0!;Br7v|2;%d-UP=1LA(Wyx5DxN4MF>MxZU3f`s;x8JK=hOVEgHZ z;~yXbok57;yfXn2jKefU=nrES_P;>{dh;+BKp|v+p#CyM5L$(~0dpG!Ap->07r0Pm zpo0ey^h*E{^h*vATyN9CJ_}sV1`*U}hX@Q1_#7}fA%gu`5F*eKg9r=|_;L_I{Y!8h z5PUwDVX8s|_d_=!f_(HL0z(q$n8I-+L49*LjwH~xhT`C@+d%~V+rxB#>jQ%4q26%Z z2O_BN2bTi^KLGXtfgcPJgdRgH(0@2^V1nz4bU2;?5!BCy2>O2w5&FZ(hy4PGp#58z zMR5EbMCcEr3}zJ^uZ9SYfA1lJcFhohejDs}!0dwA0}=Gw3lWURAVg4q1dfkFg#Iul z;P@19AOi&LrXhlMU*R~CKxZC`gFF`D^1l%j{ebJO!u9?W!FKcuuJ<>B_y0R7m>@qK zXa(}X0}f1(KRz5MgyVo9P6Ydaz$b?NzY*x3fXcysK?BzV1no}2J|OUEVgLUN!F?_p zq{9L6Dy#l}$Nwh;^>4!U0YN_cun!2z z4PY9=G=j?kLHR9+U^_B_eN%|weOf^T1_D#;qnNGl#rhe5gcFNKm_@|g9r=|_+=15K9z7B5co9^K|i$+`5`Vrg#Iwr z;P^Vs4G@G35Y*p7$V3U6Z-H$ITqLmV1k$UaGU@l@JXQ+ z)C7YHBFK{lCLK%$h@f9)*k^+X^f}--4@6*qpdLR&(C;~z0&qDXhzmjl?S)~AKm`3> zgb3P8!oCbdP+u1IIEK?J%w5P<;#T|JnFFpVLCe#~J14(!{&bbtu@ zb%Y50VYor76A=9&f^iCg2#%W(5W#kk1QF<^LInM%K?L?U0^M||9Pky)449b^fqpJT zp!WvmJBZL9Mg_D2*Bc!WLHkY+{J#j|U2q&p5buWK6cDE%0{vOI9+IHm7dQ?G#_t@{_QSB+xqr#{oe-T8Lm=&%iz-MDYGuVV?~mcpscFdEhuN%(D=IEmv#JmxJPy}ba~Y;G%quXjLIn0Vf_5r! z`TvICx=tTz2j0IiMDTf7Km>7Dh(Om3BKZ7$Ac8!7Ap!#gFN5KDC>#d_^&Z1MAn?Os zA4xDykx(3r+cSuu|3tWcGDJ|G3K8s&c@ROKg%E)O0>2dY0l|LM3K8Vp2FC$`-wykL z!0&*4K+sP&>;r;)`e7ds_(QM{2-*+BJ|K91qYy#835Y;%3L>aC4G|cUAU*?^e}>DE z1Uhq29O%x&T!iZ(3Cfq?_}>W1e?aA+zZICPFxTLAeid@&e}Kl908oP@y& zCK#W8^Tof11LNO(@t<+{H(&f`9FX(I|K^MT%pZf}1qXDo1Nd*g_|JCoZ@&1?^)GUs z7>o??!FKX*zWC2}0_KgusrTP}@t<+{H(&f`9FX(GU`qi$xW7cs6N5PL!S-Vg9&Y?! z1kYvu%@_YU-u#;{{=Io%{F^WSv%etciNVMKADox|C-cOhKBx!k|C=xV^IZ1dd@>Nvn=k&eKm407{x@F?jt5{q8C;+Jn=k%1UktVfFmDVl)&I>G|G9qp|KEJ^5$O@Q zu={^(m>7R_kHk7bIWMs#K0<8LCfvvM2?ei!|`Qx+b9M4`nO|_yiPxvh9I)`Y-oQQVs+)T!= z9_h2ktVO4N)K&)gD~cZ_7h+$AgwXlIx2wOUQS%a?jUKCu&^o-l{srp=u1<4GrM9q$ zX@*$hdt&PN619oYtu( zu&^`&83+SZ`q&7kE4HtVe%>6JJoVkjjK3(-L20+Df|T-*ymqxTvcau>@1(B>$7j>i zx(n4bHkvvzWucG|I$!Wz7qp?C^A<_k9P91c#x9oO*AF?nogNkC;lkoTNv=da{rJ^* z;}WY;^wTxB`16n@1sL2*# z9#)=9L|D~ofzqp5u8-6{l*ilcUKR}%w}~exeY$*Xcp!0MqlAg zW9u!PE)f-1Tb0C(&U1yD#N?G2Y#cLguT&Ab;5!#+LwP83(vU|E1y60)d@RmXdEB>a z{PL-ti@-MFVad;nQ!Isr2a?U#rO6B!k1cU5pOt&I%w$^eDf6m`@Qcta5lTWr=zPI< zq|t_Q{BetI(eCm<&FMs|rooKK-8eDxkh@Pt(&`ti}#`{E{Ngw_S$aY7sF#@DpM z1L5WIZjIOK-zS;%1&yjUiybER%Y55Lca;mat}zK%CR3;MpCwSW9nA8ej3Hv&!F}lwiZ^XU}ad=*>7FF~h zwKn?5kN;AmVaS(vaWf9im=|)CC%n{CL}CUyhFB50;JcG(LtXl0)6t>mK_p0iIe|Ge z-duliSdHYx!y7_B69|4@plU4O?@V?6WLL z2%RtZ?it!pl2lUU#+QrqtIk&i8a=3c-z;EwK_A$P&8CfY;uGLxFg0_{l2wnk52(1ggbA~n) zRn+~bc*b>)vWFT7uYAa#c5i=OQO66ZU@nG7|hneI?>eImRmZmL zeF>ZAzUimnUe9=?7on=Xr+;Z@tdh{4${lgK_?}Df1aUs`tVEUOh?}MOWFVHqcZ0aZ zU8dww_W-l>sR2j`6MBm13lYp+p$$dEm4!JGGl74Dw2}O?s+Si-4!Ow1igurYlA@ zyE1SUwUfu^cDZ|x;GNam9_N4r-sjq1nOMpBaMWFEW7}_*zi+@x9JK45kF|BOzW(#n z7gq*N&%-zZpotz|@LM^wp(e|}$4EpCrLWIdj`@iHJYU%&9<LppPz|%g64+zcM_X5&Th%tJRB&v%jY33CZ7AP`VmFC zbkAA)*$IA`pG>#yNxCn03Z$4=v%&wi#BW_%q^?$khkOn*lV|YF!L)^UlSva7F7pe! zi-b1#cpY0_W5uvGhFU|^Pn~ZXMePJ(wgP(d{MvuJqS9a zZtmSrP7CTJ?kNF%*M;L%YMVZo<_F?a-W$1EM^UJ8XT+iqPee25!92~eqStnAsJZ)U zy5oroZ`pChF}@bz>stt2dZg|rCbpD8Ym=*UCvvca6VsyGn)fn76cnvHgM*pXUfqkJ zV2?G$vYSss5n&xMYdd+hv19It9?`G97(bN76-Wa6F4!-?_?|}U%0H6cqse1RzVi5c zmLt?f?Q7yKb3)PqLXBpoi}#u8O-3(%&XqX$=uP4I-Ds~MYmhaSbak4;W8kHrJrDEB zI6{{JsZ04(Brkx$ab=^L%a`P4CFjb;wl==Fs_bqW}W=~ zy58A_JP~yx(tUN;fl7~a#Oul*T`Z6<_zf`HP&IlNzuf3l(9L;D)y!zmg{yAM@cxU` zh9urqRaWw>>E-li`SBi3)OBShw{>k;+M=B`cT^{>*heGzw;jG!6Tl#Sgemol5rS6m( zTIWn&t|W7skeFp zzW;Astl!%&3sP6U=*;B02GQVttwOGroNG@TK~XAh7*ptN3x9WVR(H)8D$`%wRYSkG zP)N%poojIr^V4t$UNTU5xS4%M=k&cl``aH~a1KQqDx~yL-Mg0_`b@FJMwG-W!(XWf z!z4|<;Zl1U>s_n5Q(LGf_5D-xCec?){2ya-+bNUdgbdQ8N!H}?WhTgwh&|IamBOs{ZX0>HltenOK0uz8C!81 zdW2|A9%nUrTwQe())%+_eguwTn9$J+=&~bqOW)Uj(Xv^4lzAmR3ZIF;GJK9WzCY#B z$zjEz5miI7-05>IQOi?xZ-uQS=*SdeV=!dj@Zw{faaqlA3SYnX73^bRe*?N4NZm2K z=;tNayyGj!KQXUACW{WMJ|~C49x3`}*r$H{L5oo4=W7NU8SPfw3q*wlZPr-f_yW{- z#|xLPtFj$-~3~O={JOfQ*{iiGlo@9 zF{tu+SD4RXUp0vre0Wtw{Is|8hjiM4r=7CdEphajC*}&!FQGH6x zx6o|b?Xih|aEKd99;+=ZCg9F;2qc6)5AmQyA)==7ltM?G^I{qYduh%pRI0^9E0C;( z-(_buW3IB#Q0zMXqLn?i82RB-vYZ3Ih4WMd)?5Fyv$f?GTJ?dqFY_%Uba|1wj>IJb z7L{{F6~p>PZ9Hm4Ln(CWYla&#KEVQh7qF;WCpDO4n=7R74O<7hgvWA=qaUys7kz!M z)KB(tW`m`%3!!@!sXJ0SRjJ5w%4Q>)@sS6O!`I;Ii^(j>Tx4cSpO$Y&wedS?J*Z0z z%8iabewxA4-*rRm+vzuH!k@D~%^eb!s*TJbbor3F@92hYQ8?*hWA*u}UzD=jd^27O zA9*JU96rw@>pFRqTwHXrIWCFlKtS7|s+{ul-Kw^0hT(I!lG4Z%go``xgKIMM_=4Yh zp$)}GdK`sMTUs5zdB3hjrob?PgX12qprGdNxtHS%Q9qZ>Z-+HcoUMIFkjzh(-SpyE zkLielp<}1$I+>uQ{rx5lNC>Ta4lN21bzjSdaNEM7y6}@t_mby15ms3blB=URbeA_i zm>s(}sjkbXZktKc$13j4TUFfZQ*Xe0Wiaa9m3go+eZmUMSP!8Ku36EBa$TAaA8csW(2DcZ0>jQV4uuLMSF^=N2 z$ejdnlDqS*L^Dw*o_8LqJ@~xIG~IfcAdMY)zbS;&&H1j{X6l6b#w`3`oxa{#a<6WU zK2XhXa)h=zG9uxN0MGFLjgX@@7fNPgA-;m@yV|oehpiVloU;oW%@cVA&Li>_M(W;G z(;Q=5=QA*-dU~4R%a2gnBd^z;LLcNWyBz3x=G1p{^GKdM<171l)}{7gwoc`oD4*1* z&Lo{}eg<2i=jcapu0)Rm=mTx2OoceCb04tWoQHN($7Q+ho5|TaYrkb6W}bYE$#p=# z(h%bInma}_-KOfS{m(_|8qNBtiGtC^tw|?mp&$2Kz`xmr))hgELPWi-%IsA}<-cCA zBY1Z{ly2~WItYJi0I7wPLngoSQ;ar zCe+@x6*HZDD3_oE3-~to=Q@3Gmyojoy>17s?fVJlCAh~NGxBX zDVsvn0RJ4N^wsdt?Ix^sHJkDy_S zW4ls>R8+T_KD4c6m^Vl4rE74<4_~q}Q)UUbgXB_33!SexQn$>EoA(-Kc>J>t!?*bq z)zSCzE)cw|ZF$I5>RffFUin;%?4;+iO}ApZuZZN<&Z20KBhkiGNw`@l-Se*&9|vs_ zx)Mm;-CdTwP5v8W1nsJ(0`&J;)UAYBvfn0u<_tWWE&br;f*j2=F6IQB_7ClLWPFUo z!&n4jN@J5+6W9thG>;!EnAMrOtsfMJ5?+o(tM4U^g=u@MfbA_1H45C(~ z7e?nNPvbrN5$9b(X@U9q+z2MMP2j!hJ4TN$pz8eat}}&X9YI3qagaibLPQNxRwV>` zu`$GZcFXf5=zG)Ig)H}#54(ISa_;BM(1vIVPm1bo{2huke`cOYew>T_ zX(umXDgL+TebPu>&mOG1a}TyPHFhxXbUrL>*XnGXYc=oCiSM>_PW4`xVGSoO*PlovQsi=a!;24a@x+ z@(5CWytf=Ji$e%qIizm6Ku|~n)890cPN>2hpWn!@V*k%eoDYvJinRu5}|tusr%}S+I~x(5*vy#Vm}piC1CQh ze(W6%2e*;P%bJv3Wht090*ar}S%{saNDyTwbauQ{P{6ag{K#lsC*kyu*f$_Y^!6Z+ z)UEK7Z_QOVzvHP!nXwt7;FI~9ks-x%OR?_Rn3Lej+g?W&%kdVaL4zUZVBZ|VnRzw; zh#_V^!*r(34~Mk{9jwMNX%zW#7Is{qb}RTP*?;A$h}5l8kbA^gqw0Iwy6d!- zYr5NqFA||Ld7hT{$BG3{ur!Dg_8TY?hFld_H?W#Lj^AZW-Y5FRr`JWHKifKzPkZkv zLRSf?TcNy29wg4yCE{Pg$Z7d}m<^@-O>?#sd-5{Q_gh}pt&V30swGdV?0u>kOR)(k zc-Y~5>T|i%px;#?l&k-v-@idakMCuqZmj3{bv&AB=Hr@%aq8QVY=;gk-Cf$>Hq{Hp z7{+H>>pID=AI8^l=wRf~)}+-8^u*>xiL+|um4<&?nRhD_Z$;#*jMQ~H-5gPV{$d?* zlkHwA@sD!W1h1BlhpD&&HQ)4bHF3{3*&6_Pm)@?JRiJrXvu>VV-zK+zDwtj#8V;kRq z`P^lO-n}`t@O@w3Md3p3+^_A=C>okvAtCfQTt$mQM2RbO{aj4ly;nw(K$X{az&Xyi z%Ce^r9WoKD%`9+7MQu;ejr<(`TP$fEn*-HdT0w4dcsIk0mR4VAc_7bk;GO_& zsMFmzq4cjPW4^p0#*=sOad_YH=m{&GSfyp7_LW1Dg##vz8bL+{!ESf#yLpt}Mr*YN z=X$Gb1cfxlMf%=Gs{VQY@aO(Q6)g%8wLy@OlLH~FQ? zPG0x-7_G`*sd*~KR2!g0c+5-gioH4)lt`EtI1tU)^m=a)8<*|2KyK~-y2BXwzuy# zPORPRjtr!-y@e+edWVyqx8q&^DOxiEC$ePu75o z!s^-tmv2}3)Pi%_>|Fh)5{aVVAL|Sa6QB)c8n1GSrN+0%o?d5iKB%~f zr(a1R(AkQ0j4$YAWY3E;=S;`Crr&F<^)h9j?v?x$?D~N_a5>J9Z9YZd*L%ZwCb2 z#H30SOOq>)$O4YUGrX@xZNJ-&I@6ohiA~EoYxchDemkT7SNlPBLW(xylx>gi8;OQO z=D|N7Lqh0r0QcZ%L+J)t9^DfX)grl?;nyPGeX+{y?JS||efN6jSpL8);{DSyVTZI0 z>8m$cA8k_)Gi&fLx?>68&OQ&UpmjeC!Qy~~(7L*4QHUu12iSWSsE5=~Y_+&glqGkE z?#0NRnPg>g{zjp4zRDuh-8Z9TOExn(4)gt<{_=yam5-%PmM90+YIMoI_3wPiNi@eZ~5mWTt@vn80@Z$9u z#S2Vf$aSK8UG|!(Kyb8tsA;_Asz!bPm>u^IjG)t*-(s8pJoo%_Jkv*uLPR~2lTYW1 zU}AB;P>4k}-$d$c!LJf+_pMTI8YMj`&T!ef>=-vyY28HBKy!Ok-b(=b*Nv>~oFu0W z9bE8g4NAeYICQ??Sd2DQZN1t{&%6NcBqh!p^@nvhy*ZJGIB&*Gs%rGo_IoM$t=8!s zsPUC2WqTfjQ#>)sZT)P0Soow&WP^5$*&m1$;#*WOonCn+9vZ@YCyIf7#i zS{KX>pbcg6LxjQ8+qq)vxk>GDfK;CMq}rC$<$Op-cmxG`>(Q* zAU^igSt<9*Lc^~64q>NLUu8BD_ZZFH%0uXaYYDWWio9`baTG?v$L*{|YftqurV!1Y z?|SS0%dj(BW&~TYc2`*Cmw1o~{g+@GDK+n(-tLA>nF3d>rD+V-&Z_GTfVoHX_?n_c zA)*}Dt8j!V{J)LvQ(t;VIVi)#KR)K$uST=eq;a`C#D1o#qal~zv&4JRCKlpm2h~Y( z4PB+dpc$R!r2QZ}HmeGRt{GBSxmMsD{ne}Y*}2)d<3dIBFs_i?X;wmjtE_Iq^=q5eqTwEKXaVINJW$A z0oKV=H^N>d;p2g7cJlzlfYJTP*6m zVzvv785`Veuj(_bydD&PZw8@z2dPW(S@@cH43^Aw?5L+$ecm7KdV(#LWLMlZ*62%F zpVYdp*Wb+x%i3dF?zlmgMUlH?#4*SANX|~(aY=YRt2`Y%^F@Dd;F&nuP@>CXb#!^S zTqfb;s+%ji#iy%>adPQ=+HMb@b3U^jY7`+{A22l+^orp|_9S~+!L2O%OU?GFrPL1z zLuX5SBgP;hw5}Ce6e0>2vv)YdApNF$VCUvJOEN8Pb>6D=r`pH;$K74-GCOy5 znrFk-!ZtWhUHj@YX+>YLmwE=fudA|!yF6Ekx3TWwj;NDgNPitWRksQ`7VQ8vOVrmZdJZ*nwIbt+ke~0{rYwF%N~*9mr=UP zwk z2hu=)ZuUssi`C9v(&z3(Y+sO63hb1>Qq;d^kI!U1xJE~6W973VV$uChj&WMJcjmSF z7r~m|xWdu2P2vZ4^S<-5wvl0E41fRPImD!H&kPIFHa~_CORq=BT)elX1k5Qj#PUx8fTz-@uef z6&?9|6e@~8q8j{mQ0_nFzCEG(ImA>xeQQii1$iIqgw&PPY8WA&&c5NM{yJ2b>xwyv zZPN9aUZ2_zd!G*K&xv?nNq<3cE&c>$*187mrkj@E-mfy+YxXQ%k#|WATF!GL&&SS4 z-KV15@nijU(0r4jS68Kii&e-))dJI_CAIxJ!DZhf=0Qy6M+9UNTOmy2v-$6^^GEos zUv-IQX_pj9l)k@0L5-4z7bcQu73<9mvY-^v68e zsZm*G15VnT<|GJPht>4^@6Ug{6-;a#h0q1p?r1|5T9dZTnF)>gF0}ky@x4ezu5k87 zf@MI(t?VGNyKk4eH1{s-)LsA3ksc=_Kgi2o5MT1-Lw%x~LUV1kh@?{R1SEvc_a0gl zBC5tWO>bD~Q&FjMw?a)s1Mg=Yj*k{^ax|;UC*Bwqs$HSG_=ydgiMPHqs-$~Rr`m!l zy)!6A?8@^dpiW=>qAqxLh}QK$>LxpS(!1p>W(#3q+0f>(Uc7K*-PyjJV=v9-)a^2w zeAwd~UgPi#Tj}Uty3Ti6_i1O$`!-y`@juqHF}YQuCx0Sz?;~|BHB}}rYzu$F`|Lzf z$kZI&@|s)JLOr|FCCt9x{BF4Fb1P=OU!$V_mU*ww6eoEJW1CW42svN0+*U0! zFx|KiT9N7`5sH3zkLP_c6 z@1uMQk*_yWS5U%;`t-3!Lz}T^*rNTUMCEf0^#>nqIKJ?4i4(O2F}4{+;?@LUYiymr zIw!a%RZ#zwev_(A+>!8j@U?>8C@(_S2dR6s6Tcd5An=BDlxww8)j_8{`iSm$mU($g zYjClh((AMl<)w`y0>isxY;k_aCxX9*pB&S4pHG)Mh+j;LN?}LN<2*#_dhxUjTnx;N zD0eT}o9>w&d2McIOLUd92TMSFD3JAz*kb~#_R^=1@kW1ahgh8;W=!aio4DvOukmSh z^}b`1Iu;^dU!*Qksd;S+_ptDgi>0rb1gTpl^SZ|_HAjDnuNd~|!H18&8D(}(0#o$m zsi9_jbB@fqwx}*=2U_j(k8|_B%HUNYbp4RJUnW(Y9`E=Sr3LVGrOgnfG;?_3mNoDC zBze#mhG-Yg?G}@Y{o+!g+VAV+Sa;DXY&!}nm%ObLZ1bxz@%ZTwKSK8rQWuZI;xbCh z$?~O0kyk&vdXm+{)zj7CBp%1pOvST_(d#=^hDsYrFumwCTLhA+~b!AukDmFPjDmW44wHV6(h$ku# z4|D$di<9(A?otSD6GcJHle@1*yjP}W>-FiSO`i2XiLp&1G`T#OUl7U8*V~8C1V~nYlpv4ag>Qp_&zw@) z{0ZhMqb?QWnvaJDYMrzS`nIr~X`66L%u7Bj%!Ql6g=|p(dHoBHL1;snu1NM{y|^g< zaPO81_M2VXmB#qTKTb$h&}|khciqrmaM2{tx28@Z+inPU<@mPiP(~*k7G*HLHC2*1 zA}$tF4+)|34MB@SM5)Od^xwt2VCRC_*7YzfpSj4uYHd|urO?)WCnC_0Qc&BLL5GOA`B?Mvls^l(KP^HR9NW-_D%ZMVu$jYN=*x~5_Sls-LKov= z;rU;A$06%hZ}{9Me$f12SbIVGs<}`)>R$5No3p*IKAeg;Cv$QlL+M;yG95UUqw@_z zi$X-%0Y7$FG9;JaQr?*Og6Y?@kL1YcYlNQ< zGTbuAU1m6|e|)nHpDU+iXraWUV^SYa!8$kWsGiibbkrjw1`?iA~tL z$=+&oYGm57)!{@H8-W@&Zm+}l*q4cpN9MQFZuk3OZZ`<8mW#YoXO%T1kB({gnvcV_ zFGqE`29F_h!;!kusF~t5i<$>&)y@25Q)`bS&x>1(<1{YDv&DGQoLy9TvlWWXbXDZj zgX+}22MYI=oQ?KZ> zaxi{MvcC0a#lX7oQO#I*Yg9XGMI|QZ;)Ib$C^^qdy0>f>C_i13KlLi@q6YH#5P=qj zh&s8#%s#Q{#eK;edvTjNE_B=_ze}Q2e6BjvVBINE!IEtq6>c7FJ^%1zL0t)7zQ=Rj`P;5yD8 zc;yh8Ume)v`kpQqUw5O%S42Km?4AnumdCX)$J>w8@n#UZ;F=0;D58todKs~qxvYb@ zpM{0K-IT7i@pQlG(_+}*ovQV2^QR)s$AwrnYM~#>5wRid*2%b5=F9Tg?lBedVWxrl zzTnvlI^SrtC`9OP%#1NuNt#jiRidjUBvz6bqG`a)Amp zZbX?nw_RC5h}yCW5#k>)-CE;Zn^pV1YV2RlQMfG9Bs!;w^@2Ls6H5XTLg)JoEea8J z+o=3@x3BUef?ElEIB(7|nRa}&m0Xv7X!K>~^{*J)a%}9`TM`eFW2;Q=<6@l=xs3a8 z!{cD-*78fH`XYy|o5=HVEK+yhMCT(XWq(26ZVdUlU(J`SZ={44+zQM3sGnq?8057X zw71MZnezs}Vwr7T$mZMfji9nwxkQmRNb(}S)|3j&aijB%L+TD=U*(#xSe;F7kg{lB zAe~Gw2uU&O_tH(}(dw$n{qjkH{IvPBW?Rj5b%J*PH;dN^o{n&S7yiI#lJV(w;p05y z@eJJCqYd?biDB5&auolpvdaFApo;V3<3yN;#Z2~mnvq8PLq>0Kfl4c#CWWZ z(s||7jJeRG>DRBOVQE?tzVVUwwlQThpO4G2*1fa+X0bOPm8=;AoCMeYXx#*)?yv02 zZc5b;o&^X`PV4RVH?cqWh;X;neawrK#L{I&kb{>|PZc4_OmQYZbuoOL@z+*|Hfs^{ z+nYgZlifu-_*4kpM5L}`1BdQ?M!DQLtZ&J7`{8zVZ*6FM*?*}8E~|7mk?`YIO{~rb zUvjBQp_4e$|JriQa3hwS?j`DM^^T}CJWli5d%H20tHgwCeWw$>L z38C{%MT8crV;7*JapC2TV^G*2- zr&S-0`>X?t&EEDm&AY5v>q=%t;F=7r`x2?!$T-S-cgN^`yCtDPEiu{LM%{WSL-Ea= zWs;bdA40A7Xs(Qxo||C1>o5`9mJ&I4WYI2oir}%ax~;;U22$c#YO;nnHX%3aX-h5%TI4&S98cE74EZg4%7rpGcmY77h-+;D`TltjyKzUJ#$9h@_F% z%wd7fMjE)MKK9D@0xBjpd(eN1uB-`;YjmMPI!#=@s`%Gwkk_oC5g&}xLhw=F9Br8zr^k{+(MW#>S3qcQDT}@`q^K zZ-h{Z9OtmStDtDfYcsQ6jeSeA6O)mGBj92ZeqRzlP4IYuo1yqdd+dwAB7L>JEQZ_n zkk6xXk-A{L+1eGmlN=^&(K&o^2Cxx$TGV+!|=EtbJunX{Tp%@cC$*>yz`n z>w;h>Lbm{^JO1Kztdd8Qm-Jv&xte{711oJ;;w6%w0wtxR3|EGGJjh~Gd)xQ5`d&Sr z?(vIU3VK%h-CL$G_ZX+i_TX6g3?4%F4N~`Odd6IOP_?PFM8?zetrQ}^ZigJ6^o}O0 z)h4#_ac%G-qF1k-NpK?9d!LCx?k(ZcIT1gQdMM>K=IAW(h=-90q5Br8D`RP)xmP*- z>x9dC>T@^tpUTO)l|BV;@6EncG-Ng@h&gdC+hJJve3rdt;N#{Ci$dZ)CO?ng38X5f zMqm9*C=A{^dVecK>c(bvfB9kcH8i2#)tZWkQptOeLiKwLuc=r#cApB*T*u__#I-oJ z{L@vOGfUrXSPUkbw^X(!3n@geN(k*;4h1=)b-}$6+E9-!m1=7(ePcDf{bKc|kaUNf zVqEv<^Ih7W_d>sGQ*!MtzUk^Z8}YCV6F1aZ?S%#j8!1jZsifbp{zCT+dmUR7NC>U_ z4lN21#d7DRzd^?||DKs;;$ZLg`?#ds7p2pyGHwB8?^O*z@|^dE08vIVI1#0 zr%IzW;yz)oZ$!nc!b*-I7yLCOy2@xm$Bg1=v7Gu{vEC<;BRby_r0$)$rzNeyg!104 z!2v?G4)*rhw6@%{&OFb*M99(eRX@qOpIRK_PMCw*B5=EP@RKJ$R}br4=uh+I!DpTC zMFtcQx}`|nk|_HNL71iV=Ayhb!=4qz^k%%ROlcM*86nT^|B!j&6!kuD(~Op9fIQZ| zI7me%)Ys5SX@zjZeaAGa9KV6Q4515NqYbtBBHvlg_2uzosF%j9uw5vzjK*xGH)hTA zpS9W=Q&%6Sj&AnZk&8Umu9>$RZpc4{Pm9AA>6Of^>7e#ZK(SE_5<-tdIa(AV>LzE? zSFd{(O9UEC;qLU`HiPf55#FzNGj}51Y{w=l%Ehk2i3$ju`^< zgHq&72mPr@>6H+=U=9^+s0UcHW9j1WXl7P6#dz4<8U`#atM)V&m~`^wv6oHB2gl@z z>&wc?tnRN}rC8yc&>t_^h-bBK378d(J@&e;I134(^Q}aSLPRm;IkxK>d?q2%cdzQS zE4dD)~&&+Ak#f%iSn#{dwILotY=SssH-j+?+t^c6Xix$kxV@)BD5*U-2Vygr9 zm}p(_j1FxmZvn;&b{dXDPjB{nv-nlIG|ntC`u4~*DH+fdUa-6HBEK2)gp7*0j0a)l z(9@%wU};~)>y)-n#BH_i_Mg}jL(T_OqeUU2QZq1VrjHC!xp&j68uL z$CQQ=R_T0zJ$P1x)~!YAe%>Do7Cz%SbHya~!>tnKOI|}J2hSTr9~XTL|KavcYlx7S z#89}kQf~lzfW$;gM?ObJFDL5Mx5(YulnQPBSuhuk)~!S8nzu&ttWYxeKKtI{Ia6`_ z`|IhEuyx#z`b4)%v-E%Qd}(`iH$16f@C?h^aTL#3T} zJyI7}vdhmYo4{)5ieD^bLN_VwBf z<68@7NC=(pd$cG-RI*PAbIi5o%e=wq+ZL=O*p9pr4lCt{3Gv}eJ6T5iHbj-6 z3g5Hp8_dk$@}7CvmnV%y<26M@{;IU9068xWu2Im2$_!2q|LK4;Epc60DOFBwWJ|^w z>+8Zb*|YJ}k$ha7ZS?PzEGETotF#WI%C7BOeWAk_UH^W&rM5WHui(o5j4&jG&bJvY z3K3;P)k^R<)2h%GZ%p>810uIzqP{sr%JKaw}Ntv;N`-s)MLF!uvvkdU4Db z6m>nqFWcpwAJ(h4tST;%7>$z@9L*!{ZLRIdRcFU3zj(n|P*rrDH4eGIbs%-kWQ%@{ z4J{0e&##|*>6LLq3Nu>c!RvtcrRt<+H|mq?1>Og_I>rSwH5v~O#xkCmCKBeGkDX={ zeW21(bC%;7AtGOJ4Tv^WPe@^D>}fOh&R4${cV6O}HOY z$pqpcfLUi$X-Xg?>K6z{%my_(I8U!8$8Ij}yb(_MBL6b;75u_eW&ON_91!jMcX)m%i(~ zd?VG=d3-o^qUy^fnlvLrZ|D2CU=9|YZx2#8xH1t-ugqvU%jFi+&DSPxB~T(W4?ay* zQeN=EY{sW6&N+Wg=>E1ko8M=VmSD0J9E#C>?b!9Q_Fs>*b$QA8z&Qe~+l$mKO8Y2r zQTbADVV5m+i?F!8?6T|%&b*?xB1ze6It5quI@Z;Z#LH!;7d>P7Q`DY2>6|>++z{XO zlK%4W^px%pxF<&I_91oEqEGBR^k-R^nL07hDThHDgI)Sg{f+fU1=gEag)C};qDl z^NByxW0UlTvx+Y+N%$J&x7xSMJZl8! zkIr844U3Z9<9m5(ofVPq08)2fyex^wmco^TF}ZNTCVAt=uQ|;5jI3wt!pYP#_57sW za=elnSDLXeU7B?rJ9UGG$kd&Y>o`ov>w->F`)=7Igf4h?f;N<3gw7X@69&-3O?_$8 z^-H3xF7B75gzZVvqATm(yox@;KU#3_RZRAoH~TFLe&(0v_t>Zy57KtV{B@kcmu72@u_!Y1gN`=zC+>N_K18(kh=b?9aYEw zhpM}bsw&zV226K%hcwb4-QC?K2ue31AtjB3ba!{RNOy{KcStu#`02f`-&lA3+2j0q z#vJqPvtq7w_U5s5VaVyz%`GM0QQdrZ8<=ek%R1Bq>te8=_0@QyY+`{?pnF{6%fP+@ z%Qps%X(atIC5GYGwGDO0e`~(~@-PRwk`^|hLtn&i+^h?~l(|)Np=BW0$QM%dIUXG< zS@%E+MTA)oi)rlk)`TQAa;Gn~ChWO>Q7nEfQ)DKe^k(Gg?{DFsI}f@V{DkTqN#2IH zO(~##+o)nHOu2?|B7YcCVnW#&lxcO`PRV3VJa7ILYJx1O3ax(m(Xi6=ucb z@S#yk!Jx*I2D;sdy9@mbY7^gXv{!wQ$Ea(%?1hzMOGlMxpU^PD>MUl_j$W&xJ^N<1 zQCPVV4tKG0Z@oclMYQHu2ih`x>+X5n5VfMD)!%jc#dzO(<>_Q) zaB39zwr%!=U@oId>__>87s53Ou|c2P8At%`3h1VD^4s5f#fJx|)53(Gs3-kymD;+3 z84_7|#(5&5cN#U1mblM!NOc@nDSzYJ6f0PKo?8+h_!C#OD~8nk^TZb5u7Ymag}T;? zLgDg;Z{qeFdwPe^L}5W!Mrz~LUMZNxnUY#_-0XL|H%oaa4QUL;f>YmZGQQ;$DFpSBX60lVc_z2*t=Fmyuv8Pd>j^gGrJ*5d zRG43$#E5v(!8+I9vH4$vA@B;2Ja$uHv|s4+`o3cH1vOCp?Z785o3U00Uh*#^Cqlz< zuZbX4DVVc@ahI|fhh!Lf5FjvXgvZ+?_=6b_eE&fI?%%)P;lBp+lKi@7y!7d(yt9q* zY3x|JJoYcG$9y9FZ|lTg6Jm@?AIL?hD!==9dCi|@SbZmlzxp6r1@}pDvS-v?M6O%@ zZ{PUO{W}Nz*I>@d%zmZzI%rxV+{5tKjhagMrqffvxBBTFFCP6=j5MbX@o{zI~;Z&>3A%nhMbkzaX-@f`FL9%?_aZ9qD6 zforK#U4(sRjMNn`BT`;yMa7bf+H85og8|^~gRb}ZGTdkKV7HAu7vi+S<{gKiQy;Ug z3{9M1tn%=5fjdkJedwX2h(@S_0%Pb8)a15ID%lh}RujZvgAnp5WzDFiE7bvj;R-|66X znxtpQFFmmT{Rz4=vKCLJ)!QsFRG%^ad_LKSKm`<;i*Ms6C6ZIs3s^@A5jS|v1u z6F>X9A$<_ER+5=CN{mr0TXNh0$v?a?GOCupR#nMb8X;zkL z@c8}mhG#Ack=E=XgqM=l_7C*WJ^qLOxx(a%2q3}9A}$rEhg$lpVjdyvAS>M*E0ZOC z`sI#1UQAc%+Bu^kp%FNnaNi-}8=~V+b?xthoqlWTeR@;Oeh;pXPe6Amq;=Sdn|v$0 zDU7lDaO}enWPx$Zc!UA8pH{Vm|1yl09^#8f_Dd^ysO_>Sp0hVjor?(63vgTM-f03; zk{`i&$tmcTCu56y;G}!o;-hn@6sT)_>{;zAY%3!S{7h!nr*mg+zx*6@Wblo8a^L8l z21|Q{hGBVG3xiar*-6vykI|)ieHX6Bkd$VD1z7q$wfH}A>JO^DeM#Wkv^LMAj zRdxpk#`fQ$%EP`@XPn-3e_9Y$d%fB!F*wjZ0*S9a5^#DFesyI|tEI80`@5Pd86_uJ z!A-6Ti1z|?S+U$h#S9CJ_}L`SispX78YgIt^Q7gGlS^6SeTM(mK;i8k^ z9-n-j$Pk9<+Li1Ipndw_PN?a(XL!SUlAX}c_71r|!Ch*54hx9)@0{vigP~Q85ZKFc z6-O#nbBfC)*Y?BFw&XnS6BrfQxOQ_b<0-D8+*M6b9;0S$4@}ncfHh#^TcP@8l~RcrW$-n;e?|}62SeNlYb4y{yJ^c9;JSVf5Fp#;Qjdn zn_ER;5Z-WeN~qu1-5hln=i*?E|2la^Hc#W@G?Nkc+aD3Cn{hB|`sQDHDy#fy{y_iY zz5R#&xx$bJSdu0-5{1w6Fy(6#f6@E zZMH#SrY_u4+C*USJ;`S&8F2va-`dH)26HsCpV-}u0Mqh7yb>K3!CotLXsP)d`7Do6 znAzMLOT|EYxw0iN9c49BL4GHj0Ighjbk!;P0I5U%dDK&_7og4852= zek)fZS3URut*oWP>_l2dvBio6rtgYYUwdTDJr6Rn#Oc2-+nXrjtrN)gVxVG3A2NNb zk?B{G(S2hZ2e`jLxA3Lyt>>dK;b4w3?#S$xfv@5shv)O0%rJL*Wh?W|VcwV3X(bhP zG$(%DDVIn8T5+X0QlDC`B^wx@bP2w%V7=r4bYny%Z*{WxECw}+1P6G1TjB+Z=GIJ! zHEnzKV7}{iJz>5aqE{XDbbU+S!=S)hlB#0Cv;~uRn@?{lnFLMA{C5xbUmhMox7hgR zOG|Vw3Rm_@R(C>ZNkgX&C8Gx007w6^@MWcInC6%pEaf&r8}1|`Mo@^I|M6f=yAld< zOB1^Tta8;9Sm$~I-57#L_+5B&I-wXiQoReYrC&Hni=DTNX?bu2aEo`XEVLFB81tzP zn)tp4O38_S*gn~&uv`&A{3bA7nv(;!;C{d}=$_N4aBTdHzl8ZMv|eI}XBIJI>`PNz z_B$Uze6PlyE)pT-qkrL_f0fI6U$VJ%RKG1X3JP@}=$M)rpAIeM7G?wT@Edg3=$!oA zI^Lr6pS5+LaLqY&Vw&(^CalbO4N%g%P z3hmd)rAP_@+&_u_{|)A`c2tM9d`foFE+putqlixZ7H)U-M_cL4z(tLwmX(n2 z`Q>G0i{#GYvfXOI$GIGDM@6*ZIDL(l*$_oOW9@D`&+1~B7nWwL#6Oh@xKN-ghuo6R zSbHX@s-H5g`PECO@qxLM=QvNG@fThs$3l^g`M&rY+i~A@arx1MogW=t)nmF_3P>zJ z^N>;Cf8)9;0WLJ?lD8rY8KIxy>-YX-z4ch)L4DGT)>3^-QC|X3+@XjhQmiy0v=3IM^J`PGRRHo zx)#y3if5e$x^2iS`+Si`ncpC-paoie;L|n=KN05&-&PqQ8rsC2p#Ux{=$^Aj`;8qy zh$Kk&Fst^MXOP^+@mCk!p+mIJa&+z`GrwcZbu6$g&Vf;2@Bh(V#d#iqmAe1@_Ve-s zMkQ7XQ#Ihifi9!m(4HLj3umw5z;t}~RZC4Pju^pbolT|XwdZC0!7~L2>k%ow{0T== zW#`P0Nc;;Gg@s$)PI=$0x50*i($0Vj54vb1E+H`st(22NZ9fbqQ9QrDhH)Hylghm! zFJ%8?&VLsD!S!C@r%x2L^U1J{&Ee`B43&k+*Z>^g&+oVv$$bCL_5Sq_2%xLmW`pmy z9Frwf+1yaS!d_Jvo!E=>G@X#@mQ^|MLrNm{5S0QAIln_eW2HTiV9VI^9#V~Ce(qDr z%bG|}fBF}I`=5t${`vg3!FYt#i<#J|S8KmDt^eT5&uhXqSLl5kx5=l^4d1w>M?2v0 zyyg+r;{x4n>(7jN8K93BhyTxFkD8LyMjFB4WTUy3N+9TXf4?G-P zh(pl6zPI@}ng7+2m*a|*|5T%5O<3|B$In#Eo}lY#vSh$T1>NXnG>jHxi9x8x1)KoS z>>JYs$E4@ocLW`gh~?}*Gm0Lyn9`$I7!aSg>5^8IT+vbdUB7*EDd_ky|G|c1!bTTx z(Lh(&1$T%b<0|$Xx9msc`W&Vw|F&aHNy}$=bf?$z${E==M$SLtow2eC}#nx!(5;-)rahsuuRd{P!KSK>z^%7^%1-xqNz}L>hqHqIT4A6xVyoe>C>k9m?fI@+E za)t25p91-p?Nd>ylU1XOMq#Vz6Z?A@WG)Fp?^(;tM;yi0HO7JOZaNSkMfjnS??rtg74DJj1EhYpi1szrY z&>NUJ1kI3^vhd1s^Ivzece``+K$vde~uy*hu64+31A z|8aLIt79HDzi4!{G9{uxjyvlH@?%n!6PYk6Cev9cH$jy~#RuIa z=k8|X2tQ6oMUMRU{(<3mR86UBLNydyrN@a@Q^H!rpG-b7ZN8*WoHtQbw|7~jIZ|Yt zj}{nw=Rxp&!h8jsM-hOoY=(Zy7esYh^}9P;&8GW$+l+()u`?N|g8ZmW#9Y+^w0;%F zaUW4C7DD#~+7P?C^bqssEeJKCk|ZamF)jGNbKZZyH$u>T`z$kGZns46qrKKa!>kmyTwTwVD%a{XdYq3vGR_}5EkE5yC1BczNpC>; z`t=B{E41=8Dqj4JKM*e&=zgd);Qmt|B|dOCzbZ>$#ZaC9{z@Mc%|(_$Y^ z9qvu2M`z4AG_?QL%>LaEl%UJcRaJvRA@{Da7@I&K>QTHYn0U(W(#B>BX0mW0cZAcr zRB!I=}NdaeJ0VVwo$y^rPVMz>kr z;}xSZN(G`2QCH$+U)sBhAL(+x!|3AP#?^i;m@R?C;zW+ucB_bISa7Y@d$*DR_77B` z>lOj~Mf3Z`^Ap#F@&+=b`m)0mx>yD4L9jtWekb#yLL2!Wu_M;o+=Q(dE64s8kzVTv zPwNL-v1r1B=4!_p2G$m|1GOt{sB5X`+feA2;*wF>Qsgt-!av<$cuFDWfdE{ z`0dc!nTWpz?hi45 zZs2zPWQ@a&@AuU41Q6LYDpi854?Cv@iWy-Bd88p5ti21P_)exqbH1+%ep#v_q-d$8T zhD>Y4at7jYl7|02Nz%ymv)xM;Rw%PC6xt;?IGTX_5_AuCZMWNa^e1zx-bEWqJUBXWP<}rzS=aEMO_l7^{--o?X>MV+-^s#wZ$i6e;Mte}mkD(H z4%BTZ2i6Krup705b=wN;?gD4Wl6vwwViQuuB{ngP(7jC0dA4tEh^p&^Q{G3t!!-9# zIuEQbs)DiEI8c`dTxQUft4)JgKs{8N`5fB&?pcU=M~=gXgBY23ESZo@oj}D3yE;QMx50v>%JBRkq4KNRQ+rwO0|gHq7|zw*)M@G0fXfQHi2FV# z5gsKcBea_XIg#X^qz7a>&?NAfYDb?Jv?+qp$qOZ9M}B|h8|1Qi&mtepX?L6lr?gYeLxP3!aR_XPi>=w+&gR+duiH`5Q}d=; zj}=?wc?-Lq^xLa4PAWdhtCwGF@2pt$@&K0|bR9+t@7mQy$7WHPPJ`{Usf3)pxx1PQ zu@kzGpdM(8Y4z~Y_$Mk_6Lg=V4FiSjZpmYw<8c_wd#>Y#d``O-CjplObc=rq;qNJ` zn=~@_-@!IGq~|&*-gT#Ks}-D3ply_mi$5ewy?vUWmp_hrj~Z>`&CL_e7(*`<{w>e4 zA#6c1p#X3>LH983;j?hk9_7oKQIbzr>~^(m$8tP=M=h=)b{641?VCS?o8&|`Hy=NG zHod6ll75~ysm)lAprtG=hv=4F5Uc@QF3{al&GnJx_v*b!U^HsrH)hsgY^RXa;X1DO z>Xj6}oBL^D(TUoj#I@e5u;q#*$R~f@RmGhd`SzVAoOZuagcDeYc?G)ky~eomSMZ;G z(6{_!xjrz2SLF=58RgdrWNJE+75YPyE*CCiA0(>0oYATNJw705yD)Q5M5umf9_otU z&VMcQpLqZK9uRKOm3V!b7Wv(P;k^80#~>ufnHo3x#8{9(`Q_>G{PO{m)-6r0FJ*%G z;dDsY;_h3j{4(KQgn*n7mbpMk>Ged8|JIfN`yLP;(ESG2JmqbDP)I!a;*fJMF!d!l zJ+=ETld7qg|7+~1RBPOeek_#*0qpl!Yo@kl>-k@d$C`KyJ(`15{6d<0Yc2j0FARj= zpDQouLWm9a+?qmkkKx3~QN(ias$N!c@Yx~^US+iU*R}LxAdvZRS$1lgZ-h>S5 zd9`?^_>>_wkPIE7FbiM#cQ51*_n-GZ@PRHY8*!QV9%79L$0Ev6^pn~S%kO?MGuf}Q zy3=g56DXUfzYcyKABP1{xffcJ8|E(1Uxcs1t@ne(QQ3)li zio`G^vVc0EEkc4FnRWAIb0QX=D#OTa>#K%@z zCFqpDHTA#GJ%<;|4G9{GpQi**C7cLzY{*^Q9~?@RV1|*m5D#f9gb@am6Qx)j}Z!orIBCEw*4ye^9tnW7^9Ps3?k4`pZ-t0|9x(q2Y|=EY`^BI*dQ#3uYsmFzK9-+C?4r^JAJOdP zVN#I+R}^%Gm<1_YU)U^E)IHVu8?CY=@_izlcDk5(l={JEC%IV2PMs4BtDqFI`U}^8 zk7MkYv=~ePo9X7F&AH&Zc@a>z8`n<8$d`f1R*KTDwmtsW$jYzH(i&O7UMc8fmPUkLz^^jI1*@`8Um_CYC z$`Ur{IZ3N5$%B8De7M}VC@=%R9dIQ-GB~?2_Wg{l0MC8Jy*oaK7%QoLh=;-lijRY`~QQ-E&G+d-SUybf=N` zsEVWnSYD{G4Ov0Y-y1(3u-0$GQFlrGn!GYJ%o3iV9Wtg(FZtP^(C%KzN0b-u-cq(T z-we3YpvykM*ZCo*Xf|m=yXK%#b0|vR2i< z`j%3LsbPW}v5O1tk@R~5n0~;O0bTlc>|XSlC(Zvy|w}$pIB`OZ#_E=->Ay3%Xt=YMTa#v662Y zvVJ1CoLwiz_Kl%v$#{L5J+>rUzxfnw;qsrF+ zIgP3}u||2ZiH|MCKEP^9L;Jtu(0_l&@}PSs#Z+uIQazFPn@2n-`6$EZeL&D5iALya z_&r38{scskxa1esGDx*Y;(XO=ZP`<4iwr%Nt+iTdH1m|AED6JAj|5bBN##=gmBhNr@26fVl6KQSDA5cHUgAHJ|sLws3 zF4=#m4;~cL6F+i!mwMm{DWiv40XY6E4W)jtO$t+P8noX{{!`%T|3}d$u|zdn%4=w zzow4hfa=2zW$f)_?DOsL@fZN|pbEOlCEfQ>-UV-6I@X(?ln$l47V~um+954*$lbg$ zYP={ToP0X?)oD%#S7-k`c;<2bEdzBfG&-Ga-T*Q2jdCMw@0V3&EX1$G?!94|k; z1YC8{Wi5;ObfZCjf`%U_`PgYNhT^LiZqsHc4x?h@Ijj_2T9+fiUzvqR7<{x(NSq@UR`)P=(;!|KpiQr+Ovc)yNfai@gK{r?NaOLSePeEnLXFd7* zZ}{JSC6m4C!cz>)dkz}AGoAUyXrpUV^``WEsyj0ON)3MHPTKe~TBp2jJ_M`cN)ru; zR||AY*uEso8V-ALjHa`lTrU<2mRQj?LRcuXCts^7V#yr zkeppU3_V}HycjNdAmFn z=zgO_I=XTqVhC+su^Fp@`SIKJaH*ZHPbJ3A!bkhPx2>*P2~tF;van4}erLjwEP$&E zy4tyUqgbnv^${wcDD2;wpz)i(snb^bWMNBLeEE&#s0WGC27wv(nLpvTKc65+@qvn} zh)d4=!ld76qtcWf#@{o)|N3=3&~2E8NTYal!Gd^-WK$6S6-kHPf_VzrTa|s?d63ds zdbLaQgBm8BM`HTES4t3Ou%_N=aamW_Hx;?4DS!cx1*V}LCoK}xmm0`=}1K~XsRWq>3AGB&K25^qreqSedOO-2loLayP0r47x zE@49RVdyJapTz(-fgty_b0VB(NZC?3WvGf#X=602cWH|1x>uLJ*xJ;IxFaE3Usf)K zhM0@^MT-@7K3kNqg8jp5&^4n$*oR~>yDn+!PGo_TsSXl6D0>&KP|R^WLFmFDnuFE!g+YT!YEmg zG%PSrSNEx<8Bw1vzKFVIYyV(-WfY(RhmSvLup!244f${&SO;Mf02893g9DzQGzHxs z5fW86*r;qlozF5qaZGFQiEl|97D%?8N3-$y@?`_UMqE}U;~uIPc(kyH89vYlWHyK_ z;{5nELtfS+xlW-0#QO$xS8qB`uIIaM8{|SI(ND94Bx&?$u@$X?QV>$JSl8b-C;F>w zVpDUDFjTWn=hL!FHB&d=D)}q$RBPJ6lbIeH1Fjk9;`%=MG_g2j690M0m7jYL;(UG| zuVg9T_gh!yqlZbwjq_nDaVUe)a(BZ-rDgO4W608WU&1WutWO!XLgf;W;C#{?bPs)Y zvfS|*T&jagB*q!0{iAtaa%#WDN_(@rNW9mty(jVkyLAKSZPa93#Ff;U-?d9jfe`bF}~?%7-Tv#^s8?ryUZE698eQ>evYL+$b~jH zexkhlmtTY|a*zTE%1E(f3_}ssb=FiONWuQ>9q7iUU{H+JXL%xPVg(13!dJ*pl8l}6 z@?np2(nmKYit4wBN(js3@ail6*gLe0;1(F>sXHr%f+eEb$a#7D-KY%6uLbC;+DEld z@mZY>;{0}8x1;5*bbU`%Zs@EpsX|l^(~>vIpzjQmm<4BnD$(l8D$-L+vg<3FfTWtv z9!~KoLy;ETC$j`yp?M<%Rv~#>OFyi|p2)iQKPx1&evj(73BwSzcSE7dBC4d~%}GK@ z77iUW^H!-$o0;*$T6i=n+7oCxAjy)G0`Xdb?#5aE=`vH`fD1=P8M$()ey!vtBTaF$ zxTzIE1N&o9id)2bcJOw9`B?s&m{OAoCoPcW$x~8c-FUMlh z>)#AB!%Rz}=W<-0$Qu{BexaJ;5ULpn(3&$kT-uqEJy~?-F@f&2z_4{{YafjvQzCId zN*8aAZv7;PQK4Zng{oW(gTTjpKi0=+_e9>&(@|q*RvCP z+N5J3svJ{@%h-^9_C!cZOKbcVD-LjNK{sXom@-3$30lzJVPoKZb>#W+0X>DM@=tPQ z9)a$_h*x7jQeSaomX$c}n$i6UwS!yq3(Je1xB8a!h58nq|JJ7xG+PHT#zAgNO z$=J`I$MAccc+Tm`X)294?#7b~DkQ|cx=M{T@?o}=<$7^4V$dP|K1bi!G}Af_Ly6qD zLnjMx?Ln8)ofFn(Rsxb=xO(tMTvf_@gvxDf=K$z2Vzdum0=_yXFn15Ib0vS>Y(cW; zkFhS$`RHfAT>_gVw;OQJCyxc@*8z0b75kL1Uc?FT9%B}Mz*|+nn60lxzP0b1tN)YPsc7KhPf~g{O>iF`WTh?Z{Rw>1#~kIY+(r_<|f(qFcAdn z2(DAYo;puIs7kL$IW#4I&?Au#Nbi?w+iTdq(1pA7c)1herF|SjTNdOflj8s9jcqAF zysn^&DaXplAB`(YICx-cmduHakx3;=R-}yF8E93LwacVaKubfOUg;SXCCA1Uc*+#L zC_d$odyK!`;*3pw@-rIj&)$QsW@ECm?<=?OGs^7nA3A*sn%W27mlzgmB9Bju`@gH! zA1Z}(L_Pi4_sfGr=jh{@*0#!~_6ka}^sGdU7NxtdJrJ)O=)#M=G_U&g)ASgt_r0=2 zFP0+d$WDOgVBwc>lY-QHv_Zagf;i}Sora*65{^O0fqW}gC-2ltm>OSRM93AyYg54e z0J`yP9;O2*L*K<3%5a+q7nXa)$aagdyTWTzt?y(g>D%KsojcG>83bKq&iakFJy06s zbCsx4x-MOtxTys0<-j_OJLtwT`)Y5?lBm}8nc371%9uRxv(}%o9QUwpUr?AbNsY{4 z64vBOuWehHpO`wmb6p9wv49RCm$zJ?9@#8;nNk77>jAp+cFs;pU%AP~Nkwd8Tov*u zCe|d7$aQq6xAmF{?65HLPGYP5?%IyRO5=N7!}EgG}64iH?J~(ar2$MvUg9=Z%%`IgdfT zv-Ya+N7;knpm!p>K)ha{>y~kJ>rlxRX{sBpLg2mi(uy71N-=U~rD}cPl3{@?$x}#9 z=8!DierGRxD3|-7F0(LFz}u;9uItLVaFCD&-Pu44IX<%!ST%-bnBKW zm7M9==^=_PwTv|Eb#UcJO>atuc&re$$T7NIA<;kmK0HFF8oXMI@C+ z$v*R!YVPB9B^v4b42zAAd>dCCUbBbt4qy4S)5>xPG9^t+VBF zAmkWS!aMR{yw^85a&^syFLw|Lwa$ps7}dptdQZI39@x{MK258jYc4`zUA7>mgL;J(-?=G$ObY=vweij6}xnfJ2=phy@*T26-)ZD2Q^Rs;aWtU_j znusIOW|`sPVJU@gKkCJZa^lw~9uk?jaSSRNUz_LgRxXVjuw<{4(Wpn3H+ z%AxG%9Ts13I2*MrS#9R~zLu4V1xxS1J}Jid`+=K$t>F5f4cf>j!YZI}CIMTwWy5cih<0 z=RSI!*vIh23Dmge$Xs?%2N!j4okgtl@!^licf1nrgZt3DV-=9;HeqzJEVvkOw{<+< z(q{>trws>Pp0VOQCM|CJoZ05jND;Bu$7{_Qoose>lxf_AE22uhL6H>PE2!k-ybGSB zRJLien%u6A@sC4!N>&8}<9m@&KpsAWuCM2pn?Q~mA-_QU4)PiCTSMgJY;i=Db(Di) zE;zVdVm5(l1x6tu(s3D+nBoY-C*3! z1?z~Bp!<~nGu743=BpPQMF}>c04dasKsXzsP|n%(@846e_|P4i98_#XU1Sk4cMW0H zdIh#toV8hR5#I|%qFGK3-6#O@MuBdJ6*T6FSHd24t9HPPEe#Ro?<1ke1eqz*$EVnO zFa6z=Dtm>QH2X;nwIcf_>WoK%8np=bL}`EHTNpS*$2frJHKIW`vU|iw{?BRnkkbn~ zHrDIF&*MTglxxF?^_C9h8`9&9Fw@*V--8+s2xxheWx{vjzFLM!%)Wo2%tL#RDMZ{0 zo$mhwnfhVnBDk;3GGN2W@s!9P`VoODM?LG*#E?dU;-Ci1NvT z^~#OrRoHvM;Q$Q08I6GW3Dk&Y*m$3jpcI>4dV5u=t0HQ^jRjq}KPNqzl@$g>%FNYl zQ)>5?t%ebl;7SjFDl0s1`V$gk`w~{J+-^#L_?W5|t)^WYFzn#NVqbB8a{fvHRU;of zza9s=uveu=&feWB4>mS)$L~fypbQK_?%K{Z&NAvu-wuwC;8SstLm5Bmse2P~>lZJ$ z3u+wE9#TEIug`Dz_=Un*0rAFz?iF6XPfw$jBlq3p?Jrdp!ym54cJkOR1h zpi6cuGyiq)QIyb05#sWgj($m-EdSd(%RDC3D;YS^@2!9Sl$;*R(ZV5?6CwJ<7-(pC z;&1tehiV34jN%nejtU>(CV}pWodaf1cb^-=nP(AeXE;^a3v`NS_n_Erh!3b=gx~U0 z^sC6OQ|v`YyhvPg`h7ZS>&$CnuWP>W=`iZ!)2TeTE=&ep2=;Wk0i7isElp=OB?8Wh zks`A~f}8CE2&aC{vTf+Y5tr1@oeG>^%WSvgoKA%pT|#IId1yMD!gm|QbZLzzRzSA5D@SwACRDfd zp!n{~XyNrF(SWcud+{Z6rKwzQUw(|*#H4T_7Xc7&8t9tcl(n}!SAM;veo=76LajOX z!<(V;*_2^hvu;~~P!?KRh(Wb7UUqvNo&I$ptrOoOXVk64o)8_6di#}3SN1lQzTHPxp2^WKMF>H8=;Iuo*08hS$%+)d8? z7=d~ROLF@hLFV30X%oa^r?>3dZG2pJed!6-8^OnYD9T;j=n%rbK4^8J>|zo7Z31V{SoAT75GjbRn;-;zzJOp-;yh!ylZ!Ai z(QsA-F8{vS7#uIMLHAf;m!U*kdd2Y*$F+1#P0kDL-;#tlA=w2RRpt)nQCX}N#4S6n z0}hBk%S=m*9qgH#SV9P!UKBj>lA*jMH2~+;IiP#W;GA$sC~-NnG{1qt_d)G${~+PP zk1?z&#!ImDr_0Le@#EC6&3m|X7>%J$H7~(3!P8Q_t-wmvcW4n!=iA^uZ!YL6QaM~) z_zVbRgy$c8+!Q_Xn(sJNqQo5bma2ZPs#@eR;R#8b*i63pUbCI~lH`g0GAvSLoq@bE zM9P;AYk*iYtzZuUF14zsvfjYd@?7qwARrS`KDMT99) zX=P+zmAVXh;fVd!cdJqMLQfEs_60ha!MQZ?GEI%pFue~1)i3+Lgah#wfbI>^2knfG zKuw&`2&BBeZg(wtjObDbTi(WU!BpuyeH2KCBv|8S&kMKG`lH`cqHS~A4gH{IN(T_|CE_5XuqRN+Nc+`@`lJFc5Nb!sEoHVfFTlrqM@Wj z3<}~C0o)SM<@xwY;nMh1kAJR=AOXDMxeTpOcdi^ONqDj``u%1H|HBWl=gc0o-sH*5 zM>Hoq+-5yeaz!}x>qfnX{?tZkJHY+VqU+D+zYXR=%RF-ON)&4%=X)8AOsRFvqb3xq z==z`og12$py#jr?ZzH`pG9vw@RBq{Kaf5GK=*Ll{-mM5NVj<9T#q#ffLaL^J{!vh!0(V#!hED zyRPM+p&Dz9Ke7E=7nXyrsaj~X6;X11+yGi+P4YfG{#5tS)zE-(tp#t;9sB$iI(JW5*+ z?{i$j^DW{Ia>}Yac?c&&6SP~Y@uy|iiySb6&0lv+g(Do5PPF_v#4Mw^4Y-w{Te_)g ztU0Cn=7z~vUCmQa2bo}Vm?)Ege!osVZ5(F0+7Nq$M?jImjoM|1+~p}zjza^l!{;aO z?o+|kTXr!daKEq$bUVwxNFkZgDIgS81Xj=*5e<&g*bo(^Vsg{V1fK*SB`E)%AA~R9 z@Vt0#@QNyJS-%ij?#-Vaddt>MeHc#b0@i7(L3e_oaPRP2^-C-Ril^e)%t)giygPA* zA$d2ZdCLSA5_BHP=A{$YN}>-&2u@W~gu)C_U2KT9dYjE^`}~;3N#Z~rYCxBretxVk zk{6Y%6Y^j}g1x}sp&P9>npuhXqn&4X-C-0iQuKFW#KXDs;HLfb)fXCL=rC=rE1bqy zHi$Forh#DHz7}-Hs=do!h^pu>MbSt&SX$GojhTEkX;eW~Bhgz9c0_mb0u#9IftwJejmQFW@rEeADNcsJRNwxh0Q6?ypI z@9FC1H0bSa1sWX^Vq~b}W@i^!z(kj)cH4E zF_iEa=gTAmEZka@N3e7?=#b$zA_hs*DEoB@QzyPt7J$$zaVKwlIAb<7{q77p(p%{p_33ORsLtCcd%gJ#+ zM3!_6eXdI$Qn9qg6O-H&YUn)3wv#Wnuv9oTvsF`m+(lP2M&hZHR`S<#U*HWXaz++ zOO_ec-@3^>1U7rMWg$c-FMjF*ZVTwzu0CBrz83SX`i%Phl(AorsHl9USXzu)op)zF z&SG5!ZUAqq-mcHS27xki=km^`OA*1H*5Pv?3m@kloBT{G;I@LUVt0VLGqYFWnnTKL zM}x0Z?yJKeA)g+p92(s9W#ZfGG2Hm?E8din=!WZctT`S*-Z6El{7lmq*t&b=C_Tgg z{$IKcbR7g!W=;irel&zm88EsOv--F+p8`t|=jL{Qb|92exewCWA3-EBR58CF^Z^PCqH(`Zc7fBo(ms^aN;oPj`8sMc@P85=p!?;c%@ViOo4I%8E7oZIhp%tmSCE<$cb@-3WO%98 z?BP*mJWA`RWU6GMZK}9aQOGv;m{L^8f*SSkW@(8><{dbH=mg!<(+oUf!vXL6)ph-{ z&K{Wn`(FGCc(%5;&^akw2ZuoO>8V4u_B>texCK| zI?wjSjjb4T?I7ORkxqg-al_tpctR2+}*{Glmuw6Xcbod}dI zUzV6rdH7Eu-J@mM>ZqWhg_QW@6^b)wRrkF}XlF*5BhFsC=@huzEGm;Pm^e=b$Dv-( z{kp(!smNh#qu*s!`}la4ZZaOhfUHMx(_OYjU_!x*0!3&WYx-e6kJxqcleyySugu@P z(4o_o;p+1y{R)1*F$M0kKG1bHD(j`Tj~7oO<$8lweR1$})YLk%=hMV{bR1DM%GM)& z@jhe8^S6rug4ygfY`Su4?_OUGFtAMNSFXconnL4M)_w4ocJcdgy+sDF-Zucc zGK0uc+{VKhJj1pVzjHT}p1#&w1Y`3$%~s@}Dh(5wi&AMBeZ9bUwZGC#;?9R=Tb|^n z&b<%Dw4d$PnH<^!&;JgBZgxSYAd_{$YEXPz|DPhn53;2%_k{?HGy}Y&z5#~getTlw zc;%2jjt_cS$*o!2uhA{MKP$N8m-HWmy*eBX9Rcz%1iHB^cp0wkKJey#A!I@)F(eDG zbkT)Vw-yh2^E^mr&MMl9Rb}y&%vl0Tc020U-OF=xO9^$E<1=Ru*JVky+g_N$wm*Sm5Igs ze^lLNR9#KbAZpxQLU0Sgf(Hu_+%>od3GVI|2<{r(-Q7Jn1cJM}yUV;YbH8i+VBQ78T8$Pqx);eurpnGQ9K6?_I$$9=t>niVBgSD(1jLJ`B?u-E{SVY z--x8SFKSWh-Uu5=ozZ6Nayfka_+3|lR;DB*W4SpX#np(DshPDsKyTG=zM$$3j@V^K zJPe3;40JKhA4;{72DK&g=tk10+0)z(SD{n(IRvx25P~D(%*cFC6hdZ%gjtH9Mnmv8 z`rdo;>6li&F|c4w>FUF5)vXO~W@H5buJT8~72y?pY?wpd>l%9|?Sj)U%RDc3(&6gzc^!TrZS z{nu7#2MZ05IIKSl_SmaE6=xLTl#fh|;xVBvTf;(-Gyx% z<7L8DrA3n6Qj`_`YzqqB!fNR$j0oOU==sGtJ1Tl$e}YNSg?BPX%?yD;H8D;MNVZVY zkrYfngd50;Vw=5|5tceA!D>zQSeayKUMBu&+EV8xVH8WeG3!+$4k2tr=s-rI4CHqT zba}qh`ZQm*IWET@jH^>Q=?JSSX&SW3+Anl;$)wX3h<=D0CMs6(D-Rxm`2>%~{AOfA zI$ZUcKNu;9JuoZ=fgNzCLD!MB?=l8$ZbkpYh!K&}KZ{YzFqTEOv2cc~9||tY_1|z; z=dlY@D4&K?u?kT3>cYJK-qU_Oe|yDpO-`AwQvrXEXFykAyVCvMGLdH85XSC+@M*MA z25(49%+k9VvU=&L*I_gGW!XhheQ^r3x=mg zD<)V~jIwgFaQcvMdj)($mA-NW*m=xxe`t$E0CygAo9914MrF!*oSh2sn3ZXj!baE# zzy}l4!JP5&#YV*j?7rV8wyy4_;je=?x^LzjdRD#>m$$fuIgXGR`xs`O1Go#IJ3S>- zTUt^6fNQnWzrlov%WpNe(6sGa81lxZ-sqNfoxCSP-8C?L@3qw#!Rr!y{#Iqx@T%Fz zC=u+AY^Egc2Dpo$D@yxE22G3aO{pBF|Mh!4bvG}Bkoe<;MCGvW0P$H$oZ`=HSfM)T)m0Q7esSh)}%>UaCySo_H3*F zs+QPs;3DSaKl5&=b1JWS$UbW6>3;#-RnYxTgm5mLt%C5bu*je3WNEVVi%*a{q<5{P zoE!1E>(E3%v;CYyjJEz(#iLul_jHqt#Oa%kMdQy9J~lF|hst%pT?5^0uhJ$+h3>b% z75pF1yytfc?NFiXK0$cAFv?_Nq{~dKY-ae?{`a!mH%Xx`Y^%>D=h^@A+7O8AjTC0f z`_}4t0Cyd9FDquSpb@oxde#rj@H3H3O=%0m>DWMul;Cx0B-2oG?Y%knqa({9hN%%^ zaPo}E|4m{;AY1R;y5&!+`xh(s1#mY&H%rf3313fR5F-|a=G+8^5WXOr#cEj*?WSPS z5o$zOLOunxHiy4ASCt11cPPFedz zEf;Yf9Rz$9nhLgSvoNZ@P+h%v+>()xa09~6fH*qoHYB?r1YAO z{qLUO|Fh4v1-eQL16TcxUgN9L&1M?#?%S4Y+%tD<&cTHNCbXt4jd=HhkXX?;JXm*5ihYK1Igj zS)?m;3KM0Ob=-S~QXQkH;=*c{@|dpL^F`Mfsz(aXz&{r#m=ZreyWD zuUAh%>`+j6hb&&rGKj$Rg5%*3boqiWaW;pAmz$4MCk~+YFDA%1N|}Rt*+qKPo#3tI z%d|9Tkk)eFOrBl~$9ePqwukqUO82dt>juw?@KgSqCII3+0$s&M2pG~PE2%yRBErP& zK^UpIp4(%k8xPrp%;Y9SuV&_8!rBlfy}l`bCHfqNrDWgS#;=0#Qbc%s!Um~4V5^?>P&~gNQiS%6tAm4d>*YZ&?)wkP*u1&bMH4kkM z5&wKtdAwWSZ+xAN?7IfIC!jk~Vp!~`($T<*)OS-0g?<#o}r5VefsY$K&)5h|LysBhF#L_ zkJkaVs^+@I+?B$YBjWCy`GyDDgK3QpDp8mRa9{ZXbTe7Yu-ZP3of!X?Ce+enW-^Do zkF+`H`a>1@$?0-68ja4V0sk_=EyQwhKfq|iIbvflKLulvE*pL|3VN1rJ(g~+ zlj02veMt^$mZX%k@uNCYx%kRi+GTpbn=)8$xdz==$ehmiZ67WZIc#PpKNpPkqUS%# zGmNbmz6SQ=-u^;2AmCd>eV|hF!#hm~wx3aQz96#OK;vhCWdl})(OIS;-hQAs=;ns#tfh&;vixKvB zq?c>|?W1CE@{s7^ak_}Lbz=TnA8G?U@I1^N=qm69oBx(m&(?AM;cfge0usII-O#yZ zYzyx)kyasmvO(oFW&VHN-ruc!NxV-fQuT~9v|#glt=#t1S3WADul@xR0Pa7d0@G{}jY5m}2TtWw>$=9Ud!0Pc*hL zea=b%XW)w}KAR!84Q;|kRof8_BloZ6>P~1gx-NaR@&2lR%vl_)zdeDj7^OZbJx(M+ zuaL)ged_u3K8}rHJe_}N{^+XsJwfan%wFg7mfzo6|Bg!#13tK;yikaT+<`a(4r`$|B5UqRPZGVB6&u~~9- zS*5>IFm7quj7i9JGMV~yi*cnS_ZJe+hg@?7BXx2a$$s)gD|8yB9UW};9#EY>OgQmOUv_A2 zSYqM5CR4e1?wc`m)Bfx}lSR$`_cbmdLyu_J&sBZBR1Ab^<=(c?yx*sX#l z$=Dh(m{BE1akFfYtLw_T?AgMlRDrUfxh^>-w*tR7b#> zW+KXyMYd%>yccJ93VrRf?>`rqks8qXe$v(Po#{=@JkZ1==Ofl zNzJ-WWM9GY01LV&zB~cCIXjz?o+@l6Cu|3${|M6q^M81vbmnnJUaq`^2EFV4F=*U( zy4bXrB114c&sk{IpyZ>Lay!8h@I`0=h!+lY|4a+^1{Qt8V;#S4xhD3~bQXv)kon_) zy^G1YZy^j#V>;~fEVoWl!Xhc8{o%eRx?14L#?2#I<+Vd?MiU+nTwlP0Zs=FM6N|Ks z3+MwWWKGGUDJ+G+AqLNRCu+$yfBz-2-kpKHythDNt>j7@>bCLQYDLe4+kKeFc-|d{=lPd z$r^H(@zd`7`)E8xfFYP2c79+At_u-CcTX?vo090z?G3aZa`qqQXHF+rNZDlSA59G1 zF&Q}}crsonPgI7##5P3}{c9uTTQq5=_ln!I^7|lNcyb;0dw~2Rf$pmq<{!NgEtbFV zEge16@j~A$>IanVhQE*Oyxvkm=GnIi)NBk`BJhR9ZgVh*&!oJBXh7D`bTS^QOgKSR zgn|3j$e`Q4jh$}zI?aD!gKNRda7AL9EcUsS_FjucQUp3;lO^`_SejaH0WnsQv8O7^ za=*<$x>o=paVkMUK^Ph3${!+zt)b4;I(7&v&2G$2PhQ(`%dlvJWOQYiCCiw7ZW|VTR zuQk}<3$$7HHxvt)Jf?^0-E{H7#l1@Z`g$2X#G;+hTou|uoYkd91>;=41D+v9ykXHdKucNA6tV zC)ZqsOyXC6LP5KWqA=$NxLBZTxcJXAra7V~5?dfFw4X4Cosyyk2D=K5jhSD{8nQ+K z^;cboEgP*bo;Dx9Q&z^`rc8akXuG{>Lu(?HjA~C^z{LjLwNdVo!{G(fQtjV zlI~dv4De;rlTL!9*>+a;XuV>(lMAD0I)ArfkID!mZ&^(j{k@?TI3uaVtdiDMGN~hH z9tZvYtA$UiI+%WV16*9tMVq3&rcL;zy)fYNnv<%6@}2b^3~pn!Ucd)gXGH>Orw2IC zg4K7nd`8X@+x=3XCawfX6^p~Tzt@#v^>-Q6A~2UBVjPe4ZrYAhuWE? z6d(B;6KyPNtO!-(Um<*_$%PZfsOu8(5X3PYoI5V$j94tT8M{abvUpuc21-Sa2l++%18{+1n5Fh>z%Za01df3H|EQp&!Qo zL@c0{exmU7t13UZ@-Aa)sq2otn_L9@G?0L9@H2n+KPYM>`i#c?XI3?0(l?>-9P@j;_3<$vcF%kHOWx@xOk4_cv_ zyq{qd=D(L<`J##rmxjYhyk9I z%+{!r(gozaKc|WmSj06jy+?b;n=|=Px)ll80eZE1ws-h7;Q2Eu(0z*5e8UXKw&319Zjiosp{)GU#fb+7jG52Hx9ures4MMz43s5oeJjht4}@*%#hvP zb6|b`Bj{4H^K7=u)5oGP8>@X&)k>jng4W6ozJIriUr)e^=G}om>Ky(xcI@M44-)oG zvg6IAIfIZ-#^9PDr)zto^hPT%9`VkE;Yz8z{O5KA&pr3(KERuf@JQq8tz ztuq9o2s7l8JCAQ~=jMY2dDHv;!r}{BnfmZSm^HRT&c+F6z@-7*)Qn1Q7c{utU-!YZ zHd9g1jB9u3aui|@!|#ucccW6P_X!wjWVR_Li93ceed~&;^i;je&YLo2-XlZ@C@Dph z04^=)-tQy}p1&Wa`(hxZwQPTZ*19*4m#EIsGsQ0NiQ}Ls%b!sKJ@lH+i9@lnPS1Bn zPaS!e5je$y{%kBC_9ydqE8xIR)%@EoG9(0w^P^9q8)@JBfS~il0nGOY(uPLh?HWUX-T0mwPm+m>cT+0 zpFnp?q6m)W^FReh5WPxG+|Dt_#86(MXbF`8b*F-oVK%2tR_;}9nT2+fQrvvC@6WVP z-deloe0qa8Hkxm z{gdB8%#BWZ{-z)g{W}sNyqLBMH^&Jk;W1_OKM}V(z-0v8uZvtgeYO?Oe4ES5hz_?g z7I}jElOpG9U0W;ng8COTZ2w&#@v4*`JE>BIh%t_#34&j;hakop*_CW^cp}fs0bC}~ zZ5p@c=a7`qtrCTfNv1pF3TD_;^kQbP?~2auJ2@l_$#g8N1+pu0VfV5V2ex)sg-GHFC9c;z7wWJ0ImyJK{uen z;)VG(&0ERqM-DPk>#MqBt&TZW4|`8k2QLnq_#)q0i^9{b#haE4_c%&M-9gg0af2}f z-%aAmTL+PsdJd2WHqa%C7O`NrVE#y6eD1SsKbqTN8z{ZNuoG{3#0lq z`$#Uan*J-LSEpvBNIs=eJ*=?S5E%=rKqoQaeg@rg39pwc&o$*o#W;hn#}`v1K^#o0 z!u8FNNHg##Y}kl|T}Xx4lDFq^)>uXo}j7Fqj`9eJ^>4J|JPIQ^!4sTTI6 zml2Q7zvXIlT3KhP64M?p)$o{#<8wTR0+5?ptaNWr`*Dd^7&Ftrb1Ph+tM@&?Oh1|u z$4SDKL~84(q9m67k2*SL<;^?RCjT=HcdO-bSuM1ULBywV{3HV8TR1EV2l9S?Mxh}D zhUFnbaDBlIx*HslzgY4z+ib)SPm@2syOIA((ZAV1{6X2U(kST_W&Gsk_L^Mzod#6_ ztIE)hgVk&P$h)9BBJGfXN-pPKm(M_czkqJ38pak{s39)w^lBj?l#&VJb1m!sq}b6p z@8FRIL2~?s2amu6QMhF6RokhwgsGnEm(`S=>fONh)3K5dfAH}Dmj`rHszo)3VN4bb zL}dny3vNT%p^b~7*mlQiKhiOhAWYK@6@F$gmkGv5^&#Y7Tld>d`D5Naz%k~8{uTnm z!8M)$xV)f?E(A%_){A>%9(OYQRZ(XVHn?=(P2dPOGn!%if_!1~U*2w>RKI~3ce?Xn zZdNP4^*znBL#oI%MD{wi$W8EHz~uwomW3~|Fq3k}yKM4)o3w#4mQb58n22cPL}E%R z%?PNq5gi=+w`kQ~b$@TsqWj0PPf{I)P5hzj#5O&lR*_taKS^nS`r7DBF&F7y5P`R`ifv&<#i;Y;LkOdjM(R4=#1@%58`SI$N6SY! z@Hxp3x>Do_E|-uL`A8pP1O2-vTa5EPv#aal5yb>OgwtV*l;iK@5@yv*ZUHUM_^gvDpXR69?o6 z2_J-w=41CzHnZmo5o=%U8m~;2DxMD3yppICbNQQ-IO7wEnZadDpJ`Qz;w3%1shjSRx+?@Uo2Q^_;lC(Al=?;EI4QYO6EzMgM%oU0-et400dzPBa^e}_at zS5HtD@hxH{SvKdSbhCKl-hznbq{;ig9-6GzJ2-VeJQjm~m@2UKx8R?;O`lDDZE-xQ zq^RhC)5X>))o+Y{0_!4Tpo`>jjFAv}GLjVv|DjQ*%Ps0N&gj7X7BR*P<-fYdH?HF? ze*@7|zJYEHiE2$v3Pu(aWLj@><5) z^Amlz#re?udunOhS3sIb%Gx=9efAZ;yt)ju}| z0lEo46ka6a+9@FWgH5Jhf^h6UW*8=jaeGANFN>sxkxE(~yuCn^k7{UdL&RA35>wy8 z1-Q~?-KY^El7);_BZdO;N`fw8sr#y>rE;=^!Ij%|a)Nd+YPQsevcR|{*5!DrEzvc? z?DWr5zqn}V`K!Mmo<+A*F9;495;9VQ>*<|q;nZc< z%;=an3wq2y7o44e7$3?;%7PmA9MYiEM4O&|dBBwg-HeE0fxaAg#;iO{iXxNvv(6>l z{oZ1ce}*+lPx3h}?Jha#OU2T7&Ym>Jf0JyqmpUaNq7rPRm1eU=iT-PlfDX8Fpew)F zw!kWJbJjD%%wll2G*>1!XT`wsO`}YY(`RjbE20Tysbx5}e#*1%{e_B!?q$w0)=)xo z$glMXziomVRdAfigYLA-_O)zK>NcAZ+SUC4VzB)^yOrp?%xLSB`7Og0WOhkNLuPR} zl!{#9kRIa@%(u}4yPPu?qfbenggvv0Za0B=6+oBSSfEsd^UJonUul}Zg<#z%IIa~>1>!sUs`wU6M6@uNwC2CPTxQ`Qn;&Q zMbPVgok&4J_ofc!&{oV*E7mw`81WMIW)=*iN}kO~=1-B;cc_P#_Nk%86*hReiRr)0 zjhAhUyjOI<=Z+%iF8$ZM{c5AVG-CP;$;@31V(;I}Gs(rmw2so(0kq4;2r4SPaTOGm zBf3jtD}yo5w#TM9UQnWZPcJi%^s=!>wspfpp$p_!8FT|6kvItl zd<^K8z3Z~N#G^@=lYZ~5u=TC&&AH1Ti$g%}LcjD`a8))`xsiMAym=96vK=j&7;!RQ zykL;ADhn2K(WR~v2dG<|_5BTF8Yx9C--cASDXr#JZcV@o1l5ZeSc zq*a-LiO|?UIVhgoXgSWIP+HofHU#XWr3$*gudT{MHuD3SroX|=c{6(bw~LB^Di;;` zB7BvBL_p7RMTD&1PU2>6(}sUBm67{RQ2ASV05ZOf#*h^BF%KlT4p0N#g^gKnjv`%{ zZe-IjHpPE#uMH}2buAV)Vq}wX*^^_||JoVV8^lsSw!QyUjiippYmvag_xWe-o@9i` zkn^`=4ImHdplhCWD=h1}k=ZGh-8?@Rge9ig%#5QR*XB?b=lw5)=42|O!b%Z}^mmAR zw2V#QvLedsnl_1lCpRf;bo#SS{{!G^fUc`mx(2)LPTL_q(Mg&eivycoV}e+f+v!OBbmRJoF!sYcB>FSQthWfej=3L)W4U%_(ZD9)YJ%=#72bPSM{mJ* zPPWG1Hy=|>c6E);Q1p@hN?h=qedPBkq6>jg;X~6hX$Y0l=X9t- zHexwSEr#*CIPMv=;AQt|NJz=cb_pS zNnR|H)=gN?^B}rEmG#j~$hWeSAzT=do43j&Uq8DyKn*LVKTFS8!~`8wsU`nT;;Tx>&m8fE7smiUv>{ERK@2u!-et41TSV{@bxnSS{|hy&p%@*F?5vcmEk*YU+2}hSWxm`==Vu0>8`GjGNgg6=B48LBrLu)a>sb1A@fo?RwI@i@YJ_-C z{a=ROD}HL5p{S`+*6vOQlV&@D|BEc$P7Lkq)=t51aQqsA?w5kUaVao@>-ZN&vqKBk z*Y<|vGYO7DOy-7Ac5W5LVw7@{1Fyqnn)eE;_slfP-=^IS+a{!hz4t7CzaKRZ1mFAn z19Z3bWmJ6Ni{z)^H>_b|DtHq5|Ct{{msly5prPiyWiAwaD-ew>?rvUpO*jmMp)FN# z>q%ooM~v@r5#$j>=LGLr8iDTMk6MckmN-$kOZj3(5>xj;ii%4L%}A@MN6t~ZE*^_- zXdkuv44-^Hp(ePR+f*JM_Eq*U#ngU4(ZrSZ@KtF7#+fnb;+sKsaK-+pu@Ezi=iDx_k}tHa7(N4GG%E2npXqc`j;N~<(0 zdxSULeKC4x{)Ym=OMq(%y4_cPvFs=Ad4-gjrhGj=Y#)4#Et!~mbJrs16av{L3WPcZ zybxcZH_jRD@DXSc%P4k{)`%@CV0l=<8phy5MUf9#}miwo3xL;tk%szBiURvVC_AT%CNZ z_ibMtCmG@EqX-|BC8&8Z8}wVG3hjVx_&+Uo{-^$C3A!CcKd1hClMT+aQ_QPqh-a1N z$kb{jbfjEdEEZ2G%=7R~#Jhe{-?EJNZ9bkzs=61KhD}hXgOw9^JoGmz{;AKpsvVC1- z9rXO1d2^oD!Mz8bC$I+HUwIuhSEB?tq!PlnTcwVQ+GK*^|1L1md*;U9;A=qCG~w2hMNUYv|m`N{YiN z$GkE8b!mQQbTwJ;+WtHG?c@9_%@Ga1q%Znh*aD}R_^6h&AZjG%c2KeZWDRg_L6^}H zbECnF45C+*mZ=~Yu4C;4BOxt`n~DRbYOBu;1J2i#)4j-I>|(zqF2%9oSy5c~MpeIt zWP1~0?*g*r89WDK2fFDATNpZp){#4usJni_gLUxhg6?F?`$IV}skG%k1%5I~7^Ts{ zhomFQ>e9J$Aj*#U-S0Za1dAH{9+_wA>IL^F?LoKUZ(`UUb zB_Ei@_m#sm!(3yclN9mW+sX;u_oj9%JaY7ipT&dk6BtnocuJCqEVA(SygmmB_)WmK z(GR++nb#U71Fj?JIyRLPcIw<4Ik9)n=y&blHI5IVQKhS}Xh;B&_bbRVX%hI2Qzi1p)7EW2$D*xQ4~n8&6g zg5TG%Oq+MN(EXx%M5%CA?bb+XxX$6(WV<+5PJhdHmqE=9;$T!6YyjeQ23=!ehX$*O z)AxsM#1>*oN#}1OFMa=*W^+YHcq8}zG(wZgs$!P7m8a)b?A6hck{RV(F$R+&V{?wc zNw|sra(@T7E}-j_(UE*?V#Z1+@lN$;So4frWGXAU-dKX|oK*;q#A)uBvDE{Vl*PZ- z%0|@l>EgoeTbjpN6>nk}%_cMH5@c{ZxPmU{8BS;y9*P++ugdtkEt?9diPb{_wdedN z*c_QqI^@)c(80pM&eGl8$g9(%Wo*)>Nk-2wh!Z?BRk$={6D+WP>;}5~p$ex4@qKvs zpKH*ph{?yB?W#)@B?3`@q~YBKp!_~ zyX<|+zhw&^CPrN!W3Xh@&z|RX0rKksx({`?P2Mou)d8oZdn-rF(oY!i2|anC?!z`{ z>KVFfl`Y3SbV-+sW?v6KU@$DUtz}m}2w?gJVQdbl$uyc_&;qU}=<2G)XOCNV868Gg z;+csoKv|C3s=8feVnAD*4XDuXVG)Jl4)UKNrzxz9B`dEw1Or@ z&#)s$v{%0EB=eaQE7j(0AB%j7n6Afk@)XL;U-ZLs#|A}s6J21hB}*E5yvQJDV}B=j zKfo7sxebLc_8V2FglK^oXd?+$6eHy-JxGb8i>~qbg5h1S8)uWc-cR*F!!pohIzljQ_yo8HtWDZjHSb) zf6#rQJKI=d9)cG14i+oqH$4cvBjlSb7dV9RwvgKw6K}|FP@Si9vgo+}f`Lyf zacgf>G!3mT<1;V?E{YL76$xAgaPi1($24WCbdUT%9)5vtPnt&UlxbgL2quLXdHf}c z*zW=zSX#2LIHZ~%KV|$blIf9A@Gsxk`0v#auy7>thrDdx?9F~N?$e908o|v}iU4jP z=w6+v7Vj?MeNnsK_SUF~T$(MT2qqC3T_%`7U|*CW^7QrBA)QK$#Z}Ou_8P>*du{~QZHLOjUV5-?Zph0F1xM83Rh21PDPI;+eVQIw|Vk)ns=REb( zso!ChX$T`(lU{1HLyGIBMxKn4#@$Yeh(Q0INt7`F%Ki~cMg;AnFWN_3zzqjo_OlG^ z(Cb&K!aNk=tMuAs^oR*UH<7hB4pv*ym4yGc+bch@_@o{itGkL98}#e?J)yi^tQDAj z_KjWGk97t!HJ4WyiRXz3yw!py8Hq9LKn=};=l>Bb zEm0c__rrJ?`j`+AS9OKdsGP21itW+5oQfIVE1vhAo8NkO6N6>z8{kHPZt$(t!7il@ zj`ixg>x;1qRyP83j7$*{O@Yq8C!Ugj63b9FqdYl3yVuwGuI}8GuNrw=G9XroL}z@evjLE`>=bxlfjGx&Un0bN=Dl(ez`-sdD%9p`MPJkN`CzLmj}?)p$- z^lPS?2J-J$JNf;j>V%*cV)y%lX(e&zjpgM68Cx~$J~=gq(hyM~53!(IuJNg3Xbf@E zCkHd_=skKbrKdaI&;G$MwxQ}K`IA?}grcOXuUqvg6bCYn6k;&L2#wWD{v;48nm?C# zNDMU=05=YFkIgM#oNAJ5wilr0>HQt@dP@!1$Q{Ggn!+v@AS^%0s3%!XVy(YT91Q#_&%N?3Wo2x`;tA?3KUj`;Zt-^-HN4@`o%->tNT0r6OXE z-_OSi=vu66nbNSQSSswS5-nmvGs!CsVAC`2E|W1ns_SU2-seq&96pQ;h}Hd{9MfML)1b_4Ra&@^Rz@yx^N6PDF!{I&vW0Luw#8s3Oq`Gg1|FJB=I^Kk@-uU0-(LE2Ao6 z-GO@J(xdYe8@isDSeEqT2c#RoO#)p=)$94 z6$+Atjxuvm?swL}QpPrlK*1GYG?mJjmX8>|Cmz2I9-MD6nPTHfvYGaw-W%`7XmNQ0 z++@&A3MhX5Z0R2Q($x{ruG1m6O?~7d6Met%I_#w@K`tXUb6qSit*u<0etLF^!4u9> z-z~V&L@RKuE$#@_emoAI8%zP+xTk(IZ+-*SMlIfOr0WT8XA&}xL#|SjB0TFcmV>|X z_@l@lXa+UrD+AKr$FeQ=OZ|j(9af`%*W?tGOpcTR?oX!vKkiEKOFM6AIMcfUJcQOm z@9v(8FMVe;N!7@~sy8CXYaCiWDJ5yA-zK!YannXFG{k-GDd2Zlp~Ak5jH6Zm`~c)3 z4RpCqXi3ea4$8@mZHdiKB^p`ADNB?SFL}cIO!`DH8_{UGbgJTpsLgoAambkNPk6_0lG!AQDAnAy|9k=82CI!^nEC=NHlBSK(sYd=FX zeWeuYC9g>9UHR5j8Aq!M(dq5=BXZOu<#R=NNieu>$pGDrdN=aj+;ndA<&RC&I+ zSL4*|;IOGC2si&BqH>4i{LgQ~D7d0}_T65le>?KAwpJ`XZB3JnudNFxEl4GSJY<6I z_W?TQnoWMd`BxofgWus?=My`o-bNi;rJ z(MPBb@oL7e=!)rao6O)m*T0}wt4bMpNx-oj0&X_w>O;*bCJS`^h8mjYIrm2#ffKSj zq>IyppHd1Co%W>~e^Sz8O+wg^py@Q;F^=LF@-KxGI+)mQuNlm9;<8U>5#9rt_-`ya;P)aAbbFG%GPvXn)0F*r5Q&kQJ{-A}Ry!g) z8TEgEIKN4=MwY{OvKHkZd|6qEAdWU$+{_9Ue?(=i|0d0vgv|pR8 zie{G1q>@)7!dJZEcptC(A`&x(u*xYg-a^ooTh#uXJ_pU~LQwfhgJ{T1HCJqmQ4NDb z50ThLeRm-QecZ3b9Tpavj59O!1S3$_^MPBV0!OE&LON+UIw9Q`h_?uIhnS*w88%>F z`{Bf0X>15i)@9iZ#TYmOdg0zF(hTbTy{b@!W)l$AcHQDLQ#Dc6I}uBFN(x3ZZ4cKO zNGXa2_jigxH>-FlKs$!qUAg)AIT-unpjcCS(>>WXYYQHKFVhdh&ROLbYg3F2Z zAOo&1%0M^DWrh)XDmT5=-1GNfT%hbvEyAO{bT}q#$>tO@q8ZkXObprrfiUKu*Jc^S z)+$0AezAf09dRsNMlyMD{$!g#yyc*KS`$CQa3`#tb~mgr&HRyZ8E;Nr^n{gEQ>9*i zkylg4AWS)fggPTgaKQ5}Y~c!zzU;boAdPPPdl`M$$A=YgzEc6Z?O`4Ye9AYbU*>#1 z1++?kxf5tTgRWxQjMoxTc7cCFNw=eu#kPX*qD*go4UfUp4CoLwtiQ>Y+oM$#t$%C- z_s=Rpx38gxG)YKbPU*Z6CXg`u*xvKp=~TZ>6IF|xrOH2TwDZjrl2@Of4Rx`}u`UEjmv*5Hi;dKy$LOg+8_CCrDiQkttQ$F&kD zE~TWCU49xhA;C$)U^OSjcuM)>i7x#!rT9k=bHWwjIVq6eYS8_=qw}s7F?8KuP#9_q z&VN7W_Iz-!B-c~dmM&PTop+%#fO1O0L_h+BYlkBj8d{3{I%GUB6rg=Hw z)_|@J;i}p%&(1``O&3^Mz9T*we3(O_SkB=oMj90r$*wSeWgHn)_wgY|>AP`?of5Cq z*}XwQN{xqRO#$QjyAe;otp(i`e&pzbtgUF3ZAB;>T?K&=)^Aj$!dYCR`0jZ17P ziSDJL8g+C1D#)lL<_V_*-B!?Ehv(NZQX__LC;v}fx(;;rwJ;%(S%k}tdk>>#(w3Me zw9iJJ3*$;Spv^LTVHv*6U3yFP|8_(g+W4Ilp~If5o6pT8CB{dxVIZHRk@5)kQ>X{s zJI{H?ZmgFv;iKi`;H>T_QRmIah*8q*ES(mb+Z12;mF2g4KS}*%-m=|b2>K0{uxZx6 z3eN*CyvD2NWQEY+K34#2*A^*WGH0RTFO?GNf57KtH4Ro2NwBMAj}$-+^ptSpWfc8p#Siah+QqOx zI8n<^TscfC76}F1X3%Zleg6xt16>)Cp8M>V;Dyb=V!!6OqNk-RGa*z4_pvPVr}?6V z+E_Z*`Tz3s5bRr8ZdcZwEhKNCqrr{Gg~0~6EufpV0ByCll%ldQ)z-ZAWPo)zHGpJJwCWZ3SI?)Bp_v z1-0@0dlX)S((gubYwWnm24b6z@;`Q4j9xJsJN^y8UKGp+Z64+)JPbn<$hCU znpCEaH?*HCfw{}%8ARn$@G}30Jt5XJNE>!u}nXT z!RjW3aTu-{U|&$4dYo33wSjk`IH^segUV}|Z z_cpT&d_Hu7E=2^UP?i!?eu`X1RpY4Cc00?mDrLFUCO^dQ{OP#R-BKdClND@>7?rdI z=4|q|miqdFngs+68RUc&K@_2>Ss>mn(B)k1>DOfB;}alRL(G)S4p@mPgV<3({xEl2 zZpRloh`w`!{^wA`gUhdqE|*Jy*R+_23>Y>{eM}D?GZ)( zHH0vCg_?O@9+nPgVU2c-0sy`>HxY%LtLzY$N@ z(Ki5Y59l6W{7SU2trCE?`CnAs1yhw@*e-CorMpAAySqcWQKY*&q&uahySuwVq`N~v zIs~Lag!6yToOfm&KEQ9Tx%cMTYuzgz>GbidSX%@CaoJAyLbzS4D(k;hhq7dkqng=I z)l0I8vy=($Jd!sbLlvRn9oAkykmYbZW+npe5a@m%gH-#HOj-QzV!ic;5sKq6_nF7j z`;XNoAJ~6Yf53y=mXMFLh)g8utrq1F{s-&1Wm(&!4XGL#va>lw#gGa1ONK$0yZ;xe z)I|HO|KGh}IucsFvf)g8(RX!(s@F5@aN|$i3Uu@>|5%KDcNmOo(Ng+TVJi&wUK%Kf zvQ8^teq=|1{fF1O?w6WH3`&?lJ-H576bZK4XNv zQLyEykfw~dMz%~6FJGLu{hX2hnwMHge_WA-sS9&oX6eb*=2Cu!8gqrSR$p(6?7nLR zU&S)?+wdN6M?sf8mr}COiFijfuH5n~T<}ts985rKA^v(68|u&S2xkwLvd)IsPkAxK z5l^Uhyz>>#hhc#$Uh4wCjeExoPfgkIq+UE2|wK?>Oide4&`f3t z{`AD)gX0MEHX(1~(Zs^4J6awo>GHnqYc*EiXm#rCdDIwfKSK<~6YhITL)Z6IdN;3# zIpZg8fI9)YJo(V@-%W+oC)YmVbi0!dp)d`}Xu}qg1b;2`@>TeWTg@uM;Pu|rcsvh2 zAH5o%T&mKV?lba{<5I__2#b>w5 zO#vUDFTR(kXwr;~3b3q#>am-A%v(LQrth|Cwq$_T3|!02ne{dq+4`JHsl zDz5~%)1W(eM+LVS&fC|ez-e9ernJV#dn8SO=F_I!nQ_Q zF*P4rLrzy{y1}xDy42eqp>02t(%OWpT^;ktO3{k9mWDsSrseMUja_Lw;ac}&_#x2% zcMfzN;Xd_=A+cnd%D>!oeY4+; z<&3peF>>rShbridbuj!|^;O3&Io0%mtLRtoa=4eP!`(B5^RRArWCr_9eD)zW_sX zD3Pg$dQK#g(z`pDf%AYR&<*4vq;s*z6(KYrE;Cp z^y0%Nt8bbd#v@+2Kj~3D?V_+4{Tq~}H+~I_jS*0XWzd}@lc?Pz%oHlU(QT#9S>*8d z5-O;c6h!oiZ|L6o@je#{YWN*|d1B|fhjRR1{;RmoSi90eim9Sw!H?Gna?%2Ty8^l` zjDrlbw)c9lvr2B(2w!iv<4X)VV{G2Ik2!!7SSEhOti$mC?YCb8-Hx1&e=QLIo#gN%MWq*EG>3$& zQdO~wZ5umQS1sSxjL2#Z=m@0YTzgCk4V-ofofIwu6H$a=98U9<-XeNliR ztBLG@Zx&czNAt`QQ^1l@!9yT?pCkEkDZUHrr*497pi(Jya7J^<%_Q;UFM-Sl)P%+K z!%LE!=TY}tcssOMhU}psQ^u>(LE<&Em+cJr&bR>#)WNtDnXudQeB66>Anz9Fs<}mM z)C9|?aku8-`D57WG%pe)!+gGgy6&km;3b$wB2S}|qk#OCiQS~gerR)Ntd z(sq4JpjQs}2i%{aTh?>Inkqp|)e4*Dlhn;7_F*5HQ;@5ouxYsO4Fl@rmM&-1pwod= zv;t#?V>k{{dCPil)1drnfh%FZNNaA-9B{WmH%aXRVbg>?4<&7=Ff?5r9aAjwyN!m2 zN4E5`pY>=Zr(2-7`fZ*+6=wOAd5xGQc|nb~TJ}V2(L!CgESKhYu>ZRQx`?xwvGXcD zxLvg&ozitX%z+wHu{WYjL7y>E+|1k5-Xo85ho(&>203WLa<7Z)`wMg$BGha!pL93c zT7;cqbOCvHK{x5Y7r_g%TkyH6`3YLMTzhi0d{9}S(Oe|<;4^A``ThHEff(N{?>>uusMTNM~7<-T&fw+j2d;2NG=}@G&TYjkdB=$rTP0~*eQIJBi zsT0Tuxci{%$keORPXDOol*KeI9I3rdYQ8mol5WfXt>wOHvVAj>z1x9&3(g1eVW@RT zbwaB;BU9Dxve6+9BGqBRG=CEoa1TJYV#2Py)q>)kNYDFN|1if;LqE7K48s3B z1ApI+KzF|qq07{$jJEhPeWhT|F{ffx4ZLKGn0&bd<*xjz$BV84hC-G_2|tN zd3bP|*?*Ie{Uyds1#34Y)h1QxFE184|3W^L#eke0~nSr0j@?zSVIq&!8igVAgT z5hTdxk3@)0YGH+-kAjWg0eR0rH$)0QSw{~Zqds}h$o0ft$}bWwC-0KY45nY+jQnRU zX=u2vx@=ufbQ4sE=dxXbS*R3FE`jUlJ9>T6OOrZ zH{*AhUmn=S7V^OIR#7kVzZtBzdtCjk(Drhk)x}RmrMj@*p%@F9n z=dC+2zHDFdkDBpYHsJmN-JRSkse}gzbk4d-w8tX^m&#TTEu^x*CI659&qVwYX$oUY zu~cD)zo|A(jV$NVsRY}MA09S6cNwC>|$!+rLiXZ#H3#l(3j!C4EBv~K-X?k zi4I|B#TioGfcQI4I&@T`YnHnz?E1R&F9OcbmdgK#!@glklqNyiMm&vGRAe$UtOzDP z>Z+Lu$;41woNoeoZ$Z}vy~E(6QQ>9u2d{=-{is*vEo*2+0g=tmYsEkNd#B+!a!1X` zh)+Jv)4(2^s0ByU39)|BjrwMw*H%{FFxa`($v7L{rSZy2!W9fL?A?Nic#0|%nq~H*?!^9L_6HHI@!zdy zQ&i)a-@nm;edPzx9mSRTp@;ow4y!QQ6Yp0Qw=H^hs1GNqeP&fhvyJa_j9wU4 z@sBH1-DLTiBR3_T|HzG%^=^(f{C9#F$om&`%`@3+OCOR&-tGH3aQxU2OlTgL1h zNnuA=9Z86208bEpcw6)xzHIw0T!8b0OYu%-R*@0+P|LfUG)W!&p1pzYbO@UrE&L%d zc0!?<$`~{`uKLWpK!#f1+leF+{RDyy>)z>KuGr?(ZMp?lE~jks^>M9$0!(Ag&Q~Nk z{~BrV-VTWWd5OvY`-6E$@<2Sqc%^jJf%#%O`_N^i^mQh&fZkzDJFOlW9gXzSi zH9>}E-YA7S8c{~1jDW15r;4fAYl#Ritm(>s&i|g9fdpND%d1^4I#k)V#Cd^;*pC{* zpNJ+hqhD?A;bHol+eLhh*OfFLEqslS)fZDH@v&)I3|DuIWgMoed`)e_);ZMy7YcMu ztj^{72TX>z5erOgsZv#xd+EM?3sB(aybAP;=1%3w!A911=0@4uJT@hKE9|5zR^1N2 z@rjg${H~P!6L}mQM?r%wW8~5}|9}Vb=1r*poJ@?Fq~Eq%z0p|^uB-*zrJlbmg#D61 zap)1M)urWqLLAO)?W>fC)DfBi@2yo~MaK>@kQWAYS*~feuhXVjt)$d0f=d5-?uiog z>vXt3eb3+&G4DYp#539AQI!l!?&UM-)>7b#`xm0IX-Wu&ykjcn5~#S*_{FLqf%sV>;{E(OLJ z4^ce2kS%3~g5y3o(2cyJ^65PHb$A!-20QrG@Fq-Yk}&v&)V zG=y!qC0CU=Sxqw}%ki7+L@HlCpF9fen~4B<;X!u^K4;<2*}7wT6~W}fY!S3&KujqC z_a7fw8w`l!hm!*m#mY#XM+c!~b2}f6ubbmY%iTY>M5QD6$Og~$_*|6L-js2QR~vfz^l1pz z&rc%HGfZ3oE+Xj8b$9eGAvTSMiT7$4i~kTgQkFgaW)yI>kI^*PC?!;39#psk#nB+jSzQjECx#TZLkAcPu4!rT*U} zPs4?t41ufbem>K3ys&GVf39<0Rx!s=COleREb3Ipy4XjThQ^5qfQt;e+Zjno7X%+` zSidkvptG^H<$emWnzCVETaIwVyTnxJmoGW}SJN2W$-l!KCu+kH@~rJ@`H+V5u#j{bS3PXLhfQPF+1HVg z$+8kU)5 zy$}Hx4RjF=wvN?ymU3nU5)_9oLeB(VYbZk{=bvj5`>o>-m>(4yEU;m)YB{Dz@8jAF zqfL(k^1ac&%?0}_B@HlIMvMV2I_Q$D`fb;%Z`!4Vt zD<9;Pnst--`&GF-lHvb&g?GrOkd*jh4BojX@2e{av8$TJ*RbzPk6UR~Xa;5*r zfCUnx%)D-A*S8)G-AY$D`UF zbW*RvIZ9ld=0_vVjc$6V_g7+2)K$5Wb|JmvYoatLnR@pd_h7I=R2taogo<{~{$oGMEZ=jC<_vG_eo?>yse&I#o?s8S{-h$0 zX)2h+W99_g-5*#>X6Op$U$919=BOosc=(=O>AsVlSL=!^yZVq3x=U;ZwLeyu+ z?wdW>RNmzQ=*|1?3*&ie0rC4*+ildm`{nnOY(@!s}h^zVDB#|%|8qjAx` zZF?Kxt2iJp8R*tHt;l-$x=e3PEPg1FR}Gk={p_cso5(>}V=px>a%_uwpqK0$hBdfW zo(=J6A;wg5gM;#}KE=fIatps{=kX43$w3!E_r^Goa+mLO6!Nd>234Qig^#|LNF{l` z-R@BMFb%^uCutM*&&X0u7eQPcaH~E6BT~{@m}AD@(O#FkV%$FgE(Pcw>$!F`Png+{ zt<_Cwl2d!hnNHVQjX+*oQ45cST{?BK7HCJ%)SVyUjGo=Ch*r7&CUb^3lg**(Yw?xd zGSf%_TuRW@)xD3NykAW#P=5=$oWV-@J$1(MkUcM>(VuPJwAQ2c9_BvEjoPb^#c3SD zej-H~r#>;)-gEs8QhuW2O;z<8aH&9d>JL^AhS+udaOB$SMNzR`Qo9f7xQS*JX(KZ=T| zmBcKaQeA))oG>Z39<(bS9Q6$-80BY_(Um~HXUGVG_j3qEPKt@}X*{UU)6^mR1 z&X1`W(0lep0k z_9WHjsbAJHEt0q%Ew|d);B%4&bQ|_tm4!S1yIddHRL<3A+Sjht?BZttCRTInM*^xc zVpnC!`r;_D&qESyYhwZ<5)`6PyL9vOS6IEF3YI~LHZwqdX+c+T9~bHIEBE^t2vJR! z0Eof2ihM+dJh!_~@d3OY7#Gc`547_H*7#;Z^t8v(c=ccEa1-dQcNFrWN>P5=q%`IO zE*Oq0JRj`0q!p&0y0HRrM8%9~*=GvFLBGfDr)lj(N;!@VXiGtr||c*uOK zT|+bYrap9SwowSG?1TZ_A4Lzk`|8zN0}IK;>wZuP4!>{`eLAl6HWIh$QF@W#>#?PN z&Z(ZYbS|kpI*OeBUJF$?HgNiWN?g8`h&KY2UlyE;1mtA^UDpBxag1G;44>zPZuDXQ zB>wk0XZfm2(-}LM(E10{vR}}z8KbIB^dm!O)aL%F@7k-AgjXybE00;&b2*{^Sp{51 z(EVtjNqjA8-C&#*L4?4|Zz3y{n;Z>GX($}G{&S{e@TBIqdsG+alk+z+%NjSfkRNdf zQ}w?NjOrld?&FBQQ;q^I6X?1!sUMi3BLzCiSI~2GO&ab&W3($5El&~qOgTvxHkJJN zQ5bggnK|K;6Wn!V-e2pgG;6isF4#5duJQEQ30QK#Wd>a}XoR9pgFj|g>Ba7Ed$5|# zXZbNh+6#D_weji!SxU$jT=y+LV&uH7XlS(0I_LQ{OCsKY%L2Mm zUYV|Ib=JgOKb-JoAO;ti#6%kwoYv3KX#@X+BYIn(6#Y)vr_FCkUoTurHb5WQk`0H} zp~F%(A=CRBCwdS5ZnA=IJSLvi`StS13l#&gO;?(NpGLg(fc6Z;I3CB1&*AX=mnz#d zI&!K_`Z3dA6$I3S9Xn{Qa?OX*j+#{}%iHQdfxK*>8?wk9R+cs4A1}}7A za|(%?E9HcDB?nv%&<*6s`cta(Zs)Bfwfgq8w@Si~6OrUwwUW;8Ubg8&d8+(E=@thf z5uEzsRm5k3K_)!mAOrm=E#Z~+IqDnE{yxCv1YKgghW!0i4f5F22~FPu#9&V^%!qo3 z_#5_>Q^6CscyGOQ^qYUf-X(90|AxKvYn$u)PG+Nst!}pfFT38+@KqQ%6mcit}r8^3+Jo6IBU7G4;R&Wl*+puIOZ?jQ8;77X^O6l%jh(; zAzR+i2<(pn9IwdTJHm3lTVzKy17}r$%LBTsKG3za2$XzOi?Ei`i&k~^MpPY1;hOat z|Mc*^D>+?mNO4xe))@WvbdUJ*HWEzsd42sUWgFV9%qX@$#T@((^}qcnUeNWc6?opb zgZh9iRHGrtdScPYVYgtv<5c&X{AP&u7k>t;Sp!l0h}nCr-ze^OVd)#*{d5p!!5flP zOzeG?yFW`Tq=owhUMQN zls%tOce28hj|$XcH6YJ}nLZ33Qux19;{VOd54wF(m52DU$y79*E2Z{Aa~KA&-g&PK z{q+GP&41;vD(&UI-EgZ!c)&3J%gCkdrEMepJH)Ylq9beqgQK#=*aV&@698Qm551cZ zow@H80}oF#G!*a)wYA~8O*9E5bg)sQlpTS%NW-MUf%KfWo_3y&PGZHdt{wB}IIVr^ zab_A_pX0!NHiDpA5HiI)Z<^>sua^SwrGr)|0+<{AZ zf8FcNrOX!R-eQT>3kPvvn!hqA+#DnQ#fucv$7yy!I@A{L7X|d@2=KWHgD&=4#UJ!9 z{5H~C;o?Y5Pr!$@=5zx&N;CH{)?MJ-7eJG?=x*QPT+T(j6<}jTddr=ZO&y7D_v9WdUSg$XX z4VQ0qRo|&4ke^@4^sW;+Q(%wDvA_phQPAC?&XUcVySpLZ@L<3`wmtos2Crt?R+Dpv zCoM|%QbxGi(Yy6{Okj05*%+&jw;81BIvM##Vf(}7nZ^~=Zk-w6ih(ZglF7G%%b_Oy z7z#~ntd(k%P^F*}$-!*wF^=SN?L~SYo@a*cxAf~Asmd3Bwe{-Uj?stRNmU8M5WdWx zRyrAgD-OE2sGI{Na6Dc&1q;lZ#V)I`;cUft@buni7x1y$f{n3?+$t(Zc!)_8Vkr@! zc%IlNGN{ug=F3ZN?iwSMx`R1@D*?LFPw!vG15kR$w0YaQKEnBApxKu#_+ly?&?9V79$;I%%M>66qGu*nwXNWi4nh!L;W(bjep-on(dUE8(3u35{$-^`N+M<7jYJ$%6V$lWaHGpKV6zpfM8v#k$0gcLze26Z4VV`6olB|6nmseM-otjyx*zK995 zKx<6)PrHy{+2o78Rp0_#IncG}-yXhj?>EAelHot7J+PghceZirTXd#ug@EgD~6R7mj^Tc)s5&|2b|j zB+6Ka_$Uk^P&E9PO204DH+T*YM8VJJ_ftMn(VYMOh^3!FB%#Q%TlrcPmnB2IF~Hrj2Ve%-Nzov`taCbStTb*!-L|Mwbho z$PeYxdAFY^im-HUKwd@A#jKBDqsd*_n7h7(^UYzW{BD)S>{pALkNf+z&TAk2LZI>2 zq%Hwd&&P5?7@TL$yl>M4a{DEjui@Y8ghk9o(g9ZqbSFNs@!NhG%d{~x&6kP)t5-0b ziHmMzDt5-gvv^%i9~R0ulJ+znSc-dGldxOy5?#|O#k&>Et94J`nrCV&XbQL=Kvya$ ze><$~GU5+8)Qf1{hwcLWp7MRlHZjm0rs8myY_n_qPrUdCsDzc}r+8BH zLyZ;@3exe;wLb%{GU&#m^_+`GI)4?zcafJa{i+RLLnp|n3^x~ts?>R;7|OuW-6P8L{ds|gC3YCC&A5}XA4sj8qGC_$^t z#ioW{hO+BIF46jQ7jy`-SB2s^d(kTXB6dF^bxx4Ql3LGOzsD|oW^PpKqh z2r=Kk4t=ry-DkIVs6N=cnO;}#uKsd#g8dS8&~0ysx@mt_6veherz`k?b*dFLH)|1> z{%-MJqgI$-X}x6Q?gY86ofHe}a5BjDof+YY~6LbXwh=dz?5N(ZyZptOXMPTtthPoOX%{TO4c(YsLH`MM(==||dY(x&u zL%;21CYv+RC80QebXC`bETYj`!2I#>;g3!TFhg@ zlGK=$Fu?`~VWhDm-iAYWr!UP~q)FGw?a|wuA-?+Q#39tp7b}E7UTx5wsW_YC8s@_& zo@^*}WYFfbK2b=Cgo_>yWkLQM^i$F*cQU_l_w{cwBeg8@53kjlapQ#NAQuYbuMZUg zD7UE(fU5(#G|~(F^VRzUZa1ZGrQ1;3=c#30AsUdK+ev@(DV|BM9eaCA`KK>zpW}eB}$nf8efXnPt2b$Hl2q!{9|fCeN@)8tzk`0$e@N{rtPqmObgGkRn9v z(|hirX+$wRmx^d!V?2V-|33MLDx#{*MC4d?FZNyZ{b40M@SC#`xiB5UriLcCh1#_G zcL2EhpgX7}Jho+q;rccA4=KhcnIa9Eky&|GDYeVxcqIsW@9$yS7gw_jIy~hc^^vdE zA_?Aw=cG~Ze>0@Fb^kV$egLiy8G!D`!cP->!6_ZQ;j%TPtg0d9Sbo2yw^>tgo&V6i ztIC_bdJbFPv3R%~v7ebc;DA$5Bvi@thve4v8$_h{-zOm&Df@zfn3{aC&nWJ zz4HT`Vq-jU+}}L0mQt5W@VR3Mx*z|0=UvSQlyPChYEcsz4nnT#ka#k?O?+9>Sw^GU z0tAkUqr2MXA<`gr&1sb;>cN&LkKU;|%^Vb7rMTlxYj9u1XV5KN_7lENRPmz4AHrLP z=~ly@5*iw`tW!8-U+eC`FBG_#hZ?YSK|Q#ti~HdDnO94HTa=2Ad}Mt3oo1u6bw=c=9NwG&6*W?m3CpaeUr4$9>4rirn55+I zx<-|}tA%9i*;uIyKEw!kj>;HxsUAJd_68`+X(1X~{g8Z8cI^1Q`Ik_TQBN8%s%y^8 zETdOcetng^W=)59W~(0j!s(5Dk(Ho9r`?{B!Ggqp4&*ff-L;-I_)g^o`9Ba+|Gnda zCn{u*8I}44IyOrznZdZU!eVL*-E&g{-GGO}JH?=YreBAg z1>N#i=D39a-m2m5e`ik=b*X6ssmjix0mGyi#4U?(jFBDBo$#I{aK`VHRJHyHq+ zB*|bRT(){d4))h!=Mo3tnt^WKID6dTlU4W(wxWe3Ha=e?SIpYe7`c?`y%e#B4LwX) z9)9_9*R#i}e!~x8x32o?9t`A+ugK`Wy-qmzaJZ0wYYw`rn@)D0cP%9{4H*>e zemL<#I=6^OMKJpDC}&}qYAf@7FGZE_oPsH~I~8K1oi5XzauCk>9zmO-lhNA|a4kT0 zd+mt+VR>fq!y=C8+VsD?wSilwn7EKB*2XGiplcjb7u z#nRtpuC=PTEwhOf@i(mM2WVqK{R5v zO6S&Rrw>J&7~p)!26PuXt{)rbreJ@Tiszrbqrr*i_S1R?*LW`z`S$l>X?atj6Mir2 zs_?Zq4(-F)8ooIVrY`&N(Jg|pJVQd63=`PrvIX6nCaqj}A}x2q@Mewj;MNO>`?iGI z6~^GtxD!qW5>tsK0o2<1WHe0PbU*mc;q!-he79JSmaj(Zg(O_?Q=o%@I@p1(mT#T* zy`k89qJwm@9Vd2IWrjo1wCY)A9x~;AMyTMVN(?HjE??PL&Y70c)Rqd?AMY z)^^2R`RjuPz_kZmssZc>hMg~+o61T{TU^yd^zJQ;U#rD#>XdR09o^KSoh6h)K1u9v z^~Rk+9v#hGy`f;w_CnK0pb-wNXAofjX{vxt znNG>=JW%X)#9<$2sT6P0=gKx4l&)!vnnT{Z92Zj?;QTUKh}HpH9Et9CoH5 zVDsoj7@CNAbSYAE3MAz|MHKNg z-jmfk=f)L=kwfO(4vS!5qD(fF@IDa~1e1GWXwCjDiR1VN-e=3$}X8ljyB3iT$K67#$3 zv7X-&A&5|gE(iv^e>0Ki)~y3|@C4nF;e?vAUTx2f3MB2rr$OrV7E3oS4L_SG>%{QB zFMNwME_v7CRAboB-Rxdbn(f7nrqK!<)b82)-rYaiDVo4OffwjHY$rwZ*PvS8^OyhW zUg(AOU(O4R)E(E5v7fY|{E7%qo2%E1$XptJ+$mm3Dsjy8*QIDxDjHopN^MC$0Ko)Y zC-VkfQ|XjY3#@ydzcbz1jFAm;U!Y1gL`#=%yBG*t!@zY0_J zK;8h*?Z|#ock=BD{m1e`gIn?|rc+NLCR6H}k#O26JWj5xMR@^5Kign{wV{}^OoLPx z@0m{=D#fs+q#~Oqi+_JJ8*l?bcd`ZxzGdq;Q-Z}>G4tqjTs-ve?E(MZ&}m_C%2HRT z(>N~K*yZaEqH*Gd0F0P)i(O@#?UxjDsw{!97#vi(aKH@$-QDFIDuKHARZ7mxt7TSKOABC2;t*Myey5EzurB#kQEy^nXXpGbU^)1|^AyG}dSmCIUd-5YVmKgS_^@Sj?ZOMS3L4&dvC1&BpR3)|r|!Yk@9nGi_>m zuma0>fm$?PGLXot+Whd8Ub^@hsVV|8m;PP2YHS+dhJvmwd33MiNhn2wfDDElmC!o* zmCLp$=6|=nK_q>*le~E7jP)X~*YomARfW0z5QKyjeGpKvDqU?i`zf0!>Z)4?aKk|N z2Zhr(;qgl%T|hyBuT|6P3SOabby|p>e&0&HlqBObr4a2p4}mw|%E>bq31o_@qy(bw z=8p#s0=y2B2R(`nzzql8-*d7&Hn}(y%xejDNbTj-6-Qe+6i7%4qD?@smXR$%n)=lchZA(|{WZx)XUd z%qXeX>Ae1!V;?0 zrX&6!qcU7!7QYJG9nnR7Ic!W~XQ3sNTCfiu4Z0kwy#Z_1?^oFHVzhN;rfax<<6)XM`Ap@6@S*cXY7d3x8C(f`5OMGE`Zx1dXj$@C=2flxerFY8cBR~C1d_B*%0WBZU^Mh)pi#8Vp7 zSpo^+XDO5-z>Njn`ohmEN2Fqek8)$I5>Ap_f}eM}PqcAlY*uRLx847Ac+To=U|F|! z&#ydb<^EeYP3?>5giN1uxDa>#cjY>43b=8g+tX}SO)9MhTbEknqcI#%2^U61(^_tK zh1z6+n3c}Zx+O0OcO0jOV20uSgrQ9opAeLqb1@+4FDd`bU3qPT3%K#1t0hc8RY4%V zxcetIYRE0tKB}*4&&I!>9Wyn4_TgV+ov80m+#gsKLe+jaB+}o=Yxq^C_|dv^*KF|c z6J1ZA!2aY{&=tSJZrQHGnPtX-hq@wfG}-!WY^Pq15WO}Wa>h1rwY6C~010*LT(&t) zh#LNn;s%c6ZdosKVLJK>^za+~_e`N`M7`BHBqh=4yduW5TtJB@W8%CBIy3C^OUFjw#FSy zk$|<+650xa zWY%GzzDc0_mJrM9b}JjU@HKFgpsbE20>n4^AikT5U!n%nvH$J7pE+e=1Pc>nS52MC&4rhAeu@c%6Zbe%gRI(^agT|QtH z7HIvqn7{i(cKr`CyVOx2S<*MAe>TQalkY|!*CyhWtYiA*li-Nfm1)tg_#zL*k@k;! zgAnm-B?(|NhvGEMuYeAa} zy56JHHh64eKP0sHmKkrr%+L$CX`m}Y&9~myAQ(oZ#%$0THzAZkm2&a8Eq!!AEf%)! zIz9BGk!Mxw$ue6Of zU)t7;=hS~sht&HuvvNa@RF*DB;nv%?X1ZD6i~vLa+l`LyyNEKF#~KE+0dko;f1!vP&n{v3-fqB~w4vwaeWgG~K3 zV^jZ)|GVEW8tl2@vl^2UO^GmfNKqmiN@H0*ku+YK0&XVgZnyXHmn285=Q0M*6aVFq zVwtib7%5zEuy~wMfJd9^uqGxstEO~@I7T>x$YF5UEQ~t0_EQd!K#t<2wcyzX|1YvY zS0uR}a#7SoYj*wd#n$T!C4*!~sL3`wsTK^JvE2vq72>Kmh&5*M{(SWU zxOH~z(Q>-NsEsa|;5tJ#=(e>x-6x6Y|E=dry-LnrBF!$Ktln8KK1fnm4vNwj3t!%SCh5Rghq_ng#0D~`Do@slV$N#z}ZiN8|rtJQjJlg9WCH??SrF;Lod zHWQ#2FiK=pg$&Msym_E28#M!|*{>nyhvpwZ_~C;3?eC}XSyu)MxiUv+OboeI3S2|d zOI@ve>j7?*f6f-)hJQz{NF*cw-ecdm%&!)32i$zn<(?W<$&Q$M((r5Fsj!3+$rOB7 z*tny20xushNsc?mEBFq>8qgWax56keQ+X!5f-(9` zL@g>!JlTO(=bR1)N#aI9J&Wv-jq}~m4eS#Xf$lVJFe)tx%R$_M%+!7eEpCxg=*scK zXFIZKmyL(aziH%mf>?=%&c%IE%$+_iaIlIo7%VxfI-#ATf|XjvtAaosia{4+cF5ua zV)Z&aa?w~t6XD92HueYc}qc;D^~g_-SL-OrNrcW$m94J{g-XxZW<tnGiPE7(+-yHK|iHD_?XB;`toJ%2mcQ>nUnX_X!yRw;FWugLlRG_l!QI6}CLl(sjf{xb8zO4`Y&uiKU8H zO`J||QuRifvn{9Y|41f&6Bpj^nc$I7i|8&~ZJ<{X&{zTYG1Y+Xf7e3lf3?sQ;mzr? z+kmWFX_<(ezqZ{iPR)C7#j&)OBNWJ1O_(vBH!ID4>Zrdv=t%8o+UQ-B7o`1+?qDs? z0_3d)-Jzc(;?aa%j0f+gc@3R|uC;hAe%}4OkqAfj!my+G2>o9#+c9V551tq~Wnnyj z6_rrP&DVLG5JWaR8t8>PPYb}U1Ks1?d*1KwY~x!8>c+8me!pCMQYbge2}o?g4la6a zF5;||T;C%vdOyMn#^W{P6(nMoC`AN2bnWS061<+QR)GELdeD`3G@mq^=rNmFgcJI{ zgJMBniP}ggKR>JX-S(D~G$6+^GKXgkzxy{6hEyrd4r(FE6PgXRX4lolka_k_?mh#M zw*hpoJIh@Sh2B}pcv!%mm?hu{Uq}{BUvjSo?N}2-2ii!#O=tG<{L!^z6D4a}o%ARg zT+0+>V+$ZI`Frjb@_7pE8#RLNIsum9GJbAfTu1#|NMSfE>utJTkPB^peH;gcnTbps zTAYD^JXI3^hO0wvv7GgMD5nomTAD>t!n#)Cgu@}&S8f8`t@|Vwf4OJk9&MfYs^>U6 zc%Nbv-$;K`BP`6@IN2arldqzWnXlq~s3t=OP;XmMe0g(j#K> zAn@gI5Q;^7+4s#8P9R~r4)Gv4l1}53G~$<#F9Ej|bcNMy%Y*f=a3}v=^8CRrn`46f zm+RvDI|lW<)v%xwnm#V@FBw#dnk)agBi3K-_AMr9KJLE!`+%*}XqPg%6Yz6u16^&r zS9_Vhf1$}nRivnNp&f_?_zVbmPN{H2Wm!IN?+{lb%^qncXtIlm_X6cc57iiL6yn_b zgW5l5w3Q>zMim2j+d=os{9H{*fN%MWulp(=t8?J&N^{!V(SUd3h4e2grdYi29;1|75-StW|W$*VZ z8VzpRs?ER%>3$zE-Lzl&G@pB;Te2ISre)gFuVm4&{#)jZZx{W~*H4aBFIy^Ez6U`U z<_=oYe^S+7cg>AHeBaPv4eJ*i{`6p-Mq7S(nRr2^6cN(h8Zq6>yF3f~7RcT{Yj|Ow z7B_Nbj#l&hx?_oZjG6ZE{Mgm!J7zAix8}}k_XlR><7%n4Z^=6K+3su2GWy(U()M(( zC&}XMkGKxl7BSu61UYxT>yR?e@LBcyw&|U;ROUi;Z?9Yz%x?e1--|`d-)z92&&N*q zYJB}PV?G>e)ayWvRVfeFeOLEz%Z($JPgz>H;_V3K-5xRB5^IykD!1nN{dD0OGPVu5 zRJ?lbLQe+FpWOHPp>n@}d!hf6jHn}vz1rkdFA=(70RGav60GQw+Jo@dJ(JFZn-`8aGzJ>U30`X||QX^ zv+zVcQ|`YsVP%F)sVlZgB>P)6fB)uCpZ+7ty?fcP(q}>W9DQP~Y5ncCk~5zEvCF$_2x zL`Zi}#B?_lUGziS^i3ZQUVpvQm%*`Cl%E&W#&?ka!-=oAj@gzg$Hx(h_>OtvIY~MCSM;~E8>0Wy%E#RlKn-$I}b8F&VH`$qAdMe-a2=- zTBD)4#>MIR>`~mFXDf6Hsrh=+!qdT77JNQ$;`M~%E0?cx`s|jG&%P~t@{g*m&t{1D ze969u=~n-|^GN?8j@4nA){WVa?@&trNh!OYn>2Cm*SUu~z8czdS4dc&F@-19n_j8W z?|v&HpK%XL9 z#+~aEbZ8g$b@JLJeysT8y`b>D>ozrup;e5!bhWP7{K{^LWhfW$@q-*t8j*qULyKpp?p1mmOJFKhD`zvM9NN-#$ zhl9CXxKDt;XO|G=@aZz87Z&EPuE}vKw!^_U!oqcI%WNC`XZ%2Z72?;er(dwc5t9u@ zmtWbk|9y5b`iP#|1#&n#_y_*2&k6fqCew9v^$zvZK6=Ey)0WwGk#F06VB5g<16jcK z1KS3M8`yoK-4+Z-u*XKbPqh0)yH9lOH`)Dx-5=Qf!T-+wK>Hwpe!W9g`tE+ty*XpQ z&O5}|O%5S+ZSv??=h-sb2A|>whOc=C2I&sNU|t;Mo!@Y!6KKn98~h*fgBWsX{`8(^ zL?5?hvVd(D+XjXk{I72xYUgU>3SI47$9kV8qKu#aV`LhCI|T%FP?=)zAZ!wI{IzAa zUE~{CCpjFvM;-ndq02Y^FUWLl!G0k@0X_X3L#8+!@lC(~ua+rapO6s05FdwQG%u#e zwunAiru6!FcMDJ&y7%7A+<^JNAk(#V@ekGK{v0i5xa-K4*>;gn*nVK!Kz)R_Pf&0m zVXT!72j7qj=g^k@53+$#M+UwrSwPm(l z;GfPWx&q%ItGKP&$KyG(ZJ5>mie>RmYQZf~~i zU$cws7wq3H*qaw~BR;oo%WS*g8n$oPHjo?Geqh_c_5Jnnh{(rZDeBFJm(1c}?1ksf>fzvfz~PcrF=b)M z^!$coU7U|}ICda&-Gj7c|99=8I(MyII^}hbjsKHnwx2rJA9lO^zuPWpmjCpR>1?>` zB+WJLKidMf1#Ao07O*W~Tfnw}Z2{W?wgqep*cPxYU|YbpfNcTW0=5Ng3)mL0Enr)~ zwt#H`+XA))Yzx>Huq|L)z_x&G0oww$1#Ao07O*W~Tfnw}Z2{W?wgqep*cPxYU|Ybp zfNcTW0=5Ng3)mL0Enr)~wt#H`+XA))Yzx>Huq|L)z_x&G0oww$1#Ao07O*W~Tfnw} zZ2{W?wgqep*cPxYU|YbpfNcTW0=5Ng3)mL0Enr)~wt#H`+XA))Yzx>Huq|L)z_x&G z0oww$1#Ao07O*W~Tfnw}Z2{W?wgqep*cPxYU|YbpfNcTW0=5Ng3)mL0Enr)~wt#H` z+XA))Yzx>Huq|L)z_x&G0oww$1#Ao07O*W~Tfnw}Z2{W?wgqep*cPxYU|YbpfNcTW z0=5Ng3)mL0Enr)~wt#H`+XA))Yzx>Huq|L)z_x&G0oww$1#Ao07O*W~Tfnw}Z2{W? zwgqep*cPxYU|YbpfNcTW0=5Ng3)mL0Enr*VudqP(ZPATnj_h-W@ydh*`*`{XhJ<2*AK>TNB-qcdT$aKGvxNBf^$Y5lKZ{ou?*RW!fk7SPDf;^l&OaUdl8^^s zD*fQk^*b8JsSSz4;b)$sJky%8j;1UIzjK+gPNpm-zdcP^XHyo7-=CW@e^VBl-$hMX z7gH98-(^i%S5p?3-<3^SfGN|3G&W`3Oj&$>`ua8u z0@)r#Op^l2wn9fTk3DHDNDog6;%p5!E-Wy4KbdSutkvWzfg8IavDWnUuGVMgd_%0`*zWkOcolznB&G9$}m%0`>AEXWpt z{>GTHtQ^lZ?L5|$WkWXKl#MfG*^w87jz zvNWdbTT`a?@`3Wnmu8qUqrI5t%`{~N(cj@b`PVE{R*2)|reDoAWrdNwHT__YDJz2P zy(ycEOl9^0wJEjr`Q~|QlQ%d|zO=xU72~)t$MTVdrmQ%}Z_WB$WXeh)^W<25v)GiC zexZ0Fe;P@8y%SE`xlvU*TJ?G^nTx-fIar_f9)$=-2R+-~N$n>|~lvUx_-;`}I zWmS8}w>Rr;vni{AEUg)Liz%y#EIl&$&G)9P7RMRQ^R}9@ z+Q{M|lP_tYkU8o=0yB^8rmQZqgr;nVDXWLfVaj%zvigXkBa<)fGGz@oR-2M9?KWi% zIquD|d})sl;fl3wH+{JUm%Ol;2?V+ zM88obo@VX`EA^ zrT$3cmc}WKOIaW*WP|LG19F1;h1`$_)bFePt6x{URr^!hQopQzSpBZrhT4LBU;U=~ z$>N}XQ2n0zIrVGm$K?0wr_?W1fQnEFDnk{h3hF1SLk&>7S36g`Ry&qYtDUM{svWA` z$ye2`p)$o2bw1}fQHZr z8bcFk3SWTcOwFMd$S1l&F^-Ev2`CArpfqF#&8MQ(7{_12A&w8j5m2959~wY)s0Fp54pf1PP##J_3AjXAFRQ%p8(am=E3bp*k~iTt z+=07rA0EK(@DLuu3RnrNU=6H;^{@dp!Y0@ZTi|=x3fo{gjDx<=5Bfs@KB@U?Fm!`1 zP!*~}4X6pVp$^oAdQcx4KtpH*jY0F<3Q!c%Lk7qQn&YN`BrxBDITI{|MX(r_KriSG zeV_+~z%|Ns9TvhO=m-5_01Sjd5DtT32n>Z`FdRm}moO4W!B;RE#=uw@2jgJ^^oF^# zwRxa9^eSJaL02)GLs10?XK4=}2hCChclq=#_dCZAVL#>m0S>`o&|FV*HqAdZf7QHu2`qqx&;_(c4ul}+4i`x01T2K-{C)|q z;5EE~KjAIBgZB`P^PcOhIjBD zK7fOG9uN(pLkx%su^=|Yfw&M4;zI&R2#FvuB!TMWUju4FEvOB3pf1#d25^DAeueQc z0VcvE=tdFB683-=@C7VI_8lyP<*))y!YNn_g*Yb*WCg8H)|37Q*a(|oGmHSOQ%1r- z7zA3Ql!Q`H8p=RfCF24AUs@Wx^})8(f7C#C;4}-yDQe#MQc{D+EF?ctb_#iLd4-EC9J7FKB(L zb?Yc}U%`HkHSh1oaa(8y?V%+!fu^8!PCCc{TFWGWx0K}#9ELv79|pic7zE)k7=}P+ z_z9Mh)^b<@sX^-y-&Y={Fl zIkz|@hn$cNvcm=Hpdt0w7`}iO&b+`d{;2zwE2cUJ@6c__tpb{LR ztZ!jH%!QednEFWqDIh)mu?gF5fgP|DcEKLl3;Wzg)44&{g zl;ix0kPSAH?-=sv2VcTqxJ~-n%hBG=TX+K6vpEbug7#a|gZ4zCgU(MwIikaO($_vl zZ@5GLUW8ep5EO*`j(=}#q%0niT;A&-e{ z3divYClLziwcjuve&?L)5Q}tTLtGewekcrs5uiA6h_8JC#d*m2zrbkZV_>X#&Pc*h z5RY?Pb8Z{Z+P^(`Lkl<;AzlZ5U*Pve&^rGLT!!D^xCVRvw*TP_|Jj;eYnist2DG+m z1udaDG=ndoDPT@xJ=B=vM$ix%Kz-1-SC6n7l!Ow{iwK&-hk?fD2B3a12TTXe*EMfH zOkB<3HLutFz6w+Z)ul4c2cN+Q;yi;Vpt;L5m<&I2+C;($FczAC#&3<|8qZ_GaG-h| z!w82$O^yc>hC>6;+(UDa?2r|*fclRFK$SS+6DA~#M;HfUKs0!P>^VG!2XF^8=Qs*K z!H+N%zJVbS4qD%9{~-ZrFCrNvg+ve^;sHq;dnqcj^5^m$RJjAQI%pCOs@agMOu93q zbCjNRLdTLN0qwo$SmcM#AP?k*T#yrTKz7IqnIR*j2kpmXgDj8-H3;CcB6o7)@39kJa?Ss^Ta-jX05>OV(fc9oegZ5{Nf^?D< zhhpY$9ZOytRL3=-I#h$IP#&s4WvB!dpdz^RI#=h_0+po&)HRQlu6{R%FQ5@(h?7mk&V^7RK-TC$1q z*Lg0Pe!KE^$v*zAv|T!#AMv-2BbGOW-~FK<^o3yP0j_#=g?*6eT-R^akIJXI4h7Yt z>Ri|1%DXtn%D)$cL1%E4RoA5YQoN3!c)EU-Eh-!IMBW?Z3$8LKR6NNB!2r

Gwcz z`J8^sZw7;ob>0Y=0$;*VPWVH(7Qgs=z}!UC8H-+~7yp6i_N zI9>`7&z;Zji08Y`orhey*`|I5;T({BE=WEL6xSu!@nTT=OU!UL;Xc?4dqDDCuoHHG z%Cil&!&dknw!kLX2ug1?Y=HH!4%WgNP(G_*C9Htuunc4e=@d`8&7eGXo5#`#+3I5( z{YPXj+kKoyOwv`I9EL;S`hA4sWAJkX*>R3ffX=-MH{d#)0>!%or{Nd42p8a2I0t9p z44gNQM-X0vt8fJ_!*6g4l=f}75BK0MxXP837&?ka~=6MN`NvG@4`L5qOSLYRmP!tM5LGXn9@EPQVJdh6x zz~>-cVJHG#p!1|pLRpo**5wsJ=~japkQ(_QLRTB_!*PGmUh@|a0!^U_G=@gd5HwGf zFVrX0{#adz$L~6XiYI*w2!BcUI(17Gj~^?9ucJAgN| zhqllL)c3U_Y!1yped7#JKP;K@ZV8I7e3h1DN?*sWc#_TIx6)S{Do8KpawhX% zet{Ek435B$@Gb0vy)Yl3v4!jZztRadtevrfSs@#et-k89}b$w zhY1hCPmmIRhNB=G$TpQgcFGB-;3TN+N_QF*o`iWcg%|MLJbps>2gnDEafD;Z-orb11v>XNyadIU-X(Yaj+kD=<2Ri9 zC%gs8Bc>7YJP!sNt&cV6L<9YnP2v)2k4t-zi9vgl+LO^B{F~BvB8RHATi<6XV z9Mc$-99c|A21y|aB!mPI3tDqdRfxxNXTs_bo8xMxJUYkaIo3QdKF2yw>Bfn0j!UmH zOQ&;$(sjkrH6(%>$drdGeOFwa|Bv#PK4SccWmFovKJ~4c2~&dZ4c6ti%H$ferTSmdsGisl*hK=BnfH^^snta!>JFYuq?+uBo;KgrKEKhS=+epe(c z0~Me=Xy30i=sv6N%anqWPy&jB_R6&vSqut;_FzAUaLVaQTYavMb)QCZ<)ct(Dz4Jd zZ=p0IhAz3{xc($_9V>L5uX3v#Dwon$xm`L}sQIbuII7SkQ+$=Hl9`5%U1ig?ROPp8 zE+G3=uven9y)+G_<$eiJoWkN--9{s0o@@8 zxINDV*mdq3=jWcI9!SC<@RF?aM_dsP*+G#*#Q`#!uT{sGN;5J-^TW|`_!%axexhm5Q(Dht{ z-{1;dhD&e}F2JvF4vxbiaFzE6$11~NLglYKkAbe`tSLJ|cm__xFQ77<1f8#VLgy<~ z938vT)pe=7N;9f_UHYivNG2OAzK&I=vSn1qs%w`{abz!@=PH|QuJT@w5MRG#KbK@b+5XXPR9}Fv##@X zt*)}Tt~;vV(y48|1^J2Ej4KV<>>aqu8!^7FMfDudg8Wcv z=^WWu&v&RDeVmpnjcDlAE@V4B=OLTwnGZetp?bUtsyF$eo)M8B>DiE25EEj6(pGw| z`c|ku?BhIjy%A;q?6wyR;lD&5VQ@AU&jWg~V~y zr_$B^O@&HNYnq5QhH97oK@hu@0tOSqFTkgyvBKv(DlexN$kd5fW=sp~>0 zA8^&R&gqQIA5$)Y|0V-cF zP`MTND`fJep8R%|LpIm>5wFGNv$`%e+SO z2RQHXAN-HMhljMg&$v~lRpDZ7!bZeA z3wjnQ=2_6VdzKi@t(ocC#!nn;9Gc>uQW*EZxaSZ)xYqR}-huJsqw(@AB+Wu{Wkndi zYFJ#aI=QCyFg0wZZ9tO%&3A1spT85mM0+Ej;+{ndJ9e9K9=84YN`^d(@1QB-Ng3$N zOwI1cy&e{?^7~k4YQ;T^JAOl>5@f5er|_1*vyY6_ig}XH6EyMBO#Hm%*jyXSmW<<3 zsDx*sq7+VdBI2S+yLv?C6OBjeN{e}xBr!)!?5MJ(OPMq2;Ovc$pdnHbl1qw4*E4e4 z;*)85oyc#bR?xFhA>%oUAWc!vqMpT! z{cgpHQNDiL=R51ZFyfT(Eb82?&W`3??UHF5XZBuYXbKX=*uzx`US4s8Y(G-|8XA&P z38qsv*o?73jBM8QC7P006c^uUUg?VhYr-4cy3|W? z3VKp`UVG4_B+l%@P0kJ;xJ$Lbby5+=uC46$Lv-I|nYTB~fTp;!e*8iL`~yR23r)iN ztxmG^6&lPY7tx5Bj5v?V1oX_i;HHoA@gkC!qjOkD*FMxl>W4n1OF!JS*ob4)wssIz z>KBu8_Bpol`F6!A;#t_Uu%nxQuy-fFE+MZPEQ{YWMHRDbo?gWr-Gf3xeL_MUt-XIf z^q@|h!A5Fco<#~dLc9_4*XVYouaZve+*_%67L${{HS?+U>ZgF1j}|mRQ-qqs-(u6I zxaSZ)Cuey6&xZSTh~wdfgHS!jYE(8zf2ME2W_`Ty1L71a&h?}tP9ij^-e;@2YxSvd z(PW*%g-I?C8r9tM{%LQ-+dI>NrVy>9uyfx?)~V7gL#{#764yXe(6ew+RJDkc0nLae zCl>BLHspYzDM~iR-iB&BciMpTV@{kLZKPJ58ZvqZ<#Xdm_F+4EU+RmdFs3e4%vk%Y zY#Z+{7}4ZlmS{$5g=EnwW~qlX_AC->NB2oa996c_=dqU)UhDFe()n+{9p=_(rLCS~ zS>4>bK8#7SAB~r@KiH2()+twbZ0l`zPnJSs*0xq7O6|-`@YUPvFi2dMjXQr z3Zl{V6bu-2I9UL0sD7mUbwygv1zeYois;??(mZ0;Inj5dg%^VAjdOpfl3yn&h zw?WH1BQlkXZ^V&Zo7ueEIg}Y9>O1w~<_tCd85Nv?*EP(UUg4enBID z!>|3jb@L0RY)hI~ay+~-IVKwQ2Zc%+Bb#i{A-vL?7T?>md7EdUf&t#$Xr=5u{NZ`fC-2QvMrwr>C%~__ zzmIo-qe}a?P1@W!)6f}5o~ZeQ@>v!)L*jg%nU101VrW8zD04S&e_u!0M2E*bZS|r{ zG!LVzq82oVkadD%cD;3{!h}~wYQ=P=hG{6<;Gh0DU%pY>St^0E-_e{R8F5y49sP6N zlzL0i$PeffH9Ju6zbb8+?(YLf{)&dSt=jIP3gs_#@1Y&L&3bkeje1@#NTa^`s$^43 zJUP|U>#g#ki>H2q{k(nX5*-t6pSzrP!v3$NDPcIzLE^}-ug=`HzuStXnT&i2DxXVe zWP^TbO26^{vPnmGoLgwr=F%2y`0dGP>&l?f<&e)CG-=T!=(FQeaG&G-6^ByON|H19 zt7Q$EwIzCyih--7!O~=u1&v0g@cG;FPrCHv2^y|PB`A+ZJw?7%&3oKgd`~`Ff_xZ% z>Y|Z{>}p-3Zjr;+!i;>VZ!bq%G@2juZCW|&Z%YoBMnlahwQgu+(fv2(j`F>6Y$zJ_ zNR+J)nj~l*XX@@1tIf^wXjB3;qtK*5v(j_b@x(EhPg zzAl@pBpR~?d$7qZA1$zZ>4in93;CNG+Jc_AmGAf^KTssalFxLd4BBcXsnDndjjk{5 z`gMoZU!Wlub<3H}I9Zan@f^9as;(5zQJfGum+si&%#|fa#(kKfn&Vp3!R0ecP`Y86 z2QTu5s_tW#qpu1Jqbc&2 z{$ZV($8%;njK=6gSW$%WwU0nw$B*aMw@#HTn>^l&ilc0L{#qWtzDlOtOHU4>Wik9I zn@BxU3XgJSm0Yn%?ME6q7uxB^v$Q0C(;Oq8q~x<>%pcWKJnC~Wnnykrl|SYNp?$jh zIZD0=O%T7&Jv6F?J4?SxeCAB7?$JE-o|(M% z(Ur%~c$@CBH<`WbU zBww6)Da)4)*Tq~xJ{oCh++}GzDnaMh{h2Rstd2%~2$~STZap=erL1}Li-ePMTt{QB zOKK2D{`TFr_V*SH>DV)dhtU*>(+-Vhw|igZf3^C@-{-qEp=i`6R^IgIr4!rdgv7u{ zji&9Gj7D`^KSuhD-Qw@{c5Bw4QSb7@{P)LO-MYEZtvP~5S6m}-VU_n?V_b4;?wOiR zd*-hzmh9#twpWrSjJxIBnjC1dp=sZwQJM!$w$*cMs-ek(CTonubFVa6 zd(Exsh(;~=;mDB}<5cf4#C@fG(WFAt??H#h35I?b!3Gmd&708`vkyEqJ1=B#!r%_xn4rY0=VzoWmOuj6i&I4{~BNYoDv1HB?mH1o-I z^=;yJ`L2b#<8<)$55RRCzurv{dTYxsL2eDRZa;=ON7)pO>J%HkaxNOPY_mvBBgXV= zb*@b*cBH5~&SEp4G?^;3$~HLfMl@zEga!wNb?QuQ+;95lwQAXFx#RTIC$lKQcc0y= zk+SjVmuMKYb*0v&wl2X9N}yiMYsiGGrH}2&MQUcLJ?JXs2g90<@BBP=-o0*(bw1Xq zS;zT;X{zcpe8^z0ez%`%WrVkBm%wTzQP9zWIO@s2Uwi4mhS=Hb5=Y%KHt2&!wY}%n z?L7~X6@+~Nr_{6RUtIETzKhL zr~h2Jv$j^Q+86tWsagAubv`r5ht4lN)$1WCGFM2Xe9RK8LZjL~k~M9Gj0d}Ecg57~ zMPsZA4h*@yZgs5~M%fInx{M|XadKo%m;BxOcxlm??X+(%|4<*=sYi7?$6>7E;Yk%8HoGipy(&L*61x}V2PR-h@ zqPiZnjeq2c*0#3JX9$xjd2iAC-^}m3{~9xVSDj8kqkdOeL$MV15j8F?p(d0ri|G=y4-v(FGsA<;e&t{w#2RGI4 zn)Q7IkGJj*TI4Y2iQUg!j=rSh9(QWiaja|m-`|s4m-^pdHfvjlQK72s!*gP8X_si6 zc3aIl{RWL@-up|`te45-Rt~qux)!W$U_Am_$Fc5RKBWYgb31pkdfce^+wK{gG(Rw} z^i%5RC8^=`;h$Bmwx{y2hibuQYUyaC>CtrS+V@G%_06<{VQN05)Yf*hw$5ZqpczWB z>Ms|ZeVuvczfuD0vhB_5>?!Kis+cdl-r;ZEsa-*%Ud+4Hhw5d|Ue>yJJqe(hek7$MBxM5uD;TC zyIT|G+~4R!tmk4;#`!1fg-M>y`CHs6&y$s&Ui%~^FlYGI*7=ltqMDC7=VI3VL6oZx zY|7KYlv?YoYRiWNeA)Zp0e7jb=j7Ha7VAFoQ(B4jy573|l>Xegu76nB`)tb4UGume zbIs)I*CDJEv-WXA9bQ+{9b!M;rLh?6>0v*mcd_nat>?YgwzeLft!-^R{#dvAfB(qT zEyP$SK3X-nN#@1POAm41N%?DAuyt#+wzYK$e0+k! z^c+gwdc)s+y|7w@HF|$MOLM-m`FkB$@NS!W3UfVY-CtYh^PJSwCytx9uIal0$C{Fw zxvsZfg+{sJ|0jK-^}gjXDoNL~q8;oTCNjtQSZoZf|>+CKs zw{@c4v!i)CCKE?@IF3eN+_h@aHO75fXP>whO;R+~PMl5SweipCq-Nenx8CQo-gmJs zn{}_8z{^=bmqL2)UY)m`R=Z|xXF;Qx_sw^=n?1|1HwhZEud+_fx=uf(Y*FRoY=zdT zg%ow}agG1^&G_7pJ?m3yv$jW|(Z0*$*6|=&b%FBTS~ezLgfNCs%mL&2b$~ zN;F$mU7K0$`-a@DPz0%7p-GPBykq50SyRnnzY2{JDSl~ZYCbpTh0Pi5$6kOL#gPq- z+Ry5`XDW`p(xQ$L$9S49@?C81%HwH~pVBd&KQJ`CNKGX;I_TxLxp^v`!*23kO8pfY zwUPn@GjGW9`BB$B7RC{u03J~E@eXbA>#)UzdaRZAYAhknG~(!f$(Z@^X7-r0q@tOc z+FX$WeS?C!IV#Q?7-!am-zT^=Q9grVJhx$O>rc7TtCUUigCPay?0z->;6qBlLy+>` zd*+p{O|+~;nM{A`UYl&dmHxHX!3MrO4eI#mQPQK?uH?{egxOR4wRZay>v)!Nj;E(` zUD)bb<3>HaDocSZRH#WKI2jSI*fVTu3qG^sNQAxO2ybkMS4T-&~r8<(>Ub zg7ztYjeGho_rzTq>o|!kIOBL0pHQ^+cTIfVaWbJ%%>~uVG`VyCnAO~x{AQd@4INEO z<=R=yttoA4{CuCkd%n4MWw*wmk6+`_#d)r~0TNAmRx@<6rIP$7*V}$g7y=5zV0WQrLG`iAUx0`!Mh z7Y}jo-KB|L(P`0g>E?emvSW^}ZcSP=D)q+QZ}xjFt+&ms$%{rdShS|osS(j`RdQ=e zpyAeYxL5njYai|Sjh%W|KGy!$&`j-4|66CWOVD321Vb$^eA`XT$-U~YU_U~`6}Dgbd>#6xDPSbB6q~o5rD<5l5v~G>osacoWT4Oyr|96eFb_p!!NUY@|kEj29 z7VzIybCG8b*J>Ky0gd#OXc7;XCd#)+jG1?fgK@UYL39*MaOT=rI(gZ^qli z$}jR86p>4?lT{6p_ ztSMQ~MsxNQfA@}!@g!NXy3V>OHrOU}=FFM6n* zAM#;I=ja~nAJVN$h+|FJk_Ue7`^{oQQ&^fxq^8oZxVrx7yiQ+fZ=;Cw4VTDQTH8D~ ziWQi-J{`vy>aR!L$&#*0>p%2tp1DGfT!MkDSd0>6aoQll^AzTaKXpClSn@;o^3&7S zE{brEzaX0QszmX$iF4!eM zF!Sk%CJ~y9Vcui+1ijqh)|9L79OnvDtx_V*`7a{eZ?sO$dQBU}UVvGGDE0zeyEv@2 zKH3W~_urz}3ouhNAKG(xog07o#i?pJxgK*rq@#COKq&rQV|sAjbv^n;uuf#J8i-e^ zemVy6P70Bp#vFA0m-s(0q8m>;;-e{}J1y#NC3?L??Po({r$rBWB>lDi@;l6g%m=0; zy}_5nL+9%+vcDP6hVv|DJGFyNoMkJpYDmJYrT1o5o$4KmLQ&*%oO9(f zQ9e&@tn@N9b$iI1Q||>~hTn;!-sOv_ zw->Z)S!NZQqL_#MhIeSx=3b?ru|D;fqtOkGv2uNa9pa;Ly~*jGA8h-=+3Mp4l{t8? z;kWc519|tR7!Qy}wwvqio5*%^X{>!T@_Plco0^vOu16GpZF=w2W=`*&H}GP%>H}kF z57N{`T|dgldKV`uTkDmbf{x|nqy2-&Pn&%)ZT`fYss-d|Hls<1X5OOUzS|aOR0&K? z6z#_>+kWC0GyJ(d13%0h!a56&H~heQm)d$i(RxoP>bE$Cdn6Z0I*j9F# zcQl^&caB-sb!xp&zJt^>{&eb)q{ETheT|XH`C5$im3~S-8Cp2^Ekm~@-PQE)rT19W z?8TzIXGA~77bl3L{yJ^qDQy#U`AWUNx$98|O$sz`Q~a4f_lQ65qcLl)0UE8D9+ofq zQ}37?v5u?fv#uX&zqYoUbsXz_tW&eTgKJ$2ds;dBtQYwUWk@%-fb%It!)}rL1EaU3 zF+@41D6BV-+O>9Cw8oudYxeY85X-bRUGvB9UMBGf;av4zk@M+qYOM1~(8k#sa~>&v zC9qEd_7CVNRoUzfhV+(yH@@8JdaFy}dKSm&h*9)!x*nYyWj?(3S2T(^#o9Vc9d*CM z$Ph)GDa6rT|ETlPMqlI+kWy1_Ml+uR{(;@YLZ1|UlC;vZmHX+hdD>r>`j}MJ=DG*( zPV(fND|(LD9Jh%R7RsaextgY#I<8rBJ;OvFuQ52Xtz9Evg)EthvZ#(+3r-tV7GP zs0B1F+dD_5gsEP&d_3kiy}zy6#(x*1Q9CN%_G!;ImFKK5HN|;5*VN>!n|1E~@)wgE z8u~uIP=iLJ*@M+(=cU-#G>@U-Js-X?ghq4Ns(mVbyKPU^goZ}bkbjU`OK)dtGo~L( z*Q47O=kssIP4P}>RJM#6-VK_#?DkJ)+3@S`Xf#8KJ#*rb>_czqUXHGZEy~W`Awj(Y znOl}09d}f%-;Sd(du8ieO&`0oNntZJuGHEF)~Q)r=Mnj6TzI^B&Q zZt6~#aleFbEc-ZnzS9|YUu?W#|3X7!yqELWmY`EmcjH~scjX)Yw59V@-6@vcu&6I3 zNJH7?Cz|$s*w}6CN*S8M+m0FiNa%U;$RwY>duGk0W$itz`S{xz(L74fDf8(J$8F+hwh?Xd<3k-gZ~07bTs!AgPtfQN?(F8J&P3bilZp2Z z%~Hqi==7bzg-(a2_-t}>wDjJob)7Y?6V7Y>QobaM+G#h_0LW(2tLtuU2E=BL>HI3}^{}S*Cv9X5)^vlG z)gjp1$1jAhccnNNYwNz-tPar_ufTeAcKXij5!)&+eV2%x64qHpoCIi8k}8SUED0TZ zoSl+^<_yI@q(G;Dpbm~+!#dvbtRGesjn>e%4|KRS~ zGEYsLFwAL5${wn;ea`OwV9w9*?F-*vsM@azW1P2^m%6{f0$3 z`lB&tT#ba>KJnMr+j6ju{se`A&*a3#1nLPKwKJ@q(SU-uFYXb?v{YaM-v~ByiRO?_deFyCG2930e*8Ar9VgJutHUq2BVb53FX4V5TYQ$}Ca zloDj1)N@aK_V)OFuMTEv|eQ_J`pju1z)4Ou5DTN_tueLd{^dj|)5_bHdMv{#bXhre^* z50<7k#ooGZdlUZ;e(W%r$?IN0hv2Y~&M$t>oUM7r2B-f{Lnj;ToBm{po5$lUyZ84R zeVV|1AKjH&+%V_$#C>(_POIH%Ka1wUgwyy-jz)eku7CIBTSBtYU(?)-zsUFAU8zOB zj}cK5(v$B|KQ6Q2ZI-Oujg6rB=ze_!O-Dca_^OBME^mGN*D`+>hpkH=zefk}gjY_S z@SVQw0PIwxiyiW z7>H1|?%tuD9YwCh+wgYaPlw!bikKFyo@2<{>{V(Facjz&77dve_r~MAy?%9TYM7;d z^;4bC<4%8oS4GH&$7%xnvh{vcvE=w$d))QYkT}UG^^~O}-hA`y6}tqkIMz=YU;{p| z(%r8>#@gpzPwAh=9S4n`fN=CFmSo$c;d7h1H9q8{aUoU4W~q+;adV2h1ZJEFPffT= z5c!D?mxeeWJ@XSm^HB+0&utLrqgLX2J|sIHrP{t4kn`N4T79Sm#i7xwm3&k`^hk^h zhQ{KVlE}4fU4rJlob4vEzq#T>POSrRR9e$+5p3Pn)I_$mu2d~DvJK9#r=rv%w?T+9so0Q3-8NRcx2+%{Zm|Ip;gO z6aKt(z?y*T^u6ZHF0ysdn7#7He&D(uYrB!!U+{F3%dh`oTlmPYRW{RmKWeA$Q6|#$ znAVBh7F;u-Uk5ln<@@N}d<&%eO*;#kThbOjo^3E&H|^GZptJYy+yBS(OXnP8kIOYW zN0FL2`XmYn@@ZMon%+6guz zT}GpsP}(0S#=SIr&2qy-jQLySwao9Fo1Ak${+?YD562MaIxg~@Pm^`b#w>tOXYkRU zJ28GcJM8>=*7eSBdqp)*bj9fu?A^UHA99&Iw0OQwpBLC?Sd{O3@in<1U+)m+|GTgC zEIX@YJ$;XhnH`UB7PZ#2ALeW=0cBRK3VmSA$z=nYUDS3%8TI_Fq>TMLsinjgefN%N zn)-)+f13>!jOx32g*|(F()U=Et#F}e!<{2j!k6h*=jc7uSqpd^U){Cb0Zc(0wfgLb zYIN*;Cbw2O=3RuywSXrYchn-+scWBNzzC;xrtkB9G@;qJgN8+o9on&Ibbn#)iLKpF zBnlj7Xbii}L!((h;OmmF>UL?A-q0A|OIwL1HJbgadR0p1QKcUsg*6N~x=Fa+|qYl!dq<=@tBW@rqLFG(DY4~5Dv%-Mfv(sXFFW}?)Q z*TJsuzgessDM1&W@>V{hW}aylOFdD6$>CUa!jZKN8S?5>5U3Rpobyqc~KAuuruk<4C{P4vO_Hl|h+LM~*E(L>* zO`n*00IPT1@l>e;&9bHcZqSugw>LV>ve6cLqfrgkf4y(x_81+$b!$eOskK<1{a(W1 zJ-4GVS9*~>#HrD~a!61>Pd`VVLW_>2zZ1sOsIr@?E+ot+SkI|$I?z4-jJhd<(NI5f zk(cBnf18nV^SyM#w|7OuF05)h@)&Gb6zdq96x2`0W@8_wYf-PN@=>Hh1rv>Qj+%Wh zWKMF#r$8U01Z=eNtrkXF{he&|%kU@7!_?{v8;=Dzy7PtFP&QEpJ-pwiOs!ON(3ra( znMqCU=Uzjfeho(F;YnFzg!S)9Wi#T0cJ||IM|~VmpRKsNwB4Su(L73xa<)(Fe5!rrY;$RoB@1YhJ+b;xbI$h# z8nuPp>2k)})~@|;Xw)rZgLbASbF^!18$Nv49gRNRhaYtF5A^TOL+v54CJgYIR!TdK zS_m^4>r5P7>F^}20|JJ5%tXWfftqj>->;IKI*pv#I#NkTDt$&f$E+5+9>$z``FH2& zY?R<fBy7n{9o(*0B^PYy?;o8@>&Cp1Zjv$W6qpdxeA zSBzoQk39Yv8uf4Q0zB4q&sK%i2R$sl$gaXzXPj4W`gPvh{Od*0JU(!)JTMoUc!U|V zUCq8bZnp17_Ees~BZ7;*3Y5D_#*)XhXn5pV?^4e#asoJmhnVuIq{X z<*_KP$NVyB6xU;xEsA_hTSsv{X1~*Ag0mJ{94yx8Uai*;D7Dkw98p|pN#}@Rolp2g zr*#?(4A_2g?Rwp{QVFo=Xj8MZk^jDE?@vuYWA+DHWEJS@*XLx&uJ@-Km5FQE&8Y2U zlbn8CA?1~6rFO;9-7y-1QtQ>(U!Ndy9O*Xh<%0T8wId@97WEJD>&=$DV`=_ozcx;G z@2oq{O;VGs`X2E`Gd+m=$RSN75@x85FF z8P|igqnr(z;{q?lbn+waAB&UCj5evGT0JvM)v3kIj+&cCKAJrpO;GojmKz`C)f;bo zN}6l$=p7Q;gRtkdTBi!u8{5W6jeblP4GWQh66dJer2LfptqqMa(nh`>bredi!z|}$ z_GRq6v+FKcr@dS4;g{r2BpR&&OJ}apw(q1`wGE9iSN@dt6M2R|d9L$YSAP(yM~ZVM z<`u8%L%EZ}ip^;Cedak!kZxFtR=N7FWF_I!j6|a`xaxq(nqN(rXUclM+A`?X&1 zeBQZa?(12DCKLHJivBL;rcIaoxHUhc(H+ly;fWR;T6$rqTVrjVTf|X2?Rc$p%9=s9 z3%ldIL8E!dwHJ%FgdBRE+O7GN>-m(jeM&yoSNiWy?fe4gEMV5aOq=`P8>ss-`U-g= zR{Z`NoB21(LTg0p)c-o`ISq{{91Ryb=X`&R?ik~6yMl~!1J%dVP_3`@udO-jd_KjZ z{)?Qw|BVM*)9udk?gTb)J}@2W-;>bD#2~{Q?&E)h!u-lLn$@JH*=_Q`!^iJT&gs0* zWXxTx^XcW+p}V(FSI3vNbHt1JdGGd0jqa7d{EMCIx#TkkRq@?i(Ao1Dsg*^eF)Lc7 z*Q<|KeyTCcoN-l0lM+p;)JKPIIaTYtThkhi+Ucs@l{|}Q32xxl^fmJt_~ol(i=L%> zkH)-9IvS07zP*F$T@AXjP2;vXqEADk-v9e`TPjQ%otV2Xt`aOaHIo9To_~GwZB`>S zqtrXmsMICX)$4L1X&#MPW@TR&Mkd5N~fipCh7 zuerNy-abLWf%wDum*ai6&b{#jjV3~jrzw^?TjR8^EA0PeOLOk>xc1UAqRB>TF|$ui zQ0soC67JN>qe+jZaDqRtH_3Qryj#-}jn=7iw*8X+x9+uhugzthP&9gma?k zZ0tEWOJM!I8jEj|QR-);Ca)^jHrn=I9>tL#n6+T>l^x<(fAz%iozt(|yk7n+dGX5y z-Kl-b_k^PK5Mz~N{XHQMhGtz)|2x3!`ywb92%S!!b)9K}14=Kg{ADrf!N8aKxG+e&@e z>vZibSbx#TdbVLbmh2)Qt)^Ob^~h1U@R0UyKZxQDOLJajZJkP4oTZNNzNYC_)~QAD zzNQ(+`m2Ibyvb?CiQ?T$QxnCToTes6m%W zI~+y}j+~ln)Qqfg^#|6PPqB`*S6RP9_~$z3th3e4FDBe;nw4jZT)mj}=xqJop>=O* z-6vYVk@z2ehmeJ^^*d_T-`)R|T3{+6l{p(k}Xa-h*Hdfv_z$17g( zo0P;OwJC|5n(OX=ThqkM=R}1++um1u~ z?^K@(ZcRTlS}!~<-@R7a`5oBRb(MOIsVUs_e8sh+c2Ns1%`7xp*KZ4Mv7*7aMRnbp zHE1*siC*HB=c@YCe|Bs3nwk%P=Id~GN{=7hnqSap&#qOi$&-uTnKjU@xrRm-&Drfr znxC`JVK>iJf=6bYT^(1vI9K4pRJZ0G8m*>=mAqa1S%w+A-I^%=wLxuIv5oTazu%bT zZ}~*_DtxrKo_C4tRW41GPu%>IcNHgYc6vy*85swb?eQ$PyVTZYbN-8*{Cmj0tIsUT zlyYzqkGZ6xd#%=StUbiK7OZQ|Iv;C|b?aja9b)_i=WZa?_A)w?wxUyoa3osV^# zPbtB_o{!tBtothKHfP+ulkfye@dLlUn60@W4vkmDXIPY<9td9tn24ve{*}t z>21!wH|xNlLKPApJ>u3_m%uu;k3IfhNiC|rH;XZKSiIf5)Aelkn&uOFF)mC)qA?`$ z+eGZtGiXG056tx~kjOo`OOt7b^BtR4yKnwj@Z9Lq(L9nO(TEheY_2&)f*Qr5_)g&F5$ulNBCa$b!^9G)c5UHO@4Qj*%>KhkHcz4PKL zBQ@i>r?qG_7pwRDK=XPRk1+;Q0`8#l#e&Y>0bLyq$Fb{!3)V^1`kbEcH5U9)Jb}m8 zS&b4z{z^P=pK7w<;F%U5Z^8jBT4Zeh$K(vUM`@TKif1Eqw;U62uZ<}_{&2f{-GD^p zs?XIkr#wQ0Ci1;^Z8;Q0)7{@UG}zD2aj4ShHb1l)X3VRM8|;p1d!6&%wh4CjK6>d+ zS+z#vMlt`c;>XWm=n)L%Q+%IuUiBa}Pv4TGb28_{qDJ8=qf!0D{V;J(`_QW%hDGt- zf{r?9)LNE|`rR*=cb@8K%%_`LpwV5P@oj@=zpU)D>20$Fjv+gj z{#Y&8`Ao2(d17iRwkuyf#_}1D+;Oa*m5Q=#M!yr~vtLF&Q9NU2?iT&D-`Shpt-0vx z_szoVQ36vF6zFHX71ONL@^)jBdNA+hdW>2aJ<(aG5uVA?{tiCV&A*$!Pl1kstJ_q` z9)If0d&Du_D)vEV&Fx86`gyxLlk`Ncyq7rEHn4bN4U1YlFGl+*`Gd2z^E|oyC?xgc z&E#YDRh7}mLl%8A`$5bs?JJ^DGaQ!5EfBaQ1H0FL)3pA?R!*TBBs#Wh0 zBa>^Fz|Yk5Y*skq*#7knp)su!h22yF^0BZRJw*>v(<-NX(__tVHw$KM<|;uH7B%Z9 zis#=<&EiAOow+Zj-w02fVKz@^xZW+_i6%AqB=e|w@sG`&7LboT9=p9jqq}__&zE1< zv_OfQ(L7Eg$%-WLVdqYX-NF;fY#E#~y?KwHa^M5wnuY>N1>p_(N*W8=HNmdkz z|2=>gAonfE016^h&kZxkj4iGp$R(F}4OVwoch^jHRZ+*x^oS^~3Zjd$D#Gfn2Z9HP zD2SpiD2F1TqFkZ~iVDb~u8PO1`2R*^zW4H7nbjko{ryMmdU-@-WM*V!WMpRKf}t8O z*UC!6{j*P!B7BXA9VcZ|Qg(RSt9Ltkw}1V%-2Go7hgyc3 zq{zPJ@}t``yS@FogMnd`GNCShCn>V?eb9neoUqFo_g_kij0M`DVZ@Lq}Ujb%vs&GOeI4X@A$G+JACadPTX)VT6>|^J+)u!%naSN=!u7)eZ>oEq|gU~ zt=1LiD&3)VFZ}6qPW$=|O;TtZNh!aV^AjkLK3&+j;`cXyZ)y@4D=8-G&GAaVJ+#Gx z^(}V!;w!tPSV>-fe?d;cSD|3|nlBF@vhW2m7Q{~qC%PY}9FZgqA*yeo4Nb>F8 z^`i$)y!FP5fRX;8MOSqzI@z9c&%3t#%Y~P2M2bDhyd^Ntr<{{^dHMHu|DP-1-iQ+$ zwKYxQszuLy{!xb?`P#FU0=@fEep{vBM7zw}^ul7w5v$bpTYdhk>)+EnU(3-pcqu6| ze&((@;_}XRM_@ytrptutf*H{E&rufU#Cv|W-!{9yV!gs>yM525T=DHU|N9}6mp)(1 zVWbLQ*V-ID8@gw&?>zr)FMkZHl(s14+yjjC&)2tj=+lQ?F)s2zPVu7FJYcacJ+Jcq zJ+9wITE}{9D{r00fDx`*cl8qoKJB2xF9Ak45ouXYTfUsrZ8_yI1!wo=j0|&0`Pu!w zZZmED?SDIMiyb$*lRZ7AC25=?ZU*V&3>$6v>pM@n?mp2_gldus=55ip=+xi-!0_lv zue{~x>Q+~N3m7}k{ne%%cG|Wd8(pc=(#nH&rP0Dwf9>wQ4t~blXU=>FFm_~-g6_~8 zdgArRzWeXpn_ozZa4)>`+uQ9}_~eSOKlt`hIn6C&0ggZ84pX0R?R&~w*8Xf_OAEs& znP^mdJ>IMS{zvb>dguDn!5c(`(O9y1GmXK zOHyP6H>YZydat|ne>}3|PWRt|+@X^CuVnnt3$5lDa$;!crpphx;QV#fO%)r|4GkRt z1zW*yH{SBd7f(Os$Ws#v)^h8wJu@_P!-wzr?mnlj`D=0mwj5hI%x!m>nojTg+|kdx zfAknRfoFK38ZCacZ&8v)}T~*X?(o+f9Ut>%5@z2^Mw zinCt%@xPqXIL(#=OvzkNIfb04OTBWrQ?7@Ulmi7Nb3JWPKA)D)nz#MLXdCxkb;*`v z$6dtAl@YAczkCKPtf|gf_kwL(df}MQet+%bcke7W_+&Ptb>4KJSxH>7$H^c6-?#tc zd8F7;e>N$b1M}FQM}Kn4w$GH6gq3_d-EX+}(aq=XyxCiyFItTZAZl93@jABiI_w|8 zB06P0<-dM9G;|Ot`~1xC==MAP=l(lv^qm!!YUYI{q{x2#2lsmM{y%Ci-&v_vFK6Ix@1Oq5@iqmS zSfD3bPCO47Y0>RZ`p(by-|L2z3WE?iaHw#cY8zY)%(H94Qalavpo=kZ0|6 z_d~!Dui^Uk(DsM_@}y_~ z`}2MdwgYs*GW2Y6>J^OJNGd!P#?YB{yB>+?;-HNff4;}_~7en51)Pf zNeZLf`(V` z-w-QlYxJFRY_I#JX}5p5@o&pse9Nh4+unwP9w{Q{e*2bNj(y?YfB6raLR-Iw6ybx` z1#f%!6VLtn&uj`DQqtC>Tm+0rjGI4o=9WKv+F?IQ%K0KG+mmwJ0guf&`j+jFNK$UL zDL?wxZ|t|Ye_>(C|xR;v#4=FO{9v}b3 zul{(}8;?{9bOaAQXyoaYtaJ-L~B1zaBVc;jTs|>L{s@ zA}g?k|F~+mqn~#6S8Pq8pnL?Ea1~>&#K$Mt%13<(SAnfu9vleO(mK=s@wGk9tzL5- zRLgt{za2`7$hXbz`NIcJzG^e05p%noyCS4)PRdqC-g?r(ofo{*))anwJt@K=_r7ZB zrRP+x77nq!zK? zAN;TEFIbovQlt$|zVvm!+oN%uS+&taOEyUE;3%w8t8(FZ&tJX$UDw=p9_5JT)H|~7 zw6q;Rw#8lheR!p5QLRt;_$lOg)m{GZuxabBu6yK_KRopBFSa#B`d@74`i$m4yIuY( zGk%7a{b9|=4|~@elh!Gx;H$t0A8dclx;>t}=R5Z$Fnc~?#=;}j%bvEpck!Jzg`V4w z6lv?t9(d6suXx|N&(;+BVQA=JQe-r4eB#=(A3Sf@KUx^ZcFD+suaC6leDs1%57_>W@jka|93{S+x5^h);#>fjdr(l7Ohh(?c14Of--+^n%}AZ_KV&=RV>qp^%b78Tc2*K;L3hwf6o z$|+fW(5K~8mt(u(1BMfy8b2~66wuoxBMW{jm;U82#ceHy%W`ZRLfijUIptKBSasp} zMSnC_sZ~>l{dw(AUu0JAasXneWDSViDY0mRt)z9}s*+yt%ES*s+qUTOuX^pAUB7zY zm;TktH(*NS4k;xRczI_-`ln>fm9H_N;HjjoL>{QkyWA$UA+MB_+jpK?D^9ts=3Rd> z>-v`rZ#31I{L`+=2fB+;ZgVK%H~6}w=SV5ngnk7DJHUxUw;g-K-w*qFWUHaK%WbMp zWpDf5%PFZ3w3YYV@qacgdegsL`@Ju{XT{OBMVT>5Y=g{MC0YPu?(<;Vn$~&apMF>Q z#wi!G$D;ZuDJ2>T*scR+M_`7}-(%lLUb3fHn5+%u9#Xa^pa zi!98pe}2nmS6+MR8Nk?`{Fjg-I>oP^{@q7+-1am*XJ@dz?HpsbbUbWYzmsMA&PzOc zNPgcL3h;?MG;{(Kh;98{`+a|xPb@h`cJf8KAVZ$o9qIC2-wUB&Cn#93@fW`F@G0+* z-3qZQa|(XEc51)c;MC(A7N2nD;-8;Iij`I6Ddm0oxqlchJDVQ7{EW}ddGDvSrs}z` zMC)Yyl=DHsDG=#Xc&GftK*`x-+WJ;#6F%7N%42qW&IOIW)>Q8ym+O{gtDFU8rz(;D z<$Zcb=~;sNp-njc;JrV!V5>vk`!Q`d-K{9mylI2-a{-TAY|DRi-Jd?Z)yG7OM!)CS zT*0}3zmoTbl)P$aj_!1Q{ikoe`P(xqr-6|%w{RI}L0e6jc<7+c=*s0gU9#5FM$d_# z;A!rDT=0|kU-im&JZStD>22dRV3Yl{h%~{_|7CP{_jJ>G{<|tBjCuO61-*MXc?_AVOQp)=$-v*qJ zT*5Khp>hfe&x`5lz;c)kp-sANbY^JioFBg7&AT=Kp?jgSS?%l7ytEXT=5%(Ed^-@{ zp?|7RH(d4I?O*_fHnZ4bio(P#T<_{H|LX>$99^LgBb9DFryTADF_W9>L zaliJq5Iw}Zc*=_18*cpZEg$%e>|)D_0bmL_UU#v}&l(nbsMTKRHuOo2p~wI8S=;Sf z`?v4fa-iT`XcIYl*}Y$VY>)2#yO3f}2X08u_%d(w z3(D?i(boApDRMA@grSW#ouf1Qmp}H~NA~^l=-c()u4=d2lOiVu{_W*2I__hydI6^p z={8N7N6K!b+_G@*?e2f(Zr3L%<=CDCW@})MJm?#%KD7A1WcOe65S*^$X`_FXxGguE zqo*@={qC8!JapxXrND^XS-5nl+MQ`~{A*~3FCX{WJukUuZ&H{ug|7=nz14T7SIZlX zyqxjZiLb1=<|A@PfZZQCzf&@wl2Um8R9&tfEZ-;Nn$!rb<1G6J3 z+fdUvU-|p`l{cI|mZY3Yis-OgzVy%^Z@K(&sSjdoF=X>bB=2q1x|>b?xxV-_w|$%x zE8`11vdEZ2b13(~T~1qh|7^(l>;1PdGNg6>_qII!tRp|DRO{P?`S)Cxl2_{fGJfTp z&|zO+`N21Czh6gT7KuK3EGcq6b2{{PGRrml-DQQqFc@RZ}5xs%6r*~X6NJQ z7@d5R7az0oiEEBxZj4gOVV>HQ^S3qa_xxkO_3++b{T-+FA_{8To4Nj`Tfh6rF2}AE zdle5Si4e)(e2h}w4va{bGnO6l(%t|1zYl7=X`g4KaHYAgjBO z!uforx}ht9*%p}no_=BH?)}!w>1UA?%nRQlMfmN|UA}Zt=g-&ZeL%zkQt>WQwkPGB z_y6PHPdnv34Pb~CarvlCx$VO*-0I$UJp5Tw7Ryx{m~;mt-}ZR&*q=ZD+dInGUS@7x zY;99+-T%^;{o}cBJ&qL4gb4+Ek|MOd>9==Wcj@7qTtSLx(WERPMQXa>y!%_%UT~|q zx2q|Kks>kf-hQqzHFCd+RxG>+X2Ql~5pe z9#9>=LW)TLEB|`%gLjNvdQ(!)9i;3?$~|8^XxIN-^h;JoVG1^c4_>sRX@iemz2x3c z?^AnHT1T|L1s>YU+n`9RStRCzq5ms>yO8>bq}cd~bM|}B-;RAZ^$|_T%ZXP5^BiD4 zfBLq!zUFKHIj%6u2OGkpcYsZN!mrqPhD5c=0t=z&wQWRJggaYQdJdZ|xL1%T> z%IZkvROeS85!D4cl3lrO$m*lKragN<_(Qc9?RoECPP&F(Ko=ACE%20Lea`2{=}&i_ zGxSdSvb?5+YcthU%VEk}rx2Tt(=u&v;exzZjc8;Q-o{huUtUh({Vb)Rq^(&=l=LYR z)9-dQqw(~`mp$tTV{g5YUJ$((xl>-B@|(kjGn1aD$n*7z<>qQ?A=SDHEr+>cH=~PP z@$L^l`+eX1jIm#9pJwM0-m#G>nEK?gMIO=%eGIS)EV4#Kqhz<|8a?FPFI|6qea{7F ziBA|KDe|%^N-60D)+HNqAN?RG5PkH|U;3ZbCmlYu2R$eHC{!1=s7jaeF_+c2eC-xV zq`a*rs%iPiDwme!t&{J$h=Ou#8`2Bqt&^u9(qZq}eUAFjfAHA#8y|Jz1#&hVty7-S zES#Bi#WGf}a22^V^pcG?``fC=F55ySpL$dHE7-)E_sS=(T=uCOeOWj`;N`$z+ZWn~Ns(3CiLbi#m9@uCe>o{K(=yL3vnd~b z&3lF)ec=^SQ(d(!9{LxX^4)JOT=A_RAM#R7(bq?hA!SprT{Cgx(_i=H8{}SrwH}x7 zHM>@?21eR#;=b$O`GqfhW+58myby)klld3favGa|wg0)7-zN8iZCmH5K3%A$ zKKXVdMWtnde+rp{X&pRhjaS;MC$Br?@khS(^DoQocJ9bY$}_%cth$c>9roa)oKNj( zq|5J4`|QZIuiWSz+6FqdZzM(Z#J?WDa_4*QJNrIOQ3`%cinQC*l2b08xBufG&~mhY z3d_+|!E5(29CE}L|MAoNUiZ@zfsvkrwpynnHzn5n6gpF&#$7+Hb6 z=FB_xbT-Z$4r> zr2wnPLeILsaLfU-q{v!?6-gFgk38M9^~0Y!d&|4-dU~63>?kQ;Zx>iS#99Q^1>RMJ zcbI7l?xWj%TK2%Q_t7nn&ZBiiA9Y?1T6O`+w-S+g@`0y%)&6521~k zKK%t|WUU|B?}0^?6E?Fc(6$XJvJO7zb4R}Zp`%{XOHyV?k+t(PUfekQD|4@rb%~5% zsShbq-rl!7@9JxR`mn4qY#WqgD~BoYc46oGS$Vs#HjP_f2&2(`jfSuHc0M3-b%g5vjcRt?%9AMc@0z z-U+sIEd_i3>~CBC{e=(6EmMI(CKhU3TGR4!3bh*W`BoBzD;Cvv%Jtqt>zCdoEnj_z z*?%?uYTj`|^PQE`|8^HO#mi3_SDDpE!CKS0JUnfo(Q3M1I_HcP8||?IY@+Wl8gC>; z&JrBHa`NGSy>JuJ90Z0oxSbS{*Z;U{-tikfe1X8o2*zSI-kMf-SV#Zz7dxE0=NILC zh?TH~)_tApw_jw+IrG~m|81|m4*tECqy4iX6qLi1r5u%IgBpaH=Pkc!mR}e2q~kd)2RDzCMxtV6@e0jPi8){OTR|oc{W&qmw<9v-Flt zlpjw1j``j6;BIGZm*VkGT`_ElEw~D{0 zW@}>8vF@6Ma!Tj&Ef0RvohN?e1IM5ah=z00=F2K)f9$?%CB4^OGj!)0w|wcH_wK&k zO43C_)DF6&{mxCspF2)E{~x+?^x=n|dDk`HD@qsq`yaHtdh9#?{l!`QlD3dSPWfd0 zo4Z{1u8SJB{1u-cd2-I9jZbSj1>Ak=%m4DB)jvD4FdYI9SpU=UyI*6p0?3L$7ku2bSdMBPrc}uBmee=)3WsSs`bWNJoFv*+8^xw>-|2z&)smj$iWXj zx#*C_<*(Q#`)+DBz1tt_(1k-+-PqpjLq~1jxq$O7vR?S{iEsSGt1g+l^Ct+)EPBR{ zCvNeN?H1p6IpM7c-?;3Q<6peypu2nW5HkP1{@3j;>b(EG+l|(n^CWJ|`z}2A*Snqi zl^$;;ckuD@`rGH8aoof!{(RZB2H)G3F1dBix-Fi(i(N8V+kR`~@z3o3wO8%B^jXHX zveABTKjpknZvNPjgr&c?yyyHg|9ajn|NUCRvi~vk*d4#T`FqRfpJh30`%PG8@Nd(D z`%7V^n3ER7oTSM^W8-s&hv&`Rd+)jOu<_hVZSL##tWV5+LA%o$t984>wdR`PqmTKQ zm!0ymqfeT<$MWTK_aSAxKh~?an)~eg#yH!$dhZ1b_I(r3b5A>M?o_|KdSwdpJUhVf<2)l{;x>`rvdV#Cku8mQa%ZwO9*q^=AJ!Hg>(tx5{$ztUkENCI>RoHh>-5?4pRSG6rdzA(oPzK4 z=8ac-)sYFvW{EIa>DT9(hT~HDbhSq>QbI#r`88@pjN#C3WD3(m1AdA#6WOZM{ap)e zeYxAv^!Peq2ir0>WJ;NCHL+D!YU6dPYqLwe=82@lbHfT-a7H?{Xv~IUqon~vi)t;%qbYqQrd$d*UjK@q^<2*=*_rT>UzEYMnP<_Et zp@`^itKVT535HQ#O|3P@Vbp4Wyxt;FI=a@0IX7n6gjW?ylQ9OP)1RZ&&Z*kp!&+cx ztkR>HDgg~w7A%Dk+Wj6S^jb1?m}r>Td>Bj0Xsh2GuXbikT+}g@b``#%M_WykU@Kt* z5wCXPtVV;eHr>XPF^!D}i=J@BYPj4&PWM?mR_OYA!z8ykW2b}J1kE6_-!XgJaBei2 z_#RVhuMW>kab^Iy-`T;WX>jOT)5;8|PHhSyQdy0A$#orQ>#VMfl52I>1|}zHB#nk; zr@Aohc)wBW7-3y$H5)U@NbGmzWO`E?pUWQ&9e7i{Ub}nHJXmj1JfzPIH`~*zy2Gu` z4VBRuwKO?gRspZIANdiXG2n&{qK}-(u zo2-|mZsr*UtsKX^AkTcN8GIZDoF{sROckD1lw>K4A0>DYdb4bAiOoi9FDw>H4TFkc zS>t2tc))=${;tr7V&?QWqbUI)2Wb3_ER2SPrv(*X0+UHDhGjbMuan}O!Xm&p zt*}&++Fi74{<>0FPnpyGZX;LCRLPnni=l{G8<(L{V)N8VYyr*^YRFwc#oB6jdeK6s z()8G(g_U}>Ad<2?%$~zR_Lx-+R$?5cnGG(XCxc9)W{*v)n_$wQg9YujS#6`p8fDi< zVmc=%EsTt$;3!lZA29lh&^aJD^ql5ZZjJgfu1hM`sgkb_$yhDqRtL8ko#`?Vi5o_? zN(2Xa#F`ra6p)_9pC#UuvmxZmcuNy5G5;LN8JrHWihF~|RAG@JRO9|hQnPqnVh#Qc zA~)M%BH5k-W-vIMW^l>bnhpiyUk!E7M;E_&wdTGB5{;6Ae|8N4k}!r%&}Gj^9p*V{8FB5D~WonLDYRr zm^~9xQY@`tw0w-o|03I8iU(L*4_$g_i9ttRg1Jk5`y;d z+Q2b<0PTb-dPrEB(60KY$9dsd|5U`RhT*3-#hYH8*0dN@#E8-CVnNb)Plfx<`b52^ zi%;`aVINHvw@FE4>GtS^(f-5)4`OsNY*txRv_@BX@UEGtH>Cab7vec-C@2+|A=t2; z7&y6j!&N32Szs!d`ns4zL%5@SaTyj)EhnLF_L2ccZ#pG}2d(WTMHC~H(N6Qy4xZY@ zEe%-wt*oc3Vh?1m)9O!70cJjcu{H`1jN6^2z_@o2H8Sb>sZ~Z3_AwvYHla1_DCS5Y zkK+L+{?zR2s_){!LEWRh#0Oni4{dbHKZM20MT z{WX2a(z}~UnUa$)!q$p)OdYj z0s#4842QYNIIj>0^>3lQ8=BsV8dNG3%QEd0ZD$qXA&$%dnqJEzH1QjPG3Jjv(l}|L z{zxcIv11BtDrF#QmD9nYzi^$H&a}#`>wWrvW zM$2-d05OwU(xJ+;9N?5;lLy9}hLP>W6VmyjrM@YcI&WXw|{fPw-sEj3a-0772}C4;LtSF_Ky z0*|I9W1MYY_Vsx2%N8SA=6F%Mr>{ex5FN?_;dW`7kVXgU?Ql=Y`OK7+k4y%i2iL4y zth6sYt(H;FCa7zC$vbut}#Ts|ligjUSJ`6ogc}I3-aK$}-6QIX2JL>};TsRUP#! zRRnwgtzNU$b9A|Mx(MObwceDf$Yq7;@s9GuN!BMRwJ=>oui6zdnVulkX+5JHJFl;X3raybgQMxh})_H+hsDU z&Zv>{6rLo^45yrqz?B?PsklDecA+f!01Us7VWtL3q9DIUS%Bh1o>N3QMsJL3%(aMRC4}e4bgBl;*@p)}#t5&Z6Zg^CV=8 zCq+57!|^H6S(5~94qh6b=7i~bPLhqwERJ<1w-BvzSWR;9%H-gk&~>0vlxwkWDn#3f zF+&@=4hrqyMVkEi!c6=u`=GRS^7xC|aF(TIIw@5~CN067G*`528)vm*&vNAx8*-{I zAC2b5X}x&3N2u0LF|8EYE2etW4W#2_MfA@E&U9;kR8QK#ErWMkHvvl3JBu-?_~5ux z#OAT5lEEKV(2@%08!ClApFB(YBLvz%g{U=U1FGp70 zyd7Q?B+=TvOeYCoDneIxA;U|Cms5N$7}H#mN>*-}CH`o5zkHXkZSL#7C8IWT9o0ZoMyc5|AvSzF5*zNTmI#c=DXpa&}5q*Pfnui_P_G69fVxX@5%p-9a?3VYA5w>Yr55GPt{o>MX8vY4e~fs#HCaXqA{DvEa0Hghub# zhPpAKqo`XVRI08PWpG#}Zss^&2De<%mHltS1+Y5Tq2dke!5TjUlbJ`<>@BW9U+b%%+28|)Eq z8Du=V6cBfc-c*V2b;#8Qj=Idq4HLcJ#E}85M;=4@rHx_$=qpapdWQO=h%hh~MI&YG zmXe2elKxFMfNe#?is>Yk93jD7+}vO-PSCoChHye_Jc3eW#<9+f9`~;%YDJcKn!{v} z2CZ@lXbJ2S*2SXz51^t?hQn-{-k~jDp)4ilqIHI+&SaQofADRk05YCuzaNFGef%t6Ut#Dce)>6WCaaU#kMd} z>szg9H)XK$qy;EU?L=-H=^=)B)G==Z=C5f;gDa8K8jsta2nph>_1h%XR8& zpq+?8ap+k;18t7|4YG#@p*1e08!-7Aw6v(R3}h4<7#cpYkbEgrXt1l%8-WB^^r=gN zkf|kMPGNpisC;RmOGUyYIhiLp||iRrFM0>a}-XrhL^ae^)GRHuR*LrubywN?_CVk4v48Jlv7 zMtV8O5MMKsuJGlFq;U@VI0YcrP;3*N;xFAv)*xF~atO9pkM8Y@n3`gdoq*{r`bZe7 z;E}uIDbiIMIS_+L)S~RBQ!|29N6|p+sAO0iv3v~aqYfRw*s3K+Y=kZ~2Ip*{#KAF( zGq6$n8))qc2F8NmVaTK!!Y2k=z7%+HS&?B1ONB5_!YMjX2X6~S=^_%{sQ$WsBub$AXXrHKD{v0Yi-W))tu+f-yTxfFCp3!Lor{LdB~sUL>uS zK6pdo?8gA}F-y<_rDkwGh_o47xSddxg2DMQnwcyTJ5Q-uZt(V(1qAb*!C?clq(f(0 zgFrt451%_sSV$*eTfWT2vn*Ywa|5oQvji92z|zjwV8EHBnq6yw#a()_d}mMxMP!02 zTFQYZ`jk}{O*soB$s!RnNk9rae`iwOa$XxNPZqZ$KsJisd4Q7tB5Q0L}%p(U);6z+ruI-;9)W+c?J10i~fK5V3`6WIo7qAiLSrSLuQg526BIUI5rUJNY$dC9b` z^>Pz8t_@Q|ISen0AlK^>O6iK_a?l<{z%x_nI!sP!$QVrmaG>1o@{^S<4-8`Cl@n76 zno1YTgNKe>ULgk%uu+Fx?ZlAovlXY%X=d>0FD-8OxjfXE2G2Y10!dkt__m3Rd6BoT z;Z-;gnvk12t{U z8oxk`bu%tdjjxI$K8cs4HENU9v6&}hJE@as_Y%VH9OF6|6zRAG6Gds#XpNO$)ps~ClhZ+KVH2dD z=JB*X7(JR@6Z^Pv)XdEQGa#49MXxC?GQ7ZjfG3EZEK@eYmjoLuV4ZGb5U zhZdxTf=k?G`B@Xgzj{}s@aU7`7KYCn#C{o()Bw9dxpXXV9r;V1sN&8JNd46+h8Q$n z&)LP?ET*H_FB1jYX(*0jQDTQMhdX6Ia!s{U$D5`{T-XId;_eBwNkF(;Gz)+I(}jT` z9U;9C?7c+m06q!8grY0@%eT#-jyIE5(V$LOy4I7yX8hgZw9BfN=4Yv=c@iiPCjpt~ z?JZcYn~Q@hIi5c0vj!LsjFpsA*GdUq36NBz7+(IqtO+7tj(oLBj#09xQUJxtGp#jFc94BP|R!&N3D>-4~ zD?%p>=iC;xm$}YEVm>pN^N|(U%U(Qo63212EB7Bb{K28QX>MFKYwjghyX?bhXmV~R z@-X3{sUSB*6~w9@5E?EHTue_IT^y~VGbCA(nt~+>sDUR0TikZ(7Izwzkth>MX!Ix> zC^Le|9S_13_As`mxg*)Ez^vISXypSricHj~WH^GPD0-llwS+_#ovc^ggub#Isto8D zklTel?ryNu63@QgqocXJ6qJk8SvjPS*$Nm(#|gRC@v#me%OPA7v0AB~Hs#V3>txnC5M}kUY=UfzaaYu5PbdS?=9Oblas4O+{ha> z-Fm_I^h`P<#ib)CZ~&D(RWvwKVNh5T#stDx6b7Z6nWDl9qt}uT;|0biVaU~5E61D6 zht4UX*)dffpsBFbEI7Y9w3(tKO0^?Gx~)KK6)miO4+I%^DvW$K*rx% zrfSh3QyqiD)Akr|7q*ETrcrB7_Ixhk3UX6WK`cdDld~;Po=AubwJl@L5yQfID7w`N zFhC4W9b%BAZFAVP+94*%w}@%;OfxC2kPa}}YKxspel&kbB`j(h;ESTc7dBVAZy|uk z4GuI3=U-OLIU-HTnpxXnU#2&Y8_T^Bj_q?iRW7l}iy%06_B3`E0}qWYKXBfGb~8qHGG){9 zV40=o?3&MkiYo zoSQivu$o6>)SajOWPsRO%A;b|`x2`m=2)HvA>5lv?z2nCQ?-e1Z4GTLEf{_3mBZ0QK=%xD z@5RM=z!QHfQK81HHfMAo02qDJB=!b`hrhBSG?J9y5$T_}<`=PAN#>YYpj2P{0-nMd z$8H$XWD-bWO`>Ou*34efoMVxuBR3o+vs(@$aT|uv){%)cg)@o%XV7LTHCQ)*R;xGA zpq9!cVz%_TDo7fcp>80qOmrJT5i2?>YjniNb|%>p;<=o4!|Uwkkj&r}pZN(JZjaZI zKy1HoOUab0Xl2ULllwlQnq(3jF@!42k%XSttTr({4Uc&>2$Kmc<^vgE-5YxbtfK(} z=!BT7m}j&a9&lHg1l|p51yt!g0Amee)q`fV-yLnufWUkJYIP56>9y)~m-oxO^Sql4 zEN&27VQZVKECX-VW_BXvJP)L6y533mif}iZvzP^e_w;6qfu|lxcy!_~?hg<+QnW#zIcy1}}<{##9_ft^(t*S~8 zvO1$>jmxhLY5gxyn!NsL_)f}8LPpaIX;tHk1ZokP?co!??a^(%qGREFnnN6X#1{?t zwl^2}ita)1X>L{Uk-7Xqj2<;Lm^@9M&S>a!Af>Zmlt8nX!W`g1_9G~{@C3Tu2}aD} zHLmR;U9ZAP9tw`Xm9ImhQQAu`N*Riyi#p*k3fu$G6gszXY`N)3<=O$7y6{ep zayzrD*bXXl(G{qSeCWbXK0NIReev<3_AkHx^a6c4Hvg~<$2tL(I=`o zaTW!MvZ3gGY=VB;a@T4UKMEx3gBbTb&>g)n&rNwI$aj7L+4j^R$s}mZ z%^pEM$CB zXaEUOctvP{p;B-+XaFdqS{gFVx2W0H2=42%g2_ZifjLuE?zVx3$xJ3P(}u3l0hOCI z87nee{F8J%LsG-|o8cOkwizuP_)H9g%MKBvymOJ-(_S%f3K4hDmIbxF7`=e%Izjvr z#HAZ$pC_xm>dswG_$lB5pxnYsvcXjqRF zFUr8=uF(<^=IPYvDRmmOwoa}kn6VRDrSl#-ugojy@&P+qM%L$S*=Zm1?N=q4z>ow6 zVrmK-Yq3QkI$->*!Z38|fZXZV)D4DooYD}I0MY3p&Q<8nOjrHJnK#=(>Pt>q%F`*_ zgn;pqt; zG}P=X*=iV*Db8mVvxPfkBsZhVg)DDD+X>x%QF$rJFDtW8riR!3nMy-tCPwwd&~SEI z5e!a9*(o%fbv+84S(nP{ImF6lO%I4&+z+<0$E4Su;t^NiqEA(?LZ|x2Ix}syGjzPi z-w4RCdNtN+W6x<-c!0V++N!EDk&B~+t?JopO-{1hik+@_*gTt+I%T7@?L&gBI)yM1 zL1sRn1+Pm+dlrLTmlOSREGIhuWzOO$i=b$7W>8LB)M`yO&b{ZB;!Kp3K)}=CHp~|Z zj(jAP+fyZ;R*MBDDB^FOheOk+Y986v083%KnL|QsKhooQHFlQ9#b(aId`>^WpJMt= z0$}18nhrn1>ceA5DOIFwn5+Eq*q=F@W#ceoPSzqDe00GHrH$_7;-#)kM0Cktj5#(dmz4&1~@|PqnEGBU_t|dIKGNqCRO| z3bSWzPnB`*XDT9@w<@{G6NOfFqpwD*%IiG}A4PGp9EV!6bnN!Wc=w5e=K52S3lL7U zhCQ5N!D##l0b|k%+>uqC`uJpRZLQJZ{F*%2*lyzsR*$u-M_D#D`qLbC@5(8=8WRa_ zC3xR-%x|tO$}XK>fJGQ8d@a9S7blIn|6;y$ofDD()9Wd8dOf4SUWsMnRE|?k*Ezd2 zRpZUc8SZWRMz@no=ucpfK=JVq7R0D_EQ}X>dcndvSVS_0#->9skWPs5!!uy68^Mu_ z1Cu?esArOD-D%nFALdrEr#AM)gKp_#;G!Zb5-_B2+K3Gid5Nmt7+;0r)NRXaY3ebL zAtB2vOmF>8XU2ox$py(y2x2Kt%UY@0iRvlR6ASfA4+(n^3mt89$|w57m>`O|U;Bt3 z>XjCZ9{H8Ml(1lqs}5-VT@h=uT2=LM1II+#;x*6fY9^BtbeWiBu{2t3ROM+f#}~RXw{K=w^|+RiWu(KN7}0=Q8rXwS38qF7RzP@dkiK* zBiw1Jk^xr=*toS-V*gOwUazf>bI7SZeUKX-S*bdZ8&zRjR>E!MB5&|n023NnDJ-}k zNtVoOne*bjabRSn@@O2EoF0aa^Fe?74a=gKOXCVD`0wUWpAShw7g&;jt|CmB!Diu;e)5TDibOaJ4te=Mw;>jOOz0jbg6s} zLNgKuksGSAa4G`e=rieSPlLr<9011O+M^+EE79b^} zWP@>0Iz)iZ32A?Z*xAJtH?xoxHMwfv+P5;OEP79y+8hoksP7B0Dep?ouaxJH(cirSZ59@%{OtKBH3+Mo(RK$;470A#CPACOo5>VKny*5>6 z;M0x3*dO2~SG3~J*b{`BmY(~wb}M%MjHSV7PO9GOb-?i&O)GnUo}No*1a~?@Ypd)e zjDXz)^OFag$p9AfF_1D7RmK<=oZ6X)L>=^$YjHThdJr;XXN-z=%p491Cl*;3UUAG+ zr{!d<{LsOosl)vqqGQtxK=JoL@;IVf4WvmxsWkI8yCn?_S6D^5J{5e-%?BB&u+C5+ z0UwoaEJ#l4nF#S#V(0xRC*HXFD9&md!>Y)dyo%|@(@sey&?kYx1~JAIAxdm2vF8L& zSw%!Tm2zpPwZpoaV`~GA6B=k}BsM#MlYsVfXtszs3efml1dmyw$#7nU%Lu;>QgQ$< zKQuQd`9ea|xlI9_6VhQE8WFM864>~gmJ&-dV`)URJ+YlmbcfN_dv17bwc&k3Ner|L zL93S9g;YlLQUCbNq8FmN!%~p6g_+#kJRG;enTzkxhScd)Tenj|!s*Bm5uI9J9-jrs zegS4}G#)lCAj?p1X;eN?4#JnDj798?-h0q)Hlf9V(#BA}IJIA6WGkP_b9^59GC9GR ziBY7XO|Vog;RjnqzEpIDM#y+W4m|T$dIBEvb3TSyGi>>gI9<*aBDU@c%h53N`(QePd`Iy*b|)A(ETI-VF|?_2JCik>Aq2l^{E z5Hv#Vfha@Bb5trHh%~>8$X8`cgzoAAtG_O6gnYEqT8nZX1=WdaOWyvH6XOO-ByoML z%S5mDR=96re!0ma#&0~>tn*MZRg4XqN>Y;Pz=PGL7gW0baGH1kNLz7$?N>qTRCMzR ztH`{G8e3xQD%f(91br}(r^BoG!|Uc!Wx0s~GNM+{u_dW~oT4~`U;u(mM(s6DF zy0jhbvjZr}ZY0a|7;Y^&(IY6gp^*#eL>M<9o)IU(gEp%jEkAn9YrbsE*NixTP< zS6gc7-8?FVksgg6HO}ILczgb|iZpLZ2bSaoQY=ccZCRGX13W!m0>tSRHlaQJ6M>}RJjA3>B;dkQp+$mQe;^I>iB1`s3zDu&jSNNyXv!0{KA|Ql12>K1 zNljWS$;9CqE)LAduyuxxr!_;MsrnUTS2{um*j;wX3+SX9m4Wjabd4KY((1@Lr=rpJicjt#G9(~?~3 znY?K(%pmI=<-|;U(l(g{5vwT_VVPNwJx7SaLKBA%CZ&tF!Xh83tf8ug*&7z?AgeNF z$tfSkW^A0I_c=82IR6g_w=~u<1t>42pEzpj5bTE;x9imlOt}C+U#f zI~HK{qz*)5WHZD9Il3oT^B_eoEk@i2#eDoI&T^I=;>>gQgR9lRQ3x5OM~i_OG>aARPc_!SbS|^v=GH6Q zk(@N;bnk*xhQZ6rF5+cJL`xKV9_}#2Kx4{5O_p*zs2vb$;DqSP$Yp%_&6|pv-c7)=9yf4dRsS!Fk;NGiFCSmNBD7tZ2&KfuYyz)hmGR#u}S=h%B ztu>rD=beu@Lhl1qdMJl+6!+1l(lPNJlQ@t1>q|l|xQx(loP^s6PvaYgNjpQ1`6wN& zayV#WiCV#Uy%3)l1zpjaz?)u=ds=dGXW>E*eQ`Am2&9=02kRiv+9Uw$?<`^CwioWi z>!P`0lhkUE^0l{s*WX!|kGmGxoRXheLJCt!q5RAeGUhYWB4bz0&sqMMmkMac=y=#g{#emw&% z5BTyn0Gf@tms&xkuMQtDpAAIrBC-8rC+{`&POtBMZ>VZoYl$(ZvnraH%bT4$qUCp;H+CtW_ z6dHp-K;^<$(1w}Z=;q3AuS(hK0bCSyUAc&&-sgJVgkRGzS+J^rApRDe!tUD$njS|4 zJ19t=_S9_|^)fC!!5ccpe>jGMWx=XUbI)$lI0A@1MI;O_*3mH%#qnNNO&+Sl7mS_| zj~*G|m0FqoYvXacht4<;WX9iAcQIlB{++08NV)5kZ%wnhglB|9PQwYVm2E^@IU>92*& zB&}nzgTpQWJrs*R*^xtS<^y47pQ>fn#i=1LJg|~eAE32fucn(vwdp#aokfk-WToFm z3e?nt5)CCDlssj1CdS2&uKXZ#!mX1OAoRthVeXdU3Pki>A=09ZGe^ruV` zl?j^AQ|UMlSmJM3HGKXnH86lqg%gJLlpT*TpyO|y)p-!m-)`I=dF|Fu(n?nzY)6LjHm6RXEy?KhW3hux^k(w=;*Q;q^vH<}Q`+ zT995jiELY5Xm*0iF(LT43jvP52WsCGCjgxa(`I3zo^Tc5NnoHW8u7g`K;!R<-j@zl znf;E2tfcif=^Qx;j;`RJS*QBl)pB}TfAxB7X#H1XMnU;7UN0B~YJkZ{Rub>u)_<~r zYd89n^`^p&boyOJlq?J_JWMk-g^K{L4#X}3bj#Q5_*UlX05+Cr)sQ)JBGi~YHKqb3 z&0eh_C>PO;id&MDsMiq%hCG8J9Ve$LstXp(q+@l(!o72`HglX)-5bbxMtO+izy%AI zPAnN8Kd`cJ{=!A`51hZG0JfqIZ@6F1`=e%iJKoebx&0kz^3a18HNJ|Zv5=}GHb%IK zqffn=D2uV4=2G>ZWV>y_Z062b6kgRG(GjtBswKuATmz`z6V%2|btaQ!?y^;ED|SZu z%}JcgBKZ1ay}tOB9%~@YECq4|Ec(`CGhEn0aCCY8K_9oLA@vH3dZXb*7+&z~clb|i zH;ixg={kZ@ z$}&gp{dyWO$po$>Kx5d|hzE9{K?mA_@5 z1F+~*heT+nIxTR3PDiMs5t>0A`6UPvX*Lk^v0P!LB!K#%^L>cwSabpwe-Ff@az(AG z7cuOY*bNqj7Y#c4Wl2Jb@i+BD;SX^)+Vwb#ZvgF=t}#@8$gV5)BN6#3_VKoWlPeE% zI=R5-ga*=qo#KFYLaNaPC7POhd(u^H`8IxF+&(I`BIsnrOmWBi-^B(?vZ&A zIJrRLgp|KRWM`*cP-LSLg_fJ>sDgnDs!1ln6+=+1@SMqH->FUjXFha%g&0UU6$rgu zDGH;GN3JHKsx@NPVcb?nQ4q05dYV!>#>ooWY*Zq$Ja`bo#$tf=W{@?vO#N-q%gu+D z5vn#S0lJaCjn#Lg$Fl z{sd0SJ&oz)0)rD$?g+_3VL&Y0LbCY+D!jDAji0@cX1}y)Lz2GMWvp_UO|*0UqG@D> zfO;Xu@G7eluZQI%lcISp9sXFwUb`Y<^vlhq%lbQ0-!XeTPX7^C=RstEQs|ni0kkz^p5z zS))&7x)2@dCAugmTKU)yhg!uvO_|O}>FJ2Xu;szQdPzNp03Ce}ROjj{8L0e4>_sOAqen+Ji=t0u2yGi{ zxthRqWlhb>oMOQV9BdVHhdg470VK*z2?en@%%*#5AgJfEz=ri z%?XUi33q9lS;<6(&U|^I;*@)*PaAiQZ_VUBkM~W-0*Yw%f>`FnMOKEZiLz2}j)`@x ztD#|i;9gdy!?Mb3aJD)Zk9vsNBpWUJl&ZJ!{3Vu82N5sui4~C|D)1f`)iGp7uY8Nc zqXk+;89npWmTd3Hc8vrYW-xWQU}Hma=`|h0q}^I;{b~=Zr=9Lfk^J-3m_EEj277bT zp-0adF%niT(t;Q3-zEC@0R4NQ{$1+fjZ&Pnd1;!CWMh3wzf4c`ph$@zf^1nS7LI^9se;I2HG*iA89+ zXoJq8SDbxzg`(^fP!z56b7-MZTwgF2CxBBt)%&fKxwZ#gE>5Y_iQe{wSl&I&>f{2Q z69Uly^NQQYz^D2g4B_4P4AX!k6Qfu{65QcJgCXKcX%2hAL`S!MDyuR&BWM0=&B-2* zc`)kD8r9)XcAmy+)irpb*WCvZEaPTtq^oDi*3@fjEixyZV9i9DBjAuJEQ?&xp)JRK z^ktJa#t<&!t*J%)<`5#oLbo1grbk;1u6&u(W9)Gl1743k6ESE=Ei6FaWHB5MbN1y2Hb`*F@r;8)D0n`b}APOF% zDIO~zDo)Uv=+PY9LUil0!{@YqP+pe6}JVXFv@=$R6t%U`RjC!)h z++!0+9p?%eI&YPw(-fT|eNa5<1~7=yabU8h8GkF7HnkYELkDUF8J@QPfptO(9U7=E z!yBmtl6<7%3K?%{RR_{k#EH;$V5i5gVYmP~L&jH_N>XNBO=sTC)YVB*tRZ7XiXycz zXAx@TKopiwJm4%(RNNtJMIktoT^VMQHXLsmQZOzIKA@PHx&@ca?#X1XbcJ{bMpOvE zqfg+%d+<3;A}Ze~Cw3Eiu;HvAN`L{DRaNNZKuWCvHWilTe-H!JwJMUSyA5cog+u|c z1-h8QQ;;xdPo(rcgDn*`ka-N+Vja@ph=WQ;Xruz~Dwqa^8>3dg(1ep{yC6scN=BY7jXSXoR|8wL1{i-P$x3F|e2q_KCuqG1l}8P-FylZCL@fx5CyK!9bA;LvnaS z#6qBgVN**mm@pW^6OhH=*y$29PNcGUXwtX1ti3P6VRATNrBO6#ggM*mcOy3%jTX0O;4d?uxW+>}rdO9A22Lx#sATW>^aZ2VON zf2b)bJ8Qr-6lG(TB6H<6N=lm6)2jK*;Lk^DzpE;M9YrqY4Ko47hU?aZr-WhvN-0jz zis)ox*OPdG>~Tz6v}R8!`OK7%kJNgoaFV&bJJaMQj5XsDp|>gO>(#F-s-RrjnjxA8 zP^=9-%4-m_Rc<{L1PM#B#$+W?T4ilq3KQK6TfhBcmh#|EGD)WySee&E=!7J2Qd9yf z51$C^AUa=7j)j3_7UhB7z(nN?Kx4k`4T^lcwjx_yel-P(i+b50ElNo`$KMfYW-|EF zaoV&Y`l7QOAc#JJ3-8ir#bC4p1fH^_WGr}UmIsdZoC+~ViHbYKMp@i40T>=BNdc>s z>$9LZ+Ego`H%bEU!r&q)y|EWv!2oM<0;-4XgTjr~PQ5B8B|9_Xi&o^AoPKGot*$eZ ztw|flTHdMlG403a>Uo24vX7;`g1Ox{ZKd)-P%2E_Uk2s}j!BmSrGr zAsK9Hu;oP||1d15EjSBemHf~wnQloCr6ZVW!zcHv>$oGphN?oFPu9-%0*j}K(tRq> zsZORjZ@G-j#c(_hC=Ix%jIUfO*G*jAR6S&q|hAdH@Om7%RBY_QQ?kAed zOZFg+ZuwiaP~K*3qpqV&$xFrNBH~JOL#lLZS9IXu_HG&_6-Y;jvSg1dTJK+Sz(hBi z+gOu*&LqT)rFlRae}hjt#@{V^xQ8JEVDzbyBGjEq8bJE13>+E`Zk=L4A&~y^)0plU zVABy=dT5$0=x-1e#4;1f^O1gvcw4$JKH~IOhhE68J}qu4rn@@N4M?kHXuDuD7wZz5 zi&9lCg<3N=@7MzwIxN8NO;_FsnIPC>eMa#?(IiBw!r6y$LaLVG$k07*Su zL`iLChf*sdZL7LWe3g9zkhryu5V~;*g*-pVsz_b=`#90fCI}rUIMALK^bo3oUhe(^ z9DnO;+abv$mfcY~U~fL;!n9S}Rcc^@@ITMp<>Z%?s=ouxPds5&k-`9qJ{3+Sv|bL8 z4)Rns&I5+{ThB~|#+OqiV4cuFiNW#uU~#ML9c z5?7@8+^Yt1!WF^s>lUDY@~n$2RO z{dW;#9FQD;(~e=ofcnql7IPabk33kZ6%{O8wXS62nN``F)tY3|MVDTKAql0r0B&t< zw9Re%;zUm?df_0_o9+ff=y0m1zWJL$I*|wuR{Omccep0%>%0u@W`iI% zC^C<&XjQ=NJ6)_YySZVO(ZO8jzdC)71u!*(x2ZAihZ~laq2Y zF;aV!KYzvJjgNZzuNVMwiW5}Kn5V4qEC!YIga^CaUu*E16!@Y~Mq{{PQ*{D3 z6-Ha2b3&r0sHubuF}`P7kV;{+SrXc;HVVkRe!niKS>#7mywJklf{D|>&UmM%Zq}l2 z)+MaTLRE<|!05OGY2vP$IhC10X3E&nx4HRHoUGjzTE`X?Lp@Vy#dZ6@^%ZK_Ea_F( zRi?P0tCw_jo7JGqC&_EVEYw0ebBa#nfqAUVBGn+(mBp@q@TSlhkDCUE`rQ;Wthoc= zVQ-7#R53-I+QoQ_p)4egEsSt9P|Dc6p%~ez!i905rPbq!JABFm9tV(DvTZ$3mCehV z_#Og7{Ef(w_i9z-SsG1Rm?V=gL1r)wMho}P##*h_`jlV%tqYO?`)|BDsrFa>r4-@=*ejBL%ZWLP^vBrq@=XcFwl8 zN^PzXjV&!_+CGqyF1^{Rw3>|>a|0rs)Nl5?oDerwN%Rw!GqejlR2Sq<$hC~5I2>{d zm|eFbJjIlOP?ov4$Q&&i2cp#q4q?iQx$$db%)uc~A+|1RphzlV>=ojFGh`nx^Q0hp zW>8+J#VsWOQZce%fu>xXC?~e!o|^Sy&l2_v;T^psFOMlRk;*;}+M;Zz&F4honm7#Y zsW5(dQeif4Qeh-oE-X&kV}zJ0QW+#H8NBq1qd_tH@r&YsHWuoP0*-MFq#$#HgEeCT zPVJDWV>TE|$|eT3RCqR6?ddo{q37kW3oRK`8FWSr@J>jLOF^9}FY>{XkEHj)b9cdv zVo4fH&oeAD!hWLCGKvWcT)8;IRXR*LbD^oSWqb6no^R2 zJ$QN$ywUpruCaBVMqgHNEmmtdVZ0NAs779I>l1$)90x*Ht zk^HN&glwT9xq~8GkFhb$sj-I)Cl|P!5WS6X)ID`E&6)P{BZ^wpAc{gDvZp}=&ni^9 zF%KF|2Z8)>e z7!KW7ZDDn2;%S~Ta>HxD1U&>N0i;AkvKfK81A$?3nnyKIT1`$lEGB9Z4ht&!f|ze_ z)C;Fw>%q&s5k5+99xMpTlSNIF+e;h;`iobwV)O0NeK22BI!WYr$=J ziA>3|$b{wTPiv)*tzDpuzYWHabtr947Y!t2%t(9EqQrCr91$LLa+6a$R0zHHGm(Fu z`if0A>xT>+MbRn=L$%lE0lK{zj0~W0Lb9d^%29PfhEmexGKU<$N`)zZ89e2e#0ElK z5X9gT09N?X4;%G;t--^ng;O3lJrm+!{821yz+9Zb_=|eK)y1_-WfFIsn2uX|`c4nu zsausLc8bPx2(JZ1a!haf(DBlh&6cXkD5DQkCqc47D)V%88dJ)^ zn=2=k3jJYL&tT7El$}6(iawoJJaDNEc|g;NW=rrJgRU?}sAFs~t|2c$~A%z}Ei)+KE*$YDu4P@OT`MXt`a4Wpfigb-wzyv!0M z&@$nr_w=}2ypV=b&yO)MSj&=FT;MP>Fz;X~iSUY1!93)FtqiYMEwP_?)8Qzw&G=_F zp5tO;%|1aJeG4y!Z~i(pWBb$4m#%Oet)vXdUbg+!zMAV5V|9f*>1lt>`<2Tgg9J3 z&WSz`Rx(9{G5Sp8CNbQPfQIM9Dt4d0UeaP+*ZQz{<`aL{*rU^AT9LhI< zcWi%QS5|+qpiK8*CVe)-FXB!%(sF2J_q2j>%uc*K;Kcl(myg5{W4>fI@||>wcZR_p zf5W*0yBW6x%4x=IwPg=_*)R{|>PW;6Y2Kxg=%{^^k>#~gU9ZLlS6r ztNJAs_Ejp65ZdyQ18GV*YC)BYqncrhN8A8lh`&Yj1x1TE%VL9DS~&Vlw6vlBpZ@#* E0LrBH9smFU literal 0 HcmV?d00001 From 898b67583919f02e71ed97ab22288b47063c3494 Mon Sep 17 00:00:00 2001 From: chenhe Date: Wed, 8 May 2024 01:17:22 +0800 Subject: [PATCH 009/273] t --- .../model_providers/togetherai/llm/llm.py | 3 ++- .../firecrawl/firecrawl_web_extractor.py | 2 +- api/libs/bearer_data_source.py | 18 +++++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/api/core/model_runtime/model_providers/togetherai/llm/llm.py b/api/core/model_runtime/model_providers/togetherai/llm/llm.py index 64b7341e9ff7c9..f38e4089426701 100644 --- a/api/core/model_runtime/model_providers/togetherai/llm/llm.py +++ b/api/core/model_runtime/model_providers/togetherai/llm/llm.py @@ -124,7 +124,8 @@ def get_customizable_model_schema(self, model: str, credentials: dict) -> AIMode default=float(credentials.get('presence_penalty', 0)), min=-2, max=2 - ) + ), + ParameterRule ], pricing=PriceConfig( input=Decimal(cred_with_endpoint.get('input_price', 0)), diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index 4d70ff74d4d20b..f0d03956e7461c 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -22,7 +22,7 @@ def __init__( url: str, api_key: str, base_url: str = 'https://api.firecrawl.dev', - mode: str = 'scrape', + mode: str = 'crawl', ): """Initialize with url, api_key, base_url and mode.""" self._url = url diff --git a/api/libs/bearer_data_source.py b/api/libs/bearer_data_source.py index e94f291246c16c..7108cad4f39afd 100644 --- a/api/libs/bearer_data_source.py +++ b/api/libs/bearer_data_source.py @@ -1,13 +1,21 @@ # [REVIEW] Implement if Needed? Do we need a new type of data source +from abc import abstractmethod + + class BearerDataSource: - def __init__(self, api_key: str): + def __init__(self, api_key: str, api_base_url: str): self.api_key = api_key + self.api_base_url = api_base_url - def access_token(self) -> str: - raise NotImplementedError() + @abstractmethod + def validate_bearer_data_source(self): + """ + Validate the data source + """ + class FireCrawlDataSource(BearerDataSource): - # [REVIEW] Implement if Needed? Do we need a new type of data source - pass \ No newline at end of file + + \ No newline at end of file From de558efaf11009b104774c4ad929ae373854023e Mon Sep 17 00:00:00 2001 From: chenhe Date: Wed, 8 May 2024 01:17:41 +0800 Subject: [PATCH 010/273] t --- api/libs/bearer_data_source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/libs/bearer_data_source.py b/api/libs/bearer_data_source.py index 7108cad4f39afd..95b890f0cca8b7 100644 --- a/api/libs/bearer_data_source.py +++ b/api/libs/bearer_data_source.py @@ -17,5 +17,6 @@ def validate_bearer_data_source(self): class FireCrawlDataSource(BearerDataSource): - - \ No newline at end of file + def validate_bearer_data_source(self): + + pass \ No newline at end of file From 77cc39f162d8d45cebbe07374d1770fe842dbad2 Mon Sep 17 00:00:00 2001 From: chenhe Date: Fri, 10 May 2024 14:04:14 +0800 Subject: [PATCH 011/273] c --- .../console/auth/data_source_bearer.py | 116 + .../rag/extractor/firecrawl/firecrawl_app.py | 2 +- api/libs/bearer_data_source.py | 21 +- api/models/source.py | 2 + web/yarn.lock | 7378 ----------------- 5 files changed, 139 insertions(+), 7380 deletions(-) create mode 100644 api/controllers/console/auth/data_source_bearer.py delete mode 100644 web/yarn.lock diff --git a/api/controllers/console/auth/data_source_bearer.py b/api/controllers/console/auth/data_source_bearer.py new file mode 100644 index 00000000000000..293ec1c4d341c3 --- /dev/null +++ b/api/controllers/console/auth/data_source_bearer.py @@ -0,0 +1,116 @@ +import logging + +import requests +from flask import current_app, redirect, request +from flask_login import current_user +from flask_restful import Resource +from werkzeug.exceptions import Forbidden + +from controllers.console import api +from libs.login import login_required +from libs.oauth_data_source import NotionOAuth + +from ..setup import setup_required +from ..wraps import account_initialization_required + + +def get_oauth_providers(): + with current_app.app_context(): + notion_oauth = NotionOAuth(client_id=current_app.config.get('NOTION_CLIENT_ID'), + client_secret=current_app.config.get( + 'NOTION_CLIENT_SECRET'), + redirect_uri=current_app.config.get( + 'CONSOLE_API_URL') + '/console/api/oauth/data-source/callback/notion') + + OAUTH_PROVIDERS = { + 'notion': notion_oauth + } + return OAUTH_PROVIDERS + + +class OAuthDataSource(Resource): + def get(self, provider: str): + # The role of the current user in the table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() + with current_app.app_context(): + oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) + print(vars(oauth_provider)) + if not oauth_provider: + return {'error': 'Invalid provider'}, 400 + if current_app.config.get('NOTION_INTEGRATION_TYPE') == 'internal': + internal_secret = current_app.config.get('NOTION_INTERNAL_SECRET') + oauth_provider.save_internal_access_token(internal_secret) + return { 'data': '' } + else: + auth_url = oauth_provider.get_authorization_url() + return { 'data': auth_url }, 200 + + + + +class OAuthDataSourceCallback(Resource): + def get(self, provider: str): + OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() + with current_app.app_context(): + oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) + if not oauth_provider: + return {'error': 'Invalid provider'}, 400 + if 'code' in request.args: + code = request.args.get('code') + + return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&code={code}') + elif 'error' in request.args: + error = request.args.get('error') + + return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error={error}') + else: + return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error=Access denied') + + +class OAuthDataSourceBinding(Resource): + def get(self, provider: str): + OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() + with current_app.app_context(): + oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) + if not oauth_provider: + return {'error': 'Invalid provider'}, 400 + if 'code' in request.args: + code = request.args.get('code') + try: + oauth_provider.get_access_token(code) + except requests.exceptions.HTTPError as e: + logging.exception( + f"An error occurred during the OAuthCallback process with {provider}: {e.response.text}") + return {'error': 'OAuth data source process failed'}, 400 + + return {'result': 'success'}, 200 + + +class OAuthDataSourceSync(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, provider, binding_id): + provider = str(provider) + binding_id = str(binding_id) + OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() + with current_app.app_context(): + oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) + if not oauth_provider: + return {'error': 'Invalid provider'}, 400 + try: + oauth_provider.sync_data_source(binding_id) + except requests.exceptions.HTTPError as e: + logging.exception( + f"An error occurred during the OAuthCallback process with {provider}: {e.response.text}") + return {'error': 'OAuth data source process failed'}, 400 + + return {'result': 'success'}, 200 + + +api.add_resource(OAuthDataSource, '/oauth/data-source/') +api.add_resource(OAuthDataSourceCallback, '/oauth/data-source/callback/') +api.add_resource(OAuthDataSourceBinding, '/oauth/data-source/binding/') +api.add_resource(OAuthDataSourceSync, '/oauth/data-source///sync') diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index e9cce18ffa7606..bdb7c8e247aa7d 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -8,7 +8,7 @@ class FirecrawlApp: def __init__(self, api_key=None, base_url='https://api.firecrawl.dev'): self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') self.base_url = base_url or os.getenv('FIRECRAWL_BASE_URL') - if self.api_key is None: + if self.api_key is None and self.base_url == 'https://api.firecrawl.dev': raise ValueError('No API key provided') def scrape_url(self, url, params=None): diff --git a/api/libs/bearer_data_source.py b/api/libs/bearer_data_source.py index 95b890f0cca8b7..9c14e8e2ff41cf 100644 --- a/api/libs/bearer_data_source.py +++ b/api/libs/bearer_data_source.py @@ -2,6 +2,8 @@ # [REVIEW] Implement if Needed? Do we need a new type of data source from abc import abstractmethod +import requests + class BearerDataSource: def __init__(self, api_key: str, api_base_url: str): @@ -18,5 +20,22 @@ def validate_bearer_data_source(self): class FireCrawlDataSource(BearerDataSource): def validate_bearer_data_source(self): + TEST_CRAWL_SITE_URL = "https://www.google.com" + FIRECRAWL_API_VERSION = "v0" - pass \ No newline at end of file + test_api_endpoint = self.api_base_url.rstrip('/') + f"/{FIRECRAWL_API_VERSION}/scrape" + + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + data = { + "url": TEST_CRAWL_SITE_URL, + } + + response = requests.get(test_api_endpoint, headers=headers, json=data) + + return response.json().get("status") == "success" + + diff --git a/api/models/source.py b/api/models/source.py index 97ba23a5bddbfa..70b139ed25d280 100644 --- a/api/models/source.py +++ b/api/models/source.py @@ -15,6 +15,8 @@ class DataSourceBinding(db.Model): id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) access_token = db.Column(db.String(255), nullable=False) + endpoint_url = db.Column(db.String(512), nullable=True) # For validation with endpoint + bearer key + bearer_key = db.Column(db.String(512), nullable=True) # For validation with endpoint + bearer key provider = db.Column(db.String(255), nullable=False) source_info = db.Column(JSONB, nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) diff --git a/web/yarn.lock b/web/yarn.lock deleted file mode 100644 index 50e5ac1ee36e33..00000000000000 --- a/web/yarn.lock +++ /dev/null @@ -1,7378 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@alloc/quick-lru@^5.2.0": - version "5.2.0" - resolved "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz" - integrity sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw== - -"@antfu/eslint-config-basic@0.36.0": - version "0.36.0" - resolved "https://registry.npmjs.org/@antfu/eslint-config-basic/-/eslint-config-basic-0.36.0.tgz" - integrity sha512-2b3ZB7pO00nxAERDXo82iYPjLQ4l/AOMm0CTKmGmqWbN3RB33EIQWzYheZRboSbAVzWpI1/3rg/Gu+7xYVMYHA== - dependencies: - eslint-plugin-antfu "0.36.0" - eslint-plugin-eslint-comments "^3.2.0" - eslint-plugin-html "^7.1.0" - eslint-plugin-import "^2.27.5" - eslint-plugin-jsonc "^2.6.0" - eslint-plugin-markdown "^3.0.0" - eslint-plugin-n "^15.6.1" - eslint-plugin-no-only-tests "^3.1.0" - eslint-plugin-promise "^6.1.1" - eslint-plugin-unicorn "^45.0.2" - eslint-plugin-unused-imports "^2.0.0" - eslint-plugin-yml "^1.5.0" - jsonc-eslint-parser "^2.1.0" - yaml-eslint-parser "^1.1.0" - -"@antfu/eslint-config-ts@0.36.0": - version "0.36.0" - resolved "https://registry.npmjs.org/@antfu/eslint-config-ts/-/eslint-config-ts-0.36.0.tgz" - integrity sha512-I/h2ZOPBIqgnALG2fQp6lOBsOXk51QwLDumyEayt7GRnitdP4o9D8i+YAPowrMJ8M3kU7puQUyhWuJmZLgo57A== - dependencies: - "@antfu/eslint-config-basic" "0.36.0" - "@typescript-eslint/eslint-plugin" "^5.53.0" - "@typescript-eslint/parser" "^5.53.0" - eslint-plugin-jest "^27.2.1" - -"@antfu/eslint-config-vue@0.36.0": - version "0.36.0" - resolved "https://registry.npmjs.org/@antfu/eslint-config-vue/-/eslint-config-vue-0.36.0.tgz" - integrity sha512-YuTcNlVlrEWX1ESOiPgr+e2Walfd6xt3Toa0kAKJxq2aBS1RWqIi1l3zIVGCHaX72lOrSXNmQ7bryaZyGADGDg== - dependencies: - "@antfu/eslint-config-basic" "0.36.0" - "@antfu/eslint-config-ts" "0.36.0" - eslint-plugin-vue "^9.9.0" - local-pkg "^0.4.3" - -"@antfu/eslint-config@^0.36.0": - version "0.36.0" - resolved "https://registry.npmjs.org/@antfu/eslint-config/-/eslint-config-0.36.0.tgz" - integrity sha512-otZ9PfKRT3gnGMMX1gS8URTNPMPCZ69K5jHZvLkYojru0gLBZ3IO5fCvjEZpWqOyIUHtAgg6NWELf1DbEF+NDw== - dependencies: - "@antfu/eslint-config-vue" "0.36.0" - "@typescript-eslint/eslint-plugin" "^5.53.0" - "@typescript-eslint/parser" "^5.53.0" - eslint-plugin-eslint-comments "^3.2.0" - eslint-plugin-html "^7.1.0" - eslint-plugin-import "^2.27.5" - eslint-plugin-jsonc "^2.6.0" - eslint-plugin-n "^15.6.1" - eslint-plugin-promise "^6.1.1" - eslint-plugin-unicorn "^45.0.2" - eslint-plugin-vue "^9.9.0" - eslint-plugin-yml "^1.5.0" - jsonc-eslint-parser "^2.1.0" - yaml-eslint-parser "^1.1.0" - -"@babel/code-frame@^7.0.0": - version "7.21.4" - resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.21.4.tgz" - integrity sha512-LYvhNKfwWSPpocw8GI7gpK2nq3HSDuEPC/uSYaALSJu9xjsalaaYFOq0Pwt5KmVqwEbZlDu81aLXwBOmD/Fv9g== - dependencies: - "@babel/highlight" "^7.18.6" - -"@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": - version "7.19.1" - resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz" - integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== - -"@babel/highlight@^7.18.6": - version "7.18.6" - resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz" - integrity sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g== - dependencies: - "@babel/helper-validator-identifier" "^7.18.6" - chalk "^2.0.0" - js-tokens "^4.0.0" - -"@babel/parser@^7.24.4": - version "7.24.4" - resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.24.4.tgz" - integrity sha512-zTvEBcghmeBma9QIGunWevvBAp4/Qu9Bdq+2k0Ot4fVMD6v3dsC9WOcRSKk7tRRyBM/53yKMJko9xOatGQAwSg== - -"@babel/runtime@^7.0.0", "@babel/runtime@^7.10.1", "@babel/runtime@^7.11.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.18.3", "@babel/runtime@^7.20.6", "@babel/runtime@^7.20.7", "@babel/runtime@^7.21.0", "@babel/runtime@^7.21.5", "@babel/runtime@^7.22.3", "@babel/runtime@^7.3.1": - version "7.22.3" - resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.22.3.tgz" - integrity sha512-XsDuspWKLUsxwCp6r7EhsExHtYfbe5oAGQ19kqngTdCPUoPQzOPdUbD/pB9PJiwb2ptYKQDjSJT3R6dC+EPqfQ== - dependencies: - regenerator-runtime "^0.13.11" - -"@braintree/sanitize-url@^6.0.1": - version "6.0.4" - resolved "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz" - integrity sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A== - -"@emnapi/runtime@^0.45.0": - version "0.45.0" - resolved "https://registry.npmjs.org/@emnapi/runtime/-/runtime-0.45.0.tgz#e754de04c683263f34fd0c7f32adfe718bbe4ddd" - integrity sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w== - dependencies: - tslib "^2.4.0" - -"@emoji-mart/data@^1.1.2": - version "1.1.2" - resolved "https://registry.npmjs.org/@emoji-mart/data/-/data-1.1.2.tgz" - integrity sha512-1HP8BxD2azjqWJvxIaWAMyTySeZY0Osr83ukYjltPVkNXeJvTz7yDrPLBtnrD5uqJ3tg4CcLuuBW09wahqL/fg== - -"@eslint-community/eslint-utils@^4.1.2", "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.3.0": - version "4.4.0" - resolved "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz" - integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== - dependencies: - eslint-visitor-keys "^3.3.0" - -"@eslint-community/regexpp@^4.4.0": - version "4.5.1" - resolved "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.1.tgz" - integrity sha512-Z5ba73P98O1KUYCCJTUeVpja9RcGoMdncZ6T49FCUl2lN38JtCJ+3WgIDBv0AuY4WChU5PmtJmOCTlN6FZTFKQ== - -"@eslint/eslintrc@^2.0.1": - version "2.0.3" - resolved "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz" - integrity sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ== - dependencies: - ajv "^6.12.4" - debug "^4.3.2" - espree "^9.5.2" - globals "^13.19.0" - ignore "^5.2.0" - import-fresh "^3.2.1" - js-yaml "^4.1.0" - minimatch "^3.1.2" - strip-json-comments "^3.1.1" - -"@eslint/js@8.36.0": - version "8.36.0" - resolved "https://registry.npmjs.org/@eslint/js/-/js-8.36.0.tgz" - integrity sha512-lxJ9R5ygVm8ZWgYdUweoq5ownDlJ4upvoWmO4eLxBYHdMo+vZ/Rx0EN6MbKWDJOSUGrqJy2Gt+Dyv/VKml0fjg== - -"@faker-js/faker@^7.6.0": - version "7.6.0" - resolved "https://registry.npmjs.org/@faker-js/faker/-/faker-7.6.0.tgz" - integrity sha512-XK6BTq1NDMo9Xqw/YkYyGjSsg44fbNwYRx7QK2CuoQgyy+f1rrTDHoExVM5PsyXCtfl2vs2vVJ0MN0yN6LppRw== - -"@floating-ui/core@^1.1.0", "@floating-ui/core@^1.4.1": - version "1.4.1" - resolved "https://registry.npmjs.org/@floating-ui/core/-/core-1.4.1.tgz" - integrity sha512-jk3WqquEJRlcyu7997NtR5PibI+y5bi+LS3hPmguVClypenMsCY3CBa3LAQnozRCtCrYWSEtAdiskpamuJRFOQ== - dependencies: - "@floating-ui/utils" "^0.1.1" - -"@floating-ui/dom@1.1.1": - version "1.1.1" - resolved "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.1.1.tgz" - integrity sha512-TpIO93+DIujg3g7SykEAGZMDtbJRrmnYRCNYSjJlvIbGhBjRSNTLVbNeDQBrzy9qDgUbiWdc7KA0uZHZ2tJmiw== - dependencies: - "@floating-ui/core" "^1.1.0" - -"@floating-ui/dom@^1.5.1": - version "1.5.1" - resolved "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.5.1.tgz" - integrity sha512-KwvVcPSXg6mQygvA1TjbN/gh///36kKtllIF8SUm0qpFj8+rvYrpvlYdL1JoA71SHpDqgSSdGOSoQ0Mp3uY5aw== - dependencies: - "@floating-ui/core" "^1.4.1" - "@floating-ui/utils" "^0.1.1" - -"@floating-ui/react-dom@^2.0.1": - version "2.0.2" - resolved "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.0.2.tgz" - integrity sha512-5qhlDvjaLmAst/rKb3VdlCinwTF4EYMiVxuuc/HVUjs46W0zgtbMmAZ1UTsDrRTxRmUEzl92mOtWbeeXL26lSQ== - dependencies: - "@floating-ui/dom" "^1.5.1" - -"@floating-ui/react@^0.25.2": - version "0.25.2" - resolved "https://registry.npmjs.org/@floating-ui/react/-/react-0.25.2.tgz" - integrity sha512-3e10G9LFOgl32/SMWLBOwT7oVCtB+d5zBsU2GxTSVOvRgZexwno5MlYbc0BaXr+TR5EEGpqe9tg9OUbjlrVRnQ== - dependencies: - "@floating-ui/react-dom" "^2.0.1" - "@floating-ui/utils" "^0.1.1" - tabbable "^6.0.1" - -"@floating-ui/utils@^0.1.1": - version "0.1.1" - resolved "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.1.1.tgz" - integrity sha512-m0G6wlnhm/AX0H12IOWtK8gASEMffnX08RtKkCgTdHb9JpHKGloI7icFfLg9ZmQeavcvR0PKmzxClyuFPSjKWw== - -"@formatjs/intl-localematcher@^0.5.4": - version "0.5.4" - resolved "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.5.4.tgz" - integrity sha512-zTwEpWOzZ2CiKcB93BLngUX59hQkuZjT2+SAQEscSm52peDW/getsawMcWF1rGRpMCX6D7nSJA3CzJ8gn13N/g== - dependencies: - tslib "^2.4.0" - -"@headlessui/react@^1.7.13": - version "1.7.15" - resolved "https://registry.npmjs.org/@headlessui/react/-/react-1.7.15.tgz" - integrity sha512-OTO0XtoRQ6JPB1cKNFYBZv2Q0JMqMGNhYP1CjPvcJvjz8YGokz8oAj89HIYZGN0gZzn/4kk9iUpmMF4Q21Gsqw== - dependencies: - client-only "^0.0.1" - -"@heroicons/react@^2.0.16": - version "2.0.18" - resolved "https://registry.npmjs.org/@heroicons/react/-/react-2.0.18.tgz" - integrity sha512-7TyMjRrZZMBPa+/5Y8lN0iyvUU/01PeMGX2+RE7cQWpEUIcb4QotzUObFkJDejj/HUH4qjP/eQ0gzzKs2f+6Yw== - -"@humanwhocodes/config-array@^0.11.8": - version "0.11.10" - resolved "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz" - integrity sha512-KVVjQmNUepDVGXNuoRRdmmEjruj0KfiGSbS8LVc12LMsWDQzRXJ0qdhN8L8uUigKpfEHRhlaQFY0ib1tnUbNeQ== - dependencies: - "@humanwhocodes/object-schema" "^1.2.1" - debug "^4.1.1" - minimatch "^3.0.5" - -"@humanwhocodes/module-importer@^1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz" - integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== - -"@humanwhocodes/object-schema@^1.2.1": - version "1.2.1" - resolved "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz" - integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== - -"@img/sharp-darwin-arm64@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.2.tgz" - integrity sha512-itHBs1rPmsmGF9p4qRe++CzCgd+kFYktnsoR1sbIAfsRMrJZau0Tt1AH9KVnufc2/tU02Gf6Ibujx+15qRE03w== - optionalDependencies: - "@img/sharp-libvips-darwin-arm64" "1.0.1" - -"@img/sharp-darwin-x64@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.2.tgz#982e26bb9d38a81f75915c4032539aed621d1c21" - integrity sha512-/rK/69Rrp9x5kaWBjVN07KixZanRr+W1OiyKdXcbjQD6KbW+obaTeBBtLUAtbBsnlTTmWthw99xqoOS7SsySDg== - optionalDependencies: - "@img/sharp-libvips-darwin-x64" "1.0.1" - -"@img/sharp-libvips-darwin-arm64@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.1.tgz" - integrity sha512-kQyrSNd6lmBV7O0BUiyu/OEw9yeNGFbQhbxswS1i6rMDwBBSX+e+rPzu3S+MwAiGU3HdLze3PanQ4Xkfemgzcw== - -"@img/sharp-libvips-darwin-x64@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.1.tgz#fc1fcd9d78a178819eefe2c1a1662067a83ab1d6" - integrity sha512-eVU/JYLPVjhhrd8Tk6gosl5pVlvsqiFlt50wotCvdkFGf+mDNBJxMh+bvav+Wt3EBnNZWq8Sp2I7XfSjm8siog== - -"@img/sharp-libvips-linux-arm64@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.1.tgz#26eb8c556a9b0db95f343fc444abc3effb67ebcf" - integrity sha512-bnGG+MJjdX70mAQcSLxgeJco11G+MxTz+ebxlz8Y3dxyeb3Nkl7LgLI0mXupoO+u1wRNx/iRj5yHtzA4sde1yA== - -"@img/sharp-libvips-linux-arm@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.1.tgz#2a377b959ff7dd6528deee486c25461296a4fa8b" - integrity sha512-FtdMvR4R99FTsD53IA3LxYGghQ82t3yt0ZQ93WMZ2xV3dqrb0E8zq4VHaTOuLEAuA83oDawHV3fd+BsAPadHIQ== - -"@img/sharp-libvips-linux-s390x@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.1.tgz#af28ac9ba929204467ecdf843330d791e9421e10" - integrity sha512-3+rzfAR1YpMOeA2zZNp+aYEzGNWK4zF3+sdMxuCS3ey9HhDbJ66w6hDSHDMoap32DueFwhhs3vwooAB2MaK4XQ== - -"@img/sharp-libvips-linux-x64@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.1.tgz#4273d182aa51912e655e1214ea47983d7c1f7f8d" - integrity sha512-3NR1mxFsaSgMMzz1bAnnKbSAI+lHXVTqAHgc1bgzjHuXjo4hlscpUxc0vFSAPKI3yuzdzcZOkq7nDPrP2F8Jgw== - -"@img/sharp-libvips-linuxmusl-arm64@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.1.tgz#d150c92151cea2e8d120ad168b9c358d09c77ce8" - integrity sha512-5aBRcjHDG/T6jwC3Edl3lP8nl9U2Yo8+oTl5drd1dh9Z1EBfzUKAJFUDTDisDjUwc7N4AjnPGfCA3jl3hY8uDg== - -"@img/sharp-libvips-linuxmusl-x64@1.0.1": - version "1.0.1" - resolved "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.1.tgz#e297c1a4252c670d93b0f9e51fca40a7a5b6acfd" - integrity sha512-dcT7inI9DBFK6ovfeWRe3hG30h51cBAP5JXlZfx6pzc/Mnf9HFCQDLtYf4MCBjxaaTfjCCjkBxcy3XzOAo5txw== - -"@img/sharp-linux-arm64@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.2.tgz#af3409f801a9bee1d11d0c7e971dcd6180f80022" - integrity sha512-pz0NNo882vVfqJ0yNInuG9YH71smP4gRSdeL09ukC2YLE6ZyZePAlWKEHgAzJGTiOh8Qkaov6mMIMlEhmLdKew== - optionalDependencies: - "@img/sharp-libvips-linux-arm64" "1.0.1" - -"@img/sharp-linux-arm@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.2.tgz#181f7466e6ac074042a38bfb679eb82505e17083" - integrity sha512-Fndk/4Zq3vAc4G/qyfXASbS3HBZbKrlnKZLEJzPLrXoJuipFNNwTes71+Ki1hwYW5lch26niRYoZFAtZVf3EGA== - optionalDependencies: - "@img/sharp-libvips-linux-arm" "1.0.1" - -"@img/sharp-linux-s390x@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.2.tgz#9c171f49211f96fba84410b3e237b301286fa00f" - integrity sha512-MBoInDXDppMfhSzbMmOQtGfloVAflS2rP1qPcUIiITMi36Mm5YR7r0ASND99razjQUpHTzjrU1flO76hKvP5RA== - optionalDependencies: - "@img/sharp-libvips-linux-s390x" "1.0.1" - -"@img/sharp-linux-x64@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.2.tgz#b956dfc092adc58c2bf0fae2077e6f01a8b2d5d7" - integrity sha512-xUT82H5IbXewKkeF5aiooajoO1tQV4PnKfS/OZtb5DDdxS/FCI/uXTVZ35GQ97RZXsycojz/AJ0asoz6p2/H/A== - optionalDependencies: - "@img/sharp-libvips-linux-x64" "1.0.1" - -"@img/sharp-linuxmusl-arm64@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.2.tgz#10e0ec5a79d1234c6a71df44c9f3b0bef0bc0f15" - integrity sha512-F+0z8JCu/UnMzg8IYW1TMeiViIWBVg7IWP6nE0p5S5EPQxlLd76c8jYemG21X99UzFwgkRo5yz2DS+zbrnxZeA== - optionalDependencies: - "@img/sharp-libvips-linuxmusl-arm64" "1.0.1" - -"@img/sharp-linuxmusl-x64@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.2.tgz#29e0030c24aa27c38201b1fc84e3d172899fcbe0" - integrity sha512-+ZLE3SQmSL+Fn1gmSaM8uFusW5Y3J9VOf+wMGNnTtJUMUxFhv+P4UPaYEYT8tqnyYVaOVGgMN/zsOxn9pSsO2A== - optionalDependencies: - "@img/sharp-libvips-linuxmusl-x64" "1.0.1" - -"@img/sharp-wasm32@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.2.tgz#38d7c740a22de83a60ad1e6bcfce17462b0d4230" - integrity sha512-fLbTaESVKuQcpm8ffgBD7jLb/CQLcATju/jxtTXR1XCLwbOQt+OL5zPHSDMmp2JZIeq82e18yE0Vv7zh6+6BfQ== - dependencies: - "@emnapi/runtime" "^0.45.0" - -"@img/sharp-win32-ia32@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.2.tgz#09456314e223f68e5417c283b45c399635c16202" - integrity sha512-okBpql96hIGuZ4lN3+nsAjGeggxKm7hIRu9zyec0lnfB8E7Z6p95BuRZzDDXZOl2e8UmR4RhYt631i7mfmKU8g== - -"@img/sharp-win32-x64@0.33.2": - version "0.33.2" - resolved "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.2.tgz#148e96dfd6e68747da41a311b9ee4559bb1b1471" - integrity sha512-E4magOks77DK47FwHUIGH0RYWSgRBfGdK56kIHSVeB9uIS4pPFr4N2kIVsXdQQo4LzOsENKV5KAhRlRL7eMAdg== - -"@isaacs/cliui@^8.0.2": - version "8.0.2" - resolved "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz" - integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== - dependencies: - string-width "^5.1.2" - string-width-cjs "npm:string-width@^4.2.0" - strip-ansi "^7.0.1" - strip-ansi-cjs "npm:strip-ansi@^6.0.1" - wrap-ansi "^8.1.0" - wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" - -"@jridgewell/gen-mapping@^0.3.2": - version "0.3.3" - resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz" - integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ== - dependencies: - "@jridgewell/set-array" "^1.0.1" - "@jridgewell/sourcemap-codec" "^1.4.10" - "@jridgewell/trace-mapping" "^0.3.9" - -"@jridgewell/resolve-uri@3.1.0": - version "3.1.0" - resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz" - integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== - -"@jridgewell/set-array@^1.0.1": - version "1.1.2" - resolved "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz" - integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== - -"@jridgewell/sourcemap-codec@1.4.14": - version "1.4.14" - resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz" - integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== - -"@jridgewell/sourcemap-codec@^1.4.10": - version "1.4.15" - resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz" - integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== - -"@jridgewell/trace-mapping@^0.3.9": - version "0.3.18" - resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz" - integrity sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA== - dependencies: - "@jridgewell/resolve-uri" "3.1.0" - "@jridgewell/sourcemap-codec" "1.4.14" - -"@lexical/clipboard@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/clipboard/-/clipboard-0.12.2.tgz" - integrity sha512-RldmfZquuJJJCJ5WquCyoJ1/eZ+AnNgdksqvd+G+Yn/GyJl/+O3dnHM0QVaDSPvh/PynLFcCtz/57ySLo2kQxQ== - dependencies: - "@lexical/html" "0.12.2" - "@lexical/list" "0.12.2" - "@lexical/selection" "0.12.2" - "@lexical/utils" "0.12.2" - -"@lexical/code@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/code/-/code-0.12.2.tgz" - integrity sha512-w2JeJdnMUtYnC/Fx78sL3iJBt9Ug8pFSDOcI9ay/BkMQFQV8oqq1iyuLLBBJSG4FAM8b2DXrVdGklRQ+jTfTVw== - dependencies: - "@lexical/utils" "0.12.2" - prismjs "^1.27.0" - -"@lexical/dragon@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/dragon/-/dragon-0.12.2.tgz" - integrity sha512-Mt8NLzTOt+VgQtc2DKDbHBwKeRlvKqbLqRIMYUVk60gol+YV7NpVBsP1PAMuYYjrTQLhlckBSC32H1SUHZRavA== - -"@lexical/hashtag@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/hashtag/-/hashtag-0.12.2.tgz" - integrity sha512-2vYzIu5Ldf+eYdUrNA2m80c3N3MF3vJ0fIJzpl5QyX8OdViggEWl1bh+lKtw1Ju0H0CUyDIXdDLZ2apW3WDkTA== - dependencies: - "@lexical/utils" "0.12.2" - -"@lexical/history@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/history/-/history-0.12.2.tgz" - integrity sha512-PM/EDjnUyBPMWh1UiYb7T+FLbvTk14HwUWLXvZxn72S6Kj8ExH/PfLbWZWLCFL8RfzvbP407VwfSN8S0bF5H6g== - dependencies: - "@lexical/utils" "0.12.2" - -"@lexical/html@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/html/-/html-0.12.2.tgz" - integrity sha512-LWUO6OKhDtDZa9X1spHAqzsp+4EF01exis4cz5H9y2sHi7EofogXnRCadZ+fa07NVwPVTZWsStkk5qdSe/NEzg== - dependencies: - "@lexical/selection" "0.12.2" - -"@lexical/link@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/link/-/link-0.12.2.tgz" - integrity sha512-etOIONa7uyRDmwg8GN52kDlf8thD2Zk1LOFLeocHWz1V8fe3i2unGUek5s/rNPkc6ynpPpNsHdN1VEghOLCCmw== - dependencies: - "@lexical/utils" "0.12.2" - -"@lexical/list@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/list/-/list-0.12.2.tgz" - integrity sha512-3CyWtYQC+IlK4cK/oiD8Uz1gSXD8UcKGOF2vVsDXkMU06O6zvHNmHZOnVJqA0JVNgZAoR9dMR1fi2xd4iuCAiw== - dependencies: - "@lexical/utils" "0.12.2" - -"@lexical/mark@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/mark/-/mark-0.12.2.tgz" - integrity sha512-ub+37PDfmThsqAWipRTrwqpgE+83ckqJ5C3mKQUBZvhZfVZW1rEUXZnKjFh2Q3eZK6iT7zVgoVJWJS9ZgEEyag== - dependencies: - "@lexical/utils" "0.12.2" - -"@lexical/markdown@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/markdown/-/markdown-0.12.2.tgz" - integrity sha512-F2jTFtBp7Q+yoA11BeUOEcxhROzW+HUhUGdsn20pSLhuxsWRj3oUuryWFeNKFofpzTCVoqU6dwpaMNMI2mL/sQ== - dependencies: - "@lexical/code" "0.12.2" - "@lexical/link" "0.12.2" - "@lexical/list" "0.12.2" - "@lexical/rich-text" "0.12.2" - "@lexical/text" "0.12.2" - "@lexical/utils" "0.12.2" - -"@lexical/offset@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/offset/-/offset-0.12.2.tgz" - integrity sha512-rZLZXfOBmpmM8A2UZsX3cr/CQYw5F/ou67AbaKI0WImb5sjnIgICZqzu9VFUnkKlVNUurEpplV3UG3D1YYh1OQ== - -"@lexical/overflow@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/overflow/-/overflow-0.12.2.tgz" - integrity sha512-UgE5j3ukO6qRFRpH4T7m/DvnodE9nCtImD7QinyGdsTa0hi5xlRnl0FUo605vH+vz7xEsUNAGwQXYPX9Sc/vig== - -"@lexical/plain-text@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/plain-text/-/plain-text-0.12.2.tgz" - integrity sha512-Lcg6+ngRnX70//kz34azYhID3bvW66HSHCfu5UPhCXT+vQ/Jkd/InhRKajBwWXpaJxMM1huoi3sjzVDb3luNtw== - -"@lexical/react@^0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/react/-/react-0.12.2.tgz" - integrity sha512-ZBUvf5xmhiYWBw8pPrhYmLAEwFWrbF/cd15y76TUKD9l/2zDwwPs6nJQxBzfz3ei65r2/nnavLDV8W3QfvxfUA== - dependencies: - "@lexical/clipboard" "0.12.2" - "@lexical/code" "0.12.2" - "@lexical/dragon" "0.12.2" - "@lexical/hashtag" "0.12.2" - "@lexical/history" "0.12.2" - "@lexical/link" "0.12.2" - "@lexical/list" "0.12.2" - "@lexical/mark" "0.12.2" - "@lexical/markdown" "0.12.2" - "@lexical/overflow" "0.12.2" - "@lexical/plain-text" "0.12.2" - "@lexical/rich-text" "0.12.2" - "@lexical/selection" "0.12.2" - "@lexical/table" "0.12.2" - "@lexical/text" "0.12.2" - "@lexical/utils" "0.12.2" - "@lexical/yjs" "0.12.2" - react-error-boundary "^3.1.4" - -"@lexical/rich-text@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/rich-text/-/rich-text-0.12.2.tgz" - integrity sha512-igsEuv7CwBOAj5c8jeE41cnx6zkhI/Bkbu4W7shT6S6lNA/3cnyZpAMlgixwyK5RoqjGRCT+IJK5l6yBxQfNkw== - -"@lexical/selection@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/selection/-/selection-0.12.2.tgz" - integrity sha512-h+g3oOnihHKIyLTyG6uLCEVR/DmUEVdCcZO1iAoGsuW7nwWiWNPWj6oZ3Cw5J1Mk5u62DHnkkVDQsVSZbAwmtg== - -"@lexical/table@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/table/-/table-0.12.2.tgz" - integrity sha512-tiAmTq6RKHDVER9v589Ajm9/RL+WTF1WschrH6HHVCtil6cfJfTJeJ+MF45+XEzB9fkqy2LfrScAfWxqLjVePA== - dependencies: - "@lexical/utils" "0.12.2" - -"@lexical/text@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/text/-/text-0.12.2.tgz" - integrity sha512-HyuIGuQvVi5djJKKBf+jYEBjK+0Eo9cKHf6WS7dlFozuCZvcCQEJkFy2yceWOwIVk+f2kptVQ5uO7aiZHExH2A== - -"@lexical/utils@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/utils/-/utils-0.12.2.tgz" - integrity sha512-xW4y4l2Yd37+qLwkBvBGyzsKCA9wnh1ljphBJeR2vreT193i2gaIwuku2ZKlER14VHw4192qNJF7vUoAEmwurQ== - dependencies: - "@lexical/list" "0.12.2" - "@lexical/selection" "0.12.2" - "@lexical/table" "0.12.2" - -"@lexical/yjs@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/yjs/-/yjs-0.12.2.tgz" - integrity sha512-OPJhkJD1Mp9W80mfLzASTB3OFWFMzJteUYA+eSyDgiX9zNi1VGxAqmIITTkDvnCMa+qvw4EfhGeGezpjx6Og4A== - dependencies: - "@lexical/offset" "0.12.2" - -"@mdx-js/loader@^2.3.0": - version "2.3.0" - resolved "https://registry.npmjs.org/@mdx-js/loader/-/loader-2.3.0.tgz" - integrity sha512-IqsscXh7Q3Rzb+f5DXYk0HU71PK+WuFsEhf+mSV3fOhpLcEpgsHvTQ2h0T6TlZ5gHOaBeFjkXwB52by7ypMyNg== - dependencies: - "@mdx-js/mdx" "^2.0.0" - source-map "^0.7.0" - -"@mdx-js/mdx@^2.0.0": - version "2.3.0" - resolved "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-2.3.0.tgz" - integrity sha512-jLuwRlz8DQfQNiUCJR50Y09CGPq3fLtmtUQfVrj79E0JWu3dvsVcxVIcfhR5h0iXu+/z++zDrYeiJqifRynJkA== - dependencies: - "@types/estree-jsx" "^1.0.0" - "@types/mdx" "^2.0.0" - estree-util-build-jsx "^2.0.0" - estree-util-is-identifier-name "^2.0.0" - estree-util-to-js "^1.1.0" - estree-walker "^3.0.0" - hast-util-to-estree "^2.0.0" - markdown-extensions "^1.0.0" - periscopic "^3.0.0" - remark-mdx "^2.0.0" - remark-parse "^10.0.0" - remark-rehype "^10.0.0" - unified "^10.0.0" - unist-util-position-from-estree "^1.0.0" - unist-util-stringify-position "^3.0.0" - unist-util-visit "^4.0.0" - vfile "^5.0.0" - -"@mdx-js/react@^2.3.0": - version "2.3.0" - resolved "https://registry.npmjs.org/@mdx-js/react/-/react-2.3.0.tgz" - integrity sha512-zQH//gdOmuu7nt2oJR29vFhDv88oGPmVw6BggmrHeMI+xgEkp1B2dX9/bMBSYtK0dyLX/aOmesKS09g222K1/g== - dependencies: - "@types/mdx" "^2.0.0" - "@types/react" ">=16" - -"@miragejs/pretender-node-polyfill@^0.1.0": - version "0.1.2" - resolved "https://registry.npmjs.org/@miragejs/pretender-node-polyfill/-/pretender-node-polyfill-0.1.2.tgz" - integrity sha512-M/BexG/p05C5lFfMunxo/QcgIJnMT2vDVCd00wNqK2ImZONIlEETZwWJu1QtLxtmYlSHlCFl3JNzp0tLe7OJ5g== - -"@monaco-editor/loader@^1.4.0": - version "1.4.0" - resolved "https://registry.npmjs.org/@monaco-editor/loader/-/loader-1.4.0.tgz" - integrity sha512-00ioBig0x642hytVspPl7DbQyaSWRaolYie/UFNjoTdvoKPzo6xrXLhTk9ixgIKcLH5b5vDOjVNiGyY+uDCUlg== - dependencies: - state-local "^1.0.6" - -"@monaco-editor/react@^4.6.0": - version "4.6.0" - resolved "https://registry.npmjs.org/@monaco-editor/react/-/react-4.6.0.tgz" - integrity sha512-RFkU9/i7cN2bsq/iTkurMWOEErmYcY6JiQI3Jn+WeR/FGISH8JbHERjpS9oRuSOPvDMJI0Z8nJeKkbOs9sBYQw== - dependencies: - "@monaco-editor/loader" "^1.4.0" - -"@next/env@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/env/-/env-14.1.0.tgz" - integrity sha512-Py8zIo+02ht82brwwhTg36iogzFqGLPXlRGKQw5s+qP/kMNc4MAyDeEwBKDijk6zTIbegEgu8Qy7C1LboslQAw== - -"@next/eslint-plugin-next@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.1.0.tgz" - integrity sha512-x4FavbNEeXx/baD/zC/SdrvkjSby8nBn8KcCREqk6UuwvwoAPZmaV8TFCAuo/cpovBRTIY67mHhe86MQQm/68Q== - dependencies: - glob "10.3.10" - -"@next/mdx@^14.0.4": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/mdx/-/mdx-14.1.0.tgz" - integrity sha512-YLYsViq91+H8+3oCtK1iuMWdeN14K70Hy6/tYScY+nfo5bQ84A/A+vA6UdNC9MkbWQ/373hQubx2p4JvUjlb2Q== - dependencies: - source-map "^0.7.0" - -"@next/swc-darwin-arm64@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.0.tgz" - integrity sha512-nUDn7TOGcIeyQni6lZHfzNoo9S0euXnu0jhsbMOmMJUBfgsnESdjN97kM7cBqQxZa8L/bM9om/S5/1dzCrW6wQ== - -"@next/swc-darwin-x64@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz#0863a22feae1540e83c249384b539069fef054e9" - integrity sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g== - -"@next/swc-linux-arm64-gnu@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz#893da533d3fce4aec7116fe772d4f9b95232423c" - integrity sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ== - -"@next/swc-linux-arm64-musl@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz#d81ddcf95916310b8b0e4ad32b637406564244c0" - integrity sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g== - -"@next/swc-linux-x64-gnu@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz#18967f100ec19938354332dcb0268393cbacf581" - integrity sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ== - -"@next/swc-linux-x64-musl@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz#77077cd4ba8dda8f349dc7ceb6230e68ee3293cf" - integrity sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg== - -"@next/swc-win32-arm64-msvc@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz#5f0b8cf955644104621e6d7cc923cad3a4c5365a" - integrity sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ== - -"@next/swc-win32-ia32-msvc@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz#21f4de1293ac5e5a168a412b139db5d3420a89d0" - integrity sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw== - -"@next/swc-win32-x64-msvc@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz#e561fb330466d41807123d932b365cf3d33ceba2" - integrity sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg== - -"@nodelib/fs.scandir@2.1.5": - version "2.1.5" - resolved "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz" - integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== - dependencies: - "@nodelib/fs.stat" "2.0.5" - run-parallel "^1.1.9" - -"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": - version "2.0.5" - resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz" - integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== - -"@nodelib/fs.walk@^1.2.3", "@nodelib/fs.walk@^1.2.8": - version "1.2.8" - resolved "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz" - integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== - dependencies: - "@nodelib/fs.scandir" "2.1.5" - fastq "^1.6.0" - -"@pkgjs/parseargs@^0.11.0": - version "0.11.0" - resolved "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz" - integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== - -"@pkgr/utils@^2.3.1": - version "2.4.1" - resolved "https://registry.npmjs.org/@pkgr/utils/-/utils-2.4.1.tgz" - integrity sha512-JOqwkgFEyi+OROIyq7l4Jy28h/WwhDnG/cPkXG2Z1iFbubB6jsHW1NDvmyOzTBxHr3yg68YGirmh1JUgMqa+9w== - dependencies: - cross-spawn "^7.0.3" - fast-glob "^3.2.12" - is-glob "^4.0.3" - open "^9.1.0" - picocolors "^1.0.0" - tslib "^2.5.0" - -"@reactflow/background@11.3.8": - version "11.3.8" - resolved "https://registry.npmjs.org/@reactflow/background/-/background-11.3.8.tgz" - integrity sha512-U4aI54F7PwqgYI0Knv72QFRI/wXeipPmIYAlDsA0j51+tlPxs3Nk2z7G1/4pC11GxEZkgQVfcIXro4l1Fk+bIQ== - dependencies: - "@reactflow/core" "11.10.3" - classcat "^5.0.3" - zustand "^4.4.1" - -"@reactflow/controls@11.2.8": - version "11.2.8" - resolved "https://registry.npmjs.org/@reactflow/controls/-/controls-11.2.8.tgz" - integrity sha512-Y9YVx38sRjYtbPsI/xa+B1FGN73FV1GqqajlmGfrc1TmqhJaX+gaMXMvVazT/N5haK1hMJvOApUTLQ2V/5Rdbg== - dependencies: - "@reactflow/core" "11.10.3" - classcat "^5.0.3" - zustand "^4.4.1" - -"@reactflow/core@11.10.3": - version "11.10.3" - resolved "https://registry.npmjs.org/@reactflow/core/-/core-11.10.3.tgz" - integrity sha512-nV3nep4fjBy3h8mYSnJcclGcQtj8fkUBmNkEwCZCK4ps+n3HNkXFB0BRisSnQz6GRQlYboSsi0cThEl3KdNITw== - dependencies: - "@types/d3" "^7.4.0" - "@types/d3-drag" "^3.0.1" - "@types/d3-selection" "^3.0.3" - "@types/d3-zoom" "^3.0.1" - classcat "^5.0.3" - d3-drag "^3.0.0" - d3-selection "^3.0.0" - d3-zoom "^3.0.0" - zustand "^4.4.1" - -"@reactflow/minimap@11.7.8": - version "11.7.8" - resolved "https://registry.npmjs.org/@reactflow/minimap/-/minimap-11.7.8.tgz" - integrity sha512-MwyP5q3VomC91Dhd4P6YcxEfnjDbREGYV6rRxbSJSTHiG0x7ETQCcPelYDGy7JvQej77Pa2yJ4g0FDwP7CsQEA== - dependencies: - "@reactflow/core" "11.10.3" - "@types/d3-selection" "^3.0.3" - "@types/d3-zoom" "^3.0.1" - classcat "^5.0.3" - d3-selection "^3.0.0" - d3-zoom "^3.0.0" - zustand "^4.4.1" - -"@reactflow/node-resizer@2.2.8": - version "2.2.8" - resolved "https://registry.npmjs.org/@reactflow/node-resizer/-/node-resizer-2.2.8.tgz" - integrity sha512-u/EXLpvOfAmq1sGoPYwoX4gi0PnCi0mH3eHVClHNc8JQD0WCqcV1UeVV7H3PF+1SGhhg/aOv/vPG1PcQ5fu4jQ== - dependencies: - "@reactflow/core" "11.10.3" - classcat "^5.0.4" - d3-drag "^3.0.0" - d3-selection "^3.0.0" - zustand "^4.4.1" - -"@reactflow/node-toolbar@1.3.8": - version "1.3.8" - resolved "https://registry.npmjs.org/@reactflow/node-toolbar/-/node-toolbar-1.3.8.tgz" - integrity sha512-cfvlTPeJa/ciQTosx2bGrjHT8K/UL9kznpvpOzeZFnJm5UQXmbwAYt4Vo6GfkD51mORcIL7ujQJvB9ym3ZI9KA== - dependencies: - "@reactflow/core" "11.10.3" - classcat "^5.0.3" - zustand "^4.4.1" - -"@rgrove/parse-xml@^4.1.0": - version "4.1.0" - resolved "https://registry.npmjs.org/@rgrove/parse-xml/-/parse-xml-4.1.0.tgz" - integrity sha512-pBiltENdy8SfI0AeR1e5TRpS9/9Gl0eiOEt6ful2jQfzsgvZYWqsKiBWaOCLdocQuk0wS7KOHI37n0C1pnKqTw== - -"@rushstack/eslint-patch@^1.3.3": - version "1.7.2" - resolved "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.7.2.tgz" - integrity sha512-RbhOOTCNoCrbfkRyoXODZp75MlpiHMgbE5MEBZAnnnLyQNgrigEj4p0lzsMDyc1zVsJDLrivB58tgg3emX0eEA== - -"@sentry-internal/tracing@7.54.0": - version "7.54.0" - resolved "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.54.0.tgz" - integrity sha512-JsyhZ0wWZ+VqbHJg+azqRGdYJDkcI5R9+pnkO6SzbzxrRewqMAIwzkpPee3oI7vG99uhMEkOkMjHu0nQGwkOQw== - dependencies: - "@sentry/core" "7.54.0" - "@sentry/types" "7.54.0" - "@sentry/utils" "7.54.0" - tslib "^1.9.3" - -"@sentry/browser@7.54.0": - version "7.54.0" - resolved "https://registry.npmjs.org/@sentry/browser/-/browser-7.54.0.tgz" - integrity sha512-EvLAw03N9WE2m1CMl2/1YMeIs1icw9IEOVJhWmf3uJEysNJOFWXu6ZzdtHEz1E6DiJYhc1HzDya0ExZeJxNARA== - dependencies: - "@sentry-internal/tracing" "7.54.0" - "@sentry/core" "7.54.0" - "@sentry/replay" "7.54.0" - "@sentry/types" "7.54.0" - "@sentry/utils" "7.54.0" - tslib "^1.9.3" - -"@sentry/core@7.54.0": - version "7.54.0" - resolved "https://registry.npmjs.org/@sentry/core/-/core-7.54.0.tgz" - integrity sha512-MAn0E2EwgNn1pFQn4qxhU+1kz6edullWg6VE5wCmtpXWOVw6sILBUsQpeIG5djBKMcneJCdOlz5jeqcKPrLvZQ== - dependencies: - "@sentry/types" "7.54.0" - "@sentry/utils" "7.54.0" - tslib "^1.9.3" - -"@sentry/react@^7.54.0": - version "7.54.0" - resolved "https://registry.npmjs.org/@sentry/react/-/react-7.54.0.tgz" - integrity sha512-qUbwmRRpTh05m2rbC8A2zAFQYsoHhwIpxT5UXxh0P64ZlA3cSg1/DmTTgwnd1l+7gzKrc31UikXQ4y0YDbMNKg== - dependencies: - "@sentry/browser" "7.54.0" - "@sentry/types" "7.54.0" - "@sentry/utils" "7.54.0" - hoist-non-react-statics "^3.3.2" - tslib "^1.9.3" - -"@sentry/replay@7.54.0": - version "7.54.0" - resolved "https://registry.npmjs.org/@sentry/replay/-/replay-7.54.0.tgz" - integrity sha512-C0F0568ybphzGmKGe23duB6n5wJcgM7WLYhoeqW3o2bHeqpj1dGPSka/K3s9KzGaAgzn1zeOUYXJsOs+T/XdsA== - dependencies: - "@sentry/core" "7.54.0" - "@sentry/types" "7.54.0" - "@sentry/utils" "7.54.0" - -"@sentry/types@7.54.0": - version "7.54.0" - resolved "https://registry.npmjs.org/@sentry/types/-/types-7.54.0.tgz" - integrity sha512-D+i9xogBeawvQi2r0NOrM7zYcUaPuijeME4O9eOTrDF20tj71hWtJLilK+KTGLYFtpGg1h+9bPaz7OHEIyVopg== - -"@sentry/utils@7.54.0", "@sentry/utils@^7.54.0": - version "7.54.0" - resolved "https://registry.npmjs.org/@sentry/utils/-/utils-7.54.0.tgz" - integrity sha512-3Yf5KlKjIcYLddOexSt2ovu2TWlR4Fi7M+aCK8yUTzwNzf/xwFSWOstHlD/WiDy9HvfhWAOB/ukNTuAeJmtasw== - dependencies: - "@sentry/types" "7.54.0" - tslib "^1.9.3" - -"@swc/helpers@0.5.2": - version "0.5.2" - resolved "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz" - integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== - dependencies: - tslib "^2.4.0" - -"@tailwindcss/line-clamp@^0.4.4": - version "0.4.4" - resolved "https://registry.npmjs.org/@tailwindcss/line-clamp/-/line-clamp-0.4.4.tgz" - integrity sha512-5U6SY5z8N42VtrCrKlsTAA35gy2VSyYtHWCsg1H87NU1SXnEfekTVlrga9fzUDrrHcGi2Lb5KenUWb4lRQT5/g== - -"@tailwindcss/typography@^0.5.9": - version "0.5.9" - resolved "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.9.tgz" - integrity sha512-t8Sg3DyynFysV9f4JDOVISGsjazNb48AeIYQwcL+Bsq5uf4RYL75C1giZ43KISjeDGBaTN3Kxh7Xj/vRSMJUUg== - dependencies: - lodash.castarray "^4.4.0" - lodash.isplainobject "^4.0.6" - lodash.merge "^4.6.2" - postcss-selector-parser "6.0.10" - -"@types/acorn@^4.0.0": - version "4.0.6" - resolved "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz" - integrity sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ== - dependencies: - "@types/estree" "*" - -"@types/crypto-js@^4.1.1": - version "4.1.1" - resolved "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz" - integrity sha512-BG7fQKZ689HIoc5h+6D2Dgq1fABRa0RbBWKBd9SP/MVRVXROflpm5fhwyATX5duFmbStzyzyycPB8qUYKDH3NA== - -"@types/d3-array@*": - version "3.2.1" - resolved "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz" - integrity sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg== - -"@types/d3-axis@*": - version "3.0.6" - resolved "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz" - integrity sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw== - dependencies: - "@types/d3-selection" "*" - -"@types/d3-brush@*": - version "3.0.6" - resolved "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz" - integrity sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A== - dependencies: - "@types/d3-selection" "*" - -"@types/d3-chord@*": - version "3.0.6" - resolved "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz" - integrity sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg== - -"@types/d3-color@*": - version "3.1.3" - resolved "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz" - integrity sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A== - -"@types/d3-contour@*": - version "3.0.6" - resolved "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz" - integrity sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg== - dependencies: - "@types/d3-array" "*" - "@types/geojson" "*" - -"@types/d3-delaunay@*": - version "6.0.4" - resolved "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz" - integrity sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw== - -"@types/d3-dispatch@*": - version "3.0.6" - resolved "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.6.tgz" - integrity sha512-4fvZhzMeeuBJYZXRXrRIQnvUYfyXwYmLsdiN7XXmVNQKKw1cM8a5WdID0g1hVFZDqT9ZqZEY5pD44p24VS7iZQ== - -"@types/d3-drag@*", "@types/d3-drag@^3.0.1": - version "3.0.7" - resolved "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz" - integrity sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ== - dependencies: - "@types/d3-selection" "*" - -"@types/d3-dsv@*": - version "3.0.7" - resolved "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz" - integrity sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g== - -"@types/d3-ease@*": - version "3.0.2" - resolved "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz" - integrity sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA== - -"@types/d3-fetch@*": - version "3.0.7" - resolved "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz" - integrity sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA== - dependencies: - "@types/d3-dsv" "*" - -"@types/d3-force@*": - version "3.0.9" - resolved "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.9.tgz" - integrity sha512-IKtvyFdb4Q0LWna6ymywQsEYjK/94SGhPrMfEr1TIc5OBeziTi+1jcCvttts8e0UWZIxpasjnQk9MNk/3iS+kA== - -"@types/d3-format@*": - version "3.0.4" - resolved "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz" - integrity sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g== - -"@types/d3-geo@*": - version "3.1.0" - resolved "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.0.tgz" - integrity sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ== - dependencies: - "@types/geojson" "*" - -"@types/d3-hierarchy@*": - version "3.1.6" - resolved "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.6.tgz" - integrity sha512-qlmD/8aMk5xGorUvTUWHCiumvgaUXYldYjNVOWtYoTYY/L+WwIEAmJxUmTgr9LoGNG0PPAOmqMDJVDPc7DOpPw== - -"@types/d3-interpolate@*": - version "3.0.4" - resolved "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz" - integrity sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA== - dependencies: - "@types/d3-color" "*" - -"@types/d3-path@*": - version "3.0.2" - resolved "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.0.2.tgz" - integrity sha512-WAIEVlOCdd/NKRYTsqCpOMHQHemKBEINf8YXMYOtXH0GA7SY0dqMB78P3Uhgfy+4X+/Mlw2wDtlETkN6kQUCMA== - -"@types/d3-polygon@*": - version "3.0.2" - resolved "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz" - integrity sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA== - -"@types/d3-quadtree@*": - version "3.0.6" - resolved "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz" - integrity sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg== - -"@types/d3-random@*": - version "3.0.3" - resolved "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.3.tgz" - integrity sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ== - -"@types/d3-scale-chromatic@*", "@types/d3-scale-chromatic@^3.0.0": - version "3.0.0" - resolved "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz" - integrity sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw== - -"@types/d3-scale@*", "@types/d3-scale@^4.0.3": - version "4.0.4" - resolved "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.4.tgz" - integrity sha512-eq1ZeTj0yr72L8MQk6N6heP603ubnywSDRfNpi5enouR112HzGLS6RIvExCzZTraFF4HdzNpJMwA/zGiMoHUUw== - dependencies: - "@types/d3-time" "*" - -"@types/d3-selection@*", "@types/d3-selection@^3.0.3": - version "3.0.10" - resolved "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.10.tgz" - integrity sha512-cuHoUgS/V3hLdjJOLTT691+G2QoqAjCVLmr4kJXR4ha56w1Zdu8UUQ5TxLRqudgNjwXeQxKMq4j+lyf9sWuslg== - -"@types/d3-shape@*": - version "3.1.6" - resolved "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.6.tgz" - integrity sha512-5KKk5aKGu2I+O6SONMYSNflgiP0WfZIQvVUMan50wHsLG1G94JlxEVnCpQARfTtzytuY0p/9PXXZb3I7giofIA== - dependencies: - "@types/d3-path" "*" - -"@types/d3-time-format@*": - version "4.0.3" - resolved "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz" - integrity sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg== - -"@types/d3-time@*": - version "3.0.0" - resolved "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz" - integrity sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg== - -"@types/d3-timer@*": - version "3.0.2" - resolved "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz" - integrity sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw== - -"@types/d3-transition@*": - version "3.0.8" - resolved "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.8.tgz" - integrity sha512-ew63aJfQ/ms7QQ4X7pk5NxQ9fZH/z+i24ZfJ6tJSfqxJMrYLiK01EAs2/Rtw/JreGUsS3pLPNV644qXFGnoZNQ== - dependencies: - "@types/d3-selection" "*" - -"@types/d3-zoom@*", "@types/d3-zoom@^3.0.1": - version "3.0.8" - resolved "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz" - integrity sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw== - dependencies: - "@types/d3-interpolate" "*" - "@types/d3-selection" "*" - -"@types/d3@^7.4.0": - version "7.4.3" - resolved "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz" - integrity sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww== - dependencies: - "@types/d3-array" "*" - "@types/d3-axis" "*" - "@types/d3-brush" "*" - "@types/d3-chord" "*" - "@types/d3-color" "*" - "@types/d3-contour" "*" - "@types/d3-delaunay" "*" - "@types/d3-dispatch" "*" - "@types/d3-drag" "*" - "@types/d3-dsv" "*" - "@types/d3-ease" "*" - "@types/d3-fetch" "*" - "@types/d3-force" "*" - "@types/d3-format" "*" - "@types/d3-geo" "*" - "@types/d3-hierarchy" "*" - "@types/d3-interpolate" "*" - "@types/d3-path" "*" - "@types/d3-polygon" "*" - "@types/d3-quadtree" "*" - "@types/d3-random" "*" - "@types/d3-scale" "*" - "@types/d3-scale-chromatic" "*" - "@types/d3-selection" "*" - "@types/d3-shape" "*" - "@types/d3-time" "*" - "@types/d3-time-format" "*" - "@types/d3-timer" "*" - "@types/d3-transition" "*" - "@types/d3-zoom" "*" - -"@types/dagre@^0.7.52": - version "0.7.52" - resolved "https://registry.npmjs.org/@types/dagre/-/dagre-0.7.52.tgz" - integrity sha512-XKJdy+OClLk3hketHi9Qg6gTfe1F3y+UFnHxKA2rn9Dw+oXa4Gb378Ztz9HlMgZKSxpPmn4BNVh9wgkpvrK1uw== - -"@types/debug@^4.0.0": - version "4.1.8" - resolved "https://registry.npmjs.org/@types/debug/-/debug-4.1.8.tgz" - integrity sha512-/vPO1EPOs306Cvhwv7KfVfYvOJqA/S/AXjaHQiJboCZzcNDb+TIJFN9/2C9DZ//ijSKWioNyUxD792QmDJ+HKQ== - dependencies: - "@types/ms" "*" - -"@types/estree-jsx@^1.0.0": - version "1.0.0" - resolved "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.0.tgz" - integrity sha512-3qvGd0z8F2ENTGr/GG1yViqfiKmRfrXVx5sJyHGFu3z7m5g5utCQtGp/g29JnjflhtQJBv1WDQukHiT58xPcYQ== - dependencies: - "@types/estree" "*" - -"@types/estree@*", "@types/estree@^1.0.0": - version "1.0.1" - resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz" - integrity sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA== - -"@types/geojson@*": - version "7946.0.14" - resolved "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz" - integrity sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg== - -"@types/hast@^2.0.0": - version "2.3.4" - resolved "https://registry.npmjs.org/@types/hast/-/hast-2.3.4.tgz" - integrity sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g== - dependencies: - "@types/unist" "*" - -"@types/js-cookie@^2.x.x": - version "2.2.7" - resolved "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-2.2.7.tgz" - integrity sha512-aLkWa0C0vO5b4Sr798E26QgOkss68Un0bLjs7u9qxzPT5CG+8DuNTffWES58YzJs3hrVAOs1wonycqEBqNJubA== - -"@types/js-cookie@^3.0.3": - version "3.0.3" - resolved "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-3.0.3.tgz" - integrity sha512-Xe7IImK09HP1sv2M/aI+48a20VX+TdRJucfq4vfRVy6nWN8PYPOEnlMRSgxJAgYQIXJVL8dZ4/ilAM7dWNaOww== - -"@types/json-schema@^7.0.9": - version "7.0.12" - resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz" - integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA== - -"@types/json5@^0.0.29": - version "0.0.29" - resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz" - integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== - -"@types/katex@^0.14.0": - version "0.14.0" - resolved "https://registry.npmjs.org/@types/katex/-/katex-0.14.0.tgz" - integrity sha512-+2FW2CcT0K3P+JMR8YG846bmDwplKUTsWgT2ENwdQ1UdVfRk3GQrh6Mi4sTopy30gI8Uau5CEqHTDZ6YvWIUPA== - -"@types/katex@^0.16.0": - version "0.16.0" - resolved "https://registry.npmjs.org/@types/katex/-/katex-0.16.0.tgz" - integrity sha512-hz+S3nV6Mym5xPbT9fnO8dDhBFQguMYpY0Ipxv06JMi1ORgnEM4M1ymWDUhUNer3ElLmT583opRo4RzxKmh9jw== - -"@types/lodash-es@^4.17.7": - version "4.17.7" - resolved "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.7.tgz" - integrity sha512-z0ptr6UI10VlU6l5MYhGwS4mC8DZyYer2mCoyysZtSF7p26zOX8UpbrV0YpNYLGS8K4PUFIyEr62IMFFjveSiQ== - dependencies: - "@types/lodash" "*" - -"@types/lodash@*": - version "4.14.195" - resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.195.tgz" - integrity sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg== - -"@types/mdast@^3.0.0": - version "3.0.11" - resolved "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.11.tgz" - integrity sha512-Y/uImid8aAwrEA24/1tcRZwpxX3pIFTSilcNDKSPn+Y2iDywSEachzRuvgAYYLR3wpGXAsMbv5lvKLDZLeYPAw== - dependencies: - "@types/unist" "*" - -"@types/mdx@^2.0.0": - version "2.0.5" - resolved "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.5.tgz" - integrity sha512-76CqzuD6Q7LC+AtbPqrvD9AqsN0k8bsYo2bM2J8pmNldP1aIPAbzUQ7QbobyXL4eLr1wK5x8FZFe8eF/ubRuBg== - -"@types/ms@*": - version "0.7.31" - resolved "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz" - integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== - -"@types/negotiator@^0.6.1": - version "0.6.1" - resolved "https://registry.npmjs.org/@types/negotiator/-/negotiator-0.6.1.tgz" - integrity sha512-c4mvXFByghezQ/eVGN5HvH/jI63vm3B7FiE81BUzDAWmuiohRecCO6ddU60dfq29oKUMiQujsoB2h0JQC7JHKA== - -"@types/node@*", "@types/node@18.15.0": - version "18.15.0" - resolved "https://registry.npmjs.org/@types/node/-/node-18.15.0.tgz" - integrity sha512-z6nr0TTEOBGkzLGmbypWOGnpSpSIBorEhC4L+4HeQ2iezKCi4f77kyslRwvHeNitymGQ+oFyIWGP96l/DPSV9w== - -"@types/normalize-package-data@^2.4.0": - version "2.4.1" - resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz" - integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw== - -"@types/papaparse@^5.3.1": - version "5.3.7" - resolved "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.7.tgz" - integrity sha512-f2HKmlnPdCvS0WI33WtCs5GD7X1cxzzS/aduaxSu3I7TbhWlENjSPs6z5TaB9K0J+BH1jbmqTaM+ja5puis4wg== - dependencies: - "@types/node" "*" - -"@types/prop-types@*", "@types/prop-types@^15.0.0": - version "15.7.5" - resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz" - integrity sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w== - -"@types/qs@^6.9.7": - version "6.9.7" - resolved "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz" - integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== - -"@types/react-dom@18.0.11": - version "18.0.11" - resolved "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.11.tgz" - integrity sha512-O38bPbI2CWtgw/OoQoY+BRelw7uysmXbWvw3nLWO21H1HSh+GOlqPuXshJfjmpNlKiiSDG9cc1JZAaMmVdcTlw== - dependencies: - "@types/react" "*" - -"@types/react-slider@^1.3.1": - version "1.3.1" - resolved "https://registry.npmjs.org/@types/react-slider/-/react-slider-1.3.1.tgz" - integrity sha512-4X2yK7RyCIy643YCFL+bc6XNmcnBtt8n88uuyihvcn5G7Lut23eNQU3q3KmwF7MWIfKfsW5NxCjw0SeDZRtgaA== - dependencies: - "@types/react" "*" - -"@types/react-syntax-highlighter@^15.5.6": - version "15.5.7" - resolved "https://registry.npmjs.org/@types/react-syntax-highlighter/-/react-syntax-highlighter-15.5.7.tgz" - integrity sha512-bo5fEO5toQeyCp0zVHBeggclqf5SQ/Z5blfFmjwO5dkMVGPgmiwZsJh9nu/Bo5L7IHTuGWrja6LxJVE2uB5ZrQ== - dependencies: - "@types/react" "*" - -"@types/react-window-infinite-loader@^1.0.6": - version "1.0.6" - resolved "https://registry.npmjs.org/@types/react-window-infinite-loader/-/react-window-infinite-loader-1.0.6.tgz" - integrity sha512-V8g8sBDLVeJJAfEENJS7VXZK+DRJ+jzPNtk8jpj2G+obhf+iqGNUDGwNWCbBhLiD+KpHhf3kWQlKBRi0tAeU4Q== - dependencies: - "@types/react" "*" - "@types/react-window" "*" - -"@types/react-window@*", "@types/react-window@^1.8.5": - version "1.8.5" - resolved "https://registry.npmjs.org/@types/react-window/-/react-window-1.8.5.tgz" - integrity sha512-V9q3CvhC9Jk9bWBOysPGaWy/Z0lxYcTXLtLipkt2cnRj1JOSFNF7wqGpkScSXMgBwC+fnVRg/7shwgddBG5ICw== - dependencies: - "@types/react" "*" - -"@types/react@*", "@types/react@>=16": - version "18.0.28" - resolved "https://registry.npmjs.org/@types/react/-/react-18.0.28.tgz" - integrity sha512-RD0ivG1kEztNBdoAK7lekI9M+azSnitIn85h4iOiaLjaTrMjzslhaqCGaI4IyCJ1RljWiLCEu4jyrLLgqxBTew== - dependencies: - "@types/prop-types" "*" - "@types/scheduler" "*" - csstype "^3.0.2" - -"@types/react@^18.3.1": - version "18.3.1" - resolved "https://registry.npmjs.org/@types/react/-/react-18.3.1.tgz#fed43985caa834a2084d002e4771e15dfcbdbe8e" - integrity sha512-V0kuGBX3+prX+DQ/7r2qsv1NsdfnCLnTgnRJ1pYnxykBhGMz+qj+box5lq7XsO5mtZsBqpjwwTu/7wszPfMBcw== - dependencies: - "@types/prop-types" "*" - csstype "^3.0.2" - -"@types/recordrtc@^5.6.11": - version "5.6.11" - resolved "https://registry.npmjs.org/@types/recordrtc/-/recordrtc-5.6.11.tgz" - integrity sha512-X4XD5nltz0cjmyzsPNegQReOPF+C5ARTfSPAPhqnKV7SsfRta/M4FBJ5AtSInCaEveL71FLLSVQE9mg8Uuo++w== - -"@types/scheduler@*": - version "0.16.3" - resolved "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz" - integrity sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ== - -"@types/semver@^7.3.12": - version "7.5.0" - resolved "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz" - integrity sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw== - -"@types/sortablejs@^1.15.1": - version "1.15.1" - resolved "https://registry.npmjs.org/@types/sortablejs/-/sortablejs-1.15.1.tgz" - integrity sha512-g/JwBNToh6oCTAwNS8UGVmjO7NLDKsejVhvE4x1eWiPTC3uCuNsa/TD4ssvX3du+MLiM+SHPNDuijp8y76JzLQ== - -"@types/unist@*", "@types/unist@^2.0.0", "@types/unist@^2.0.2": - version "2.0.6" - resolved "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz" - integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== - -"@types/uuid@^9.0.8": - version "9.0.8" - resolved "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz" - integrity sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA== - -"@typescript-eslint/eslint-plugin@^5.53.0": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.59.9.tgz" - integrity sha512-4uQIBq1ffXd2YvF7MAvehWKW3zVv/w+mSfRAu+8cKbfj3nwzyqJLNcZJpQ/WZ1HLbJDiowwmQ6NO+63nCA+fqA== - dependencies: - "@eslint-community/regexpp" "^4.4.0" - "@typescript-eslint/scope-manager" "5.59.9" - "@typescript-eslint/type-utils" "5.59.9" - "@typescript-eslint/utils" "5.59.9" - debug "^4.3.4" - grapheme-splitter "^1.0.4" - ignore "^5.2.0" - natural-compare-lite "^1.4.0" - semver "^7.3.7" - tsutils "^3.21.0" - -"@typescript-eslint/parser@^5.4.2 || ^6.0.0", "@typescript-eslint/parser@^5.53.0": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.59.9.tgz" - integrity sha512-FsPkRvBtcLQ/eVK1ivDiNYBjn3TGJdXy2fhXX+rc7czWl4ARwnpArwbihSOHI2Peg9WbtGHrbThfBUkZZGTtvQ== - dependencies: - "@typescript-eslint/scope-manager" "5.59.9" - "@typescript-eslint/types" "5.59.9" - "@typescript-eslint/typescript-estree" "5.59.9" - debug "^4.3.4" - -"@typescript-eslint/scope-manager@5.59.9": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.59.9.tgz" - integrity sha512-8RA+E+w78z1+2dzvK/tGZ2cpGigBZ58VMEHDZtpE1v+LLjzrYGc8mMaTONSxKyEkz3IuXFM0IqYiGHlCsmlZxQ== - dependencies: - "@typescript-eslint/types" "5.59.9" - "@typescript-eslint/visitor-keys" "5.59.9" - -"@typescript-eslint/type-utils@5.59.9": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.59.9.tgz" - integrity sha512-ksEsT0/mEHg9e3qZu98AlSrONAQtrSTljL3ow9CGej8eRo7pe+yaC/mvTjptp23Xo/xIf2mLZKC6KPv4Sji26Q== - dependencies: - "@typescript-eslint/typescript-estree" "5.59.9" - "@typescript-eslint/utils" "5.59.9" - debug "^4.3.4" - tsutils "^3.21.0" - -"@typescript-eslint/types@5.59.9": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.59.9.tgz" - integrity sha512-uW8H5NRgTVneSVTfiCVffBb8AbwWSKg7qcA4Ot3JI3MPCJGsB4Db4BhvAODIIYE5mNj7Q+VJkK7JxmRhk2Lyjw== - -"@typescript-eslint/typescript-estree@5.59.9": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.59.9.tgz" - integrity sha512-pmM0/VQ7kUhd1QyIxgS+aRvMgw+ZljB3eDb+jYyp6d2bC0mQWLzUDF+DLwCTkQ3tlNyVsvZRXjFyV0LkU/aXjA== - dependencies: - "@typescript-eslint/types" "5.59.9" - "@typescript-eslint/visitor-keys" "5.59.9" - debug "^4.3.4" - globby "^11.1.0" - is-glob "^4.0.3" - semver "^7.3.7" - tsutils "^3.21.0" - -"@typescript-eslint/utils@5.59.9", "@typescript-eslint/utils@^5.10.0", "@typescript-eslint/utils@^5.53.0": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.59.9.tgz" - integrity sha512-1PuMYsju/38I5Ggblaeb98TOoUvjhRvLpLa1DoTOFaLWqaXl/1iQ1eGurTXgBY58NUdtfTXKP5xBq7q9NDaLKg== - dependencies: - "@eslint-community/eslint-utils" "^4.2.0" - "@types/json-schema" "^7.0.9" - "@types/semver" "^7.3.12" - "@typescript-eslint/scope-manager" "5.59.9" - "@typescript-eslint/types" "5.59.9" - "@typescript-eslint/typescript-estree" "5.59.9" - eslint-scope "^5.1.1" - semver "^7.3.7" - -"@typescript-eslint/visitor-keys@5.59.9": - version "5.59.9" - resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.59.9.tgz" - integrity sha512-bT7s0td97KMaLwpEBckbzj/YohnvXtqbe2XgqNvTl6RJVakY5mvENOTPvw5u66nljfZxthESpDozs86U+oLY8Q== - dependencies: - "@typescript-eslint/types" "5.59.9" - eslint-visitor-keys "^3.3.0" - -"@vue/compiler-core@3.4.25": - version "3.4.25" - resolved "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.4.25.tgz" - integrity sha512-Y2pLLopaElgWnMNolgG8w3C5nNUVev80L7hdQ5iIKPtMJvhVpG0zhnBG/g3UajJmZdvW0fktyZTotEHD1Srhbg== - dependencies: - "@babel/parser" "^7.24.4" - "@vue/shared" "3.4.25" - entities "^4.5.0" - estree-walker "^2.0.2" - source-map-js "^1.2.0" - -"@vue/compiler-dom@^3.2.47": - version "3.4.25" - resolved "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.4.25.tgz" - integrity sha512-Ugz5DusW57+HjllAugLci19NsDK+VyjGvmbB2TXaTcSlQxwL++2PETHx/+Qv6qFwNLzSt7HKepPe4DcTE3pBWg== - dependencies: - "@vue/compiler-core" "3.4.25" - "@vue/shared" "3.4.25" - -"@vue/shared@3.4.25": - version "3.4.25" - resolved "https://registry.npmjs.org/@vue/shared/-/shared-3.4.25.tgz" - integrity sha512-k0yappJ77g2+KNrIaF0FFnzwLvUBLUYr8VOwz+/6vLsmItFp51AcxLL7Ey3iPd7BIRyWPOcqUjMnm7OkahXllA== - -acorn-jsx@^5.0.0, acorn-jsx@^5.3.2: - version "5.3.2" - resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" - integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== - -acorn@^8.0.0, acorn@^8.5.0, acorn@^8.8.0: - version "8.8.2" - resolved "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz" - integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== - -aggregate-error@^3.0.0: - version "3.1.0" - resolved "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz" - integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== - dependencies: - clean-stack "^2.0.0" - indent-string "^4.0.0" - -ahooks-v3-count@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/ahooks-v3-count/-/ahooks-v3-count-1.0.0.tgz" - integrity sha512-V7uUvAwnimu6eh/PED4mCDjE7tokeZQLKlxg9lCTMPhN+NjsSbtdacByVlR1oluXQzD3MOw55wylDmQo4+S9ZQ== - -ahooks@^3.7.5: - version "3.7.7" - resolved "https://registry.npmjs.org/ahooks/-/ahooks-3.7.7.tgz" - integrity sha512-5e5WlPq81Y84UnTLOKIQeq2cJw4aa7yj8fR2Nb/oMmXPrWMjIMCbPS1o+fpxSfCaNA3AzOnnMc8AehWRZltkJQ== - dependencies: - "@babel/runtime" "^7.21.0" - "@types/js-cookie" "^2.x.x" - ahooks-v3-count "^1.0.0" - dayjs "^1.9.1" - intersection-observer "^0.12.0" - js-cookie "^2.x.x" - lodash "^4.17.21" - resize-observer-polyfill "^1.5.1" - screenfull "^5.0.0" - tslib "^2.4.1" - -ajv@^6.10.0, ajv@^6.12.4: - version "6.12.6" - resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz" - integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== - dependencies: - fast-deep-equal "^3.1.1" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.4.1" - uri-js "^4.2.2" - -ansi-escapes@^4.3.0: - version "4.3.2" - resolved "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz" - integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== - dependencies: - type-fest "^0.21.3" - -ansi-regex@^5.0.1: - version "5.0.1" - resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz" - integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== - -ansi-regex@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz" - integrity sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA== - -ansi-styles@^3.2.1: - version "3.2.1" - resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz" - integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== - dependencies: - color-convert "^1.9.0" - -ansi-styles@^4.0.0, ansi-styles@^4.1.0: - version "4.3.0" - resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz" - integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== - dependencies: - color-convert "^2.0.1" - -ansi-styles@^6.0.0, ansi-styles@^6.1.0: - version "6.2.1" - resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz" - integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== - -any-promise@^1.0.0: - version "1.3.0" - resolved "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz" - integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A== - -anymatch@~3.1.2: - version "3.1.3" - resolved "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz" - integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw== - dependencies: - normalize-path "^3.0.0" - picomatch "^2.0.4" - -arg@^5.0.2: - version "5.0.2" - resolved "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz" - integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== - -argparse@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz" - integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== - -aria-query@^5.1.3: - version "5.1.3" - resolved "https://registry.npmjs.org/aria-query/-/aria-query-5.1.3.tgz" - integrity sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ== - dependencies: - deep-equal "^2.0.5" - -array-buffer-byte-length@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz" - integrity sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A== - dependencies: - call-bind "^1.0.2" - is-array-buffer "^3.0.1" - -array-includes@^3.1.5, array-includes@^3.1.6, array-includes@^3.1.7: - version "3.1.7" - resolved "https://registry.npmjs.org/array-includes/-/array-includes-3.1.7.tgz" - integrity sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" - is-string "^1.0.7" - -array-union@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz" - integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== - -array.prototype.findlastindex@^1.2.3: - version "1.2.3" - resolved "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.3.tgz" - integrity sha512-LzLoiOMAxvy+Gd3BAq3B7VeIgPdo+Q8hthvKtXybMvRV0jrXfJM/t8mw7nNlpEcVlVUnCnM2KSX4XU5HmpodOA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - es-shim-unscopables "^1.0.0" - get-intrinsic "^1.2.1" - -array.prototype.flat@^1.3.2: - version "1.3.2" - resolved "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz" - integrity sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - es-shim-unscopables "^1.0.0" - -array.prototype.flatmap@^1.3.1, array.prototype.flatmap@^1.3.2: - version "1.3.2" - resolved "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz" - integrity sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - es-shim-unscopables "^1.0.0" - -array.prototype.tosorted@^1.1.1: - version "1.1.2" - resolved "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.2.tgz" - integrity sha512-HuQCHOlk1Weat5jzStICBCd83NxiIMwqDg/dHEsoefabn/hJRj5pVdWcPUSpRrwhwxZOsQassMpgN/xRYFBMIg== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - es-shim-unscopables "^1.0.0" - get-intrinsic "^1.2.1" - -arraybuffer.prototype.slice@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz" - integrity sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw== - dependencies: - array-buffer-byte-length "^1.0.0" - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" - is-array-buffer "^3.0.2" - is-shared-array-buffer "^1.0.2" - -ast-types-flow@^0.0.7: - version "0.0.7" - resolved "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz" - integrity sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag== - -astral-regex@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz" - integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== - -astring@^1.8.0: - version "1.8.6" - resolved "https://registry.npmjs.org/astring/-/astring-1.8.6.tgz" - integrity sha512-ISvCdHdlTDlH5IpxQJIex7BWBywFWgjJSVdwst+/iQCoEYnyOaQ95+X1JGshuBjGp6nxKUy1jMgE3zPqN7fQdg== - -async@^2.6.4: - version "2.6.4" - resolved "https://registry.npmjs.org/async/-/async-2.6.4.tgz" - integrity sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA== - dependencies: - lodash "^4.17.14" - -asynciterator.prototype@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/asynciterator.prototype/-/asynciterator.prototype-1.0.0.tgz" - integrity sha512-wwHYEIS0Q80f5mosx3L/dfG5t5rjEa9Ft51GTaNt862EnpyGHpgz2RkZvLPp1oF5TnAiTohkEKVEu8pQPJI7Vg== - dependencies: - has-symbols "^1.0.3" - -autoprefixer@^10.4.14: - version "10.4.14" - resolved "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz" - integrity sha512-FQzyfOsTlwVzjHxKEqRIAdJx9niO6VCBCoEwax/VLSoQF29ggECcPuBqUMZ+u8jCZOPSy8b8/8KnuFbp0SaFZQ== - dependencies: - browserslist "^4.21.5" - caniuse-lite "^1.0.30001464" - fraction.js "^4.2.0" - normalize-range "^0.1.2" - picocolors "^1.0.0" - postcss-value-parser "^4.2.0" - -available-typed-arrays@^1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz" - integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== - -axe-core@^4.6.2: - version "4.7.2" - resolved "https://registry.npmjs.org/axe-core/-/axe-core-4.7.2.tgz" - integrity sha512-zIURGIS1E1Q4pcrMjp+nnEh+16G56eG/MUllJH8yEvw7asDo7Ac9uhC9KIH5jzpITueEZolfYglnCGIuSBz39g== - -axobject-query@^3.1.1: - version "3.1.1" - resolved "https://registry.npmjs.org/axobject-query/-/axobject-query-3.1.1.tgz" - integrity sha512-goKlv8DZrK9hUh975fnHzhNIO4jUnFCfv/dszV5VwUGDFjI6vQ2VwoyjYjYNEbBE8AH87TduWP5uyDR1D+Iteg== - dependencies: - deep-equal "^2.0.5" - -bail@^2.0.0: - version "2.0.2" - resolved "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz" - integrity sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw== - -balanced-match@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" - integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== - -big-integer@^1.6.44: - version "1.6.51" - resolved "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz" - integrity sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg== - -binary-extensions@^2.0.0: - version "2.2.0" - resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz" - integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== - -boolbase@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz" - integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== - -bplist-parser@^0.2.0: - version "0.2.0" - resolved "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz" - integrity sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw== - dependencies: - big-integer "^1.6.44" - -brace-expansion@^1.1.7: - version "1.1.11" - resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" - integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== - dependencies: - balanced-match "^1.0.0" - concat-map "0.0.1" - -brace-expansion@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz" - integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== - dependencies: - balanced-match "^1.0.0" - -braces@^3.0.2, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== - dependencies: - fill-range "^7.0.1" - -browserslist@^4.21.5: - version "4.21.7" - resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.21.7.tgz" - integrity sha512-BauCXrQ7I2ftSqd2mvKHGo85XR0u7Ru3C/Hxsy/0TkfCtjrmAbPdzLGasmoiBxplpDXlPvdjX9u7srIMfgasNA== - dependencies: - caniuse-lite "^1.0.30001489" - electron-to-chromium "^1.4.411" - node-releases "^2.0.12" - update-browserslist-db "^1.0.11" - -builtin-modules@^3.3.0: - version "3.3.0" - resolved "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz" - integrity sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw== - -builtins@^5.0.1: - version "5.0.1" - resolved "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz" - integrity sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ== - dependencies: - semver "^7.0.0" - -bundle-name@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/bundle-name/-/bundle-name-3.0.0.tgz" - integrity sha512-PKA4BeSvBpQKQ8iPOGCSiell+N8P+Tf1DlwqmYhpe2gAhKPHn8EYOxVT+ShuGmhg8lN8XiSlS80yiExKXrURlw== - dependencies: - run-applescript "^5.0.0" - -busboy@1.6.0: - version "1.6.0" - resolved "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz" - integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA== - dependencies: - streamsearch "^1.1.0" - -call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz" - integrity sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ== - dependencies: - function-bind "^1.1.2" - get-intrinsic "^1.2.1" - set-function-length "^1.1.1" - -callsites@^3.0.0: - version "3.1.0" - resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz" - integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== - -camelcase-css@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz" - integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA== - -caniuse-lite@^1.0.30001464, caniuse-lite@^1.0.30001489, caniuse-lite@^1.0.30001579: - version "1.0.30001581" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001581.tgz" - integrity sha512-whlTkwhqV2tUmP3oYhtNfaWGYHDdS3JYFQBKXxcUR9qqPWsRhFHhoISO2Xnl/g0xyKzht9mI1LZpiNWfMzHixQ== - -ccount@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz" - integrity sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg== - -chalk@4.1.1, chalk@^4.1.1: - version "4.1.1" - resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz" - integrity sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - -chalk@5.2.0: - version "5.2.0" - resolved "https://registry.npmjs.org/chalk/-/chalk-5.2.0.tgz" - integrity sha512-ree3Gqw/nazQAPuJJEy+avdl7QfZMcUvmHIKgEZkGL+xOBzRvup5Hxo6LHuMceSxOabuJLJm5Yp/92R9eMmMvA== - -chalk@^2.0.0: - version "2.4.2" - resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - -chalk@^4.0.0: - version "4.1.2" - resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz" - integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - -character-entities-html4@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz" - integrity sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA== - -character-entities-legacy@^1.0.0: - version "1.1.4" - resolved "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz" - integrity sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA== - -character-entities-legacy@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz" - integrity sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ== - -character-entities@^1.0.0: - version "1.2.4" - resolved "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz" - integrity sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw== - -character-entities@^2.0.0: - version "2.0.2" - resolved "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz" - integrity sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ== - -character-reference-invalid@^1.0.0: - version "1.1.4" - resolved "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz" - integrity sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg== - -character-reference-invalid@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz" - integrity sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw== - -"chokidar@>=3.0.0 <4.0.0", chokidar@^3.5.3: - version "3.5.3" - resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" - integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== - dependencies: - anymatch "~3.1.2" - braces "~3.0.2" - glob-parent "~5.1.2" - is-binary-path "~2.1.0" - is-glob "~4.0.1" - normalize-path "~3.0.0" - readdirp "~3.6.0" - optionalDependencies: - fsevents "~2.3.2" - -ci-info@^3.6.1: - version "3.8.0" - resolved "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz" - integrity sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw== - -classcat@^5.0.3, classcat@^5.0.4: - version "5.0.4" - resolved "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz" - integrity sha512-sbpkOw6z413p+HDGcBENe498WM9woqWHiJxCq7nvmxe9WmrUmqfAcxpIwAiMtM5Q3AhYkzXcNQHqsWq0mND51g== - -classnames@2.3.1: - version "2.3.1" - resolved "https://registry.npmjs.org/classnames/-/classnames-2.3.1.tgz" - integrity sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA== - -classnames@^2.2.1, classnames@^2.3.2: - version "2.3.2" - resolved "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz" - integrity sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw== - -clean-regexp@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/clean-regexp/-/clean-regexp-1.0.0.tgz" - integrity sha512-GfisEZEJvzKrmGWkvfhgzcz/BllN1USeqD2V6tg14OAOgaCD2Z/PUEuxnAZ/nPvmaHRG7a8y77p1T/IRQ4D1Hw== - dependencies: - escape-string-regexp "^1.0.5" - -clean-stack@^2.0.0: - version "2.2.0" - resolved "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz" - integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== - -cli-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz" - integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== - dependencies: - restore-cursor "^3.1.0" - -cli-truncate@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/cli-truncate/-/cli-truncate-2.1.0.tgz" - integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== - dependencies: - slice-ansi "^3.0.0" - string-width "^4.2.0" - -cli-truncate@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/cli-truncate/-/cli-truncate-3.1.0.tgz" - integrity sha512-wfOBkjXteqSnI59oPcJkcPl/ZmwvMMOj340qUIY1SKZCv0B9Cf4D4fAucRkIKQmsIuYK3x1rrgU7MeGRruiuiA== - dependencies: - slice-ansi "^5.0.0" - string-width "^5.0.0" - -client-only@0.0.1, client-only@^0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz" - integrity sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA== - -code-inspector-core@0.13.0: - version "0.13.0" - resolved "https://registry.npmjs.org/code-inspector-core/-/code-inspector-core-0.13.0.tgz" - integrity sha512-oYPNLdJjn3SY50YtF3IuxZOKLBNwzXSRPOqiXVnZFceMz9Ar6ugP3+zj7HszouxrsLFb2dVtlv//5wr4+cq62A== - dependencies: - "@vue/compiler-dom" "^3.2.47" - chalk "^4.1.1" - portfinder "^1.0.28" - -code-inspector-plugin@^0.13.0: - version "0.13.0" - resolved "https://registry.npmjs.org/code-inspector-plugin/-/code-inspector-plugin-0.13.0.tgz" - integrity sha512-v4mq5hhHkyMmutembTzREVsFeZ/+KsCwfx20+0gTqm1Il/M1T4d2BCv9mZ4ivie3GvvDMt/pVz1iBBVP3SuzJA== - dependencies: - chalk "4.1.1" - code-inspector-core "0.13.0" - vite-code-inspector-plugin "0.13.0" - webpack-code-inspector-plugin "0.13.0" - -color-convert@^1.9.0: - version "1.9.3" - resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" - integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== - dependencies: - color-name "1.1.3" - -color-convert@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz" - integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== - dependencies: - color-name "~1.1.4" - -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" - integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== - -color-name@^1.0.0, color-name@~1.1.4: - version "1.1.4" - resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" - integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== - -color-string@^1.9.0: - version "1.9.1" - resolved "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz" - integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg== - dependencies: - color-name "^1.0.0" - simple-swizzle "^0.2.2" - -color@^4.2.3: - version "4.2.3" - resolved "https://registry.npmjs.org/color/-/color-4.2.3.tgz" - integrity sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A== - dependencies: - color-convert "^2.0.1" - color-string "^1.9.0" - -colorette@^2.0.19: - version "2.0.20" - resolved "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz" - integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== - -comma-separated-tokens@^1.0.0: - version "1.0.8" - resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz" - integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== - -comma-separated-tokens@^2.0.0: - version "2.0.3" - resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz" - integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg== - -commander@7: - version "7.2.0" - resolved "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz" - integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== - -commander@^10.0.0: - version "10.0.1" - resolved "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz" - integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== - -commander@^4.0.0: - version "4.1.1" - resolved "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz" - integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA== - -commander@^8.3.0: - version "8.3.0" - resolved "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz" - integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww== - -concat-map@0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" - integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== - -copy-to-clipboard@^3.3.3: - version "3.3.3" - resolved "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz" - integrity sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA== - dependencies: - toggle-selection "^1.0.6" - -cose-base@^1.0.0: - version "1.0.3" - resolved "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz" - integrity sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg== - dependencies: - layout-base "^1.0.0" - -cose-base@^2.2.0: - version "2.2.0" - resolved "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz" - integrity sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g== - dependencies: - layout-base "^2.0.0" - -cross-env@^7.0.3: - version "7.0.3" - resolved "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz" - integrity sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw== - dependencies: - cross-spawn "^7.0.1" - -cross-spawn@^7.0.0, cross-spawn@^7.0.1, cross-spawn@^7.0.2, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== - dependencies: - path-key "^3.1.0" - shebang-command "^2.0.0" - which "^2.0.1" - -crypto-js@^4.2.0: - version "4.2.0" - resolved "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz" - integrity sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q== - -cssesc@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz" - integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg== - -csstype@^3.0.2: - version "3.1.2" - resolved "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz" - integrity sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ== - -cytoscape-cose-bilkent@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz" - integrity sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ== - dependencies: - cose-base "^1.0.0" - -cytoscape-fcose@^2.1.0: - version "2.2.0" - resolved "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz" - integrity sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ== - dependencies: - cose-base "^2.2.0" - -cytoscape@^3.23.0: - version "3.26.0" - resolved "https://registry.npmjs.org/cytoscape/-/cytoscape-3.26.0.tgz" - integrity sha512-IV+crL+KBcrCnVVUCZW+zRRRFUZQcrtdOPXki+o4CFUWLdAEYvuZLcBSJC9EBK++suamERKzeY7roq2hdovV3w== - dependencies: - heap "^0.2.6" - lodash "^4.17.21" - -"d3-array@1 - 2": - version "2.12.1" - resolved "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz" - integrity sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ== - dependencies: - internmap "^1.0.0" - -"d3-array@2 - 3", "d3-array@2.10.0 - 3", "d3-array@2.5.0 - 3", d3-array@3, d3-array@^3.2.0: - version "3.2.4" - resolved "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz" - integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg== - dependencies: - internmap "1 - 2" - -d3-axis@3: - version "3.0.0" - resolved "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz" - integrity sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw== - -d3-brush@3: - version "3.0.0" - resolved "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz" - integrity sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ== - dependencies: - d3-dispatch "1 - 3" - d3-drag "2 - 3" - d3-interpolate "1 - 3" - d3-selection "3" - d3-transition "3" - -d3-chord@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz" - integrity sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g== - dependencies: - d3-path "1 - 3" - -"d3-color@1 - 3", d3-color@3: - version "3.1.0" - resolved "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz" - integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA== - -d3-contour@4: - version "4.0.2" - resolved "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz" - integrity sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA== - dependencies: - d3-array "^3.2.0" - -d3-delaunay@6: - version "6.0.4" - resolved "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz" - integrity sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A== - dependencies: - delaunator "5" - -"d3-dispatch@1 - 3", d3-dispatch@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz" - integrity sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg== - -"d3-drag@2 - 3", d3-drag@3, d3-drag@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz" - integrity sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg== - dependencies: - d3-dispatch "1 - 3" - d3-selection "3" - -"d3-dsv@1 - 3", d3-dsv@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz" - integrity sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q== - dependencies: - commander "7" - iconv-lite "0.6" - rw "1" - -"d3-ease@1 - 3", d3-ease@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz" - integrity sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w== - -d3-fetch@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz" - integrity sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw== - dependencies: - d3-dsv "1 - 3" - -d3-force@3: - version "3.0.0" - resolved "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz" - integrity sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg== - dependencies: - d3-dispatch "1 - 3" - d3-quadtree "1 - 3" - d3-timer "1 - 3" - -"d3-format@1 - 3", d3-format@3: - version "3.1.0" - resolved "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz" - integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA== - -d3-geo@3: - version "3.1.0" - resolved "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.0.tgz" - integrity sha512-JEo5HxXDdDYXCaWdwLRt79y7giK8SbhZJbFWXqbRTolCHFI5jRqteLzCsq51NKbUoX0PjBVSohxrx+NoOUujYA== - dependencies: - d3-array "2.5.0 - 3" - -d3-hierarchy@3: - version "3.1.2" - resolved "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz" - integrity sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA== - -"d3-interpolate@1 - 3", "d3-interpolate@1.2.0 - 3", d3-interpolate@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz" - integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g== - dependencies: - d3-color "1 - 3" - -d3-path@1: - version "1.0.9" - resolved "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz" - integrity sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg== - -"d3-path@1 - 3", d3-path@3, d3-path@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz" - integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ== - -d3-polygon@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz" - integrity sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg== - -"d3-quadtree@1 - 3", d3-quadtree@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz" - integrity sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw== - -d3-random@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz" - integrity sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ== - -d3-sankey@^0.12.3: - version "0.12.3" - resolved "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz" - integrity sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ== - dependencies: - d3-array "1 - 2" - d3-shape "^1.2.0" - -d3-scale-chromatic@3: - version "3.0.0" - resolved "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz" - integrity sha512-Lx9thtxAKrO2Pq6OO2Ua474opeziKr279P/TKZsMAhYyNDD3EnCffdbgeSYN5O7m2ByQsxtuP2CSDczNUIZ22g== - dependencies: - d3-color "1 - 3" - d3-interpolate "1 - 3" - -d3-scale@4: - version "4.0.2" - resolved "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz" - integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ== - dependencies: - d3-array "2.10.0 - 3" - d3-format "1 - 3" - d3-interpolate "1.2.0 - 3" - d3-time "2.1.1 - 3" - d3-time-format "2 - 4" - -"d3-selection@2 - 3", d3-selection@3, d3-selection@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz" - integrity sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ== - -d3-shape@3: - version "3.2.0" - resolved "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz" - integrity sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA== - dependencies: - d3-path "^3.1.0" - -d3-shape@^1.2.0: - version "1.3.7" - resolved "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz" - integrity sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw== - dependencies: - d3-path "1" - -"d3-time-format@2 - 4", d3-time-format@4: - version "4.1.0" - resolved "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz" - integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg== - dependencies: - d3-time "1 - 3" - -"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@3: - version "3.1.0" - resolved "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz" - integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q== - dependencies: - d3-array "2 - 3" - -"d3-timer@1 - 3", d3-timer@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz" - integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA== - -"d3-transition@2 - 3", d3-transition@3: - version "3.0.1" - resolved "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz" - integrity sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w== - dependencies: - d3-color "1 - 3" - d3-dispatch "1 - 3" - d3-ease "1 - 3" - d3-interpolate "1 - 3" - d3-timer "1 - 3" - -d3-zoom@3, d3-zoom@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz" - integrity sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw== - dependencies: - d3-dispatch "1 - 3" - d3-drag "2 - 3" - d3-interpolate "1 - 3" - d3-selection "2 - 3" - d3-transition "2 - 3" - -d3@^7.4.0, d3@^7.8.2: - version "7.8.5" - resolved "https://registry.npmjs.org/d3/-/d3-7.8.5.tgz" - integrity sha512-JgoahDG51ncUfJu6wX/1vWQEqOflgXyl4MaHqlcSruTez7yhaRKR9i8VjjcQGeS2en/jnFivXuaIMnseMMt0XA== - dependencies: - d3-array "3" - d3-axis "3" - d3-brush "3" - d3-chord "3" - d3-color "3" - d3-contour "4" - d3-delaunay "6" - d3-dispatch "3" - d3-drag "3" - d3-dsv "3" - d3-ease "3" - d3-fetch "3" - d3-force "3" - d3-format "3" - d3-geo "3" - d3-hierarchy "3" - d3-interpolate "3" - d3-path "3" - d3-polygon "3" - d3-quadtree "3" - d3-random "3" - d3-scale "4" - d3-scale-chromatic "3" - d3-selection "3" - d3-shape "3" - d3-time "3" - d3-time-format "4" - d3-timer "3" - d3-transition "3" - d3-zoom "3" - -dagre-d3-es@7.0.10: - version "7.0.10" - resolved "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.10.tgz" - integrity sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A== - dependencies: - d3 "^7.8.2" - lodash-es "^4.17.21" - -dagre@^0.8.5: - version "0.8.5" - resolved "https://registry.npmjs.org/dagre/-/dagre-0.8.5.tgz" - integrity sha512-/aTqmnRta7x7MCCpExk7HQL2O4owCT2h8NT//9I1OQ9vt29Pa0BzSAkR5lwFUcQ7491yVi/3CXU9jQ5o0Mn2Sw== - dependencies: - graphlib "^2.1.8" - lodash "^4.17.15" - -damerau-levenshtein@^1.0.8: - version "1.0.8" - resolved "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz" - integrity sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA== - -dayjs@^1.11.7, dayjs@^1.9.1: - version "1.11.8" - resolved "https://registry.npmjs.org/dayjs/-/dayjs-1.11.8.tgz" - integrity sha512-LcgxzFoWMEPO7ggRv1Y2N31hUf2R0Vj7fuy/m+Bg1K8rr+KAs1AEy4y9jd5DXe8pbHgX+srkHNS7TH6Q6ZhYeQ== - -debug@^3.2.7: - version "3.2.7" - resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" - integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== - dependencies: - ms "^2.1.1" - -debug@^4.0.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - -decode-named-character-reference@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz" - integrity sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg== - dependencies: - character-entities "^2.0.0" - -deep-equal@^2.0.5: - version "2.2.1" - resolved "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz" - integrity sha512-lKdkdV6EOGoVn65XaOsPdH4rMxTZOnmFyuIkMjM1i5HHCbfjC97dawgTAy0deYNfuqUqW+Q5VrVaQYtUpSd6yQ== - dependencies: - array-buffer-byte-length "^1.0.0" - call-bind "^1.0.2" - es-get-iterator "^1.1.3" - get-intrinsic "^1.2.0" - is-arguments "^1.1.1" - is-array-buffer "^3.0.2" - is-date-object "^1.0.5" - is-regex "^1.1.4" - is-shared-array-buffer "^1.0.2" - isarray "^2.0.5" - object-is "^1.1.5" - object-keys "^1.1.1" - object.assign "^4.1.4" - regexp.prototype.flags "^1.5.0" - side-channel "^1.0.4" - which-boxed-primitive "^1.0.2" - which-collection "^1.0.1" - which-typed-array "^1.1.9" - -deep-is@^0.1.3: - version "0.1.4" - resolved "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz" - integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== - -default-browser-id@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/default-browser-id/-/default-browser-id-3.0.0.tgz" - integrity sha512-OZ1y3y0SqSICtE8DE4S8YOE9UZOJ8wO16fKWVP5J1Qz42kV9jcnMVFrEE/noXb/ss3Q4pZIH79kxofzyNNtUNA== - dependencies: - bplist-parser "^0.2.0" - untildify "^4.0.0" - -default-browser@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/default-browser/-/default-browser-4.0.0.tgz" - integrity sha512-wX5pXO1+BrhMkSbROFsyxUm0i/cJEScyNhA4PPxc41ICuv05ZZB/MX28s8aZx6xjmatvebIapF6hLEKEcpneUA== - dependencies: - bundle-name "^3.0.0" - default-browser-id "^3.0.0" - execa "^7.1.1" - titleize "^3.0.0" - -define-data-property@^1.0.1, define-data-property@^1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz" - integrity sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ== - dependencies: - get-intrinsic "^1.2.1" - gopd "^1.0.1" - has-property-descriptors "^1.0.0" - -define-lazy-prop@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz" - integrity sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg== - -define-properties@^1.1.3, define-properties@^1.1.4, define-properties@^1.2.0, define-properties@^1.2.1: - version "1.2.1" - resolved "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz" - integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg== - dependencies: - define-data-property "^1.0.1" - has-property-descriptors "^1.0.0" - object-keys "^1.1.1" - -delaunator@5: - version "5.0.0" - resolved "https://registry.npmjs.org/delaunator/-/delaunator-5.0.0.tgz" - integrity sha512-AyLvtyJdbv/U1GkiS6gUUzclRoAY4Gs75qkMygJJhU75LW4DNuSF2RMzpxs9jw9Oz1BobHjTdkG3zdP55VxAqw== - dependencies: - robust-predicates "^3.0.0" - -dequal@^2.0.0: - version "2.0.3" - resolved "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz" - integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA== - -detect-libc@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz" - integrity sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw== - -didyoumean@^1.2.2: - version "1.2.2" - resolved "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz" - integrity sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw== - -diff@^5.0.0: - version "5.1.0" - resolved "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz" - integrity sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw== - -dir-glob@^3.0.1: - version "3.0.1" - resolved "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz" - integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== - dependencies: - path-type "^4.0.0" - -dlv@^1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz" - integrity sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA== - -doctrine@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz" - integrity sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw== - dependencies: - esutils "^2.0.2" - -doctrine@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz" - integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w== - dependencies: - esutils "^2.0.2" - -dom-serializer@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz" - integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== - dependencies: - domelementtype "^2.3.0" - domhandler "^5.0.2" - entities "^4.2.0" - -domelementtype@^2.3.0: - version "2.3.0" - resolved "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz" - integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== - -domhandler@^5.0.2, domhandler@^5.0.3: - version "5.0.3" - resolved "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz" - integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== - dependencies: - domelementtype "^2.3.0" - -dompurify@^3.0.5: - version "3.0.5" - resolved "https://registry.npmjs.org/dompurify/-/dompurify-3.0.5.tgz" - integrity sha512-F9e6wPGtY+8KNMRAVfxeCOHU0/NPWMSENNq4pQctuXRqqdEPW7q3CrLbR5Nse044WwacyjHGOMlvNsBe1y6z9A== - -domutils@^3.0.1: - version "3.1.0" - resolved "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz" - integrity sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA== - dependencies: - dom-serializer "^2.0.0" - domelementtype "^2.3.0" - domhandler "^5.0.3" - -eastasianwidth@^0.2.0: - version "0.2.0" - resolved "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz" - integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== - -echarts-for-react@^3.0.2: - version "3.0.2" - resolved "https://registry.npmjs.org/echarts-for-react/-/echarts-for-react-3.0.2.tgz" - integrity sha512-DRwIiTzx8JfwPOVgGttDytBqdp5VzCSyMRIxubgU/g2n9y3VLUmF2FK7Icmg/sNVkv4+rktmrLN9w22U2yy3fA== - dependencies: - fast-deep-equal "^3.1.3" - size-sensor "^1.0.1" - -echarts@^5.4.1: - version "5.4.2" - resolved "https://registry.npmjs.org/echarts/-/echarts-5.4.2.tgz" - integrity sha512-2W3vw3oI2tWJdyAz+b8DuWS0nfXtSDqlDmqgin/lfzbkB01cuMEN66KWBlmur3YMp5nEDEEt5s23pllnAzB4EA== - dependencies: - tslib "2.3.0" - zrender "5.4.3" - -electron-to-chromium@^1.4.411: - version "1.4.423" - resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.423.tgz" - integrity sha512-y4A7YfQcDGPAeSWM1IuoWzXpg9RY1nwHzHSwRtCSQFp9FgAVDgdWlFf0RbdWfLWQ2WUI+bddUgk5RgTjqRE6FQ== - -elkjs@^0.8.2: - version "0.8.2" - resolved "https://registry.npmjs.org/elkjs/-/elkjs-0.8.2.tgz" - integrity sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ== - -emoji-mart@^5.5.2: - version "5.5.2" - resolved "https://registry.npmjs.org/emoji-mart/-/emoji-mart-5.5.2.tgz" - integrity sha512-Sqc/nso4cjxhOwWJsp9xkVm8OF5c+mJLZJFoFfzRuKO+yWiN7K8c96xmtughYb0d/fZ8UC6cLIQ/p4BR6Pv3/A== - -emoji-regex@^8.0.0: - version "8.0.0" - resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" - integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== - -emoji-regex@^9.2.2: - version "9.2.2" - resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz" - integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== - -enhanced-resolve@^5.12.0: - version "5.14.1" - resolved "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.14.1.tgz" - integrity sha512-Vklwq2vDKtl0y/vtwjSesgJ5MYS7Etuk5txS8VdKL4AOS1aUlD96zqIfsOSLQsdv3xgMRbtkWM8eG9XDfKUPow== - dependencies: - graceful-fs "^4.2.4" - tapable "^2.2.0" - -entities@^4.2.0, entities@^4.4.0, entities@^4.5.0: - version "4.5.0" - resolved "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz" - integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== - -error-ex@^1.3.1: - version "1.3.2" - resolved "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz" - integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== - dependencies: - is-arrayish "^0.2.1" - -es-abstract@^1.20.4, es-abstract@^1.22.1: - version "1.22.3" - resolved "https://registry.npmjs.org/es-abstract/-/es-abstract-1.22.3.tgz" - integrity sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA== - dependencies: - array-buffer-byte-length "^1.0.0" - arraybuffer.prototype.slice "^1.0.2" - available-typed-arrays "^1.0.5" - call-bind "^1.0.5" - es-set-tostringtag "^2.0.1" - es-to-primitive "^1.2.1" - function.prototype.name "^1.1.6" - get-intrinsic "^1.2.2" - get-symbol-description "^1.0.0" - globalthis "^1.0.3" - gopd "^1.0.1" - has-property-descriptors "^1.0.0" - has-proto "^1.0.1" - has-symbols "^1.0.3" - hasown "^2.0.0" - internal-slot "^1.0.5" - is-array-buffer "^3.0.2" - is-callable "^1.2.7" - is-negative-zero "^2.0.2" - is-regex "^1.1.4" - is-shared-array-buffer "^1.0.2" - is-string "^1.0.7" - is-typed-array "^1.1.12" - is-weakref "^1.0.2" - object-inspect "^1.13.1" - object-keys "^1.1.1" - object.assign "^4.1.4" - regexp.prototype.flags "^1.5.1" - safe-array-concat "^1.0.1" - safe-regex-test "^1.0.0" - string.prototype.trim "^1.2.8" - string.prototype.trimend "^1.0.7" - string.prototype.trimstart "^1.0.7" - typed-array-buffer "^1.0.0" - typed-array-byte-length "^1.0.0" - typed-array-byte-offset "^1.0.0" - typed-array-length "^1.0.4" - unbox-primitive "^1.0.2" - which-typed-array "^1.1.13" - -es-get-iterator@^1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz" - integrity sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.3" - has-symbols "^1.0.3" - is-arguments "^1.1.1" - is-map "^2.0.2" - is-set "^2.0.2" - is-string "^1.0.7" - isarray "^2.0.5" - stop-iteration-iterator "^1.0.0" - -es-iterator-helpers@^1.0.12: - version "1.0.15" - resolved "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.0.15.tgz" - integrity sha512-GhoY8uYqd6iwUl2kgjTm4CZAf6oo5mHK7BPqx3rKgx893YSsy0LGHV6gfqqQvZt/8xM8xeOnfXBCfqclMKkJ5g== - dependencies: - asynciterator.prototype "^1.0.0" - call-bind "^1.0.2" - define-properties "^1.2.1" - es-abstract "^1.22.1" - es-set-tostringtag "^2.0.1" - function-bind "^1.1.1" - get-intrinsic "^1.2.1" - globalthis "^1.0.3" - has-property-descriptors "^1.0.0" - has-proto "^1.0.1" - has-symbols "^1.0.3" - internal-slot "^1.0.5" - iterator.prototype "^1.1.2" - safe-array-concat "^1.0.1" - -es-set-tostringtag@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz" - integrity sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg== - dependencies: - get-intrinsic "^1.1.3" - has "^1.0.3" - has-tostringtag "^1.0.0" - -es-shim-unscopables@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz" - integrity sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w== - dependencies: - has "^1.0.3" - -es-to-primitive@^1.2.1: - version "1.2.1" - resolved "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz" - integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== - dependencies: - is-callable "^1.1.4" - is-date-object "^1.0.1" - is-symbol "^1.0.2" - -escalade@^3.1.1: - version "3.1.1" - resolved "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz" - integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== - -escape-string-regexp@^1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" - integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== - -escape-string-regexp@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" - integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== - -escape-string-regexp@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz" - integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== - -eslint-config-next@^14.0.4: - version "14.1.0" - resolved "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.1.0.tgz" - integrity sha512-SBX2ed7DoRFXC6CQSLc/SbLY9Ut6HxNB2wPTcoIWjUMd7aF7O/SIE7111L8FdZ9TXsNV4pulUDnfthpyPtbFUg== - dependencies: - "@next/eslint-plugin-next" "14.1.0" - "@rushstack/eslint-patch" "^1.3.3" - "@typescript-eslint/parser" "^5.4.2 || ^6.0.0" - eslint-import-resolver-node "^0.3.6" - eslint-import-resolver-typescript "^3.5.2" - eslint-plugin-import "^2.28.1" - eslint-plugin-jsx-a11y "^6.7.1" - eslint-plugin-react "^7.33.2" - eslint-plugin-react-hooks "^4.5.0 || 5.0.0-canary-7118f5dd7-20230705" - -eslint-import-resolver-node@^0.3.6, eslint-import-resolver-node@^0.3.9: - version "0.3.9" - resolved "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz" - integrity sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g== - dependencies: - debug "^3.2.7" - is-core-module "^2.13.0" - resolve "^1.22.4" - -eslint-import-resolver-typescript@^3.5.2: - version "3.5.5" - resolved "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.5.5.tgz" - integrity sha512-TdJqPHs2lW5J9Zpe17DZNQuDnox4xo2o+0tE7Pggain9Rbc19ik8kFtXdxZ250FVx2kF4vlt2RSf4qlUpG7bhw== - dependencies: - debug "^4.3.4" - enhanced-resolve "^5.12.0" - eslint-module-utils "^2.7.4" - get-tsconfig "^4.5.0" - globby "^13.1.3" - is-core-module "^2.11.0" - is-glob "^4.0.3" - synckit "^0.8.5" - -eslint-module-utils@^2.7.4, eslint-module-utils@^2.8.0: - version "2.8.0" - resolved "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz" - integrity sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw== - dependencies: - debug "^3.2.7" - -eslint-plugin-antfu@0.36.0: - version "0.36.0" - resolved "https://registry.npmjs.org/eslint-plugin-antfu/-/eslint-plugin-antfu-0.36.0.tgz" - integrity sha512-qLYtjZC2y6d1fvVtG4nvVGoBUDEuUwQsS4E1RwjoEZyONZAkHYDPfeoeULDlPS0IqumSW8uGR6zGSAXi5rrVMg== - dependencies: - "@typescript-eslint/utils" "^5.53.0" - -eslint-plugin-es@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-4.1.0.tgz" - integrity sha512-GILhQTnjYE2WorX5Jyi5i4dz5ALWxBIdQECVQavL6s7cI76IZTDWleTHkxz/QT3kvcs2QlGHvKLYsSlPOlPXnQ== - dependencies: - eslint-utils "^2.0.0" - regexpp "^3.0.0" - -eslint-plugin-eslint-comments@^3.2.0: - version "3.2.0" - resolved "https://registry.npmjs.org/eslint-plugin-eslint-comments/-/eslint-plugin-eslint-comments-3.2.0.tgz" - integrity sha512-0jkOl0hfojIHHmEHgmNdqv4fmh7300NdpA9FFpF7zaoLvB/QeXOGNLIo86oAveJFrfB1p05kC8hpEMHM8DwWVQ== - dependencies: - escape-string-regexp "^1.0.5" - ignore "^5.0.5" - -eslint-plugin-html@^7.1.0: - version "7.1.0" - resolved "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-7.1.0.tgz" - integrity sha512-fNLRraV/e6j8e3XYOC9xgND4j+U7b1Rq+OygMlLcMg+wI/IpVbF+ubQa3R78EjKB9njT6TQOlcK5rFKBVVtdfg== - dependencies: - htmlparser2 "^8.0.1" - -eslint-plugin-import@^2.27.5, eslint-plugin-import@^2.28.1: - version "2.29.1" - resolved "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz" - integrity sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw== - dependencies: - array-includes "^3.1.7" - array.prototype.findlastindex "^1.2.3" - array.prototype.flat "^1.3.2" - array.prototype.flatmap "^1.3.2" - debug "^3.2.7" - doctrine "^2.1.0" - eslint-import-resolver-node "^0.3.9" - eslint-module-utils "^2.8.0" - hasown "^2.0.0" - is-core-module "^2.13.1" - is-glob "^4.0.3" - minimatch "^3.1.2" - object.fromentries "^2.0.7" - object.groupby "^1.0.1" - object.values "^1.1.7" - semver "^6.3.1" - tsconfig-paths "^3.15.0" - -eslint-plugin-jest@^27.2.1: - version "27.2.1" - resolved "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-27.2.1.tgz" - integrity sha512-l067Uxx7ZT8cO9NJuf+eJHvt6bqJyz2Z29wykyEdz/OtmcELQl2MQGQLX8J94O1cSJWAwUSEvCjwjA7KEK3Hmg== - dependencies: - "@typescript-eslint/utils" "^5.10.0" - -eslint-plugin-jsonc@^2.6.0: - version "2.8.0" - resolved "https://registry.npmjs.org/eslint-plugin-jsonc/-/eslint-plugin-jsonc-2.8.0.tgz" - integrity sha512-K4VsnztnNwpm+V49CcCu5laq8VjclJpuhfI9LFkOrOyK+BKdQHMzkWo43B4X4rYaVrChm4U9kw/tTU5RHh5Wtg== - dependencies: - "@eslint-community/eslint-utils" "^4.2.0" - jsonc-eslint-parser "^2.0.4" - natural-compare "^1.4.0" - -eslint-plugin-jsx-a11y@^6.7.1: - version "6.7.1" - resolved "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.7.1.tgz" - integrity sha512-63Bog4iIethyo8smBklORknVjB0T2dwB8Mr/hIC+fBS0uyHdYYpzM/Ed+YC8VxTjlXHEWFOdmgwcDn1U2L9VCA== - dependencies: - "@babel/runtime" "^7.20.7" - aria-query "^5.1.3" - array-includes "^3.1.6" - array.prototype.flatmap "^1.3.1" - ast-types-flow "^0.0.7" - axe-core "^4.6.2" - axobject-query "^3.1.1" - damerau-levenshtein "^1.0.8" - emoji-regex "^9.2.2" - has "^1.0.3" - jsx-ast-utils "^3.3.3" - language-tags "=1.0.5" - minimatch "^3.1.2" - object.entries "^1.1.6" - object.fromentries "^2.0.6" - semver "^6.3.0" - -eslint-plugin-markdown@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/eslint-plugin-markdown/-/eslint-plugin-markdown-3.0.0.tgz" - integrity sha512-hRs5RUJGbeHDLfS7ELanT0e29Ocyssf/7kBM+p7KluY5AwngGkDf8Oyu4658/NZSGTTq05FZeWbkxXtbVyHPwg== - dependencies: - mdast-util-from-markdown "^0.8.5" - -eslint-plugin-n@^15.6.1: - version "15.7.0" - resolved "https://registry.npmjs.org/eslint-plugin-n/-/eslint-plugin-n-15.7.0.tgz" - integrity sha512-jDex9s7D/Qial8AGVIHq4W7NswpUD5DPDL2RH8Lzd9EloWUuvUkHfv4FRLMipH5q2UtyurorBkPeNi1wVWNh3Q== - dependencies: - builtins "^5.0.1" - eslint-plugin-es "^4.1.0" - eslint-utils "^3.0.0" - ignore "^5.1.1" - is-core-module "^2.11.0" - minimatch "^3.1.2" - resolve "^1.22.1" - semver "^7.3.8" - -eslint-plugin-no-only-tests@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/eslint-plugin-no-only-tests/-/eslint-plugin-no-only-tests-3.1.0.tgz" - integrity sha512-Lf4YW/bL6Un1R6A76pRZyE1dl1vr31G/ev8UzIc/geCgFWyrKil8hVjYqWVKGB/UIGmb6Slzs9T0wNezdSVegw== - -eslint-plugin-promise@^6.1.1: - version "6.1.1" - resolved "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-6.1.1.tgz" - integrity sha512-tjqWDwVZQo7UIPMeDReOpUgHCmCiH+ePnVT+5zVapL0uuHnegBUs2smM13CzOs2Xb5+MHMRFTs9v24yjba4Oig== - -"eslint-plugin-react-hooks@^4.5.0 || 5.0.0-canary-7118f5dd7-20230705": - version "5.0.0-canary-7118f5dd7-20230705" - resolved "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-5.0.0-canary-7118f5dd7-20230705.tgz" - integrity sha512-AZYbMo/NW9chdL7vk6HQzQhT+PvTAEVqWk9ziruUoW2kAOcN5qNyelv70e0F1VNQAbvutOC9oc+xfWycI9FxDw== - -eslint-plugin-react@^7.33.2: - version "7.33.2" - resolved "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.33.2.tgz" - integrity sha512-73QQMKALArI8/7xGLNI/3LylrEYrlKZSb5C9+q3OtOewTnMQi5cT+aE9E41sLCmli3I9PGGmD1yiZydyo4FEPw== - dependencies: - array-includes "^3.1.6" - array.prototype.flatmap "^1.3.1" - array.prototype.tosorted "^1.1.1" - doctrine "^2.1.0" - es-iterator-helpers "^1.0.12" - estraverse "^5.3.0" - jsx-ast-utils "^2.4.1 || ^3.0.0" - minimatch "^3.1.2" - object.entries "^1.1.6" - object.fromentries "^2.0.6" - object.hasown "^1.1.2" - object.values "^1.1.6" - prop-types "^15.8.1" - resolve "^2.0.0-next.4" - semver "^6.3.1" - string.prototype.matchall "^4.0.8" - -eslint-plugin-unicorn@^45.0.2: - version "45.0.2" - resolved "https://registry.npmjs.org/eslint-plugin-unicorn/-/eslint-plugin-unicorn-45.0.2.tgz" - integrity sha512-Y0WUDXRyGDMcKLiwgL3zSMpHrXI00xmdyixEGIg90gHnj0PcHY4moNv3Ppje/kDivdAy5vUeUr7z211ImPv2gw== - dependencies: - "@babel/helper-validator-identifier" "^7.19.1" - "@eslint-community/eslint-utils" "^4.1.2" - ci-info "^3.6.1" - clean-regexp "^1.0.0" - esquery "^1.4.0" - indent-string "^4.0.0" - is-builtin-module "^3.2.0" - jsesc "^3.0.2" - lodash "^4.17.21" - pluralize "^8.0.0" - read-pkg-up "^7.0.1" - regexp-tree "^0.1.24" - regjsparser "^0.9.1" - safe-regex "^2.1.1" - semver "^7.3.8" - strip-indent "^3.0.0" - -eslint-plugin-unused-imports@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-2.0.0.tgz" - integrity sha512-3APeS/tQlTrFa167ThtP0Zm0vctjr4M44HMpeg1P4bK6wItarumq0Ma82xorMKdFsWpphQBlRPzw/pxiVELX1A== - dependencies: - eslint-rule-composer "^0.3.0" - -eslint-plugin-vue@^9.9.0: - version "9.14.1" - resolved "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-9.14.1.tgz" - integrity sha512-LQazDB1qkNEKejLe/b5a9VfEbtbczcOaui5lQ4Qw0tbRBbQYREyxxOV5BQgNDTqGPs9pxqiEpbMi9ywuIaF7vw== - dependencies: - "@eslint-community/eslint-utils" "^4.3.0" - natural-compare "^1.4.0" - nth-check "^2.0.1" - postcss-selector-parser "^6.0.9" - semver "^7.3.5" - vue-eslint-parser "^9.3.0" - xml-name-validator "^4.0.0" - -eslint-plugin-yml@^1.5.0: - version "1.7.0" - resolved "https://registry.npmjs.org/eslint-plugin-yml/-/eslint-plugin-yml-1.7.0.tgz" - integrity sha512-qq61FQJk+qIgWl0R06bec7UQQEIBrUH22jS+MroTbFUKu+3/iVlGRpZd8mjpOAm/+H/WEDFwy4x/+kKgVGbsWw== - dependencies: - debug "^4.3.2" - lodash "^4.17.21" - natural-compare "^1.4.0" - yaml-eslint-parser "^1.2.1" - -eslint-rule-composer@^0.3.0: - version "0.3.0" - resolved "https://registry.npmjs.org/eslint-rule-composer/-/eslint-rule-composer-0.3.0.tgz" - integrity sha512-bt+Sh8CtDmn2OajxvNO+BX7Wn4CIWMpTRm3MaiKPCQcnnlm0CS2mhui6QaoeQugs+3Kj2ESKEEGJUdVafwhiCg== - -eslint-scope@^5.1.1: - version "5.1.1" - resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz" - integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw== - dependencies: - esrecurse "^4.3.0" - estraverse "^4.1.1" - -eslint-scope@^7.1.1: - version "7.2.0" - resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz" - integrity sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw== - dependencies: - esrecurse "^4.3.0" - estraverse "^5.2.0" - -eslint-utils@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz" - integrity sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg== - dependencies: - eslint-visitor-keys "^1.1.0" - -eslint-utils@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz" - integrity sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA== - dependencies: - eslint-visitor-keys "^2.0.0" - -eslint-visitor-keys@^1.1.0: - version "1.3.0" - resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz" - integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== - -eslint-visitor-keys@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz" - integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== - -eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1: - version "3.4.1" - resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz" - integrity sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA== - -eslint@^8.36.0: - version "8.36.0" - resolved "https://registry.npmjs.org/eslint/-/eslint-8.36.0.tgz" - integrity sha512-Y956lmS7vDqomxlaaQAHVmeb4tNMp2FWIvU/RnU5BD3IKMD/MJPr76xdyr68P8tV1iNMvN2mRK0yy3c+UjL+bw== - dependencies: - "@eslint-community/eslint-utils" "^4.2.0" - "@eslint-community/regexpp" "^4.4.0" - "@eslint/eslintrc" "^2.0.1" - "@eslint/js" "8.36.0" - "@humanwhocodes/config-array" "^0.11.8" - "@humanwhocodes/module-importer" "^1.0.1" - "@nodelib/fs.walk" "^1.2.8" - ajv "^6.10.0" - chalk "^4.0.0" - cross-spawn "^7.0.2" - debug "^4.3.2" - doctrine "^3.0.0" - escape-string-regexp "^4.0.0" - eslint-scope "^7.1.1" - eslint-visitor-keys "^3.3.0" - espree "^9.5.0" - esquery "^1.4.2" - esutils "^2.0.2" - fast-deep-equal "^3.1.3" - file-entry-cache "^6.0.1" - find-up "^5.0.0" - glob-parent "^6.0.2" - globals "^13.19.0" - grapheme-splitter "^1.0.4" - ignore "^5.2.0" - import-fresh "^3.0.0" - imurmurhash "^0.1.4" - is-glob "^4.0.0" - is-path-inside "^3.0.3" - js-sdsl "^4.1.4" - js-yaml "^4.1.0" - json-stable-stringify-without-jsonify "^1.0.1" - levn "^0.4.1" - lodash.merge "^4.6.2" - minimatch "^3.1.2" - natural-compare "^1.4.0" - optionator "^0.9.1" - strip-ansi "^6.0.1" - strip-json-comments "^3.1.0" - text-table "^0.2.0" - -espree@^9.0.0, espree@^9.3.1, espree@^9.5.0, espree@^9.5.2: - version "9.5.2" - resolved "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz" - integrity sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw== - dependencies: - acorn "^8.8.0" - acorn-jsx "^5.3.2" - eslint-visitor-keys "^3.4.1" - -esquery@^1.4.0, esquery@^1.4.2: - version "1.5.0" - resolved "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz" - integrity sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg== - dependencies: - estraverse "^5.1.0" - -esrecurse@^4.3.0: - version "4.3.0" - resolved "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz" - integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== - dependencies: - estraverse "^5.2.0" - -estraverse@^4.1.1: - version "4.3.0" - resolved "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz" - integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== - -estraverse@^5.1.0, estraverse@^5.2.0, estraverse@^5.3.0: - version "5.3.0" - resolved "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz" - integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== - -estree-util-attach-comments@^2.0.0: - version "2.1.1" - resolved "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-2.1.1.tgz" - integrity sha512-+5Ba/xGGS6mnwFbXIuQiDPTbuTxuMCooq3arVv7gPZtYpjp+VXH/NkHAP35OOefPhNG/UGqU3vt/LTABwcHX0w== - dependencies: - "@types/estree" "^1.0.0" - -estree-util-build-jsx@^2.0.0: - version "2.2.2" - resolved "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-2.2.2.tgz" - integrity sha512-m56vOXcOBuaF+Igpb9OPAy7f9w9OIkb5yhjsZuaPm7HoGi4oTOQi0h2+yZ+AtKklYFZ+rPC4n0wYCJCEU1ONqg== - dependencies: - "@types/estree-jsx" "^1.0.0" - estree-util-is-identifier-name "^2.0.0" - estree-walker "^3.0.0" - -estree-util-is-identifier-name@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-2.1.0.tgz" - integrity sha512-bEN9VHRyXAUOjkKVQVvArFym08BTWB0aJPppZZr0UNyAqWsLaVfAqP7hbaTJjzHifmB5ebnR8Wm7r7yGN/HonQ== - -estree-util-to-js@^1.1.0: - version "1.2.0" - resolved "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-1.2.0.tgz" - integrity sha512-IzU74r1PK5IMMGZXUVZbmiu4A1uhiPgW5hm1GjcOfr4ZzHaMPpLNJjR7HjXiIOzi25nZDrgFTobHTkV5Q6ITjA== - dependencies: - "@types/estree-jsx" "^1.0.0" - astring "^1.8.0" - source-map "^0.7.0" - -estree-util-visit@^1.0.0: - version "1.2.1" - resolved "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-1.2.1.tgz" - integrity sha512-xbgqcrkIVbIG+lI/gzbvd9SGTJL4zqJKBFttUl5pP27KhAjtMKbX/mQXJ7qgyXpMgVy/zvpm0xoQQaGL8OloOw== - dependencies: - "@types/estree-jsx" "^1.0.0" - "@types/unist" "^2.0.0" - -estree-walker@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz" - integrity sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w== - -estree-walker@^3.0.0: - version "3.0.3" - resolved "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz" - integrity sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g== - dependencies: - "@types/estree" "^1.0.0" - -esutils@^2.0.2: - version "2.0.3" - resolved "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz" - integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== - -execa@^5.0.0: - version "5.1.1" - resolved "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz" - integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== - dependencies: - cross-spawn "^7.0.3" - get-stream "^6.0.0" - human-signals "^2.1.0" - is-stream "^2.0.0" - merge-stream "^2.0.0" - npm-run-path "^4.0.1" - onetime "^5.1.2" - signal-exit "^3.0.3" - strip-final-newline "^2.0.0" - -execa@^7.0.0, execa@^7.1.1: - version "7.1.1" - resolved "https://registry.npmjs.org/execa/-/execa-7.1.1.tgz" - integrity sha512-wH0eMf/UXckdUYnO21+HDztteVv05rq2GXksxT4fCGeHkBhw1DROXh40wcjMcRqDOWE7iPJ4n3M7e2+YFP+76Q== - dependencies: - cross-spawn "^7.0.3" - get-stream "^6.0.1" - human-signals "^4.3.0" - is-stream "^3.0.0" - merge-stream "^2.0.0" - npm-run-path "^5.1.0" - onetime "^6.0.0" - signal-exit "^3.0.7" - strip-final-newline "^3.0.0" - -extend@^3.0.0: - version "3.0.2" - resolved "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" - integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== - -fake-xml-http-request@^2.1.2: - version "2.1.2" - resolved "https://registry.npmjs.org/fake-xml-http-request/-/fake-xml-http-request-2.1.2.tgz" - integrity sha512-HaFMBi7r+oEC9iJNpc3bvcW7Z7iLmM26hPDmlb0mFwyANSsOQAtJxbdWsXITKOzZUyMYK0zYCv3h5yDj9TsiXg== - -fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: - version "3.1.3" - resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz" - integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== - -fast-glob@^3.2.11, fast-glob@^3.2.12, fast-glob@^3.2.9: - version "3.2.12" - resolved "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz" - integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== - dependencies: - "@nodelib/fs.stat" "^2.0.2" - "@nodelib/fs.walk" "^1.2.3" - glob-parent "^5.1.2" - merge2 "^1.3.0" - micromatch "^4.0.4" - -fast-json-stable-stringify@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz" - integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== - -fast-levenshtein@^2.0.6: - version "2.0.6" - resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz" - integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== - -fastq@^1.6.0: - version "1.15.0" - resolved "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz" - integrity sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw== - dependencies: - reusify "^1.0.4" - -fault@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz" - integrity sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA== - dependencies: - format "^0.2.0" - -file-entry-cache@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz" - integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg== - dependencies: - flat-cache "^3.0.4" - -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== - dependencies: - to-regex-range "^5.0.1" - -find-up@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" - integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== - dependencies: - locate-path "^5.0.0" - path-exists "^4.0.0" - -find-up@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz" - integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== - dependencies: - locate-path "^6.0.0" - path-exists "^4.0.0" - -flat-cache@^3.0.4: - version "3.0.4" - resolved "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz" - integrity sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg== - dependencies: - flatted "^3.1.0" - rimraf "^3.0.2" - -flatted@^3.1.0: - version "3.2.7" - resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz" - integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ== - -for-each@^0.3.3: - version "0.3.3" - resolved "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz" - integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw== - dependencies: - is-callable "^1.1.3" - -foreground-child@^3.1.0: - version "3.1.1" - resolved "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz" - integrity sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg== - dependencies: - cross-spawn "^7.0.0" - signal-exit "^4.0.1" - -format@^0.2.0: - version "0.2.2" - resolved "https://registry.npmjs.org/format/-/format-0.2.2.tgz" - integrity sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww== - -fraction.js@^4.2.0: - version "4.2.0" - resolved "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz" - integrity sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA== - -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" - integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== - -fsevents@~2.3.2: - version "2.3.2" - resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - -function-bind@^1.1.1, function-bind@^1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz" - integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== - -function.prototype.name@^1.1.5, function.prototype.name@^1.1.6: - version "1.1.6" - resolved "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz" - integrity sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - functions-have-names "^1.2.3" - -functions-have-names@^1.2.3: - version "1.2.3" - resolved "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz" - integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== - -get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.0, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2: - version "1.2.2" - resolved "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz" - integrity sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA== - dependencies: - function-bind "^1.1.2" - has-proto "^1.0.1" - has-symbols "^1.0.3" - hasown "^2.0.0" - -get-stream@^6.0.0, get-stream@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz" - integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== - -get-symbol-description@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz" - integrity sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.1" - -get-tsconfig@^4.5.0: - version "4.6.0" - resolved "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.6.0.tgz" - integrity sha512-lgbo68hHTQnFddybKbbs/RDRJnJT5YyGy2kQzVwbq+g67X73i+5MVTval34QxGkOe9X5Ujf1UYpCaphLyltjEg== - dependencies: - resolve-pkg-maps "^1.0.0" - -glob-parent@^5.1.2, glob-parent@~5.1.2: - version "5.1.2" - resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz" - integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== - dependencies: - is-glob "^4.0.1" - -glob-parent@^6.0.2: - version "6.0.2" - resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz" - integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== - dependencies: - is-glob "^4.0.3" - -glob@10.3.10: - version "10.3.10" - resolved "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz" - integrity sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g== - dependencies: - foreground-child "^3.1.0" - jackspeak "^2.3.5" - minimatch "^9.0.1" - minipass "^5.0.0 || ^6.0.2 || ^7.0.0" - path-scurry "^1.10.1" - -glob@7.1.6: - version "7.1.6" - resolved "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz" - integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - -glob@^7.1.3: - version "7.2.3" - resolved "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz" - integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.1.1" - once "^1.3.0" - path-is-absolute "^1.0.0" - -globals@^13.19.0: - version "13.20.0" - resolved "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz" - integrity sha512-Qg5QtVkCy/kv3FUSlu4ukeZDVf9ee0iXLAUYX13gbR17bnejFTzr4iS9bY7kwCf1NztRNm1t91fjOiyx4CSwPQ== - dependencies: - type-fest "^0.20.2" - -globalthis@^1.0.3: - version "1.0.3" - resolved "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz" - integrity sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA== - dependencies: - define-properties "^1.1.3" - -globby@^11.1.0: - version "11.1.0" - resolved "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz" - integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== - dependencies: - array-union "^2.1.0" - dir-glob "^3.0.1" - fast-glob "^3.2.9" - ignore "^5.2.0" - merge2 "^1.4.1" - slash "^3.0.0" - -globby@^13.1.3: - version "13.1.4" - resolved "https://registry.npmjs.org/globby/-/globby-13.1.4.tgz" - integrity sha512-iui/IiiW+QrJ1X1hKH5qwlMQyv34wJAYwH1vrf8b9kBA4sNiif3gKsMHa+BrdnOpEudWjpotfa7LrTzB1ERS/g== - dependencies: - dir-glob "^3.0.1" - fast-glob "^3.2.11" - ignore "^5.2.0" - merge2 "^1.4.1" - slash "^4.0.0" - -gopd@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz" - integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA== - dependencies: - get-intrinsic "^1.1.3" - -graceful-fs@^4.2.11, graceful-fs@^4.2.4: - version "4.2.11" - resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz" - integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== - -grapheme-splitter@^1.0.4: - version "1.0.4" - resolved "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz" - integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== - -graphlib@^2.1.8: - version "2.1.8" - resolved "https://registry.npmjs.org/graphlib/-/graphlib-2.1.8.tgz" - integrity sha512-jcLLfkpoVGmH7/InMC/1hIvOPSUh38oJtGhvrOFGzioE1DZ+0YW16RgmOJhHiuWTvGiJQ9Z1Ik43JvkRPRvE+A== - dependencies: - lodash "^4.17.15" - -has-bigints@^1.0.1, has-bigints@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz" - integrity sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ== - -has-flag@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" - integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== - -has-flag@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz" - integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== - -has-property-descriptors@^1.0.0, has-property-descriptors@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz" - integrity sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg== - dependencies: - get-intrinsic "^1.2.2" - -has-proto@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz" - integrity sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg== - -has-symbols@^1.0.2, has-symbols@^1.0.3: - version "1.0.3" - resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz" - integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== - -has-tostringtag@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz" - integrity sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ== - dependencies: - has-symbols "^1.0.2" - -has@^1.0.3: - version "1.0.3" - resolved "https://registry.npmjs.org/has/-/has-1.0.3.tgz" - integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== - dependencies: - function-bind "^1.1.1" - -hasown@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz" - integrity sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA== - dependencies: - function-bind "^1.1.2" - -hast-util-from-dom@^4.0.0: - version "4.2.0" - resolved "https://registry.npmjs.org/hast-util-from-dom/-/hast-util-from-dom-4.2.0.tgz" - integrity sha512-t1RJW/OpJbCAJQeKi3Qrj1cAOLA0+av/iPFori112+0X7R3wng+jxLA+kXec8K4szqPRGI8vPxbbpEYvvpwaeQ== - dependencies: - hastscript "^7.0.0" - web-namespaces "^2.0.0" - -hast-util-from-html-isomorphic@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/hast-util-from-html-isomorphic/-/hast-util-from-html-isomorphic-1.0.0.tgz" - integrity sha512-Yu480AKeOEN/+l5LA674a+7BmIvtDj24GvOt7MtQWuhzUwlaaRWdEPXAh3Qm5vhuthpAipFb2vTetKXWOjmTvw== - dependencies: - "@types/hast" "^2.0.0" - hast-util-from-dom "^4.0.0" - hast-util-from-html "^1.0.0" - unist-util-remove-position "^4.0.0" - -hast-util-from-html@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-1.0.2.tgz" - integrity sha512-LhrTA2gfCbLOGJq2u/asp4kwuG0y6NhWTXiPKP+n0qNukKy7hc10whqqCFfyvIA1Q5U5d0sp9HhNim9gglEH4A== - dependencies: - "@types/hast" "^2.0.0" - hast-util-from-parse5 "^7.0.0" - parse5 "^7.0.0" - vfile "^5.0.0" - vfile-message "^3.0.0" - -hast-util-from-parse5@^7.0.0: - version "7.1.2" - resolved "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-7.1.2.tgz" - integrity sha512-Nz7FfPBuljzsN3tCQ4kCBKqdNhQE2l0Tn+X1ubgKBPRoiDIu1mL08Cfw4k7q71+Duyaw7DXDN+VTAp4Vh3oCOw== - dependencies: - "@types/hast" "^2.0.0" - "@types/unist" "^2.0.0" - hastscript "^7.0.0" - property-information "^6.0.0" - vfile "^5.0.0" - vfile-location "^4.0.0" - web-namespaces "^2.0.0" - -hast-util-is-element@^2.0.0: - version "2.1.3" - resolved "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-2.1.3.tgz" - integrity sha512-O1bKah6mhgEq2WtVMk+Ta5K7pPMqsBBlmzysLdcwKVrqzZQ0CHqUPiIVspNhAG1rvxpvJjtGee17XfauZYKqVA== - dependencies: - "@types/hast" "^2.0.0" - "@types/unist" "^2.0.0" - -hast-util-parse-selector@^2.0.0: - version "2.2.5" - resolved "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz" - integrity sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ== - -hast-util-parse-selector@^3.0.0: - version "3.1.1" - resolved "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz" - integrity sha512-jdlwBjEexy1oGz0aJ2f4GKMaVKkA9jwjr4MjAAI22E5fM/TXVZHuS5OpONtdeIkRKqAaryQ2E9xNQxijoThSZA== - dependencies: - "@types/hast" "^2.0.0" - -hast-util-to-estree@^2.0.0: - version "2.3.3" - resolved "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-2.3.3.tgz" - integrity sha512-ihhPIUPxN0v0w6M5+IiAZZrn0LH2uZomeWwhn7uP7avZC6TE7lIiEh2yBMPr5+zi1aUCXq6VoYRgs2Bw9xmycQ== - dependencies: - "@types/estree" "^1.0.0" - "@types/estree-jsx" "^1.0.0" - "@types/hast" "^2.0.0" - "@types/unist" "^2.0.0" - comma-separated-tokens "^2.0.0" - estree-util-attach-comments "^2.0.0" - estree-util-is-identifier-name "^2.0.0" - hast-util-whitespace "^2.0.0" - mdast-util-mdx-expression "^1.0.0" - mdast-util-mdxjs-esm "^1.0.0" - property-information "^6.0.0" - space-separated-tokens "^2.0.0" - style-to-object "^0.4.1" - unist-util-position "^4.0.0" - zwitch "^2.0.0" - -hast-util-to-text@^3.1.0: - version "3.1.2" - resolved "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-3.1.2.tgz" - integrity sha512-tcllLfp23dJJ+ju5wCCZHVpzsQQ43+moJbqVX3jNWPB7z/KFC4FyZD6R7y94cHL6MQ33YtMZL8Z0aIXXI4XFTw== - dependencies: - "@types/hast" "^2.0.0" - "@types/unist" "^2.0.0" - hast-util-is-element "^2.0.0" - unist-util-find-after "^4.0.0" - -hast-util-whitespace@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.1.tgz" - integrity sha512-nAxA0v8+vXSBDt3AnRUNjyRIQ0rD+ntpbAp4LnPkumc5M9yUbSMa4XDU9Q6etY4f1Wp4bNgvc1yjiZtsTTrSng== - -hastscript@^6.0.0: - version "6.0.0" - resolved "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz" - integrity sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w== - dependencies: - "@types/hast" "^2.0.0" - comma-separated-tokens "^1.0.0" - hast-util-parse-selector "^2.0.0" - property-information "^5.0.0" - space-separated-tokens "^1.0.0" - -hastscript@^7.0.0: - version "7.2.0" - resolved "https://registry.npmjs.org/hastscript/-/hastscript-7.2.0.tgz" - integrity sha512-TtYPq24IldU8iKoJQqvZOuhi5CyCQRAbvDOX0x1eW6rsHSxa/1i2CCiptNTotGHJ3VoHRGmqiv6/D3q113ikkw== - dependencies: - "@types/hast" "^2.0.0" - comma-separated-tokens "^2.0.0" - hast-util-parse-selector "^3.0.0" - property-information "^6.0.0" - space-separated-tokens "^2.0.0" - -heap@^0.2.6: - version "0.2.7" - resolved "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz" - integrity sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg== - -highlight.js@^10.4.1, highlight.js@~10.7.0: - version "10.7.3" - resolved "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz" - integrity sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A== - -hoist-non-react-statics@^3.3.2: - version "3.3.2" - resolved "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz" - integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw== - dependencies: - react-is "^16.7.0" - -hosted-git-info@^2.1.4: - version "2.8.9" - resolved "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz" - integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== - -html-parse-stringify@^3.0.1: - version "3.0.1" - resolved "https://registry.npmjs.org/html-parse-stringify/-/html-parse-stringify-3.0.1.tgz" - integrity sha512-KknJ50kTInJ7qIScF3jeaFRpMpE8/lfiTdzf/twXyPBLAGrLRTmkz3AdTnKeh40X8k9L2fdYwEp/42WGXIRGcg== - dependencies: - void-elements "3.1.0" - -htmlparser2@^8.0.1: - version "8.0.2" - resolved "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz" - integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA== - dependencies: - domelementtype "^2.3.0" - domhandler "^5.0.3" - domutils "^3.0.1" - entities "^4.4.0" - -human-signals@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz" - integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== - -human-signals@^4.3.0: - version "4.3.1" - resolved "https://registry.npmjs.org/human-signals/-/human-signals-4.3.1.tgz" - integrity sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ== - -husky@^8.0.3: - version "8.0.3" - resolved "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz" - integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== - -i18next-resources-to-backend@^1.1.3: - version "1.1.4" - resolved "https://registry.npmjs.org/i18next-resources-to-backend/-/i18next-resources-to-backend-1.1.4.tgz" - integrity sha512-hMyr9AOmIea17AOaVe1srNxK/l3mbk81P7Uf3fdcjlw3ehZy3UNTd0OP3EEi6yu4J02kf9jzhCcjokz6AFlEOg== - dependencies: - "@babel/runtime" "^7.21.5" - -i18next@^22.4.13: - version "22.5.1" - resolved "https://registry.npmjs.org/i18next/-/i18next-22.5.1.tgz" - integrity sha512-8TGPgM3pAD+VRsMtUMNknRz3kzqwp/gPALrWMsDnmC1mKqJwpWyooQRLMcbTwq8z8YwSmuj+ZYvc+xCuEpkssA== - dependencies: - "@babel/runtime" "^7.20.6" - -iconv-lite@0.6: - version "0.6.3" - resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz" - integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== - dependencies: - safer-buffer ">= 2.1.2 < 3.0.0" - -ignore@^5.0.5, ignore@^5.1.1, ignore@^5.2.0: - version "5.2.4" - resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz" - integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== - -immer@^9.0.19: - version "9.0.21" - resolved "https://registry.npmjs.org/immer/-/immer-9.0.21.tgz" - integrity sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA== - -immutable@^4.0.0: - version "4.3.0" - resolved "https://registry.npmjs.org/immutable/-/immutable-4.3.0.tgz" - integrity sha512-0AOCmOip+xgJwEVTQj1EfiDDOkPmuyllDuTuEX+DDXUgapLAsBIfkg3sxCYyCEA8mQqZrrxPUGjcOQ2JS3WLkg== - -import-fresh@^3.0.0, import-fresh@^3.2.1: - version "3.3.0" - resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz" - integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== - dependencies: - parent-module "^1.0.0" - resolve-from "^4.0.0" - -imurmurhash@^0.1.4: - version "0.1.4" - resolved "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz" - integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== - -indent-string@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz" - integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== - -inflected@^2.0.4: - version "2.1.0" - resolved "https://registry.npmjs.org/inflected/-/inflected-2.1.0.tgz" - integrity sha512-hAEKNxvHf2Iq3H60oMBHkB4wl5jn3TPF3+fXek/sRwAB5gP9xWs4r7aweSF95f99HFoz69pnZTcu8f0SIHV18w== - -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" - integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2: - version "2.0.4" - resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" - integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== - -inline-style-parser@0.1.1: - version "0.1.1" - resolved "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz" - integrity sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q== - -internal-slot@^1.0.4, internal-slot@^1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz" - integrity sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ== - dependencies: - get-intrinsic "^1.2.0" - has "^1.0.3" - side-channel "^1.0.4" - -"internmap@1 - 2": - version "2.0.3" - resolved "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz" - integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== - -internmap@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz" - integrity sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw== - -intersection-observer@^0.12.0: - version "0.12.2" - resolved "https://registry.npmjs.org/intersection-observer/-/intersection-observer-0.12.2.tgz" - integrity sha512-7m1vEcPCxXYI8HqnL8CKI6siDyD+eIWSwgB3DZA+ZTogxk9I4CDnj4wilt9x/+/QbHI4YG5YZNmC6458/e9Ktg== - -is-alphabetical@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz" - integrity sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg== - -is-alphabetical@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz" - integrity sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ== - -is-alphanumerical@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz" - integrity sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A== - dependencies: - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - -is-alphanumerical@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz" - integrity sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw== - dependencies: - is-alphabetical "^2.0.0" - is-decimal "^2.0.0" - -is-arguments@^1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz" - integrity sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA== - dependencies: - call-bind "^1.0.2" - has-tostringtag "^1.0.0" - -is-array-buffer@^3.0.1, is-array-buffer@^3.0.2: - version "3.0.2" - resolved "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz" - integrity sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.2.0" - is-typed-array "^1.1.10" - -is-arrayish@^0.2.1: - version "0.2.1" - resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz" - integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== - -is-arrayish@^0.3.1: - version "0.3.2" - resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz" - integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ== - -is-async-function@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz" - integrity sha512-Y1JXKrfykRJGdlDwdKlLpLyMIiWqWvuSd17TvZk68PLAOGOoF4Xyav1z0Xhoi+gCYjZVeC5SI+hYFOfvXmGRCA== - dependencies: - has-tostringtag "^1.0.0" - -is-bigint@^1.0.1: - version "1.0.4" - resolved "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz" - integrity sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg== - dependencies: - has-bigints "^1.0.1" - -is-binary-path@~2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz" - integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw== - dependencies: - binary-extensions "^2.0.0" - -is-boolean-object@^1.1.0: - version "1.1.2" - resolved "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz" - integrity sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA== - dependencies: - call-bind "^1.0.2" - has-tostringtag "^1.0.0" - -is-buffer@^2.0.0: - version "2.0.5" - resolved "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz" - integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ== - -is-builtin-module@^3.2.0: - version "3.2.1" - resolved "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz" - integrity sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A== - dependencies: - builtin-modules "^3.3.0" - -is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: - version "1.2.7" - resolved "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz" - integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== - -is-core-module@^2.11.0, is-core-module@^2.13.0, is-core-module@^2.13.1: - version "2.13.1" - resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz" - integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw== - dependencies: - hasown "^2.0.0" - -is-date-object@^1.0.1, is-date-object@^1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz" - integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ== - dependencies: - has-tostringtag "^1.0.0" - -is-decimal@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz" - integrity sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw== - -is-decimal@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz" - integrity sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A== - -is-docker@^2.0.0: - version "2.2.1" - resolved "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz" - integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== - -is-docker@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz" - integrity sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ== - -is-extglob@^2.1.1: - version "2.1.1" - resolved "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz" - integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== - -is-finalizationregistry@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.0.2.tgz" - integrity sha512-0by5vtUJs8iFQb5TYUHHPudOR+qXYIMKtiUzvLIZITZUjknFmziyBJuLhVRc+Ds0dREFlskDNJKYIdIzu/9pfw== - dependencies: - call-bind "^1.0.2" - -is-fullwidth-code-point@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz" - integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== - -is-fullwidth-code-point@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz" - integrity sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ== - -is-generator-function@^1.0.10: - version "1.0.10" - resolved "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz" - integrity sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A== - dependencies: - has-tostringtag "^1.0.0" - -is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: - version "4.0.3" - resolved "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz" - integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== - dependencies: - is-extglob "^2.1.1" - -is-hexadecimal@^1.0.0: - version "1.0.4" - resolved "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz" - integrity sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw== - -is-hexadecimal@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz" - integrity sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg== - -is-inside-container@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz" - integrity sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA== - dependencies: - is-docker "^3.0.0" - -is-map@^2.0.1, is-map@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz" - integrity sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg== - -is-negative-zero@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz" - integrity sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA== - -is-number-object@^1.0.4: - version "1.0.7" - resolved "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz" - integrity sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ== - dependencies: - has-tostringtag "^1.0.0" - -is-number@^7.0.0: - version "7.0.0" - resolved "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz" - integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== - -is-path-inside@^3.0.3: - version "3.0.3" - resolved "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz" - integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== - -is-plain-obj@^4.0.0: - version "4.1.0" - resolved "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz" - integrity sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg== - -is-reference@^3.0.0: - version "3.0.1" - resolved "https://registry.npmjs.org/is-reference/-/is-reference-3.0.1.tgz" - integrity sha512-baJJdQLiYaJdvFbJqXrcGv3WU3QCzBlUcI5QhbesIm6/xPsvmO+2CDoi/GMOFBQEQm+PXkwOPrp9KK5ozZsp2w== - dependencies: - "@types/estree" "*" - -is-regex@^1.1.4: - version "1.1.4" - resolved "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz" - integrity sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg== - dependencies: - call-bind "^1.0.2" - has-tostringtag "^1.0.0" - -is-set@^2.0.1, is-set@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz" - integrity sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g== - -is-shared-array-buffer@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz" - integrity sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA== - dependencies: - call-bind "^1.0.2" - -is-stream@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz" - integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== - -is-stream@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz" - integrity sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA== - -is-string@^1.0.5, is-string@^1.0.7: - version "1.0.7" - resolved "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz" - integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg== - dependencies: - has-tostringtag "^1.0.0" - -is-symbol@^1.0.2, is-symbol@^1.0.3: - version "1.0.4" - resolved "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz" - integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg== - dependencies: - has-symbols "^1.0.2" - -is-typed-array@^1.1.10, is-typed-array@^1.1.12, is-typed-array@^1.1.9: - version "1.1.12" - resolved "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz" - integrity sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg== - dependencies: - which-typed-array "^1.1.11" - -is-weakmap@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz" - integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA== - -is-weakref@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz" - integrity sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ== - dependencies: - call-bind "^1.0.2" - -is-weakset@^2.0.1: - version "2.0.2" - resolved "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz" - integrity sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.1" - -is-wsl@^2.2.0: - version "2.2.0" - resolved "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz" - integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww== - dependencies: - is-docker "^2.0.0" - -isarray@^2.0.5: - version "2.0.5" - resolved "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz" - integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== - -isexe@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" - integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== - -iterator.prototype@^1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz" - integrity sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w== - dependencies: - define-properties "^1.2.1" - get-intrinsic "^1.2.1" - has-symbols "^1.0.3" - reflect.getprototypeof "^1.0.4" - set-function-name "^2.0.1" - -jackspeak@^2.3.5: - version "2.3.6" - resolved "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz" - integrity sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ== - dependencies: - "@isaacs/cliui" "^8.0.2" - optionalDependencies: - "@pkgjs/parseargs" "^0.11.0" - -jiti@^1.18.2: - version "1.18.2" - resolved "https://registry.npmjs.org/jiti/-/jiti-1.18.2.tgz" - integrity sha512-QAdOptna2NYiSSpv0O/BwoHBSmz4YhpzJHyi+fnMRTXFjp7B8i/YG5Z8IfusxB1ufjcD2Sre1F3R+nX3fvy7gg== - -js-audio-recorder@^1.0.7: - version "1.0.7" - resolved "https://registry.npmjs.org/js-audio-recorder/-/js-audio-recorder-1.0.7.tgz" - integrity sha512-JiDODCElVHGrFyjGYwYyNi7zCbKk9va9C77w+zCPMmi4C6ix7zsX2h3ddHugmo4dOTOTCym9++b/wVW9nC0IaA== - -js-cookie@^2.x.x: - version "2.2.1" - resolved "https://registry.npmjs.org/js-cookie/-/js-cookie-2.2.1.tgz" - integrity sha512-HvdH2LzI/EAZcUwA8+0nKNtWHqS+ZmijLA30RwZA0bo7ToCckjK5MkGhjED9KoRcXO6BaGI3I9UIzSA1FKFPOQ== - -js-cookie@^3.0.1: - version "3.0.5" - resolved "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz" - integrity sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw== - -js-sdsl@^4.1.4: - version "4.4.0" - resolved "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.0.tgz" - integrity sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg== - -"js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" - integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== - -js-yaml@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz" - integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== - dependencies: - argparse "^2.0.1" - -jsesc@^3.0.2: - version "3.0.2" - resolved "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz" - integrity sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g== - -jsesc@~0.5.0: - version "0.5.0" - resolved "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz" - integrity sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA== - -json-parse-even-better-errors@^2.3.0: - version "2.3.1" - resolved "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz" - integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== - -json-schema-traverse@^0.4.1: - version "0.4.1" - resolved "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz" - integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== - -json-stable-stringify-without-jsonify@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz" - integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== - -json5@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz" - integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== - dependencies: - minimist "^1.2.0" - -jsonc-eslint-parser@^2.0.4, jsonc-eslint-parser@^2.1.0: - version "2.3.0" - resolved "https://registry.npmjs.org/jsonc-eslint-parser/-/jsonc-eslint-parser-2.3.0.tgz" - integrity sha512-9xZPKVYp9DxnM3sd1yAsh/d59iIaswDkai8oTxbursfKYbg/ibjX0IzFt35+VZ8iEW453TVTXztnRvYUQlAfUQ== - dependencies: - acorn "^8.5.0" - eslint-visitor-keys "^3.0.0" - espree "^9.0.0" - semver "^7.3.5" - -"jsx-ast-utils@^2.4.1 || ^3.0.0", jsx-ast-utils@^3.3.3: - version "3.3.3" - resolved "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz" - integrity sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw== - dependencies: - array-includes "^3.1.5" - object.assign "^4.1.3" - -katex@^0.16.0, katex@^0.16.10: - version "0.16.10" - resolved "https://registry.npmjs.org/katex/-/katex-0.16.10.tgz" - integrity sha512-ZiqaC04tp2O5utMsl2TEZTXxa6WSC4yo0fv5ML++D3QZv/vx2Mct0mTlRx3O+uUkjfuAgOkzsCmq5MiUEsDDdA== - dependencies: - commander "^8.3.0" - -khroma@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/khroma/-/khroma-2.0.0.tgz" - integrity sha512-2J8rDNlQWbtiNYThZRvmMv5yt44ZakX+Tz5ZIp/mN1pt4snn+m030Va5Z4v8xA0cQFDXBwO/8i42xL4QPsVk3g== - -kleur@^4.0.3: - version "4.1.5" - resolved "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz" - integrity sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ== - -lamejs@^1.2.1: - version "1.2.1" - resolved "https://registry.npmjs.org/lamejs/-/lamejs-1.2.1.tgz" - integrity sha512-s7bxvjvYthw6oPLCm5pFxvA84wUROODB8jEO2+CE1adhKgrIvVOlmMgY8zyugxGrvRaDHNJanOiS21/emty6dQ== - dependencies: - use-strict "1.0.1" - -language-subtag-registry@~0.3.2: - version "0.3.22" - resolved "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz" - integrity sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w== - -language-tags@=1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/language-tags/-/language-tags-1.0.5.tgz" - integrity sha512-qJhlO9cGXi6hBGKoxEG/sKZDAHD5Hnu9Hs4WbOY3pCWXDhw0N8x1NenNzm2EnNLkLkk7J2SdxAkDSbb6ftT+UQ== - dependencies: - language-subtag-registry "~0.3.2" - -layout-base@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz" - integrity sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg== - -layout-base@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz" - integrity sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg== - -levn@^0.4.1: - version "0.4.1" - resolved "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz" - integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== - dependencies: - prelude-ls "^1.2.1" - type-check "~0.4.0" - -lexical@^0.12.2: - version "0.12.2" - resolved "https://registry.npmjs.org/lexical/-/lexical-0.12.2.tgz" - integrity sha512-Kxavd+ETjxtVwG/hvPd6WZfXD44sLOKe9Vlkwxy7lBQ1qZArS+rZfs+u5iXwXe6tX9f2PIM0u3RHsrCEDDE0fw== - -lilconfig@2.1.0, lilconfig@^2.0.5, lilconfig@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz" - integrity sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ== - -lines-and-columns@^1.1.6: - version "1.2.4" - resolved "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz" - integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== - -lint-staged@^13.2.2: - version "13.2.2" - resolved "https://registry.npmjs.org/lint-staged/-/lint-staged-13.2.2.tgz" - integrity sha512-71gSwXKy649VrSU09s10uAT0rWCcY3aewhMaHyl2N84oBk4Xs9HgxvUp3AYu+bNsK4NrOYYxvSgg7FyGJ+jGcA== - dependencies: - chalk "5.2.0" - cli-truncate "^3.1.0" - commander "^10.0.0" - debug "^4.3.4" - execa "^7.0.0" - lilconfig "2.1.0" - listr2 "^5.0.7" - micromatch "^4.0.5" - normalize-path "^3.0.0" - object-inspect "^1.12.3" - pidtree "^0.6.0" - string-argv "^0.3.1" - yaml "^2.2.2" - -listr2@^5.0.7: - version "5.0.8" - resolved "https://registry.npmjs.org/listr2/-/listr2-5.0.8.tgz" - integrity sha512-mC73LitKHj9w6v30nLNGPetZIlfpUniNSsxxrbaPcWOjDb92SHPzJPi/t+v1YC/lxKz/AJ9egOjww0qUuFxBpA== - dependencies: - cli-truncate "^2.1.0" - colorette "^2.0.19" - log-update "^4.0.0" - p-map "^4.0.0" - rfdc "^1.3.0" - rxjs "^7.8.0" - through "^2.3.8" - wrap-ansi "^7.0.0" - -local-pkg@^0.4.3: - version "0.4.3" - resolved "https://registry.npmjs.org/local-pkg/-/local-pkg-0.4.3.tgz" - integrity sha512-SFppqq5p42fe2qcZQqqEOiVRXl+WCP1MdT6k7BDEW1j++sp5fIY+/fdRQitvKgB5BrBcmrs5m/L0v2FrU5MY1g== - -locate-path@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz" - integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== - dependencies: - p-locate "^4.1.0" - -locate-path@^6.0.0: - version "6.0.0" - resolved "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz" - integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== - dependencies: - p-locate "^5.0.0" - -lodash-es@^4.17.21: - version "4.17.21" - resolved "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz" - integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw== - -lodash.assign@^4.2.0: - version "4.2.0" - resolved "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz" - integrity sha512-hFuH8TY+Yji7Eja3mGiuAxBqLagejScbG8GbG0j6o9vzn0YL14My+ktnqtZgFTosKymC9/44wP6s7xyuLfnClw== - -lodash.camelcase@^4.3.0: - version "4.3.0" - resolved "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz" - integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA== - -lodash.castarray@^4.4.0: - version "4.4.0" - resolved "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz" - integrity sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q== - -lodash.clonedeep@^4.5.0: - version "4.5.0" - resolved "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz" - integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ== - -lodash.compact@^3.0.1: - version "3.0.1" - resolved "https://registry.npmjs.org/lodash.compact/-/lodash.compact-3.0.1.tgz" - integrity sha512-2ozeiPi+5eBXW1CLtzjk8XQFhQOEMwwfxblqeq6EGyTxZJ1bPATqilY0e6g2SLQpP4KuMeuioBhEnWz5Pr7ICQ== - -lodash.find@^4.6.0: - version "4.6.0" - resolved "https://registry.npmjs.org/lodash.find/-/lodash.find-4.6.0.tgz" - integrity sha512-yaRZoAV3Xq28F1iafWN1+a0rflOej93l1DQUejs3SZ41h2O9UJBoS9aueGjPDgAl4B6tPC0NuuchLKaDQQ3Isg== - -lodash.flatten@^4.4.0: - version "4.4.0" - resolved "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz" - integrity sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g== - -lodash.forin@^4.4.0: - version "4.4.0" - resolved "https://registry.npmjs.org/lodash.forin/-/lodash.forin-4.4.0.tgz" - integrity sha512-APldePP4yvGhMcplVxv9L+exdLHMRHRhH1Q9O70zRJMm9HbTm6zxaihXtNl+ICOBApeFWoH7jNmFr/L4XfWeiQ== - -lodash.get@^4.4.2: - version "4.4.2" - resolved "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz" - integrity sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ== - -lodash.has@^4.5.2: - version "4.5.2" - resolved "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz" - integrity sha512-rnYUdIo6xRCJnQmbVFEwcxF144erlD+M3YcJUVesflU9paQaE8p+fJDcIQrlMYbxoANFL+AB9hZrzSBBk5PL+g== - -lodash.invokemap@^4.6.0: - version "4.6.0" - resolved "https://registry.npmjs.org/lodash.invokemap/-/lodash.invokemap-4.6.0.tgz" - integrity sha512-CfkycNtMqgUlfjfdh2BhKO/ZXrP8ePOX5lEU/g0R3ItJcnuxWDwokMGKx1hWcfOikmyOVx6X9IwWnDGlgKl61w== - -lodash.isempty@^4.4.0: - version "4.4.0" - resolved "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz" - integrity sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg== - -lodash.isequal@^4.5.0: - version "4.5.0" - resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz" - integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== - -lodash.isfunction@^3.0.9: - version "3.0.9" - resolved "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz" - integrity sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw== - -lodash.isinteger@^4.0.4: - version "4.0.4" - resolved "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz" - integrity sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA== - -lodash.isplainobject@^4.0.6: - version "4.0.6" - resolved "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz" - integrity sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA== - -lodash.lowerfirst@^4.3.1: - version "4.3.1" - resolved "https://registry.npmjs.org/lodash.lowerfirst/-/lodash.lowerfirst-4.3.1.tgz" - integrity sha512-UUKX7VhP1/JL54NXg2aq/E1Sfnjjes8fNYTNkPU8ZmsaVeBvPHKdbNaN79Re5XRL01u6wbq3j0cbYZj71Fcu5w== - -lodash.map@^4.6.0: - version "4.6.0" - resolved "https://registry.npmjs.org/lodash.map/-/lodash.map-4.6.0.tgz" - integrity sha512-worNHGKLDetmcEYDvh2stPCrrQRkP20E4l0iIS7F8EvzMqBBi7ltvFN5m1HvTf1P7Jk1txKhvFcmYsCr8O2F1Q== - -lodash.mapvalues@^4.6.0: - version "4.6.0" - resolved "https://registry.npmjs.org/lodash.mapvalues/-/lodash.mapvalues-4.6.0.tgz" - integrity sha512-JPFqXFeZQ7BfS00H58kClY7SPVeHertPE0lNuCyZ26/XlN8TvakYD7b9bGyNmXbT/D3BbtPAAmq90gPWqLkxlQ== - -lodash.merge@^4.6.2: - version "4.6.2" - resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz" - integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== - -lodash.pick@^4.4.0: - version "4.4.0" - resolved "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz" - integrity sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q== - -lodash.snakecase@^4.1.1: - version "4.1.1" - resolved "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz" - integrity sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw== - -lodash.uniq@^4.5.0: - version "4.5.0" - resolved "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz" - integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== - -lodash.uniqby@^4.7.0: - version "4.7.0" - resolved "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz" - integrity sha512-e/zcLx6CSbmaEgFHCA7BnoQKyCtKMxnuWrJygbwPs/AIn+IMKl66L8/s+wBUn5LRw2pZx3bUHibiV1b6aTWIww== - -lodash.values@^4.3.0: - version "4.3.0" - resolved "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz" - integrity sha512-r0RwvdCv8id9TUblb/O7rYPwVy6lerCbcawrfdo9iC/1t1wsNMJknO79WNBgwkH0hIeJ08jmvvESbFpNb4jH0Q== - -lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.21: - version "4.17.21" - resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - -log-update@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/log-update/-/log-update-4.0.0.tgz" - integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== - dependencies: - ansi-escapes "^4.3.0" - cli-cursor "^3.1.0" - slice-ansi "^4.0.0" - wrap-ansi "^6.2.0" - -longest-streak@^3.0.0: - version "3.1.0" - resolved "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz" - integrity sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g== - -loose-envify@^1.1.0, loose-envify@^1.4.0: - version "1.4.0" - resolved "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz" - integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q== - dependencies: - js-tokens "^3.0.0 || ^4.0.0" - -lowlight@^1.17.0: - version "1.20.0" - resolved "https://registry.npmjs.org/lowlight/-/lowlight-1.20.0.tgz" - integrity sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw== - dependencies: - fault "^1.0.0" - highlight.js "~10.7.0" - -lru-cache@^6.0.0: - version "6.0.0" - resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz" - integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== - dependencies: - yallist "^4.0.0" - -"lru-cache@^9.1.1 || ^10.0.0": - version "10.2.0" - resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz" - integrity sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q== - -markdown-extensions@^1.0.0: - version "1.1.1" - resolved "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-1.1.1.tgz" - integrity sha512-WWC0ZuMzCyDHYCasEGs4IPvLyTGftYwh6wIEOULOF0HXcqZlhwRzrK0w2VUlxWA98xnvb/jszw4ZSkJ6ADpM6Q== - -markdown-table@^3.0.0: - version "3.0.3" - resolved "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz" - integrity sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw== - -mdast-util-definitions@^5.0.0: - version "5.1.2" - resolved "https://registry.npmjs.org/mdast-util-definitions/-/mdast-util-definitions-5.1.2.tgz" - integrity sha512-8SVPMuHqlPME/z3gqVwWY4zVXn8lqKv/pAhC57FuJ40ImXyBpmO5ukh98zB2v7Blql2FiHjHv9LVztSIqjY+MA== - dependencies: - "@types/mdast" "^3.0.0" - "@types/unist" "^2.0.0" - unist-util-visit "^4.0.0" - -mdast-util-find-and-replace@^2.0.0: - version "2.2.2" - resolved "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-2.2.2.tgz" - integrity sha512-MTtdFRz/eMDHXzeK6W3dO7mXUlF82Gom4y0oOgvHhh/HXZAGvIQDUvQ0SuUx+j2tv44b8xTHOm8K/9OoRFnXKw== - dependencies: - "@types/mdast" "^3.0.0" - escape-string-regexp "^5.0.0" - unist-util-is "^5.0.0" - unist-util-visit-parents "^5.0.0" - -mdast-util-from-markdown@^0.8.5: - version "0.8.5" - resolved "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-0.8.5.tgz" - integrity sha512-2hkTXtYYnr+NubD/g6KGBS/0mFmBcifAsI0yIWRiRo0PjVs6SSOSOdtzbp6kSGnShDN6G5aWZpKQ2lWRy27mWQ== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-to-string "^2.0.0" - micromark "~2.11.0" - parse-entities "^2.0.0" - unist-util-stringify-position "^2.0.0" - -mdast-util-from-markdown@^1.0.0, mdast-util-from-markdown@^1.1.0, mdast-util-from-markdown@^1.3.0: - version "1.3.1" - resolved "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-1.3.1.tgz" - integrity sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww== - dependencies: - "@types/mdast" "^3.0.0" - "@types/unist" "^2.0.0" - decode-named-character-reference "^1.0.0" - mdast-util-to-string "^3.1.0" - micromark "^3.0.0" - micromark-util-decode-numeric-character-reference "^1.0.0" - micromark-util-decode-string "^1.0.0" - micromark-util-normalize-identifier "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - unist-util-stringify-position "^3.0.0" - uvu "^0.5.0" - -mdast-util-gfm-autolink-literal@^1.0.0: - version "1.0.3" - resolved "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-1.0.3.tgz" - integrity sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA== - dependencies: - "@types/mdast" "^3.0.0" - ccount "^2.0.0" - mdast-util-find-and-replace "^2.0.0" - micromark-util-character "^1.0.0" - -mdast-util-gfm-footnote@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-1.0.2.tgz" - integrity sha512-56D19KOGbE00uKVj3sgIykpwKL179QsVFwx/DCW0u/0+URsryacI4MAdNJl0dh+u2PSsD9FtxPFbHCzJ78qJFQ== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-to-markdown "^1.3.0" - micromark-util-normalize-identifier "^1.0.0" - -mdast-util-gfm-strikethrough@^1.0.0: - version "1.0.3" - resolved "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-1.0.3.tgz" - integrity sha512-DAPhYzTYrRcXdMjUtUjKvW9z/FNAMTdU0ORyMcbmkwYNbKocDpdk+PX1L1dQgOID/+vVs1uBQ7ElrBQfZ0cuiQ== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-to-markdown "^1.3.0" - -mdast-util-gfm-table@^1.0.0: - version "1.0.7" - resolved "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-1.0.7.tgz" - integrity sha512-jjcpmNnQvrmN5Vx7y7lEc2iIOEytYv7rTvu+MeyAsSHTASGCCRA79Igg2uKssgOs1i1po8s3plW0sTu1wkkLGg== - dependencies: - "@types/mdast" "^3.0.0" - markdown-table "^3.0.0" - mdast-util-from-markdown "^1.0.0" - mdast-util-to-markdown "^1.3.0" - -mdast-util-gfm-task-list-item@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-1.0.2.tgz" - integrity sha512-PFTA1gzfp1B1UaiJVyhJZA1rm0+Tzn690frc/L8vNX1Jop4STZgOE6bxUhnzdVSB+vm2GU1tIsuQcA9bxTQpMQ== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-to-markdown "^1.3.0" - -mdast-util-gfm@^2.0.0: - version "2.0.2" - resolved "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-2.0.2.tgz" - integrity sha512-qvZ608nBppZ4icQlhQQIAdc6S3Ffj9RGmzwUKUWuEICFnd1LVkN3EktF7ZHAgfcEdvZB5owU9tQgt99e2TlLjg== - dependencies: - mdast-util-from-markdown "^1.0.0" - mdast-util-gfm-autolink-literal "^1.0.0" - mdast-util-gfm-footnote "^1.0.0" - mdast-util-gfm-strikethrough "^1.0.0" - mdast-util-gfm-table "^1.0.0" - mdast-util-gfm-task-list-item "^1.0.0" - mdast-util-to-markdown "^1.0.0" - -mdast-util-math@^2.0.0: - version "2.0.2" - resolved "https://registry.npmjs.org/mdast-util-math/-/mdast-util-math-2.0.2.tgz" - integrity sha512-8gmkKVp9v6+Tgjtq6SYx9kGPpTf6FVYRa53/DLh479aldR9AyP48qeVOgNZ5X7QUK7nOy4yw7vg6mbiGcs9jWQ== - dependencies: - "@types/mdast" "^3.0.0" - longest-streak "^3.0.0" - mdast-util-to-markdown "^1.3.0" - -mdast-util-mdx-expression@^1.0.0: - version "1.3.2" - resolved "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-1.3.2.tgz" - integrity sha512-xIPmR5ReJDu/DHH1OoIT1HkuybIfRGYRywC+gJtI7qHjCJp/M9jrmBEJW22O8lskDWm562BX2W8TiAwRTb0rKA== - dependencies: - "@types/estree-jsx" "^1.0.0" - "@types/hast" "^2.0.0" - "@types/mdast" "^3.0.0" - mdast-util-from-markdown "^1.0.0" - mdast-util-to-markdown "^1.0.0" - -mdast-util-mdx-jsx@^2.0.0: - version "2.1.4" - resolved "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-2.1.4.tgz" - integrity sha512-DtMn9CmVhVzZx3f+optVDF8yFgQVt7FghCRNdlIaS3X5Bnym3hZwPbg/XW86vdpKjlc1PVj26SpnLGeJBXD3JA== - dependencies: - "@types/estree-jsx" "^1.0.0" - "@types/hast" "^2.0.0" - "@types/mdast" "^3.0.0" - "@types/unist" "^2.0.0" - ccount "^2.0.0" - mdast-util-from-markdown "^1.1.0" - mdast-util-to-markdown "^1.3.0" - parse-entities "^4.0.0" - stringify-entities "^4.0.0" - unist-util-remove-position "^4.0.0" - unist-util-stringify-position "^3.0.0" - vfile-message "^3.0.0" - -mdast-util-mdx@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-2.0.1.tgz" - integrity sha512-38w5y+r8nyKlGvNjSEqWrhG0w5PmnRA+wnBvm+ulYCct7nsGYhFVb0lljS9bQav4psDAS1eGkP2LMVcZBi/aqw== - dependencies: - mdast-util-from-markdown "^1.0.0" - mdast-util-mdx-expression "^1.0.0" - mdast-util-mdx-jsx "^2.0.0" - mdast-util-mdxjs-esm "^1.0.0" - mdast-util-to-markdown "^1.0.0" - -mdast-util-mdxjs-esm@^1.0.0: - version "1.3.1" - resolved "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-1.3.1.tgz" - integrity sha512-SXqglS0HrEvSdUEfoXFtcg7DRl7S2cwOXc7jkuusG472Mmjag34DUDeOJUZtl+BVnyeO1frIgVpHlNRWc2gk/w== - dependencies: - "@types/estree-jsx" "^1.0.0" - "@types/hast" "^2.0.0" - "@types/mdast" "^3.0.0" - mdast-util-from-markdown "^1.0.0" - mdast-util-to-markdown "^1.0.0" - -mdast-util-newline-to-break@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/mdast-util-newline-to-break/-/mdast-util-newline-to-break-1.0.0.tgz" - integrity sha512-491LcYv3gbGhhCrLoeALncQmega2xPh+m3gbsIhVsOX4sw85+ShLFPvPyibxc1Swx/6GtzxgVodq+cGa/47ULg== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-find-and-replace "^2.0.0" - -mdast-util-phrasing@^3.0.0: - version "3.0.1" - resolved "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-3.0.1.tgz" - integrity sha512-WmI1gTXUBJo4/ZmSk79Wcb2HcjPJBzM1nlI/OUWA8yk2X9ik3ffNbBGsU+09BFmXaL1IBb9fiuvq6/KMiNycSg== - dependencies: - "@types/mdast" "^3.0.0" - unist-util-is "^5.0.0" - -mdast-util-to-hast@^12.1.0: - version "12.3.0" - resolved "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-12.3.0.tgz" - integrity sha512-pits93r8PhnIoU4Vy9bjW39M2jJ6/tdHyja9rrot9uujkN7UTU9SDnE6WNJz/IGyQk3XHX6yNNtrBH6cQzm8Hw== - dependencies: - "@types/hast" "^2.0.0" - "@types/mdast" "^3.0.0" - mdast-util-definitions "^5.0.0" - micromark-util-sanitize-uri "^1.1.0" - trim-lines "^3.0.0" - unist-util-generated "^2.0.0" - unist-util-position "^4.0.0" - unist-util-visit "^4.0.0" - -mdast-util-to-markdown@^1.0.0, mdast-util-to-markdown@^1.3.0: - version "1.5.0" - resolved "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-1.5.0.tgz" - integrity sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A== - dependencies: - "@types/mdast" "^3.0.0" - "@types/unist" "^2.0.0" - longest-streak "^3.0.0" - mdast-util-phrasing "^3.0.0" - mdast-util-to-string "^3.0.0" - micromark-util-decode-string "^1.0.0" - unist-util-visit "^4.0.0" - zwitch "^2.0.0" - -mdast-util-to-string@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-2.0.0.tgz" - integrity sha512-AW4DRS3QbBayY/jJmD8437V1Gombjf8RSOUCMFBuo5iHi58AGEgVCKQ+ezHkZZDpAQS75hcBMpLqjpJTjtUL7w== - -mdast-util-to-string@^3.0.0, mdast-util-to-string@^3.1.0: - version "3.2.0" - resolved "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-3.2.0.tgz" - integrity sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg== - dependencies: - "@types/mdast" "^3.0.0" - -"memoize-one@>=3.1.1 <6": - version "5.2.1" - resolved "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz" - integrity sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q== - -merge-stream@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz" - integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== - -merge2@^1.3.0, merge2@^1.4.1: - version "1.4.1" - resolved "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz" - integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== - -mermaid@10.4.0: - version "10.4.0" - resolved "https://registry.npmjs.org/mermaid/-/mermaid-10.4.0.tgz" - integrity sha512-4QCQLp79lvz7UZxow5HUX7uWTPJOaQBVExduo91tliXC7v78i6kssZOPHxLL+Xs30KU72cpPn3g3imw/xm/gaw== - dependencies: - "@braintree/sanitize-url" "^6.0.1" - "@types/d3-scale" "^4.0.3" - "@types/d3-scale-chromatic" "^3.0.0" - cytoscape "^3.23.0" - cytoscape-cose-bilkent "^4.1.0" - cytoscape-fcose "^2.1.0" - d3 "^7.4.0" - d3-sankey "^0.12.3" - dagre-d3-es "7.0.10" - dayjs "^1.11.7" - dompurify "^3.0.5" - elkjs "^0.8.2" - khroma "^2.0.0" - lodash-es "^4.17.21" - mdast-util-from-markdown "^1.3.0" - non-layered-tidy-tree-layout "^2.0.2" - stylis "^4.1.3" - ts-dedent "^2.2.0" - uuid "^9.0.0" - web-worker "^1.2.0" - -micromark-core-commonmark@^1.0.0, micromark-core-commonmark@^1.0.1: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-1.1.0.tgz" - integrity sha512-BgHO1aRbolh2hcrzL2d1La37V0Aoz73ymF8rAcKnohLy93titmv62E0gP8Hrx9PKcKrqCZ1BbLGbP3bEhoXYlw== - dependencies: - decode-named-character-reference "^1.0.0" - micromark-factory-destination "^1.0.0" - micromark-factory-label "^1.0.0" - micromark-factory-space "^1.0.0" - micromark-factory-title "^1.0.0" - micromark-factory-whitespace "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-chunked "^1.0.0" - micromark-util-classify-character "^1.0.0" - micromark-util-html-tag-name "^1.0.0" - micromark-util-normalize-identifier "^1.0.0" - micromark-util-resolve-all "^1.0.0" - micromark-util-subtokenize "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.1" - uvu "^0.5.0" - -micromark-extension-gfm-autolink-literal@^1.0.0: - version "1.0.5" - resolved "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-1.0.5.tgz" - integrity sha512-z3wJSLrDf8kRDOh2qBtoTRD53vJ+CWIyo7uyZuxf/JAbNJjiHsOpG1y5wxk8drtv3ETAHutCu6N3thkOOgueWg== - dependencies: - micromark-util-character "^1.0.0" - micromark-util-sanitize-uri "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-extension-gfm-footnote@^1.0.0: - version "1.1.2" - resolved "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-1.1.2.tgz" - integrity sha512-Yxn7z7SxgyGWRNa4wzf8AhYYWNrwl5q1Z8ii+CSTTIqVkmGZF1CElX2JI8g5yGoM3GAman9/PVCUFUSJ0kB/8Q== - dependencies: - micromark-core-commonmark "^1.0.0" - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-normalize-identifier "^1.0.0" - micromark-util-sanitize-uri "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-extension-gfm-strikethrough@^1.0.0: - version "1.0.7" - resolved "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-1.0.7.tgz" - integrity sha512-sX0FawVE1o3abGk3vRjOH50L5TTLr3b5XMqnP9YDRb34M0v5OoZhG+OHFz1OffZ9dlwgpTBKaT4XW/AsUVnSDw== - dependencies: - micromark-util-chunked "^1.0.0" - micromark-util-classify-character "^1.0.0" - micromark-util-resolve-all "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-extension-gfm-table@^1.0.0: - version "1.0.7" - resolved "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-1.0.7.tgz" - integrity sha512-3ZORTHtcSnMQEKtAOsBQ9/oHp9096pI/UvdPtN7ehKvrmZZ2+bbWhi0ln+I9drmwXMt5boocn6OlwQzNXeVeqw== - dependencies: - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-extension-gfm-tagfilter@^1.0.0: - version "1.0.2" - resolved "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-1.0.2.tgz" - integrity sha512-5XWB9GbAUSHTn8VPU8/1DBXMuKYT5uOgEjJb8gN3mW0PNW5OPHpSdojoqf+iq1xo7vWzw/P8bAHY0n6ijpXF7g== - dependencies: - micromark-util-types "^1.0.0" - -micromark-extension-gfm-task-list-item@^1.0.0: - version "1.0.5" - resolved "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-1.0.5.tgz" - integrity sha512-RMFXl2uQ0pNQy6Lun2YBYT9g9INXtWJULgbt01D/x8/6yJ2qpKyzdZD3pi6UIkzF++Da49xAelVKUeUMqd5eIQ== - dependencies: - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-extension-gfm@^2.0.0: - version "2.0.3" - resolved "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-2.0.3.tgz" - integrity sha512-vb9OoHqrhCmbRidQv/2+Bc6pkP0FrtlhurxZofvOEy5o8RtuuvTq+RQ1Vw5ZDNrVraQZu3HixESqbG+0iKk/MQ== - dependencies: - micromark-extension-gfm-autolink-literal "^1.0.0" - micromark-extension-gfm-footnote "^1.0.0" - micromark-extension-gfm-strikethrough "^1.0.0" - micromark-extension-gfm-table "^1.0.0" - micromark-extension-gfm-tagfilter "^1.0.0" - micromark-extension-gfm-task-list-item "^1.0.0" - micromark-util-combine-extensions "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-extension-math@^2.0.0: - version "2.1.2" - resolved "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-2.1.2.tgz" - integrity sha512-es0CcOV89VNS9wFmyn+wyFTKweXGW4CEvdaAca6SWRWPyYCbBisnjaHLjWO4Nszuiud84jCpkHsqAJoa768Pvg== - dependencies: - "@types/katex" "^0.16.0" - katex "^0.16.0" - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-extension-mdx-expression@^1.0.0: - version "1.0.8" - resolved "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-1.0.8.tgz" - integrity sha512-zZpeQtc5wfWKdzDsHRBY003H2Smg+PUi2REhqgIhdzAa5xonhP03FcXxqFSerFiNUr5AWmHpaNPQTBVOS4lrXw== - dependencies: - "@types/estree" "^1.0.0" - micromark-factory-mdx-expression "^1.0.0" - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-events-to-acorn "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-extension-mdx-jsx@^1.0.0: - version "1.0.5" - resolved "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-1.0.5.tgz" - integrity sha512-gPH+9ZdmDflbu19Xkb8+gheqEDqkSpdCEubQyxuz/Hn8DOXiXvrXeikOoBA71+e8Pfi0/UYmU3wW3H58kr7akA== - dependencies: - "@types/acorn" "^4.0.0" - "@types/estree" "^1.0.0" - estree-util-is-identifier-name "^2.0.0" - micromark-factory-mdx-expression "^1.0.0" - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - vfile-message "^3.0.0" - -micromark-extension-mdx-md@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-1.0.1.tgz" - integrity sha512-7MSuj2S7xjOQXAjjkbjBsHkMtb+mDGVW6uI2dBL9snOBCbZmoNgDAeZ0nSn9j3T42UE/g2xVNMn18PJxZvkBEA== - dependencies: - micromark-util-types "^1.0.0" - -micromark-extension-mdxjs-esm@^1.0.0: - version "1.0.5" - resolved "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-1.0.5.tgz" - integrity sha512-xNRBw4aoURcyz/S69B19WnZAkWJMxHMT5hE36GtDAyhoyn/8TuAeqjFJQlwk+MKQsUD7b3l7kFX+vlfVWgcX1w== - dependencies: - "@types/estree" "^1.0.0" - micromark-core-commonmark "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-events-to-acorn "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - unist-util-position-from-estree "^1.1.0" - uvu "^0.5.0" - vfile-message "^3.0.0" - -micromark-extension-mdxjs@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-1.0.1.tgz" - integrity sha512-7YA7hF6i5eKOfFUzZ+0z6avRG52GpWR8DL+kN47y3f2KhxbBZMhmxe7auOeaTBrW2DenbbZTf1ea9tA2hDpC2Q== - dependencies: - acorn "^8.0.0" - acorn-jsx "^5.0.0" - micromark-extension-mdx-expression "^1.0.0" - micromark-extension-mdx-jsx "^1.0.0" - micromark-extension-mdx-md "^1.0.0" - micromark-extension-mdxjs-esm "^1.0.0" - micromark-util-combine-extensions "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-factory-destination@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-1.1.0.tgz" - integrity sha512-XaNDROBgx9SgSChd69pjiGKbV+nfHGDPVYFs5dOoDd7ZnMAE+Cuu91BCpsY8RT2NP9vo/B8pds2VQNCLiu0zhg== - dependencies: - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-factory-label@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-1.1.0.tgz" - integrity sha512-OLtyez4vZo/1NjxGhcpDSbHQ+m0IIGnT8BoPamh+7jVlzLJBH98zzuCoUeMxvM6WsNeh8wx8cKvqLiPHEACn0w== - dependencies: - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-factory-mdx-expression@^1.0.0: - version "1.0.9" - resolved "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-1.0.9.tgz" - integrity sha512-jGIWzSmNfdnkJq05c7b0+Wv0Kfz3NJ3N4cBjnbO4zjXIlxJr+f8lk+5ZmwFvqdAbUy2q6B5rCY//g0QAAaXDWA== - dependencies: - "@types/estree" "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-events-to-acorn "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - unist-util-position-from-estree "^1.0.0" - uvu "^0.5.0" - vfile-message "^3.0.0" - -micromark-factory-space@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-1.1.0.tgz" - integrity sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ== - dependencies: - micromark-util-character "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-factory-title@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-1.1.0.tgz" - integrity sha512-J7n9R3vMmgjDOCY8NPw55jiyaQnH5kBdV2/UXCtZIpnHH3P6nHUKaH7XXEYuWwx/xUJcawa8plLBEjMPU24HzQ== - dependencies: - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-factory-whitespace@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-1.1.0.tgz" - integrity sha512-v2WlmiymVSp5oMg+1Q0N1Lxmt6pMhIHD457whWM7/GUlEks1hI9xj5w3zbc4uuMKXGisksZk8DzP2UyGbGqNsQ== - dependencies: - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-util-character@^1.0.0: - version "1.2.0" - resolved "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-1.2.0.tgz" - integrity sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg== - dependencies: - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-util-chunked@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-1.1.0.tgz" - integrity sha512-Ye01HXpkZPNcV6FiyoW2fGZDUw4Yc7vT0E9Sad83+bEDiCJ1uXu0S3mr8WLpsz3HaG3x2q0HM6CTuPdcZcluFQ== - dependencies: - micromark-util-symbol "^1.0.0" - -micromark-util-classify-character@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-1.1.0.tgz" - integrity sha512-SL0wLxtKSnklKSUplok1WQFoGhUdWYKggKUiqhX+Swala+BtptGCu5iPRc+xvzJ4PXE/hwM3FNXsfEVgoZsWbw== - dependencies: - micromark-util-character "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-util-combine-extensions@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-1.1.0.tgz" - integrity sha512-Q20sp4mfNf9yEqDL50WwuWZHUrCO4fEyeDCnMGmG5Pr0Cz15Uo7KBs6jq+dq0EgX4DPwwrh9m0X+zPV1ypFvUA== - dependencies: - micromark-util-chunked "^1.0.0" - micromark-util-types "^1.0.0" - -micromark-util-decode-numeric-character-reference@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-1.1.0.tgz" - integrity sha512-m9V0ExGv0jB1OT21mrWcuf4QhP46pH1KkfWy9ZEezqHKAxkj4mPCy3nIH1rkbdMlChLHX531eOrymlwyZIf2iw== - dependencies: - micromark-util-symbol "^1.0.0" - -micromark-util-decode-string@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-1.1.0.tgz" - integrity sha512-YphLGCK8gM1tG1bd54azwyrQRjCFcmgj2S2GoJDNnh4vYtnL38JS8M4gpxzOPNyHdNEpheyWXCTnnTDY3N+NVQ== - dependencies: - decode-named-character-reference "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-decode-numeric-character-reference "^1.0.0" - micromark-util-symbol "^1.0.0" - -micromark-util-encode@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-1.1.0.tgz" - integrity sha512-EuEzTWSTAj9PA5GOAs992GzNh2dGQO52UvAbtSOMvXTxv3Criqb6IOzJUBCmEqrrXSblJIJBbFFv6zPxpreiJw== - -micromark-util-events-to-acorn@^1.0.0: - version "1.2.3" - resolved "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-1.2.3.tgz" - integrity sha512-ij4X7Wuc4fED6UoLWkmo0xJQhsktfNh1J0m8g4PbIMPlx+ek/4YdW5mvbye8z/aZvAPUoxgXHrwVlXAPKMRp1w== - dependencies: - "@types/acorn" "^4.0.0" - "@types/estree" "^1.0.0" - "@types/unist" "^2.0.0" - estree-util-visit "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - vfile-message "^3.0.0" - -micromark-util-html-tag-name@^1.0.0: - version "1.2.0" - resolved "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-1.2.0.tgz" - integrity sha512-VTQzcuQgFUD7yYztuQFKXT49KghjtETQ+Wv/zUjGSGBioZnkA4P1XXZPT1FHeJA6RwRXSF47yvJ1tsJdoxwO+Q== - -micromark-util-normalize-identifier@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-1.1.0.tgz" - integrity sha512-N+w5vhqrBihhjdpM8+5Xsxy71QWqGn7HYNUvch71iV2PM7+E3uWGox1Qp90loa1ephtCxG2ftRV/Conitc6P2Q== - dependencies: - micromark-util-symbol "^1.0.0" - -micromark-util-resolve-all@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-1.1.0.tgz" - integrity sha512-b/G6BTMSg+bX+xVCshPTPyAu2tmA0E4X98NSR7eIbeC6ycCqCeE7wjfDIgzEbkzdEVJXRtOG4FbEm/uGbCRouA== - dependencies: - micromark-util-types "^1.0.0" - -micromark-util-sanitize-uri@^1.0.0, micromark-util-sanitize-uri@^1.1.0: - version "1.2.0" - resolved "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-1.2.0.tgz" - integrity sha512-QO4GXv0XZfWey4pYFndLUKEAktKkG5kZTdUNaTAkzbuJxn2tNBOr+QtxR2XpWaMhbImT2dPzyLrPXLlPhph34A== - dependencies: - micromark-util-character "^1.0.0" - micromark-util-encode "^1.0.0" - micromark-util-symbol "^1.0.0" - -micromark-util-subtokenize@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-1.1.0.tgz" - integrity sha512-kUQHyzRoxvZO2PuLzMt2P/dwVsTiivCK8icYTeR+3WgbuPqfHgPPy7nFKbeqRivBvn/3N3GBiNC+JRTMSxEC7A== - dependencies: - micromark-util-chunked "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.0" - uvu "^0.5.0" - -micromark-util-symbol@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-1.1.0.tgz" - integrity sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag== - -micromark-util-types@^1.0.0, micromark-util-types@^1.0.1: - version "1.1.0" - resolved "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-1.1.0.tgz" - integrity sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg== - -micromark@^3.0.0: - version "3.2.0" - resolved "https://registry.npmjs.org/micromark/-/micromark-3.2.0.tgz" - integrity sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA== - dependencies: - "@types/debug" "^4.0.0" - debug "^4.0.0" - decode-named-character-reference "^1.0.0" - micromark-core-commonmark "^1.0.1" - micromark-factory-space "^1.0.0" - micromark-util-character "^1.0.0" - micromark-util-chunked "^1.0.0" - micromark-util-combine-extensions "^1.0.0" - micromark-util-decode-numeric-character-reference "^1.0.0" - micromark-util-encode "^1.0.0" - micromark-util-normalize-identifier "^1.0.0" - micromark-util-resolve-all "^1.0.0" - micromark-util-sanitize-uri "^1.0.0" - micromark-util-subtokenize "^1.0.0" - micromark-util-symbol "^1.0.0" - micromark-util-types "^1.0.1" - uvu "^0.5.0" - -micromark@~2.11.0: - version "2.11.4" - resolved "https://registry.npmjs.org/micromark/-/micromark-2.11.4.tgz" - integrity sha512-+WoovN/ppKolQOFIAajxi7Lu9kInbPxFuTBVEavFcL8eAfVstoc5MocPmqBeAdBOJV00uaVjegzH4+MA0DN/uA== - dependencies: - debug "^4.0.0" - parse-entities "^2.0.0" - -micromatch@^4.0.4, micromatch@^4.0.5: - version "4.0.5" - resolved "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz" - integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== - dependencies: - braces "^3.0.2" - picomatch "^2.3.1" - -mimic-fn@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" - integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== - -mimic-fn@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz" - integrity sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw== - -min-indent@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz" - integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== - -minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: - version "3.1.2" - resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" - integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== - dependencies: - brace-expansion "^1.1.7" - -minimatch@^9.0.1: - version "9.0.3" - resolved "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz" - integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== - dependencies: - brace-expansion "^2.0.1" - -minimist@^1.2.0, minimist@^1.2.6: - version "1.2.8" - resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" - integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== - -"minipass@^5.0.0 || ^6.0.2 || ^7.0.0": - version "7.0.4" - resolved "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz" - integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== - -miragejs@^0.1.47: - version "0.1.47" - resolved "https://registry.npmjs.org/miragejs/-/miragejs-0.1.47.tgz" - integrity sha512-99tuCbIAlMhNhyF3s5d3+5/FdJ7O4jSq/5e3e+sDv7L8dZdwJuwutXe0pobJ7hm6yRChTDjK+Nn8dZZd175wbg== - dependencies: - "@miragejs/pretender-node-polyfill" "^0.1.0" - inflected "^2.0.4" - lodash.assign "^4.2.0" - lodash.camelcase "^4.3.0" - lodash.clonedeep "^4.5.0" - lodash.compact "^3.0.1" - lodash.find "^4.6.0" - lodash.flatten "^4.4.0" - lodash.forin "^4.4.0" - lodash.get "^4.4.2" - lodash.has "^4.5.2" - lodash.invokemap "^4.6.0" - lodash.isempty "^4.4.0" - lodash.isequal "^4.5.0" - lodash.isfunction "^3.0.9" - lodash.isinteger "^4.0.4" - lodash.isplainobject "^4.0.6" - lodash.lowerfirst "^4.3.1" - lodash.map "^4.6.0" - lodash.mapvalues "^4.6.0" - lodash.pick "^4.4.0" - lodash.snakecase "^4.1.1" - lodash.uniq "^4.5.0" - lodash.uniqby "^4.7.0" - lodash.values "^4.3.0" - pretender "^3.4.7" - -mkdirp@^0.5.6: - version "0.5.6" - resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" - integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== - dependencies: - minimist "^1.2.6" - -mri@^1.1.0: - version "1.2.0" - resolved "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz" - integrity sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA== - -ms@2.1.2, ms@^2.1.1: - version "2.1.2" - resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" - integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== - -mz@^2.7.0: - version "2.7.0" - resolved "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz" - integrity sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q== - dependencies: - any-promise "^1.0.0" - object-assign "^4.0.1" - thenify-all "^1.0.0" - -nanoid@^3.3.6: - version "3.3.6" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz" - integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== - -natural-compare-lite@^1.4.0: - version "1.4.0" - resolved "https://registry.npmjs.org/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz" - integrity sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g== - -natural-compare@^1.4.0: - version "1.4.0" - resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" - integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== - -negotiator@^0.6.3: - version "0.6.3" - resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" - integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== - -next-nprogress-bar@^2.3.8: - version "2.3.11" - resolved "https://registry.npmjs.org/next-nprogress-bar/-/next-nprogress-bar-2.3.11.tgz" - integrity sha512-OjSvsQwgSWa2qBMYO478QreGG9Jt82tr4wTQptmiyzNqqjzHCyKZNkhANnzPrjuFAoelIvmruJuakODofSnvTQ== - dependencies: - nprogress "^0.2.0" - -next@^14.0.4: - version "14.1.0" - resolved "https://registry.npmjs.org/next/-/next-14.1.0.tgz" - integrity sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q== - dependencies: - "@next/env" "14.1.0" - "@swc/helpers" "0.5.2" - busboy "1.6.0" - caniuse-lite "^1.0.30001579" - graceful-fs "^4.2.11" - postcss "8.4.31" - styled-jsx "5.1.1" - optionalDependencies: - "@next/swc-darwin-arm64" "14.1.0" - "@next/swc-darwin-x64" "14.1.0" - "@next/swc-linux-arm64-gnu" "14.1.0" - "@next/swc-linux-arm64-musl" "14.1.0" - "@next/swc-linux-x64-gnu" "14.1.0" - "@next/swc-linux-x64-musl" "14.1.0" - "@next/swc-win32-arm64-msvc" "14.1.0" - "@next/swc-win32-ia32-msvc" "14.1.0" - "@next/swc-win32-x64-msvc" "14.1.0" - -node-releases@^2.0.12: - version "2.0.12" - resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.12.tgz" - integrity sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ== - -non-layered-tidy-tree-layout@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/non-layered-tidy-tree-layout/-/non-layered-tidy-tree-layout-2.0.2.tgz" - integrity sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw== - -normalize-package-data@^2.5.0: - version "2.5.0" - resolved "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz" - integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== - dependencies: - hosted-git-info "^2.1.4" - resolve "^1.10.0" - semver "2 || 3 || 4 || 5" - validate-npm-package-license "^3.0.1" - -normalize-path@^3.0.0, normalize-path@~3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz" - integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== - -normalize-range@^0.1.2: - version "0.1.2" - resolved "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz" - integrity sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA== - -npm-run-path@^4.0.1: - version "4.0.1" - resolved "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz" - integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== - dependencies: - path-key "^3.0.0" - -npm-run-path@^5.1.0: - version "5.1.0" - resolved "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.1.0.tgz" - integrity sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q== - dependencies: - path-key "^4.0.0" - -nprogress@^0.2.0: - version "0.2.0" - resolved "https://registry.npmjs.org/nprogress/-/nprogress-0.2.0.tgz" - integrity sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA== - -nth-check@^2.0.1: - version "2.1.1" - resolved "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz" - integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== - dependencies: - boolbase "^1.0.0" - -object-assign@^4.0.1, object-assign@^4.1.1: - version "4.1.1" - resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz" - integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== - -object-hash@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz" - integrity sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw== - -object-inspect@^1.12.3, object-inspect@^1.13.1, object-inspect@^1.9.0: - version "1.13.1" - resolved "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz" - integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== - -object-is@^1.1.5: - version "1.1.5" - resolved "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz" - integrity sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" - -object-keys@^1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz" - integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== - -object.assign@^4.1.3, object.assign@^4.1.4: - version "4.1.4" - resolved "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz" - integrity sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - has-symbols "^1.0.3" - object-keys "^1.1.1" - -object.entries@^1.1.6: - version "1.1.6" - resolved "https://registry.npmjs.org/object.entries/-/object.entries-1.1.6.tgz" - integrity sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w== - dependencies: - call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" - -object.fromentries@^2.0.6, object.fromentries@^2.0.7: - version "2.0.7" - resolved "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.7.tgz" - integrity sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - -object.groupby@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/object.groupby/-/object.groupby-1.0.1.tgz" - integrity sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" - -object.hasown@^1.1.2: - version "1.1.3" - resolved "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.3.tgz" - integrity sha512-fFI4VcYpRHvSLXxP7yiZOMAd331cPfd2p7PFDVbgUsYOfCT3tICVqXWngbjr4m49OvsBwUBQ6O2uQoJvy3RexA== - dependencies: - define-properties "^1.2.0" - es-abstract "^1.22.1" - -object.values@^1.1.6, object.values@^1.1.7: - version "1.1.7" - resolved "https://registry.npmjs.org/object.values/-/object.values-1.1.7.tgz" - integrity sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - -once@^1.3.0: - version "1.4.0" - resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" - integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== - dependencies: - wrappy "1" - -onetime@^5.1.0, onetime@^5.1.2: - version "5.1.2" - resolved "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz" - integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== - dependencies: - mimic-fn "^2.1.0" - -onetime@^6.0.0: - version "6.0.0" - resolved "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz" - integrity sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ== - dependencies: - mimic-fn "^4.0.0" - -open@^9.1.0: - version "9.1.0" - resolved "https://registry.npmjs.org/open/-/open-9.1.0.tgz" - integrity sha512-OS+QTnw1/4vrf+9hh1jc1jnYjzSG4ttTBB8UxOwAnInG3Uo4ssetzC1ihqaIHjLJnA5GGlRl6QlZXOTQhRBUvg== - dependencies: - default-browser "^4.0.0" - define-lazy-prop "^3.0.0" - is-inside-container "^1.0.0" - is-wsl "^2.2.0" - -optionator@^0.9.1: - version "0.9.1" - resolved "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz" - integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== - dependencies: - deep-is "^0.1.3" - fast-levenshtein "^2.0.6" - levn "^0.4.1" - prelude-ls "^1.2.1" - type-check "^0.4.0" - word-wrap "^1.2.3" - -p-limit@^2.2.0: - version "2.3.0" - resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz" - integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== - dependencies: - p-try "^2.0.0" - -p-limit@^3.0.2: - version "3.1.0" - resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz" - integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== - dependencies: - yocto-queue "^0.1.0" - -p-locate@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz" - integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== - dependencies: - p-limit "^2.2.0" - -p-locate@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz" - integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== - dependencies: - p-limit "^3.0.2" - -p-map@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz" - integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== - dependencies: - aggregate-error "^3.0.0" - -p-try@^2.0.0: - version "2.2.0" - resolved "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" - integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== - -papaparse@^5.3.1: - version "5.4.1" - resolved "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz" - integrity sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw== - -parent-module@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz" - integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== - dependencies: - callsites "^3.0.0" - -parse-entities@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz" - integrity sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ== - dependencies: - character-entities "^1.0.0" - character-entities-legacy "^1.0.0" - character-reference-invalid "^1.0.0" - is-alphanumerical "^1.0.0" - is-decimal "^1.0.0" - is-hexadecimal "^1.0.0" - -parse-entities@^4.0.0: - version "4.0.1" - resolved "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.1.tgz" - integrity sha512-SWzvYcSJh4d/SGLIOQfZ/CoNv6BTlI6YEQ7Nj82oDVnRpwe/Z/F1EMx42x3JAOwGBlCjeCH0BRJQbQ/opHL17w== - dependencies: - "@types/unist" "^2.0.0" - character-entities "^2.0.0" - character-entities-legacy "^3.0.0" - character-reference-invalid "^2.0.0" - decode-named-character-reference "^1.0.0" - is-alphanumerical "^2.0.0" - is-decimal "^2.0.0" - is-hexadecimal "^2.0.0" - -parse-json@^5.0.0: - version "5.2.0" - resolved "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz" - integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== - dependencies: - "@babel/code-frame" "^7.0.0" - error-ex "^1.3.1" - json-parse-even-better-errors "^2.3.0" - lines-and-columns "^1.1.6" - -parse5@^7.0.0: - version "7.1.2" - resolved "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz" - integrity sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw== - dependencies: - entities "^4.4.0" - -path-exists@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz" - integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== - -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" - integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== - -path-key@^3.0.0, path-key@^3.1.0: - version "3.1.1" - resolved "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz" - integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== - -path-key@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz" - integrity sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ== - -path-parse@^1.0.7: - version "1.0.7" - resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" - integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== - -path-scurry@^1.10.1: - version "1.10.1" - resolved "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz" - integrity sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ== - dependencies: - lru-cache "^9.1.1 || ^10.0.0" - minipass "^5.0.0 || ^6.0.2 || ^7.0.0" - -path-type@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz" - integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== - -periscopic@^3.0.0: - version "3.1.0" - resolved "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz" - integrity sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw== - dependencies: - "@types/estree" "^1.0.0" - estree-walker "^3.0.0" - is-reference "^3.0.0" - -picocolors@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz" - integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== - -picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: - version "2.3.1" - resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz" - integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== - -pidtree@^0.6.0: - version "0.6.0" - resolved "https://registry.npmjs.org/pidtree/-/pidtree-0.6.0.tgz" - integrity sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g== - -pify@^2.3.0: - version "2.3.0" - resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" - integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== - -pirates@^4.0.1: - version "4.0.5" - resolved "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz" - integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ== - -pluralize@^8.0.0: - version "8.0.0" - resolved "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz" - integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== - -portfinder@^1.0.28: - version "1.0.32" - resolved "https://registry.npmjs.org/portfinder/-/portfinder-1.0.32.tgz" - integrity sha512-on2ZJVVDXRADWE6jnQaX0ioEylzgBpQk8r55NE4wjXW1ZxO+BgDlY6DXwj20i0V8eB4SenDQ00WEaxfiIQPcxg== - dependencies: - async "^2.6.4" - debug "^3.2.7" - mkdirp "^0.5.6" - -postcss-import@^15.1.0: - version "15.1.0" - resolved "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz" - integrity sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew== - dependencies: - postcss-value-parser "^4.0.0" - read-cache "^1.0.0" - resolve "^1.1.7" - -postcss-js@^4.0.1: - version "4.0.1" - resolved "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.1.tgz" - integrity sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw== - dependencies: - camelcase-css "^2.0.1" - -postcss-load-config@^4.0.1: - version "4.0.1" - resolved "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.1.tgz" - integrity sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA== - dependencies: - lilconfig "^2.0.5" - yaml "^2.1.1" - -postcss-nested@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.0.1.tgz" - integrity sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ== - dependencies: - postcss-selector-parser "^6.0.11" - -postcss-selector-parser@6.0.10, postcss-selector-parser@^6.0.9: - version "6.0.10" - resolved "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz" - integrity sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w== - dependencies: - cssesc "^3.0.0" - util-deprecate "^1.0.2" - -postcss-selector-parser@^6.0.11: - version "6.0.13" - resolved "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.13.tgz" - integrity sha512-EaV1Gl4mUEV4ddhDnv/xtj7sxwrwxdetHdWUGnT4VJQf+4d05v6lHYZr8N573k5Z0BViss7BDhfWtKS3+sfAqQ== - dependencies: - cssesc "^3.0.0" - util-deprecate "^1.0.2" - -postcss-value-parser@^4.0.0, postcss-value-parser@^4.2.0: - version "4.2.0" - resolved "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz" - integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== - -postcss@8.4.31, postcss@^8.4.23, postcss@^8.4.31: - version "8.4.31" - resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz" - integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== - dependencies: - nanoid "^3.3.6" - picocolors "^1.0.0" - source-map-js "^1.0.2" - -prelude-ls@^1.2.1: - version "1.2.1" - resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" - integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== - -pretender@^3.4.7: - version "3.4.7" - resolved "https://registry.npmjs.org/pretender/-/pretender-3.4.7.tgz" - integrity sha512-jkPAvt1BfRi0RKamweJdEcnjkeu7Es8yix3bJ+KgBC5VpG/Ln4JE3hYN6vJym4qprm8Xo5adhWpm3HCoft1dOw== - dependencies: - fake-xml-http-request "^2.1.2" - route-recognizer "^0.3.3" - -prismjs@^1.27.0: - version "1.29.0" - resolved "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz" - integrity sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q== - -prismjs@~1.27.0: - version "1.27.0" - resolved "https://registry.npmjs.org/prismjs/-/prismjs-1.27.0.tgz" - integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA== - -prop-types@^15.0.0, prop-types@^15.5.8, prop-types@^15.8.1: - version "15.8.1" - resolved "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz" - integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== - dependencies: - loose-envify "^1.4.0" - object-assign "^4.1.1" - react-is "^16.13.1" - -property-information@^5.0.0: - version "5.6.0" - resolved "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz" - integrity sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA== - dependencies: - xtend "^4.0.0" - -property-information@^6.0.0: - version "6.2.0" - resolved "https://registry.npmjs.org/property-information/-/property-information-6.2.0.tgz" - integrity sha512-kma4U7AFCTwpqq5twzC1YVIDXSqg6qQK6JN0smOw8fgRy1OkMi0CYSzFmsy6dnqSenamAtj0CyXMUJ1Mf6oROg== - -punycode@^2.1.0: - version "2.3.0" - resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz" - integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA== - -qrcode.react@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/qrcode.react/-/qrcode.react-3.1.0.tgz" - integrity sha512-oyF+Urr3oAMUG/OiOuONL3HXM+53wvuH3mtIWQrYmsXoAq0DkvZp2RYUWFSMFtbdOpuS++9v+WAkzNVkMlNW6Q== - -qs@^6.11.1: - version "6.11.2" - resolved "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz" - integrity sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA== - dependencies: - side-channel "^1.0.4" - -queue-microtask@^1.2.2: - version "1.2.3" - resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz" - integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== - -rc-input@~1.3.5: - version "1.3.6" - resolved "https://registry.npmjs.org/rc-input/-/rc-input-1.3.6.tgz" - integrity sha512-/HjTaKi8/Ts4zNbYaB5oWCquxFyFQO4Co1MnMgoCeGJlpe7k8Eir2HN0a0F9IHDmmo+GYiGgPpz7w/d/krzsJA== - dependencies: - "@babel/runtime" "^7.11.1" - classnames "^2.2.1" - rc-util "^5.18.1" - -rc-resize-observer@^1.0.0: - version "1.4.0" - resolved "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.4.0.tgz" - integrity sha512-PnMVyRid9JLxFavTjeDXEXo65HCRqbmLBw9xX9gfC4BZiSzbLXKzW3jPz+J0P71pLbD5tBMTT+mkstV5gD0c9Q== - dependencies: - "@babel/runtime" "^7.20.7" - classnames "^2.2.1" - rc-util "^5.38.0" - resize-observer-polyfill "^1.5.1" - -rc-textarea@^1.5.2: - version "1.5.2" - resolved "https://registry.npmjs.org/rc-textarea/-/rc-textarea-1.5.2.tgz" - integrity sha512-VVwKYtkp5whZVhP+llX8zM8TtI3dv+BDA0FUbmBMGLaW/tuBJ7Yh35yPabO63V+Bi68xv17eI4hy+/4p2G0gFg== - dependencies: - "@babel/runtime" "^7.10.1" - classnames "^2.2.1" - rc-input "~1.3.5" - rc-resize-observer "^1.0.0" - rc-util "^5.27.0" - -rc-util@^5.18.1, rc-util@^5.27.0, rc-util@^5.38.0: - version "5.38.1" - resolved "https://registry.npmjs.org/rc-util/-/rc-util-5.38.1.tgz" - integrity sha512-e4ZMs7q9XqwTuhIK7zBIVFltUtMSjphuPPQXHoHlzRzNdOwUxDejo0Zls5HYaJfRKNURcsS/ceKVULlhjBrxng== - dependencies: - "@babel/runtime" "^7.18.3" - react-is "^18.2.0" - -react-18-input-autosize@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/react-18-input-autosize/-/react-18-input-autosize-3.0.0.tgz" - integrity sha512-7tsUc9PJWg6Vsp8qYuzlKKBf7hbCoTBdNfjYZSprEPbxf3meuhjklg9QPBe9rIyoR3uDAzmG7NpoJ1+kP5ns+w== - dependencies: - prop-types "^15.5.8" - -react-dom@^18.2.0: - version "18.2.0" - resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz" - integrity sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g== - dependencies: - loose-envify "^1.1.0" - scheduler "^0.23.0" - -react-error-boundary@^3.1.4: - version "3.1.4" - resolved "https://registry.npmjs.org/react-error-boundary/-/react-error-boundary-3.1.4.tgz" - integrity sha512-uM9uPzZJTF6wRQORmSrvOIgt4lJ9MC1sNgEOj2XGsDTRE4kmpWxg7ENK9EWNKJRMAOY9z0MuF4yIfl6gp4sotA== - dependencies: - "@babel/runtime" "^7.12.5" - -react-error-boundary@^4.0.2: - version "4.0.9" - resolved "https://registry.npmjs.org/react-error-boundary/-/react-error-boundary-4.0.9.tgz" - integrity sha512-f6DcHVdTDZmc9ixmRmuLDZpkdghYR/HKZdUzMLHD58s4cR2C4R6y4ktYztCosM6pyeK4/C8IofwqxgID25W6kw== - dependencies: - "@babel/runtime" "^7.12.5" - -react-headless-pagination@^1.1.4: - version "1.1.4" - resolved "https://registry.npmjs.org/react-headless-pagination/-/react-headless-pagination-1.1.4.tgz" - integrity sha512-Z5d55g3gM2BQMvHJUGm1jbbQ5Bgtq54kNlI5ca1NTwdVR8ZNunN0EdOtNKNobsFRKuZGkQ24VTIu6ulNq190Iw== - dependencies: - classnames "2.3.1" - -react-i18next@^12.2.0: - version "12.3.1" - resolved "https://registry.npmjs.org/react-i18next/-/react-i18next-12.3.1.tgz" - integrity sha512-5v8E2XjZDFzK7K87eSwC7AJcAkcLt5xYZ4+yTPDAW1i7C93oOY1dnr4BaQM7un4Hm+GmghuiPvevWwlca5PwDA== - dependencies: - "@babel/runtime" "^7.20.6" - html-parse-stringify "^3.0.1" - -react-infinite-scroll-component@^6.1.0: - version "6.1.0" - resolved "https://registry.npmjs.org/react-infinite-scroll-component/-/react-infinite-scroll-component-6.1.0.tgz" - integrity sha512-SQu5nCqy8DxQWpnUVLx7V7b7LcA37aM7tvoWjTLZp1dk6EJibM5/4EJKzOnl07/BsM1Y40sKLuqjCwwH/xV0TQ== - dependencies: - throttle-debounce "^2.1.0" - -react-is@^16.13.1, react-is@^16.7.0: - version "16.13.1" - resolved "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz" - integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== - -react-is@^18.0.0, react-is@^18.2.0: - version "18.2.0" - resolved "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz" - integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== - -react-markdown@^8.0.6: - version "8.0.7" - resolved "https://registry.npmjs.org/react-markdown/-/react-markdown-8.0.7.tgz" - integrity sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ== - dependencies: - "@types/hast" "^2.0.0" - "@types/prop-types" "^15.0.0" - "@types/unist" "^2.0.0" - comma-separated-tokens "^2.0.0" - hast-util-whitespace "^2.0.0" - prop-types "^15.0.0" - property-information "^6.0.0" - react-is "^18.0.0" - remark-parse "^10.0.0" - remark-rehype "^10.0.0" - space-separated-tokens "^2.0.0" - style-to-object "^0.4.0" - unified "^10.0.0" - unist-util-visit "^4.0.0" - vfile "^5.0.0" - -react-multi-email@^1.0.14: - version "1.0.16" - resolved "https://registry.npmjs.org/react-multi-email/-/react-multi-email-1.0.16.tgz" - integrity sha512-dgg4TY3P5FWz6c4ghgxH1bjZOgYL3S/HN+EUNe6dqHbLMVzeyud1ztDUlqvft4NX1sUxKx2IF2zDq1yAJQA5yQ== - -react-papaparse@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/react-papaparse/-/react-papaparse-4.1.0.tgz" - integrity sha512-sGJqK+OE2rVVQPxQUCCDW2prLIglv9kTdizhNe2awXvKo0gLShmhpRN3BwA+ujw5M2gSJ/KGNEwtgII0OsLgkg== - dependencies: - "@types/papaparse" "^5.3.1" - papaparse "^5.3.1" - -react-slider@^2.0.4: - version "2.0.5" - resolved "https://registry.npmjs.org/react-slider/-/react-slider-2.0.5.tgz" - integrity sha512-MU5gaK1yYCKnbDDN3CMiVcgkKZwMvdqK2xUEW7fFU37NAzRgS1FZbF9N7vP08E3XXNVhiuZnwVzUa3PYQAZIMg== - dependencies: - prop-types "^15.8.1" - -react-sortablejs@^6.1.4: - version "6.1.4" - resolved "https://registry.npmjs.org/react-sortablejs/-/react-sortablejs-6.1.4.tgz" - integrity sha512-fc7cBosfhnbh53Mbm6a45W+F735jwZ1UFIYSrIqcO/gRIFoDyZeMtgKlpV4DdyQfbCzdh5LoALLTDRxhMpTyXQ== - dependencies: - classnames "2.3.1" - tiny-invariant "1.2.0" - -react-syntax-highlighter@^15.5.0: - version "15.5.0" - resolved "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.5.0.tgz" - integrity sha512-+zq2myprEnQmH5yw6Gqc8lD55QHnpKaU8TOcFeC/Lg/MQSs8UknEA0JC4nTZGFAXC2J2Hyj/ijJ7NlabyPi2gg== - dependencies: - "@babel/runtime" "^7.3.1" - highlight.js "^10.4.1" - lowlight "^1.17.0" - prismjs "^1.27.0" - refractor "^3.6.0" - -react-tooltip@5.8.3: - version "5.8.3" - resolved "https://registry.npmjs.org/react-tooltip/-/react-tooltip-5.8.3.tgz" - integrity sha512-h7maAlm2Xeymc14gWKhhrzsENeB83N65EzZ+AcQIGrOpNE0yefVRJIHhNcWHEJ0FEtf7VZXxtsj5glVXKxEtvA== - dependencies: - "@floating-ui/dom" "1.1.1" - classnames "^2.3.2" - -react-window-infinite-loader@^1.0.9: - version "1.0.9" - resolved "https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.9.tgz" - integrity sha512-5Hg89IdU4Vrp0RT8kZYKeTIxWZYhNkVXeI1HbKo01Vm/Z7qztDvXljwx16sMzsa9yapRJQW3ODZfMUw38SOWHw== - -react-window@^1.8.9: - version "1.8.9" - resolved "https://registry.npmjs.org/react-window/-/react-window-1.8.9.tgz" - integrity sha512-+Eqx/fj1Aa5WnhRfj9dJg4VYATGwIUP2ItwItiJ6zboKWA6EX3lYDAXfGF2hyNqplEprhbtjbipiADEcwQ823Q== - dependencies: - "@babel/runtime" "^7.0.0" - memoize-one ">=3.1.1 <6" - -react@^18.2.0: - version "18.2.0" - resolved "https://registry.npmjs.org/react/-/react-18.2.0.tgz" - integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ== - dependencies: - loose-envify "^1.1.0" - -reactflow@^11.10.3: - version "11.10.3" - resolved "https://registry.npmjs.org/reactflow/-/reactflow-11.10.3.tgz" - integrity sha512-DGNrTdkWjZtPOhj5MV8fiWWGkJo+otMVdoJ9l67bQL+Xf+8NkJ4AHmRXoYIxtgcENzwTr5WTAIJlswV9i91cyw== - dependencies: - "@reactflow/background" "11.3.8" - "@reactflow/controls" "11.2.8" - "@reactflow/core" "11.10.3" - "@reactflow/minimap" "11.7.8" - "@reactflow/node-resizer" "2.2.8" - "@reactflow/node-toolbar" "1.3.8" - -read-cache@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz" - integrity sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA== - dependencies: - pify "^2.3.0" - -read-pkg-up@^7.0.1: - version "7.0.1" - resolved "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-7.0.1.tgz" - integrity sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg== - dependencies: - find-up "^4.1.0" - read-pkg "^5.2.0" - type-fest "^0.8.1" - -read-pkg@^5.2.0: - version "5.2.0" - resolved "https://registry.npmjs.org/read-pkg/-/read-pkg-5.2.0.tgz" - integrity sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg== - dependencies: - "@types/normalize-package-data" "^2.4.0" - normalize-package-data "^2.5.0" - parse-json "^5.0.0" - type-fest "^0.6.0" - -readdirp@~3.6.0: - version "3.6.0" - resolved "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz" - integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== - dependencies: - picomatch "^2.2.1" - -recordrtc@^5.6.2: - version "5.6.2" - resolved "https://registry.npmjs.org/recordrtc/-/recordrtc-5.6.2.tgz" - integrity sha512-1QNKKNtl7+KcwD1lyOgP3ZlbiJ1d0HtXnypUy7yq49xEERxk31PHvE9RCciDrulPCY7WJ+oz0R9hpNxgsIurGQ== - -reflect.getprototypeof@^1.0.4: - version "1.0.4" - resolved "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz" - integrity sha512-ECkTw8TmJwW60lOTR+ZkODISW6RQ8+2CL3COqtiJKLd6MmB45hN51HprHFziKLGkAuTGQhBb91V8cy+KHlaCjw== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" - globalthis "^1.0.3" - which-builtin-type "^1.1.3" - -refractor@^3.6.0: - version "3.6.0" - resolved "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz" - integrity sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA== - dependencies: - hastscript "^6.0.0" - parse-entities "^2.0.0" - prismjs "~1.27.0" - -regenerator-runtime@^0.13.11: - version "0.13.11" - resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz" - integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== - -regexp-tree@^0.1.24, regexp-tree@~0.1.1: - version "0.1.27" - resolved "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.27.tgz" - integrity sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA== - -regexp.prototype.flags@^1.5.0, regexp.prototype.flags@^1.5.1: - version "1.5.1" - resolved "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz" - integrity sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - set-function-name "^2.0.0" - -regexpp@^3.0.0: - version "3.2.0" - resolved "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz" - integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== - -regjsparser@^0.9.1: - version "0.9.1" - resolved "https://registry.npmjs.org/regjsparser/-/regjsparser-0.9.1.tgz" - integrity sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ== - dependencies: - jsesc "~0.5.0" - -rehype-katex@^6.0.2: - version "6.0.3" - resolved "https://registry.npmjs.org/rehype-katex/-/rehype-katex-6.0.3.tgz" - integrity sha512-ByZlRwRUcWegNbF70CVRm2h/7xy7jQ3R9LaY4VVSvjnoVWwWVhNL60DiZsBpC5tSzYQOCvDbzncIpIjPZWodZA== - dependencies: - "@types/hast" "^2.0.0" - "@types/katex" "^0.14.0" - hast-util-from-html-isomorphic "^1.0.0" - hast-util-to-text "^3.1.0" - katex "^0.16.0" - unist-util-visit "^4.0.0" - -remark-breaks@^3.0.2: - version "3.0.3" - resolved "https://registry.npmjs.org/remark-breaks/-/remark-breaks-3.0.3.tgz" - integrity sha512-C7VkvcUp1TPUc2eAYzsPdaUh8Xj4FSbQnYA5A9f80diApLZscTDeG7efiWP65W8hV2sEy3JuGVU0i6qr5D8Hug== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-newline-to-break "^1.0.0" - unified "^10.0.0" - -remark-gfm@^3.0.1: - version "3.0.1" - resolved "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz" - integrity sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-gfm "^2.0.0" - micromark-extension-gfm "^2.0.0" - unified "^10.0.0" - -remark-math@^5.1.1: - version "5.1.1" - resolved "https://registry.npmjs.org/remark-math/-/remark-math-5.1.1.tgz" - integrity sha512-cE5T2R/xLVtfFI4cCePtiRn+e6jKMtFDR3P8V3qpv8wpKjwvHoBA4eJzvX+nVrnlNy0911bdGmuspCSwetfYHw== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-math "^2.0.0" - micromark-extension-math "^2.0.0" - unified "^10.0.0" - -remark-mdx@^2.0.0: - version "2.3.0" - resolved "https://registry.npmjs.org/remark-mdx/-/remark-mdx-2.3.0.tgz" - integrity sha512-g53hMkpM0I98MU266IzDFMrTD980gNF3BJnkyFcmN+dD873mQeD5rdMO3Y2X+x8umQfbSE0PcoEDl7ledSA+2g== - dependencies: - mdast-util-mdx "^2.0.0" - micromark-extension-mdxjs "^1.0.0" - -remark-parse@^10.0.0: - version "10.0.2" - resolved "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.2.tgz" - integrity sha512-3ydxgHa/ZQzG8LvC7jTXccARYDcRld3VfcgIIFs7bI6vbRSxJJmzgLEIIoYKyrfhaY+ujuWaf/PJiMZXoiCXgw== - dependencies: - "@types/mdast" "^3.0.0" - mdast-util-from-markdown "^1.0.0" - unified "^10.0.0" - -remark-rehype@^10.0.0: - version "10.1.0" - resolved "https://registry.npmjs.org/remark-rehype/-/remark-rehype-10.1.0.tgz" - integrity sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw== - dependencies: - "@types/hast" "^2.0.0" - "@types/mdast" "^3.0.0" - mdast-util-to-hast "^12.1.0" - unified "^10.0.0" - -resize-observer-polyfill@^1.5.1: - version "1.5.1" - resolved "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz" - integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg== - -resolve-from@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz" - integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== - -resolve-pkg-maps@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz" - integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== - -resolve@^1.1.7, resolve@^1.10.0, resolve@^1.22.1, resolve@^1.22.2: - version "1.22.2" - resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz" - integrity sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g== - dependencies: - is-core-module "^2.11.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - -resolve@^1.22.4: - version "1.22.8" - resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz" - integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== - dependencies: - is-core-module "^2.13.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - -resolve@^2.0.0-next.4: - version "2.0.0-next.5" - resolved "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz" - integrity sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA== - dependencies: - is-core-module "^2.13.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - -restore-cursor@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz" - integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== - dependencies: - onetime "^5.1.0" - signal-exit "^3.0.2" - -reusify@^1.0.4: - version "1.0.4" - resolved "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz" - integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== - -rfdc@^1.3.0: - version "1.3.0" - resolved "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz" - integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== - -rimraf@^3.0.2: - version "3.0.2" - resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz" - integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== - dependencies: - glob "^7.1.3" - -robust-predicates@^3.0.0: - version "3.0.2" - resolved "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz" - integrity sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg== - -route-recognizer@^0.3.3: - version "0.3.4" - resolved "https://registry.npmjs.org/route-recognizer/-/route-recognizer-0.3.4.tgz" - integrity sha512-2+MhsfPhvauN1O8KaXpXAOfR/fwe8dnUXVM+xw7yt40lJRfPVQxV6yryZm0cgRvAj5fMF/mdRZbL2ptwbs5i2g== - -run-applescript@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/run-applescript/-/run-applescript-5.0.0.tgz" - integrity sha512-XcT5rBksx1QdIhlFOCtgZkB99ZEouFZ1E2Kc2LHqNW13U3/74YGdkQRmThTwxy4QIyookibDKYZOPqX//6BlAg== - dependencies: - execa "^5.0.0" - -run-parallel@^1.1.9: - version "1.2.0" - resolved "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz" - integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== - dependencies: - queue-microtask "^1.2.2" - -rw@1: - version "1.3.3" - resolved "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz" - integrity sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ== - -rxjs@^7.8.0: - version "7.8.1" - resolved "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz" - integrity sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg== - dependencies: - tslib "^2.1.0" - -sade@^1.7.3: - version "1.8.1" - resolved "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz" - integrity sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A== - dependencies: - mri "^1.1.0" - -safe-array-concat@^1.0.1: - version "1.1.0" - resolved "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.0.tgz" - integrity sha512-ZdQ0Jeb9Ofti4hbt5lX3T2JcAamT9hfzYU1MNB+z/jaEbB6wfFfPIR/zEORmZqobkCCJhSjodobH6WHNmJ97dg== - dependencies: - call-bind "^1.0.5" - get-intrinsic "^1.2.2" - has-symbols "^1.0.3" - isarray "^2.0.5" - -safe-regex-test@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz" - integrity sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.3" - is-regex "^1.1.4" - -safe-regex@^2.1.1: - version "2.1.1" - resolved "https://registry.npmjs.org/safe-regex/-/safe-regex-2.1.1.tgz" - integrity sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A== - dependencies: - regexp-tree "~0.1.1" - -"safer-buffer@>= 2.1.2 < 3.0.0": - version "2.1.2" - resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" - integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== - -sass@^1.61.0: - version "1.62.1" - resolved "https://registry.npmjs.org/sass/-/sass-1.62.1.tgz" - integrity sha512-NHpxIzN29MXvWiuswfc1W3I0N8SXBd8UR26WntmDlRYf0bSADnwnOjsyMZ3lMezSlArD33Vs3YFhp7dWvL770A== - dependencies: - chokidar ">=3.0.0 <4.0.0" - immutable "^4.0.0" - source-map-js ">=0.6.2 <2.0.0" - -scheduler@^0.23.0: - version "0.23.0" - resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz" - integrity sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw== - dependencies: - loose-envify "^1.1.0" - -screenfull@^5.0.0: - version "5.2.0" - resolved "https://registry.npmjs.org/screenfull/-/screenfull-5.2.0.tgz" - integrity sha512-9BakfsO2aUQN2K9Fdbj87RJIEZ82Q9IGim7FqM5OsebfoFC6ZHXgDq/KvniuLTPdeM8wY2o6Dj3WQ7KeQCj3cA== - -"semver@2 || 3 || 4 || 5": - version "5.7.1" - resolved "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== - -semver@^6.3.0: - version "6.3.0" - resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== - -semver@^6.3.1: - version "6.3.1" - resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.0.0, semver@^7.3.5, semver@^7.3.6, semver@^7.3.7, semver@^7.3.8, semver@^7.5.4: - version "7.5.4" - resolved "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz" - integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== - dependencies: - lru-cache "^6.0.0" - -server-only@^0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz" - integrity sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA== - -set-function-length@^1.1.1: - version "1.2.0" - resolved "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz" - integrity sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w== - dependencies: - define-data-property "^1.1.1" - function-bind "^1.1.2" - get-intrinsic "^1.2.2" - gopd "^1.0.1" - has-property-descriptors "^1.0.1" - -set-function-name@^2.0.0, set-function-name@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz" - integrity sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA== - dependencies: - define-data-property "^1.0.1" - functions-have-names "^1.2.3" - has-property-descriptors "^1.0.0" - -sharp@^0.33.2: - version "0.33.2" - resolved "https://registry.npmjs.org/sharp/-/sharp-0.33.2.tgz" - integrity sha512-WlYOPyyPDiiM07j/UO+E720ju6gtNtHjEGg5vovUk1Lgxyjm2LFO+37Nt/UI3MMh2l6hxTWQWi7qk3cXJTutcQ== - dependencies: - color "^4.2.3" - detect-libc "^2.0.2" - semver "^7.5.4" - optionalDependencies: - "@img/sharp-darwin-arm64" "0.33.2" - "@img/sharp-darwin-x64" "0.33.2" - "@img/sharp-libvips-darwin-arm64" "1.0.1" - "@img/sharp-libvips-darwin-x64" "1.0.1" - "@img/sharp-libvips-linux-arm" "1.0.1" - "@img/sharp-libvips-linux-arm64" "1.0.1" - "@img/sharp-libvips-linux-s390x" "1.0.1" - "@img/sharp-libvips-linux-x64" "1.0.1" - "@img/sharp-libvips-linuxmusl-arm64" "1.0.1" - "@img/sharp-libvips-linuxmusl-x64" "1.0.1" - "@img/sharp-linux-arm" "0.33.2" - "@img/sharp-linux-arm64" "0.33.2" - "@img/sharp-linux-s390x" "0.33.2" - "@img/sharp-linux-x64" "0.33.2" - "@img/sharp-linuxmusl-arm64" "0.33.2" - "@img/sharp-linuxmusl-x64" "0.33.2" - "@img/sharp-wasm32" "0.33.2" - "@img/sharp-win32-ia32" "0.33.2" - "@img/sharp-win32-x64" "0.33.2" - -shebang-command@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz" - integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== - dependencies: - shebang-regex "^3.0.0" - -shebang-regex@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" - integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== - -side-channel@^1.0.4: - version "1.0.4" - resolved "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz" - integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== - dependencies: - call-bind "^1.0.0" - get-intrinsic "^1.0.2" - object-inspect "^1.9.0" - -signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: - version "3.0.7" - resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz" - integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== - -signal-exit@^4.0.1: - version "4.1.0" - resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz" - integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== - -simple-swizzle@^0.2.2: - version "0.2.2" - resolved "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz" - integrity sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg== - dependencies: - is-arrayish "^0.3.1" - -size-sensor@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/size-sensor/-/size-sensor-1.0.1.tgz" - integrity sha512-QTy7MnuugCFXIedXRpUSk9gUnyNiaxIdxGfUjr8xxXOqIB3QvBUYP9+b51oCg2C4dnhaeNk/h57TxjbvoJrJUA== - -slash@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz" - integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== - -slash@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz" - integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew== - -slice-ansi@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/slice-ansi/-/slice-ansi-3.0.0.tgz" - integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - -slice-ansi@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz" - integrity sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - -slice-ansi@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/slice-ansi/-/slice-ansi-5.0.0.tgz" - integrity sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ== - dependencies: - ansi-styles "^6.0.0" - is-fullwidth-code-point "^4.0.0" - -sortablejs@^1.15.0: - version "1.15.0" - resolved "https://registry.npmjs.org/sortablejs/-/sortablejs-1.15.0.tgz" - integrity sha512-bv9qgVMjUMf89wAvM6AxVvS/4MX3sPeN0+agqShejLU5z5GX4C75ow1O2e5k4L6XItUyAK3gH6AxSbXrOM5e8w== - -"source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.2, source-map-js@^1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz" - integrity sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg== - -source-map@^0.7.0: - version "0.7.4" - resolved "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz" - integrity sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA== - -space-separated-tokens@^1.0.0: - version "1.1.5" - resolved "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz" - integrity sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA== - -space-separated-tokens@^2.0.0: - version "2.0.2" - resolved "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz" - integrity sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q== - -spdx-correct@^3.0.0: - version "3.2.0" - resolved "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz" - integrity sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA== - dependencies: - spdx-expression-parse "^3.0.0" - spdx-license-ids "^3.0.0" - -spdx-exceptions@^2.1.0: - version "2.3.0" - resolved "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz" - integrity sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A== - -spdx-expression-parse@^3.0.0: - version "3.0.1" - resolved "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz" - integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q== - dependencies: - spdx-exceptions "^2.1.0" - spdx-license-ids "^3.0.0" - -spdx-license-ids@^3.0.0: - version "3.0.13" - resolved "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.13.tgz" - integrity sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w== - -state-local@^1.0.6: - version "1.0.7" - resolved "https://registry.npmjs.org/state-local/-/state-local-1.0.7.tgz" - integrity sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w== - -stop-iteration-iterator@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz" - integrity sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ== - dependencies: - internal-slot "^1.0.4" - -streamsearch@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz" - integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg== - -string-argv@^0.3.1: - version "0.3.2" - resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz" - integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== - -"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0: - version "4.2.3" - resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^5.0.0, string-width@^5.0.1, string-width@^5.1.2: - version "5.1.2" - resolved "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz" - integrity sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA== - dependencies: - eastasianwidth "^0.2.0" - emoji-regex "^9.2.2" - strip-ansi "^7.0.1" - -string.prototype.matchall@^4.0.8: - version "4.0.10" - resolved "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.10.tgz" - integrity sha512-rGXbGmOEosIQi6Qva94HUjgPs9vKW+dkG7Y8Q5O2OYkWL6wFaTRZO8zM4mhP94uX55wgyrXzfS2aGtGzUL7EJQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - get-intrinsic "^1.2.1" - has-symbols "^1.0.3" - internal-slot "^1.0.5" - regexp.prototype.flags "^1.5.0" - set-function-name "^2.0.0" - side-channel "^1.0.4" - -string.prototype.trim@^1.2.8: - version "1.2.8" - resolved "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz" - integrity sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - -string.prototype.trimend@^1.0.7: - version "1.0.7" - resolved "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.7.tgz" - integrity sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - -string.prototype.trimstart@^1.0.7: - version "1.0.7" - resolved "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz" - integrity sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg== - dependencies: - call-bind "^1.0.2" - define-properties "^1.2.0" - es-abstract "^1.22.1" - -stringify-entities@^4.0.0: - version "4.0.3" - resolved "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.3.tgz" - integrity sha512-BP9nNHMhhfcMbiuQKCqMjhDP5yBCAxsPu4pHFFzJ6Alo9dZgY4VLDPutXqIjpRiMoKdp7Av85Gr73Q5uH9k7+g== - dependencies: - character-entities-html4 "^2.0.0" - character-entities-legacy "^3.0.0" - -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^7.0.1: - version "7.1.0" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz" - integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== - dependencies: - ansi-regex "^6.0.1" - -strip-bom@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz" - integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== - -strip-final-newline@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz" - integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== - -strip-final-newline@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz" - integrity sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw== - -strip-indent@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz" - integrity sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ== - dependencies: - min-indent "^1.0.0" - -strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: - version "3.1.1" - resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz" - integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== - -style-to-object@^0.4.0, style-to-object@^0.4.1: - version "0.4.1" - resolved "https://registry.npmjs.org/style-to-object/-/style-to-object-0.4.1.tgz" - integrity sha512-HFpbb5gr2ypci7Qw+IOhnP2zOU7e77b+rzM+wTzXzfi1PrtBCX0E7Pk4wL4iTLnhzZ+JgEGAhX81ebTg/aYjQw== - dependencies: - inline-style-parser "0.1.1" - -styled-jsx@5.1.1: - version "5.1.1" - resolved "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.1.tgz" - integrity sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw== - dependencies: - client-only "0.0.1" - -stylis@^4.1.3: - version "4.3.0" - resolved "https://registry.npmjs.org/stylis/-/stylis-4.3.0.tgz" - integrity sha512-E87pIogpwUsUwXw7dNyU4QDjdgVMy52m+XEOPEKUn161cCzWjjhPSQhByfd1CcNvrOLnXQ6OnnZDwnJrz/Z4YQ== - -sucrase@^3.32.0: - version "3.32.0" - resolved "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz" - integrity sha512-ydQOU34rpSyj2TGyz4D2p8rbktIOZ8QY9s+DGLvFU1i5pWJE8vkpruCjGCMHsdXwnD7JDcS+noSwM/a7zyNFDQ== - dependencies: - "@jridgewell/gen-mapping" "^0.3.2" - commander "^4.0.0" - glob "7.1.6" - lines-and-columns "^1.1.6" - mz "^2.7.0" - pirates "^4.0.1" - ts-interface-checker "^0.1.9" - -supports-color@^5.3.0: - version "5.5.0" - resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" - integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== - dependencies: - has-flag "^3.0.0" - -supports-color@^7.1.0: - version "7.2.0" - resolved "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz" - integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== - dependencies: - has-flag "^4.0.0" - -supports-preserve-symlinks-flag@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz" - integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== - -swr@^2.1.0: - version "2.1.5" - resolved "https://registry.npmjs.org/swr/-/swr-2.1.5.tgz" - integrity sha512-/OhfZMcEpuz77KavXST5q6XE9nrOBOVcBLWjMT+oAE/kQHyE3PASrevXCtQDZ8aamntOfFkbVJp7Il9tNBQWrw== - dependencies: - use-sync-external-store "^1.2.0" - -synckit@^0.8.5: - version "0.8.5" - resolved "https://registry.npmjs.org/synckit/-/synckit-0.8.5.tgz" - integrity sha512-L1dapNV6vu2s/4Sputv8xGsCdAVlb5nRDMFU/E27D44l5U6cw1g0dGd45uLc+OXjNMmF4ntiMdCimzcjFKQI8Q== - dependencies: - "@pkgr/utils" "^2.3.1" - tslib "^2.5.0" - -tabbable@^6.0.1: - version "6.2.0" - resolved "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz" - integrity sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew== - -tailwindcss@^3.3.3: - version "3.3.3" - resolved "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.3.3.tgz" - integrity sha512-A0KgSkef7eE4Mf+nKJ83i75TMyq8HqY3qmFIJSWy8bNt0v1lG7jUcpGpoTFxAwYcWOphcTBLPPJg+bDfhDf52w== - dependencies: - "@alloc/quick-lru" "^5.2.0" - arg "^5.0.2" - chokidar "^3.5.3" - didyoumean "^1.2.2" - dlv "^1.1.3" - fast-glob "^3.2.12" - glob-parent "^6.0.2" - is-glob "^4.0.3" - jiti "^1.18.2" - lilconfig "^2.1.0" - micromatch "^4.0.5" - normalize-path "^3.0.0" - object-hash "^3.0.0" - picocolors "^1.0.0" - postcss "^8.4.23" - postcss-import "^15.1.0" - postcss-js "^4.0.1" - postcss-load-config "^4.0.1" - postcss-nested "^6.0.1" - postcss-selector-parser "^6.0.11" - resolve "^1.22.2" - sucrase "^3.32.0" - -tapable@^2.2.0: - version "2.2.1" - resolved "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz" - integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== - -text-table@^0.2.0: - version "0.2.0" - resolved "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" - integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== - -thenify-all@^1.0.0: - version "1.6.0" - resolved "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz" - integrity sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA== - dependencies: - thenify ">= 3.1.0 < 4" - -"thenify@>= 3.1.0 < 4": - version "3.3.1" - resolved "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz" - integrity sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw== - dependencies: - any-promise "^1.0.0" - -throttle-debounce@^2.1.0: - version "2.3.0" - resolved "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-2.3.0.tgz" - integrity sha512-H7oLPV0P7+jgvrk+6mwwwBDmxTaxnu9HMXmloNLXwnNO0ZxZ31Orah2n8lU1eMPvsaowP2CX+USCgyovXfdOFQ== - -through@^2.3.8: - version "2.3.8" - resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" - integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== - -tiny-invariant@1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.2.0.tgz" - integrity sha512-1Uhn/aqw5C6RI4KejVeTg6mIS7IqxnLJ8Mv2tV5rTc0qWobay7pDUz6Wi392Cnc8ak1H0F2cjoRzb2/AW4+Fvg== - -titleize@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/titleize/-/titleize-3.0.0.tgz" - integrity sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ== - -to-regex-range@^5.0.1: - version "5.0.1" - resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz" - integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== - dependencies: - is-number "^7.0.0" - -toggle-selection@^1.0.6: - version "1.0.6" - resolved "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz" - integrity sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ== - -trim-lines@^3.0.0: - version "3.0.1" - resolved "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz" - integrity sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg== - -trough@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/trough/-/trough-2.1.0.tgz" - integrity sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g== - -ts-dedent@^2.2.0: - version "2.2.0" - resolved "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz" - integrity sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ== - -ts-interface-checker@^0.1.9: - version "0.1.13" - resolved "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz" - integrity sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA== - -tsconfig-paths@^3.15.0: - version "3.15.0" - resolved "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz" - integrity sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg== - dependencies: - "@types/json5" "^0.0.29" - json5 "^1.0.2" - minimist "^1.2.6" - strip-bom "^3.0.0" - -tslib@2.3.0: - version "2.3.0" - resolved "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz" - integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== - -tslib@^1.8.1, tslib@^1.9.3: - version "1.14.1" - resolved "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz" - integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== - -tslib@^2.1.0, tslib@^2.4.0, tslib@^2.4.1, tslib@^2.5.0: - version "2.5.3" - resolved "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz" - integrity sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w== - -tsutils@^3.21.0: - version "3.21.0" - resolved "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz" - integrity sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA== - dependencies: - tslib "^1.8.1" - -type-check@^0.4.0, type-check@~0.4.0: - version "0.4.0" - resolved "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz" - integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== - dependencies: - prelude-ls "^1.2.1" - -type-fest@^0.20.2: - version "0.20.2" - resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz" - integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== - -type-fest@^0.21.3: - version "0.21.3" - resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz" - integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== - -type-fest@^0.6.0: - version "0.6.0" - resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.6.0.tgz" - integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== - -type-fest@^0.8.1: - version "0.8.1" - resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz" - integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== - -typed-array-buffer@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.0.tgz" - integrity sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw== - dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.2.1" - is-typed-array "^1.1.10" - -typed-array-byte-length@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.0.tgz" - integrity sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA== - dependencies: - call-bind "^1.0.2" - for-each "^0.3.3" - has-proto "^1.0.1" - is-typed-array "^1.1.10" - -typed-array-byte-offset@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.0.tgz" - integrity sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg== - dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.2" - for-each "^0.3.3" - has-proto "^1.0.1" - is-typed-array "^1.1.10" - -typed-array-length@^1.0.4: - version "1.0.4" - resolved "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz" - integrity sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng== - dependencies: - call-bind "^1.0.2" - for-each "^0.3.3" - is-typed-array "^1.1.9" - -typescript@4.9.5: - version "4.9.5" - resolved "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz" - integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== - -uglify-js@^3.17.4: - version "3.17.4" - resolved "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz" - integrity sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g== - -unbox-primitive@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz" - integrity sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw== - dependencies: - call-bind "^1.0.2" - has-bigints "^1.0.2" - has-symbols "^1.0.3" - which-boxed-primitive "^1.0.2" - -unified@^10.0.0: - version "10.1.2" - resolved "https://registry.npmjs.org/unified/-/unified-10.1.2.tgz" - integrity sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q== - dependencies: - "@types/unist" "^2.0.0" - bail "^2.0.0" - extend "^3.0.0" - is-buffer "^2.0.0" - is-plain-obj "^4.0.0" - trough "^2.0.0" - vfile "^5.0.0" - -unist-util-find-after@^4.0.0: - version "4.0.1" - resolved "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-4.0.1.tgz" - integrity sha512-QO/PuPMm2ERxC6vFXEPtmAutOopy5PknD+Oq64gGwxKtk4xwo9Z97t9Av1obPmGU0IyTa6EKYUfTrK2QJS3Ozw== - dependencies: - "@types/unist" "^2.0.0" - unist-util-is "^5.0.0" - -unist-util-generated@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/unist-util-generated/-/unist-util-generated-2.0.1.tgz" - integrity sha512-qF72kLmPxAw0oN2fwpWIqbXAVyEqUzDHMsbtPvOudIlUzXYFIeQIuxXQCRCFh22B7cixvU0MG7m3MW8FTq/S+A== - -unist-util-is@^5.0.0: - version "5.2.1" - resolved "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.2.1.tgz" - integrity sha512-u9njyyfEh43npf1M+yGKDGVPbY/JWEemg5nH05ncKPfi+kBbKBJoTdsogMu33uhytuLlv9y0O7GH7fEdwLdLQw== - dependencies: - "@types/unist" "^2.0.0" - -unist-util-position-from-estree@^1.0.0, unist-util-position-from-estree@^1.1.0: - version "1.1.2" - resolved "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-1.1.2.tgz" - integrity sha512-poZa0eXpS+/XpoQwGwl79UUdea4ol2ZuCYguVaJS4qzIOMDzbqz8a3erUCOmubSZkaOuGamb3tX790iwOIROww== - dependencies: - "@types/unist" "^2.0.0" - -unist-util-position@^4.0.0: - version "4.0.4" - resolved "https://registry.npmjs.org/unist-util-position/-/unist-util-position-4.0.4.tgz" - integrity sha512-kUBE91efOWfIVBo8xzh/uZQ7p9ffYRtUbMRZBNFYwf0RK8koUMx6dGUfwylLOKmaT2cs4wSW96QoYUSXAyEtpg== - dependencies: - "@types/unist" "^2.0.0" - -unist-util-remove-position@^4.0.0: - version "4.0.2" - resolved "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-4.0.2.tgz" - integrity sha512-TkBb0HABNmxzAcfLf4qsIbFbaPDvMO6wa3b3j4VcEzFVaw1LBKwnW4/sRJ/atSLSzoIg41JWEdnE7N6DIhGDGQ== - dependencies: - "@types/unist" "^2.0.0" - unist-util-visit "^4.0.0" - -unist-util-stringify-position@^2.0.0: - version "2.0.3" - resolved "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz" - integrity sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g== - dependencies: - "@types/unist" "^2.0.2" - -unist-util-stringify-position@^3.0.0: - version "3.0.3" - resolved "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.3.tgz" - integrity sha512-k5GzIBZ/QatR8N5X2y+drfpWG8IDBzdnVj6OInRNWm1oXrzydiaAT2OQiA8DPRRZyAKb9b6I2a6PxYklZD0gKg== - dependencies: - "@types/unist" "^2.0.0" - -unist-util-visit-parents@^5.0.0, unist-util-visit-parents@^5.1.1: - version "5.1.3" - resolved "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-5.1.3.tgz" - integrity sha512-x6+y8g7wWMyQhL1iZfhIPhDAs7Xwbn9nRosDXl7qoPTSCy0yNxnKc+hWokFifWQIDGi154rdUqKvbCa4+1kLhg== - dependencies: - "@types/unist" "^2.0.0" - unist-util-is "^5.0.0" - -unist-util-visit@^4.0.0: - version "4.1.2" - resolved "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-4.1.2.tgz" - integrity sha512-MSd8OUGISqHdVvfY9TPhyK2VdUrPgxkUtWSuMHF6XAAFuL4LokseigBnZtPnJMu+FbynTkFNnFlyjxpVKujMRg== - dependencies: - "@types/unist" "^2.0.0" - unist-util-is "^5.0.0" - unist-util-visit-parents "^5.1.1" - -untildify@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz" - integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== - -update-browserslist-db@^1.0.11: - version "1.0.11" - resolved "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz" - integrity sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA== - dependencies: - escalade "^3.1.1" - picocolors "^1.0.0" - -uri-js@^4.2.2: - version "4.4.1" - resolved "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz" - integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== - dependencies: - punycode "^2.1.0" - -use-context-selector@^1.4.1: - version "1.4.1" - resolved "https://registry.npmjs.org/use-context-selector/-/use-context-selector-1.4.1.tgz" - integrity sha512-Io2ArvcRO+6MWIhkdfMFt+WKQX+Vb++W8DS2l03z/Vw/rz3BclKpM0ynr4LYGyU85Eke+Yx5oIhTY++QR0ZDoA== - -use-strict@1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/use-strict/-/use-strict-1.0.1.tgz" - integrity sha512-IeiWvvEXfW5ltKVMkxq6FvNf2LojMKvB2OCeja6+ct24S1XOmQw2dGr2JyndwACWAGJva9B7yPHwAmeA9QCqAQ== - -use-sync-external-store@1.2.0, use-sync-external-store@^1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz" - integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== - -util-deprecate@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" - integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== - -uuid@^9.0.0, uuid@^9.0.1: - version "9.0.1" - resolved "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz" - integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== - -uvu@^0.5.0: - version "0.5.6" - resolved "https://registry.npmjs.org/uvu/-/uvu-0.5.6.tgz" - integrity sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA== - dependencies: - dequal "^2.0.0" - diff "^5.0.0" - kleur "^4.0.3" - sade "^1.7.3" - -validate-npm-package-license@^3.0.1: - version "3.0.4" - resolved "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz" - integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== - dependencies: - spdx-correct "^3.0.0" - spdx-expression-parse "^3.0.0" - -vfile-location@^4.0.0: - version "4.1.0" - resolved "https://registry.npmjs.org/vfile-location/-/vfile-location-4.1.0.tgz" - integrity sha512-YF23YMyASIIJXpktBa4vIGLJ5Gs88UB/XePgqPmTa7cDA+JeO3yclbpheQYCHjVHBn/yePzrXuygIL+xbvRYHw== - dependencies: - "@types/unist" "^2.0.0" - vfile "^5.0.0" - -vfile-message@^3.0.0: - version "3.1.4" - resolved "https://registry.npmjs.org/vfile-message/-/vfile-message-3.1.4.tgz" - integrity sha512-fa0Z6P8HUrQN4BZaX05SIVXic+7kE3b05PWAtPuYP9QLHsLKYR7/AlLW3NtOrpXRLeawpDLMsVkmk5DG0NXgWw== - dependencies: - "@types/unist" "^2.0.0" - unist-util-stringify-position "^3.0.0" - -vfile@^5.0.0: - version "5.3.7" - resolved "https://registry.npmjs.org/vfile/-/vfile-5.3.7.tgz" - integrity sha512-r7qlzkgErKjobAmyNIkkSpizsFPYiUPuJb5pNW1RB4JcYVZhs4lIbVqk8XPk033CV/1z8ss5pkax8SuhGpcG8g== - dependencies: - "@types/unist" "^2.0.0" - is-buffer "^2.0.0" - unist-util-stringify-position "^3.0.0" - vfile-message "^3.0.0" - -vite-code-inspector-plugin@0.13.0: - version "0.13.0" - resolved "https://registry.npmjs.org/vite-code-inspector-plugin/-/vite-code-inspector-plugin-0.13.0.tgz" - integrity sha512-hvIn9G+IFzQHVVynWh2wGTBHo51CBJRqQBzYryeuuaL0BK0w8my2/tlpSAae5ofQxOBXBMhyXC2gWgYUJnNWrA== - dependencies: - code-inspector-core "0.13.0" - -void-elements@3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/void-elements/-/void-elements-3.1.0.tgz" - integrity sha512-Dhxzh5HZuiHQhbvTW9AMetFfBHDMYpo23Uo9btPXgdYP+3T5S+p+jgNy7spra+veYhBP2dCSgxR/i2Y02h5/6w== - -vue-eslint-parser@^9.3.0: - version "9.3.0" - resolved "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-9.3.0.tgz" - integrity sha512-48IxT9d0+wArT1+3wNIy0tascRoywqSUe2E1YalIC1L8jsUGe5aJQItWfRok7DVFGz3UYvzEI7n5wiTXsCMAcQ== - dependencies: - debug "^4.3.4" - eslint-scope "^7.1.1" - eslint-visitor-keys "^3.3.0" - espree "^9.3.1" - esquery "^1.4.0" - lodash "^4.17.21" - semver "^7.3.6" - -web-namespaces@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz" - integrity sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ== - -web-worker@^1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/web-worker/-/web-worker-1.2.0.tgz" - integrity sha512-PgF341avzqyx60neE9DD+XS26MMNMoUQRz9NOZwW32nPQrF6p77f1htcnjBSEV8BGMKZ16choqUG4hyI0Hx7mA== - -webpack-code-inspector-plugin@0.13.0: - version "0.13.0" - resolved "https://registry.npmjs.org/webpack-code-inspector-plugin/-/webpack-code-inspector-plugin-0.13.0.tgz" - integrity sha512-T3ZZ84NX0cVmwff5zyYhB9OuroZYsyaQpSgFicgiuYAWCsQePYApM/R3bHdvcECkBXO50hAVtr9SjWRTu1+Ntg== - dependencies: - code-inspector-core "0.13.0" - -which-boxed-primitive@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz" - integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg== - dependencies: - is-bigint "^1.0.1" - is-boolean-object "^1.1.0" - is-number-object "^1.0.4" - is-string "^1.0.5" - is-symbol "^1.0.3" - -which-builtin-type@^1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.1.3.tgz" - integrity sha512-YmjsSMDBYsM1CaFiayOVT06+KJeXf0o5M/CAd4o1lTadFAtacTUM49zoYxr/oroopFDfhvN6iEcBxUyc3gvKmw== - dependencies: - function.prototype.name "^1.1.5" - has-tostringtag "^1.0.0" - is-async-function "^2.0.0" - is-date-object "^1.0.5" - is-finalizationregistry "^1.0.2" - is-generator-function "^1.0.10" - is-regex "^1.1.4" - is-weakref "^1.0.2" - isarray "^2.0.5" - which-boxed-primitive "^1.0.2" - which-collection "^1.0.1" - which-typed-array "^1.1.9" - -which-collection@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz" - integrity sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A== - dependencies: - is-map "^2.0.1" - is-set "^2.0.1" - is-weakmap "^2.0.1" - is-weakset "^2.0.1" - -which-typed-array@^1.1.11, which-typed-array@^1.1.13, which-typed-array@^1.1.9: - version "1.1.13" - resolved "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz" - integrity sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow== - dependencies: - available-typed-arrays "^1.0.5" - call-bind "^1.0.4" - for-each "^0.3.3" - gopd "^1.0.1" - has-tostringtag "^1.0.0" - -which@^2.0.1: - version "2.0.2" - resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz" - integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== - dependencies: - isexe "^2.0.0" - -word-wrap@^1.2.3: - version "1.2.3" - resolved "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== - -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^6.2.0: - version "6.2.0" - resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz" - integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^8.1.0: - version "8.1.0" - resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz" - integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ== - dependencies: - ansi-styles "^6.1.0" - string-width "^5.0.1" - strip-ansi "^7.0.1" - -wrappy@1: - version "1.0.2" - resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" - integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== - -xml-name-validator@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz" - integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw== - -xtend@^4.0.0: - version "4.0.2" - resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz" - integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== - -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - -yaml-eslint-parser@^1.1.0, yaml-eslint-parser@^1.2.1: - version "1.2.2" - resolved "https://registry.npmjs.org/yaml-eslint-parser/-/yaml-eslint-parser-1.2.2.tgz" - integrity sha512-pEwzfsKbTrB8G3xc/sN7aw1v6A6c/pKxLAkjclnAyo5g5qOh6eL9WGu0o3cSDQZKrTNk4KL4lQSwZW+nBkANEg== - dependencies: - eslint-visitor-keys "^3.0.0" - lodash "^4.17.21" - yaml "^2.0.0" - -yaml@^2.0.0, yaml@^2.1.1, yaml@^2.2.2: - version "2.3.1" - resolved "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz" - integrity sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ== - -yocto-queue@^0.1.0: - version "0.1.0" - resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" - integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== - -zrender@5.4.3: - version "5.4.3" - resolved "https://registry.npmjs.org/zrender/-/zrender-5.4.3.tgz" - integrity sha512-DRUM4ZLnoaT0PBVvGBDO9oWIDBKFdAVieNWxWwK0niYzJCMwGchRk21/hsE+RKkIveH3XHCyvXcJDkgLVvfizQ== - dependencies: - tslib "2.3.0" - -zustand@^4.4.1, zustand@^4.5.1: - version "4.5.1" - resolved "https://registry.npmjs.org/zustand/-/zustand-4.5.1.tgz" - integrity sha512-XlauQmH64xXSC1qGYNv00ODaQ3B+tNPoy22jv2diYiP4eoDKr9LA+Bh5Bc3gplTrFdb6JVI+N4kc1DZ/tbtfPg== - dependencies: - use-sync-external-store "1.2.0" - -zwitch@^2.0.0: - version "2.0.4" - resolved "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz" - integrity sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A== From b151a5bbcfce1e54f26706e64d066ba94054d821 Mon Sep 17 00:00:00 2001 From: chenhe Date: Fri, 10 May 2024 17:22:54 +0800 Subject: [PATCH 012/273] add auth for bearer key --- .../console/datasets/data_source.py | 22 ++++++------- api/core/rag/extractor/notion_extractor.py | 12 +++---- api/libs/bearer_data_source.py | 26 +++++++++++++++ api/libs/oauth_data_source.py | 32 +++++++++---------- api/models/source.py | 21 ++++++++++-- api/services/dataset_service.py | 22 ++++++------- api/tasks/document_indexing_sync_task.py | 12 +++---- 7 files changed, 94 insertions(+), 53 deletions(-) diff --git a/api/controllers/console/datasets/data_source.py b/api/controllers/console/datasets/data_source.py index 8b210cc756bc04..0ca0f0a85653dc 100644 --- a/api/controllers/console/datasets/data_source.py +++ b/api/controllers/console/datasets/data_source.py @@ -16,7 +16,7 @@ from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields from libs.login import login_required from models.dataset import Document -from models.source import DataSourceBinding +from models.source import DataSourceOauthBinding from services.dataset_service import DatasetService, DocumentService from tasks.document_indexing_sync_task import document_indexing_sync_task @@ -29,9 +29,9 @@ class DataSourceApi(Resource): @marshal_with(integrate_list_fields) def get(self): # get workspace data source integrates - data_source_integrates = db.session.query(DataSourceBinding).filter( - DataSourceBinding.tenant_id == current_user.current_tenant_id, - DataSourceBinding.disabled == False + data_source_integrates = db.session.query(DataSourceOauthBinding).filter( + DataSourceOauthBinding.tenant_id == current_user.current_tenant_id, + DataSourceOauthBinding.disabled == False ).all() base_url = request.url_root.rstrip('/') @@ -71,7 +71,7 @@ def get(self): def patch(self, binding_id, action): binding_id = str(binding_id) action = str(action) - data_source_binding = DataSourceBinding.query.filter_by( + data_source_binding = DataSourceOauthBinding.query.filter_by( id=binding_id ).first() if data_source_binding is None: @@ -124,7 +124,7 @@ def get(self): data_source_info = json.loads(document.data_source_info) exist_page_ids.append(data_source_info['notion_page_id']) # get all authorized pages - data_source_bindings = DataSourceBinding.query.filter_by( + data_source_bindings = DataSourceOauthBinding.query.filter_by( tenant_id=current_user.current_tenant_id, provider='notion', disabled=False @@ -163,12 +163,12 @@ class DataSourceNotionApi(Resource): def get(self, workspace_id, page_id, page_type): workspace_id = str(workspace_id) page_id = str(page_id) - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == current_user.current_tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.disabled == False, - DataSourceBinding.source_info['workspace_id'] == f'"{workspace_id}"' + DataSourceOauthBinding.tenant_id == current_user.current_tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.disabled == False, + DataSourceOauthBinding.source_info['workspace_id'] == f'"{workspace_id}"' ) ).first() if not data_source_binding: diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py index c40064fd1d8caf..1d2990166b2158 100644 --- a/api/core/rag/extractor/notion_extractor.py +++ b/api/core/rag/extractor/notion_extractor.py @@ -9,7 +9,7 @@ from core.rag.models.document import Document from extensions.ext_database import db from models.dataset import Document as DocumentModel -from models.source import DataSourceBinding +from models.source import DataSourceOauthBinding logger = logging.getLogger(__name__) @@ -351,12 +351,12 @@ def get_notion_last_edited_time(self) -> str: @classmethod def _get_access_token(cls, tenant_id: str, notion_workspace_id: str) -> str: - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.disabled == False, - DataSourceBinding.source_info['workspace_id'] == f'"{notion_workspace_id}"' + DataSourceOauthBinding.tenant_id == tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.disabled == False, + DataSourceOauthBinding.source_info['workspace_id'] == f'"{notion_workspace_id}"' ) ).first() diff --git a/api/libs/bearer_data_source.py b/api/libs/bearer_data_source.py index 9c14e8e2ff41cf..61d37b23a2ced4 100644 --- a/api/libs/bearer_data_source.py +++ b/api/libs/bearer_data_source.py @@ -3,6 +3,10 @@ from abc import abstractmethod import requests +from api.models.source import DataSourceBearerBinding +from flask_login import current_user + +from extensions.ext_database import db class BearerDataSource: @@ -39,3 +43,25 @@ def validate_bearer_data_source(self): return response.json().get("status") == "success" + def save_credentials(self): + # save data source binding + data_source_binding = DataSourceBearerBinding.query.filter( + db.and_( + DataSourceBearerBinding.tenant_id == current_user.current_tenant_id, + DataSourceBearerBinding.provider == 'firecrawl', + DataSourceBearerBinding.endpoint_url == self.api_base_url, + DataSourceBearerBinding.bearer_key == self.api_key + ) + ).first() + if data_source_binding: + data_source_binding.disabled = False + db.session.commit() + else: + new_data_source_binding = DataSourceBearerBinding( + tenant_id=current_user.current_tenant_id, + provider='firecrawl', + endpoint_url=self.api_base_url, + bearer_key=self.api_key + ) + db.session.add(new_data_source_binding) + db.session.commit() diff --git a/api/libs/oauth_data_source.py b/api/libs/oauth_data_source.py index a865ee85ab5c65..3f2889adbefa36 100644 --- a/api/libs/oauth_data_source.py +++ b/api/libs/oauth_data_source.py @@ -4,7 +4,7 @@ from flask_login import current_user from extensions.ext_database import db -from models.source import DataSourceBinding +from models.source import DataSourceOauthBinding class OAuthDataSource: @@ -63,11 +63,11 @@ def get_access_token(self, code: str): 'total': len(pages) } # save data source binding - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == current_user.current_tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.access_token == access_token + DataSourceOauthBinding.tenant_id == current_user.current_tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.access_token == access_token ) ).first() if data_source_binding: @@ -75,7 +75,7 @@ def get_access_token(self, code: str): data_source_binding.disabled = False db.session.commit() else: - new_data_source_binding = DataSourceBinding( + new_data_source_binding = DataSourceOauthBinding( tenant_id=current_user.current_tenant_id, access_token=access_token, source_info=source_info, @@ -98,11 +98,11 @@ def save_internal_access_token(self, access_token: str): 'total': len(pages) } # save data source binding - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == current_user.current_tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.access_token == access_token + DataSourceOauthBinding.tenant_id == current_user.current_tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.access_token == access_token ) ).first() if data_source_binding: @@ -110,7 +110,7 @@ def save_internal_access_token(self, access_token: str): data_source_binding.disabled = False db.session.commit() else: - new_data_source_binding = DataSourceBinding( + new_data_source_binding = DataSourceOauthBinding( tenant_id=current_user.current_tenant_id, access_token=access_token, source_info=source_info, @@ -121,12 +121,12 @@ def save_internal_access_token(self, access_token: str): def sync_data_source(self, binding_id: str): # save data source binding - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == current_user.current_tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.id == binding_id, - DataSourceBinding.disabled == False + DataSourceOauthBinding.tenant_id == current_user.current_tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.id == binding_id, + DataSourceOauthBinding.disabled == False ) ).first() if data_source_binding: diff --git a/api/models/source.py b/api/models/source.py index 70b139ed25d280..671ff56ad38a89 100644 --- a/api/models/source.py +++ b/api/models/source.py @@ -4,7 +4,7 @@ from models import StringUUID -class DataSourceBinding(db.Model): +class DataSourceOauthBinding(db.Model): __tablename__ = 'data_source_bindings' __table_args__ = ( db.PrimaryKeyConstraint('id', name='source_binding_pkey'), @@ -15,10 +15,25 @@ class DataSourceBinding(db.Model): id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) access_token = db.Column(db.String(255), nullable=False) - endpoint_url = db.Column(db.String(512), nullable=True) # For validation with endpoint + bearer key - bearer_key = db.Column(db.String(512), nullable=True) # For validation with endpoint + bearer key provider = db.Column(db.String(255), nullable=False) source_info = db.Column(JSONB, nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) disabled = db.Column(db.Boolean, nullable=True, server_default=db.text('false')) + + +class DataSourceBearerBinding(db.Model): + __tablename__ = 'data_source_bearer_bindings' + __table_args__ = ( + db.PrimaryKeyConstraint('id', name='source_bearer_binding_pkey'), + db.Index('source_bearer_binding_tenant_id_idx', 'tenant_id'), + ) + + id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) + tenant_id = db.Column(StringUUID, nullable=False) + provider = db.Column(db.String(255), nullable=False) + endpoint_url = db.Column(db.String(512), nullable=False) # For validation with endpoint + bearer key + bearer_key = db.Column(db.String(512), nullable=True) # For validation with endpoint + bearer key + created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + disabled = db.Column(db.Boolean, nullable=True, server_default=db.text('false')) \ No newline at end of file diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index fa2ed808852f4e..5dee72eac6eca8 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -32,7 +32,7 @@ DocumentSegment, ) from models.model import UploadFile -from models.source import DataSourceBinding +from models.source import DataSourceOauthBinding from services.errors.account import NoPermissionError from services.errors.dataset import DatasetNameDuplicateError from services.errors.document import DocumentIndexingError @@ -664,12 +664,12 @@ def save_document_with_dataset_id(dataset: Dataset, document_data: dict, exist_document[data_source_info['notion_page_id']] = document.id for notion_info in notion_info_list: workspace_id = notion_info['workspace_id'] - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == current_user.current_tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.disabled == False, - DataSourceBinding.source_info['workspace_id'] == f'"{workspace_id}"' + DataSourceOauthBinding.tenant_id == current_user.current_tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.disabled == False, + DataSourceOauthBinding.source_info['workspace_id'] == f'"{workspace_id}"' ) ).first() if not data_source_binding: @@ -799,12 +799,12 @@ def update_document_with_dataset_id(dataset: Dataset, document_data: dict, notion_info_list = document_data["data_source"]['info_list']['notion_info_list'] for notion_info in notion_info_list: workspace_id = notion_info['workspace_id'] - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == current_user.current_tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.disabled == False, - DataSourceBinding.source_info['workspace_id'] == f'"{workspace_id}"' + DataSourceOauthBinding.tenant_id == current_user.current_tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.disabled == False, + DataSourceOauthBinding.source_info['workspace_id'] == f'"{workspace_id}"' ) ).first() if not data_source_binding: diff --git a/api/tasks/document_indexing_sync_task.py b/api/tasks/document_indexing_sync_task.py index c35c18799a6039..4cced36ecdd856 100644 --- a/api/tasks/document_indexing_sync_task.py +++ b/api/tasks/document_indexing_sync_task.py @@ -11,7 +11,7 @@ from core.rag.index_processor.index_processor_factory import IndexProcessorFactory from extensions.ext_database import db from models.dataset import Dataset, Document, DocumentSegment -from models.source import DataSourceBinding +from models.source import DataSourceOauthBinding @shared_task(queue='dataset') @@ -43,12 +43,12 @@ def document_indexing_sync_task(dataset_id: str, document_id: str): page_id = data_source_info['notion_page_id'] page_type = data_source_info['type'] page_edited_time = data_source_info['last_edited_time'] - data_source_binding = DataSourceBinding.query.filter( + data_source_binding = DataSourceOauthBinding.query.filter( db.and_( - DataSourceBinding.tenant_id == document.tenant_id, - DataSourceBinding.provider == 'notion', - DataSourceBinding.disabled == False, - DataSourceBinding.source_info['workspace_id'] == f'"{workspace_id}"' + DataSourceOauthBinding.tenant_id == document.tenant_id, + DataSourceOauthBinding.provider == 'notion', + DataSourceOauthBinding.disabled == False, + DataSourceOauthBinding.source_info['workspace_id'] == f'"{workspace_id}"' ) ).first() if not data_source_binding: From 716b8c164615dd9071e8cdb7e4f52ca4d94267b6 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Sat, 11 May 2024 17:08:17 +0800 Subject: [PATCH 013/273] add source bearer auth --- .../console/auth/data_source_bearer.py | 116 ------------------ .../console/auth/data_source_bearer_auth.py | 51 ++++++++ api/models/source.py | 31 +++-- api/services/auth/__init__.py | 0 api/services/auth/api_key_auth_service.py | 34 +++++ api/services/auth/firecrawl.py | 75 +++++++++++ api/tests/unit_tests/oss/__init__.py | 0 api/tests/unit_tests/oss/local/__init__.py | 0 8 files changed, 183 insertions(+), 124 deletions(-) delete mode 100644 api/controllers/console/auth/data_source_bearer.py create mode 100644 api/controllers/console/auth/data_source_bearer_auth.py create mode 100644 api/services/auth/__init__.py create mode 100644 api/services/auth/api_key_auth_service.py create mode 100644 api/services/auth/firecrawl.py create mode 100644 api/tests/unit_tests/oss/__init__.py create mode 100644 api/tests/unit_tests/oss/local/__init__.py diff --git a/api/controllers/console/auth/data_source_bearer.py b/api/controllers/console/auth/data_source_bearer.py deleted file mode 100644 index 293ec1c4d341c3..00000000000000 --- a/api/controllers/console/auth/data_source_bearer.py +++ /dev/null @@ -1,116 +0,0 @@ -import logging - -import requests -from flask import current_app, redirect, request -from flask_login import current_user -from flask_restful import Resource -from werkzeug.exceptions import Forbidden - -from controllers.console import api -from libs.login import login_required -from libs.oauth_data_source import NotionOAuth - -from ..setup import setup_required -from ..wraps import account_initialization_required - - -def get_oauth_providers(): - with current_app.app_context(): - notion_oauth = NotionOAuth(client_id=current_app.config.get('NOTION_CLIENT_ID'), - client_secret=current_app.config.get( - 'NOTION_CLIENT_SECRET'), - redirect_uri=current_app.config.get( - 'CONSOLE_API_URL') + '/console/api/oauth/data-source/callback/notion') - - OAUTH_PROVIDERS = { - 'notion': notion_oauth - } - return OAUTH_PROVIDERS - - -class OAuthDataSource(Resource): - def get(self, provider: str): - # The role of the current user in the table must be admin or owner - if not current_user.is_admin_or_owner: - raise Forbidden() - OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() - with current_app.app_context(): - oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) - print(vars(oauth_provider)) - if not oauth_provider: - return {'error': 'Invalid provider'}, 400 - if current_app.config.get('NOTION_INTEGRATION_TYPE') == 'internal': - internal_secret = current_app.config.get('NOTION_INTERNAL_SECRET') - oauth_provider.save_internal_access_token(internal_secret) - return { 'data': '' } - else: - auth_url = oauth_provider.get_authorization_url() - return { 'data': auth_url }, 200 - - - - -class OAuthDataSourceCallback(Resource): - def get(self, provider: str): - OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() - with current_app.app_context(): - oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) - if not oauth_provider: - return {'error': 'Invalid provider'}, 400 - if 'code' in request.args: - code = request.args.get('code') - - return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&code={code}') - elif 'error' in request.args: - error = request.args.get('error') - - return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error={error}') - else: - return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error=Access denied') - - -class OAuthDataSourceBinding(Resource): - def get(self, provider: str): - OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() - with current_app.app_context(): - oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) - if not oauth_provider: - return {'error': 'Invalid provider'}, 400 - if 'code' in request.args: - code = request.args.get('code') - try: - oauth_provider.get_access_token(code) - except requests.exceptions.HTTPError as e: - logging.exception( - f"An error occurred during the OAuthCallback process with {provider}: {e.response.text}") - return {'error': 'OAuth data source process failed'}, 400 - - return {'result': 'success'}, 200 - - -class OAuthDataSourceSync(Resource): - @setup_required - @login_required - @account_initialization_required - def get(self, provider, binding_id): - provider = str(provider) - binding_id = str(binding_id) - OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers() - with current_app.app_context(): - oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider) - if not oauth_provider: - return {'error': 'Invalid provider'}, 400 - try: - oauth_provider.sync_data_source(binding_id) - except requests.exceptions.HTTPError as e: - logging.exception( - f"An error occurred during the OAuthCallback process with {provider}: {e.response.text}") - return {'error': 'OAuth data source process failed'}, 400 - - return {'result': 'success'}, 200 - - -api.add_resource(OAuthDataSource, '/oauth/data-source/') -api.add_resource(OAuthDataSourceCallback, '/oauth/data-source/callback/') -api.add_resource(OAuthDataSourceBinding, '/oauth/data-source/binding/') -api.add_resource(OAuthDataSourceSync, '/oauth/data-source///sync') diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py new file mode 100644 index 00000000000000..01cf81e6726968 --- /dev/null +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -0,0 +1,51 @@ +import logging + +import requests +from flask import current_app, redirect, request +from flask_login import current_user +from flask_restful import Resource, reqparse +from werkzeug.exceptions import Forbidden + +from controllers.console import api +from libs.login import login_required +from libs.oauth_data_source import NotionOAuth +from services.auth.api_key_auth_service import ApiKeyAuthService + +from ..setup import setup_required +from ..wraps import account_initialization_required + + +class ApiKeyAuthDataSource(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self): + # The role of the current user in the table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + data_source_api_key_bindings = ApiKeyAuthService.get_provider_auth_list(current_user.tenant_id) + if data_source_api_key_bindings: + return { + 'settings': [data_source_api_key_binding.to_dict() for data_source_api_key_binding in + data_source_api_key_bindings]} + return {'settings': []} + + +class ApiKeyAuthDataSourceBinding(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self): + # The role of the current user in the table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + parser = reqparse.RequestParser() + parser.add_argument('category', type=str, required=True, nullable=False, location='json') + parser.add_argument('provider', type=str, required=True, nullable=False, location='json') + parser.add_argument('credential', type=dict, required=True, nullable=False, location='json') + args = parser.parse_args() + data_source_api_key_binding = ApiKeyAuthService.create_provider_auth(current_user.tenant_id, args) + + +api.add_resource(ApiKeyAuthDataSource, '/api-key-auth/data-source') +api.add_resource(ApiKeyAuthDataSourceBinding, '/api-key-auth/data-source/binding') diff --git a/api/models/source.py b/api/models/source.py index 671ff56ad38a89..b870f1fa37db74 100644 --- a/api/models/source.py +++ b/api/models/source.py @@ -1,3 +1,5 @@ +import json + from sqlalchemy.dialects.postgresql import JSONB from extensions.ext_database import db @@ -5,7 +7,7 @@ class DataSourceOauthBinding(db.Model): - __tablename__ = 'data_source_bindings' + __tablename__ = 'data_source_oauth_bindings' __table_args__ = ( db.PrimaryKeyConstraint('id', name='source_binding_pkey'), db.Index('source_binding_tenant_id_idx', 'tenant_id'), @@ -22,18 +24,31 @@ class DataSourceOauthBinding(db.Model): disabled = db.Column(db.Boolean, nullable=True, server_default=db.text('false')) -class DataSourceBearerBinding(db.Model): - __tablename__ = 'data_source_bearer_bindings' +class DataSourceApiKeyAuthBinding(db.Model): + __tablename__ = 'data_source_api_key_auth_bindings' __table_args__ = ( - db.PrimaryKeyConstraint('id', name='source_bearer_binding_pkey'), - db.Index('source_bearer_binding_tenant_id_idx', 'tenant_id'), + db.PrimaryKeyConstraint('id', name='data_source_api_key_auth_binding_pkey'), + db.Index('data_source_api_key_auth_binding_tenant_id_idx', 'tenant_id'), + db.Index('data_source_api_key_auth_binding_provider_idx', 'provider'), ) id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) + category = db.Column(db.String(255), nullable=False) provider = db.Column(db.String(255), nullable=False) - endpoint_url = db.Column(db.String(512), nullable=False) # For validation with endpoint + bearer key - bearer_key = db.Column(db.String(512), nullable=True) # For validation with endpoint + bearer key + credentials = db.Column(db.Text, nullable=True) # JSON created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) - disabled = db.Column(db.Boolean, nullable=True, server_default=db.text('false')) \ No newline at end of file + disabled = db.Column(db.Boolean, nullable=True, server_default=db.text('false')) + + def to_dict(self): + return { + 'id': self.id, + 'tenant_id': self.tenant_id, + 'category': self.category, + 'provider': self.provider, + 'credentials': json.dumps(self.credentials, ensure_ascii=False), + 'created_at': self.created_at, + 'updated_at': self.updated_at, + 'disabled': self.disabled + } diff --git a/api/services/auth/__init__.py b/api/services/auth/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/services/auth/api_key_auth_service.py b/api/services/auth/api_key_auth_service.py new file mode 100644 index 00000000000000..8630eb6615ff3f --- /dev/null +++ b/api/services/auth/api_key_auth_service.py @@ -0,0 +1,34 @@ +import uuid +from typing import List + +from flask_login import current_user +from sqlalchemy import func +from werkzeug.exceptions import NotFound + +from extensions.ext_database import db +from models.dataset import Dataset +from models.model import App, Tag, TagBinding +from models.source import DataSourceApiKeyAuthBinding + + +class ApiKeyAuthService: + + @staticmethod + def get_provider_auth_list(tenant_id: str) -> list: + data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( + DataSourceApiKeyAuthBinding.tenant_id == tenant_id, + DataSourceApiKeyAuthBinding.disabled.is_(False) + ).all() + return data_source_api_key_bindings + + @staticmethod + def create_provider_auth(tenant_id: str, args: dict) -> DataSourceApiKeyAuthBinding: + data_source_api_key_binding = DataSourceApiKeyAuthBinding() + data_source_api_key_binding.tenant_id = tenant_id + data_source_api_key_binding.category = args['category'] + data_source_api_key_binding.provider = args['provider'] + data_source_api_key_binding.credentials = args['credential'] + db.session.add(data_source_api_key_binding) + db.session.commit() + return data_source_api_key_binding + pass diff --git a/api/services/auth/firecrawl.py b/api/services/auth/firecrawl.py new file mode 100644 index 00000000000000..f76b976739c226 --- /dev/null +++ b/api/services/auth/firecrawl.py @@ -0,0 +1,75 @@ +import os +import requests + + +class FirecrawlAuth: + def __init__(self, api_key=None, endpoint=None): + self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') + if self.api_key is None: + raise ValueError('No API key provided') + + def _validate_credentials(self): + headers = self._prepare_headers() + options = { + 'url': 'https://example.com', + 'crawlerOptions': { + 'excludes': [], + 'includes': [], + 'limit': 1 + }, + 'pageOptions': { + 'onlyMainContent': True + } + } + response = self._post_request('https://api.firecrawl.dev/v0/crawl', options, headers) + if response.status_code == 200: + return True + else: + self._handle_error(response, 'start crawl job') + + def check_crawl_status(self, job_id): + headers = self._prepare_headers() + response = self._get_request(f'https://api.firecrawl.dev/v0/crawl/status/{job_id}', headers) + if response.status_code == 200: + return response.json() + else: + self._handle_error(response, 'check crawl status') + + def _prepare_headers(self): + return { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.api_key}' + } + + def _post_request(self, url, data, headers): + return requests.post(url, headers=headers, json=data) + + def _get_request(self, url, headers): + return requests.get(url, headers=headers) + + def _monitor_job_status(self, job_id, headers, timeout): + import time + while True: + status_response = self._get_request(f'https://api.firecrawl.dev/v0/crawl/status/{job_id}', headers) + if status_response.status_code == 200: + status_data = status_response.json() + if status_data['status'] == 'completed': + if 'data' in status_data: + return status_data['data'] + else: + raise Exception('Crawl job completed but no data was returned') + elif status_data['status'] in ['active', 'paused', 'pending', 'queued']: + if timeout < 2: + timeout = 2 + time.sleep(timeout) # Wait for the specified timeout before checking again + else: + raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}') + else: + self._handle_error(status_response, 'check crawl status') + + def _handle_error(self, response, action): + if response.status_code in [402, 409, 500]: + error_message = response.json().get('error', 'Unknown error occurred') + raise Exception(f'Failed to {action}. Status code: {response.status_code}. Error: {error_message}') + else: + raise Exception(f'Unexpected error occurred while trying to {action}. Status code: {response.status_code}') diff --git a/api/tests/unit_tests/oss/__init__.py b/api/tests/unit_tests/oss/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/unit_tests/oss/local/__init__.py b/api/tests/unit_tests/oss/local/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 From f9984f854667704219c128db51110574540d7622 Mon Sep 17 00:00:00 2001 From: nite-knite Date: Sat, 11 May 2024 15:46:23 +0800 Subject: [PATCH 014/273] chore: rename variables --- .../components/app/configuration/debug/index.tsx | 6 +++--- web/app/components/app/configuration/index.tsx | 8 ++++---- .../app/overview/apikey-info-panel/index.tsx | 4 ++-- web/app/components/datasets/create/index.tsx | 2 +- .../datasets/create/step-two/index.tsx | 16 ++++++++-------- .../datasets/documents/detail/settings/index.tsx | 2 +- .../model-parameter-modal/index.tsx | 4 ++-- web/context/debug-configuration.ts | 6 +++--- web/context/provider-context.tsx | 6 +++--- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/web/app/components/app/configuration/debug/index.tsx b/web/app/components/app/configuration/debug/index.tsx index dba0b6d05fabf6..3717bab88ec7a6 100644 --- a/web/app/components/app/configuration/debug/index.tsx +++ b/web/app/components/app/configuration/debug/index.tsx @@ -41,7 +41,7 @@ import PromptLogModal from '@/app/components/base/prompt-log-modal' import { useStore as useAppStore } from '@/app/components/app/store' type IDebug = { - hasSetAPIKEY: boolean + isAPIKeySet: boolean onSetting: () => void inputs: Inputs modelParameterParams: Pick @@ -51,7 +51,7 @@ type IDebug = { } const Debug: FC = ({ - hasSetAPIKEY = true, + isAPIKeySet = true, onSetting, inputs, modelParameterParams, @@ -503,7 +503,7 @@ const Debug: FC = ({ onCancel={handleCancel} /> )} - {!hasSetAPIKEY && ()} + {!isAPIKeySet && ()} ) } diff --git a/web/app/components/app/configuration/index.tsx b/web/app/components/app/configuration/index.tsx index 259f6b2deb3650..3b61a0b7fe62bc 100644 --- a/web/app/components/app/configuration/index.tsx +++ b/web/app/components/app/configuration/index.tsx @@ -255,7 +255,7 @@ const Configuration: FC = () => { }) } - const { hasSettedApiKey } = useProviderContext() + const { isAPIKeySet } = useProviderContext() const { currentModel: currModel, textGenerationModelList, @@ -678,7 +678,7 @@ const Configuration: FC = () => { return ( { {!isMobile &&

setShowAccountSettingModal({ payload: 'provider' })} inputs={inputs} modelParameterParams={{ @@ -881,7 +881,7 @@ const Configuration: FC = () => { {isMobile && ( setShowAccountSettingModal({ payload: 'provider' })} inputs={inputs} modelParameterParams={{ diff --git a/web/app/components/app/overview/apikey-info-panel/index.tsx b/web/app/components/app/overview/apikey-info-panel/index.tsx index 555be997a1acfe..b91bfd5aaf61e7 100644 --- a/web/app/components/app/overview/apikey-info-panel/index.tsx +++ b/web/app/components/app/overview/apikey-info-panel/index.tsx @@ -12,14 +12,14 @@ import { useModalContext } from '@/context/modal-context' const APIKeyInfoPanel: FC = () => { const isCloud = !IS_CE_EDITION - const { hasSettedApiKey } = useProviderContext() + const { isAPIKeySet } = useProviderContext() const { setShowAccountSettingModal } = useModalContext() const { t } = useTranslation() const [isShow, setIsShow] = useState(true) - if (hasSettedApiKey) + if (isAPIKeySet) return null if (!(isShow)) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index ce32f29fc528fd..da5de3eb2187f3 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -123,7 +123,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { onStepChange={nextStep} />} {(step === 2 && (!datasetId || (datasetId && !!detail))) && setShowAccountSettingModal({ payload: 'provider' })} indexingType={detail?.indexing_technique} datasetId={datasetId} diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 90b798025ed2f3..988b4798a1d885 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -49,7 +49,7 @@ type ValueOf = T[keyof T] type StepTwoProps = { isSetting?: boolean documentDetail?: FullDocumentDetail - hasSetAPIKEY: boolean + isAPIKeySet: boolean onSetting: () => void datasetId?: string indexingType?: ValueOf @@ -75,7 +75,7 @@ enum IndexingType { const StepTwo = ({ isSetting, documentDetail, - hasSetAPIKEY, + isAPIKeySet, onSetting, datasetId, indexingType, @@ -107,7 +107,7 @@ const StepTwo = ({ const hasSetIndexType = !!indexingType const [indexType, setIndexType] = useState>( (indexingType - || hasSetAPIKEY) + || isAPIKeySet) ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL, ) @@ -480,8 +480,8 @@ const StepTwo = ({ setIndexType(indexingType as IndexingType) else - setIndexType(hasSetAPIKEY ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL) - }, [hasSetAPIKEY, indexingType, datasetId]) + setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL) + }, [isAPIKeySet, indexingType, datasetId]) useEffect(() => { if (segmentationType === SegmentType.AUTO) { @@ -636,13 +636,13 @@ const StepTwo = ({ className={cn( s.radioItem, s.indexItem, - !hasSetAPIKEY && s.disabled, + !isAPIKeySet && s.disabled, !hasSetIndexType && indexType === IndexingType.QUALIFIED && s.active, hasSetIndexType && s.disabled, hasSetIndexType && '!w-full', )} onClick={() => { - if (hasSetAPIKEY) + if (isAPIKeySet) setIndexType(IndexingType.QUALIFIED) }} > @@ -665,7 +665,7 @@ const StepTwo = ({ ) }
- {!hasSetAPIKEY && ( + {!isAPIKeySet && (
{t('datasetCreation.stepTwo.warning')}  {t('datasetCreation.stepTwo.click')} diff --git a/web/app/components/datasets/documents/detail/settings/index.tsx b/web/app/components/datasets/documents/detail/settings/index.tsx index 891e284c95888a..cab0c5d4007c21 100644 --- a/web/app/components/datasets/documents/detail/settings/index.tsx +++ b/web/app/components/datasets/documents/detail/settings/index.tsx @@ -68,7 +68,7 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => { {!documentDetail && } {dataset && documentDetail && ( = ({ isInWorkflow, }) => { const { t } = useTranslation() - const { hasSettedApiKey } = useProviderContext() + const { isAPIKeySet } = useProviderContext() const [open, setOpen] = useState(false) const { data: parameterRulesData, isLoading } = useSWR((provider && modelId) ? `/workspaces/current/model-providers/${provider}/models/parameter-rules?model=${modelId}` : null, fetchModelParameterRules) const { @@ -99,7 +99,7 @@ const ModelParameterModal: FC = ({ const hasDeprecated = !currentProvider || !currentModel const modelDisabled = currentModel?.status !== ModelStatusEnum.active - const disabled = !hasSettedApiKey || hasDeprecated || modelDisabled + const disabled = !isAPIKeySet || hasDeprecated || modelDisabled const parameterRules: ModelParameterRule[] = useMemo(() => { return parameterRulesData?.data || [] diff --git a/web/context/debug-configuration.ts b/web/context/debug-configuration.ts index e9f18e9e3fa40d..fc92cc7d10f84d 100644 --- a/web/context/debug-configuration.ts +++ b/web/context/debug-configuration.ts @@ -28,7 +28,7 @@ import type { Collection } from '@/app/components/tools/types' type IDebugConfiguration = { appId: string - hasSetAPIKEY: boolean + isAPIKeySet: boolean isTrailFinished: boolean mode: string modelModeType: ModelModeType @@ -101,7 +101,7 @@ type IDebugConfiguration = { const DebugConfigurationContext = createContext({ appId: '', - hasSetAPIKEY: false, + isAPIKeySet: false, isTrailFinished: false, mode: '', modelModeType: ModelModeType.chat, @@ -134,7 +134,7 @@ const DebugConfigurationContext = createContext({ introduction: '', setIntroduction: () => { }, suggestedQuestions: [], - setSuggestedQuestions: () => {}, + setSuggestedQuestions: () => { }, controlClearChatMessage: 0, setControlClearChatMessage: () => { }, prevPromptConfig: { diff --git a/web/context/provider-context.tsx b/web/context/provider-context.tsx index 40ea4ac597e65f..71f9bc3ffc9742 100644 --- a/web/context/provider-context.tsx +++ b/web/context/provider-context.tsx @@ -23,7 +23,7 @@ const ProviderContext = createContext<{ modelProviders: ModelProvider[] textGenerationModelList: Model[] supportRetrievalMethods: RETRIEVE_METHOD[] - hasSettedApiKey: boolean + isAPIKeySet: boolean plan: { type: Plan usage: UsagePlanInfo @@ -37,7 +37,7 @@ const ProviderContext = createContext<{ modelProviders: [], textGenerationModelList: [], supportRetrievalMethods: [], - hasSettedApiKey: true, + isAPIKeySet: true, plan: { type: Plan.sandbox, usage: { @@ -95,7 +95,7 @@ export const ProviderContextProvider = ({ model.status === ModelStatusEnum.active), + isAPIKeySet: !!textGenerationModelList?.data.some(model => model.status === ModelStatusEnum.active), supportRetrievalMethods: supportRetrievalMethods?.retrieval_method || [], plan, isFetchedPlan, From f5812b265d59b380946f822e67d438bbdb04410d Mon Sep 17 00:00:00 2001 From: nite-knite Date: Sat, 11 May 2024 16:00:27 +0800 Subject: [PATCH 015/273] feat: add switch for model load balancing --- web/app/components/billing/type.ts | 1 + web/context/provider-context.tsx | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/web/app/components/billing/type.ts b/web/app/components/billing/type.ts index 964cf7c14ffdd7..fb782c0cefdfef 100644 --- a/web/app/components/billing/type.ts +++ b/web/app/components/billing/type.ts @@ -61,6 +61,7 @@ export type CurrentPlanInfoBackend = { } docs_processing: DocumentProcessingPriority can_replace_logo: boolean + model_load_balancing_enabled: boolean } export type SubscriptionItem = { diff --git a/web/context/provider-context.tsx b/web/context/provider-context.tsx index 71f9bc3ffc9742..4d7fe393abd4f5 100644 --- a/web/context/provider-context.tsx +++ b/web/context/provider-context.tsx @@ -33,6 +33,7 @@ const ProviderContext = createContext<{ enableBilling: boolean onPlanInfoChanged: () => void enableReplaceWebAppLogo: boolean + enableModelLoadBalancing: boolean }>({ modelProviders: [], textGenerationModelList: [], @@ -57,6 +58,7 @@ const ProviderContext = createContext<{ enableBilling: false, onPlanInfoChanged: () => { }, enableReplaceWebAppLogo: false, + enableModelLoadBalancing: false, }) export const useProviderContext = () => useContext(ProviderContext) @@ -76,6 +78,7 @@ export const ProviderContextProvider = ({ const [isFetchedPlan, setIsFetchedPlan] = useState(false) const [enableBilling, setEnableBilling] = useState(true) const [enableReplaceWebAppLogo, setEnableReplaceWebAppLogo] = useState(false) + const [enableModelLoadBalancing, setEnableModelLoadBalancing] = useState(false) const fetchPlan = async () => { const data = await fetchCurrentPlanInfo() @@ -86,6 +89,8 @@ export const ProviderContextProvider = ({ setPlan(parseCurrentPlan(data)) setIsFetchedPlan(true) } + if (data.model_load_balancing_enabled) + setEnableModelLoadBalancing(true) } useEffect(() => { fetchPlan() @@ -102,6 +107,7 @@ export const ProviderContextProvider = ({ enableBilling, onPlanInfoChanged: fetchPlan, enableReplaceWebAppLogo, + enableModelLoadBalancing, }}> {children} From d5abcf4f301723704d3524bf671ab692cb834f81 Mon Sep 17 00:00:00 2001 From: nite-knite Date: Sat, 11 May 2024 18:55:47 +0800 Subject: [PATCH 016/273] feat: model switch and balancing styling --- .../line/financeAndECommerce/balance.svg | 3 ++ .../line/financeAndECommerce/Balance.json | 29 +++++++++++++ .../line/financeAndECommerce/Balance.tsx | 16 ++++++++ .../vender/line/financeAndECommerce/index.ts | 1 + web/app/components/base/switch/index.tsx | 4 +- .../model-provider-page/declarations.ts | 2 + .../model-provider-page/model-badge/index.tsx | 10 ++--- .../model-provider-page/model-name/index.tsx | 17 ++++---- .../provider-added-card/model-list.tsx | 32 +++++++++++---- web/i18n/en-US/common.ts | 2 + web/i18n/zh-Hans/common.ts | 2 + web/tailwind.config.js | 41 ++++++++++--------- 12 files changed, 119 insertions(+), 40 deletions(-) create mode 100644 web/app/components/base/icons/assets/vender/line/financeAndECommerce/balance.svg create mode 100644 web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.json create mode 100644 web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.tsx diff --git a/web/app/components/base/icons/assets/vender/line/financeAndECommerce/balance.svg b/web/app/components/base/icons/assets/vender/line/financeAndECommerce/balance.svg new file mode 100644 index 00000000000000..428967a697d2be --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/financeAndECommerce/balance.svg @@ -0,0 +1,3 @@ + + + diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.json b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.json new file mode 100644 index 00000000000000..c04fcda517b3ae --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.json @@ -0,0 +1,29 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "24", + "height": "24", + "viewBox": "0 0 24 24", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M12 3V20M12 20H6.99999M12 20H17M2.99999 6H7.52785C7.83834 6 8.14457 5.92771 8.42228 5.78885L9.5777 5.21115C9.85541 5.07229 10.1616 5 10.4721 5H13.5279C13.8384 5 14.1446 5.07229 14.4223 5.21115L15.5777 5.78885C15.8554 5.92771 16.1616 6 16.4721 6H21M5.49999 6L3.02043 13.4387C2.71807 14.3458 3.08918 15.3834 4.0053 15.657C5.0117 15.9577 5.98828 15.9577 6.99468 15.657C7.9108 15.3834 8.28191 14.3457 7.97955 13.4387L5.49999 6ZM18.5 6L16.0204 13.4387C15.7181 14.3458 16.0892 15.3834 17.0053 15.657C18.0117 15.9577 18.9883 15.9577 19.9947 15.657C20.9108 15.3834 21.2819 14.3457 20.9796 13.4387L18.5 6Z", + "stroke": "currentColor", + "stroke-width": "2", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + }, + "name": "Balance" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.tsx b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.tsx new file mode 100644 index 00000000000000..7743f0bd6a9101 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/Balance.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Balance.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Balance' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts b/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts index 65c6ef70f3eff3..2223daa1d50c54 100644 --- a/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts +++ b/web/app/components/base/icons/src/vender/line/financeAndECommerce/index.ts @@ -1,3 +1,4 @@ +export { default as Balance } from './Balance' export { default as CoinsStacked01 } from './CoinsStacked01' export { default as GoldCoin } from './GoldCoin' export { default as ReceiptList } from './ReceiptList' diff --git a/web/app/components/base/switch/index.tsx b/web/app/components/base/switch/index.tsx index 6794e51efdb30f..8b7d1216992e2d 100644 --- a/web/app/components/base/switch/index.tsx +++ b/web/app/components/base/switch/index.tsx @@ -4,7 +4,7 @@ import classNames from 'classnames' import { Switch as OriginalSwitch } from '@headlessui/react' type SwitchProps = { - onChange: (value: boolean) => void + onChange?: (value: boolean) => void size?: 'sm' | 'md' | 'lg' | 'l' defaultValue?: boolean disabled?: boolean @@ -42,7 +42,7 @@ const Switch = ({ onChange, size = 'lg', defaultValue = false, disabled = false if (disabled) return setEnabled(checked) - onChange(checked) + onChange?.(checked) }} className={classNames( wrapStyle[size], diff --git a/web/app/components/header/account-setting/model-provider-page/declarations.ts b/web/app/components/header/account-setting/model-provider-page/declarations.ts index e7d799ff9b4d70..de763600b31ffc 100644 --- a/web/app/components/header/account-setting/model-provider-page/declarations.ts +++ b/web/app/components/header/account-setting/model-provider-page/declarations.ts @@ -63,6 +63,7 @@ export enum ModelStatusEnum { noConfigure = 'no-configure', quotaExceeded = 'quota-exceeded', noPermission = 'no-permission', + disabled = 'disabled', } export const MODEL_STATUS_TEXT: { [k: string]: TypeWithI18N } = { @@ -116,6 +117,7 @@ export type ModelItem = { fetch_from: ConfigurateMethodEnum status: ModelStatusEnum model_properties: Record + load_balancing_enabled: boolean deprecated?: boolean } diff --git a/web/app/components/header/account-setting/model-provider-page/model-badge/index.tsx b/web/app/components/header/account-setting/model-provider-page/model-badge/index.tsx index e0ea74abe0eb36..118378b0b14678 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-badge/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-badge/index.tsx @@ -1,3 +1,4 @@ +import classNames from 'classnames' import type { FC, ReactNode } from 'react' type ModelBadgeProps = { @@ -9,11 +10,10 @@ const ModelBadge: FC = ({ children, }) => { return ( -
+
{children}
) diff --git a/web/app/components/header/account-setting/model-provider-page/model-name/index.tsx b/web/app/components/header/account-setting/model-provider-page/model-name/index.tsx index d828c703db7ac3..e4337e96c8e3ec 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-name/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-name/index.tsx @@ -1,4 +1,5 @@ -import type { FC } from 'react' +import type { FC, PropsWithChildren } from 'react' +import classNames from 'classnames' import { modelTypeFormat, sizeFormat, @@ -8,7 +9,7 @@ import type { ModelItem } from '../declarations' import ModelBadge from '../model-badge' import FeatureIcon from '../model-selector/feature-icon' -type ModelNameProps = { +type ModelNameProps = PropsWithChildren<{ modelItem: ModelItem className?: string showModelType?: boolean @@ -18,7 +19,7 @@ type ModelNameProps = { showFeatures?: boolean featuresClassName?: string showContextSize?: boolean -} +}> const ModelName: FC = ({ modelItem, className, @@ -29,6 +30,7 @@ const ModelName: FC = ({ showFeatures, featuresClassName, showContextSize, + children, }) => { const language = useLanguage() @@ -42,21 +44,21 @@ const ModelName: FC = ({ `} >
{modelItem.label[language] || modelItem.label.en_US}
{ showModelType && modelItem.model_type && ( - + {modelTypeFormat(modelItem.model_type)} ) } { modelItem.model_properties.mode && showMode && ( - + {(modelItem.model_properties.mode as string).toLocaleUpperCase()} ) @@ -72,11 +74,12 @@ const ModelName: FC = ({ } { showContextSize && modelItem.model_properties.context_size && ( - + {sizeFormat(modelItem.model_properties.context_size as number)} ) } + {children}
) } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx index e5c8a7aa6bd322..7ff0a9642799d6 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx @@ -13,11 +13,13 @@ import { useLanguage } from '../hooks' import ModelIcon from '../model-icon' import ModelName from '../model-name' // import Tab from './tab' +import ModelBadge from '../model-badge' import AddModelButton from './add-model-button' -import Indicator from '@/app/components/header/indicator' import { Settings01 } from '@/app/components/base/icons/src/vender/line/general' import { ChevronDownDouble } from '@/app/components/base/icons/src/vender/line/arrows' import Button from '@/app/components/base/button' +import Switch from '@/app/components/base/switch' +import { Balance } from '@/app/components/base/icons/src/vender/line/financeAndECommerce' type ModelListProps = { provider: ModelProvider @@ -46,10 +48,7 @@ const ModelList: FC = ({ {t('common.modelProvider.modelsNum', { num: models.length })}
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index 586ca28df96a51..539ea23dab295d 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -334,6 +334,8 @@ const translation = { quotaTip: 'Remaining available free tokens', loadPresets: 'Load Presents', parameters: 'PARAMETERS', + loadBalancing: 'Load Balancing', + configLoadBalancing: 'Config Load Balancing', }, dataSource: { add: 'Add a data source', diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index 0e6feb7cf562c0..c972dd85482dd5 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -334,6 +334,8 @@ const translation = { quotaTip: '剩余免费额度', loadPresets: '加载预设', parameters: '参数', + loadBalancing: '负载均衡', + configLoadBalancing: '设置负载均衡', }, dataSource: { add: '添加数据源', diff --git a/web/tailwind.config.js b/web/tailwind.config.js index bfd64e75d1e3f0..aa8290da8511f2 100644 --- a/web/tailwind.config.js +++ b/web/tailwind.config.js @@ -9,27 +9,30 @@ module.exports = { extend: { colors: { gray: { - 25: '#FCFCFD', - 50: '#F9FAFB', - 100: '#F3F4F6', - 200: '#E5E7EB', - 300: '#D1D5DB', - 400: '#9CA3AF', - 500: '#6B7280', - 700: '#374151', - 800: '#1F2A37', - 900: '#111928', + 25: '#fcfcfd', + 50: '#f9fafb', + 100: '#f2f4f7', + 200: '#eaecf0', + 300: '#d0d5dd', + 400: '#98a2b3', + 500: '#667085', + 700: '#475467', + 600: '#344054', + 800: '#1d2939', + 900: '#101828', }, primary: { - 25: '#F5F8FF', - 50: '#EBF5FF', - 100: '#E1EFFE', - 200: '#C3DDFD', - 300: '#A4CAFE', - 400: '#528BFF', - 500: '#2970FF', - 600: '#1C64F2', - 700: '#1A56DB', + 25: '#f5f8ff', + 50: '#eff4ff', + 100: '#d1e0ff', + 200: '#b2ccff', + 300: '#84adff', + 400: '#528bff', + 500: '#2970ff', + 600: '#155eef', + 700: '#004eeb', + 800: '#0040c1', + 900: '#00359e', }, blue: { 500: '#E1EFFE', From 457526eb7a638ad46f0e9dac88a59ec96be5cf09 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Tue, 14 May 2024 19:45:25 +0800 Subject: [PATCH 017/273] firecrawl web extract --- api/controllers/console/__init__.py | 4 +- .../console/auth/data_source_bearer_auth.py | 13 +- api/controllers/console/datasets/datasets.py | 1 + .../console/datasets/datasets_document.py | 2 +- api/controllers/console/datasets/website.py | 42 ++++++ api/core/indexing_runner.py | 14 ++ .../rag/extractor/entity/datasource_type.py | 2 +- .../rag/extractor/entity/extract_setting.py | 16 +-- api/core/rag/extractor/extract_processor.py | 16 ++- .../rag/extractor/firecrawl/firecrawl_app.py | 64 +-------- .../firecrawl/firecrawl_web_extractor.py | 42 ++---- .../7b45942e39bb_add_api_key_auth_binding.py | 67 +++++++++ api/services/auth/api_key_auth_base.py | 9 ++ api/services/auth/api_key_auth_factory.py | 14 ++ api/services/auth/api_key_auth_service.py | 44 +++--- api/services/auth/firecrawl.py | 60 +++----- api/services/dataset_service.py | 39 ++++++ api/services/website_service.py | 128 ++++++++++++++++++ 18 files changed, 396 insertions(+), 181 deletions(-) create mode 100644 api/controllers/console/datasets/website.py create mode 100644 api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py create mode 100644 api/services/auth/api_key_auth_base.py create mode 100644 api/services/auth/api_key_auth_factory.py create mode 100644 api/services/website_service.py diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 498557cd512e6f..e6ccee079c5909 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -29,13 +29,13 @@ ) # Import auth controllers -from .auth import activate, data_source_oauth, login, oauth +from .auth import activate, data_source_oauth, login, oauth, data_source_bearer_auth # Import billing controllers from .billing import billing # Import datasets controllers -from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing +from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing, website # Import enterprise controllers from .enterprise import enterprise_sso diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index 01cf81e6726968..ab0cdbdc23126c 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -1,14 +1,9 @@ -import logging - -import requests -from flask import current_app, redirect, request from flask_login import current_user from flask_restful import Resource, reqparse from werkzeug.exceptions import Forbidden from controllers.console import api from libs.login import login_required -from libs.oauth_data_source import NotionOAuth from services.auth.api_key_auth_service import ApiKeyAuthService from ..setup import setup_required @@ -23,7 +18,7 @@ def get(self): # The role of the current user in the table must be admin or owner if not current_user.is_admin_or_owner: raise Forbidden() - data_source_api_key_bindings = ApiKeyAuthService.get_provider_auth_list(current_user.tenant_id) + data_source_api_key_bindings = ApiKeyAuthService.get_provider_auth_list(current_user.current_tenant_id) if data_source_api_key_bindings: return { 'settings': [data_source_api_key_binding.to_dict() for data_source_api_key_binding in @@ -42,9 +37,11 @@ def post(self): parser = reqparse.RequestParser() parser.add_argument('category', type=str, required=True, nullable=False, location='json') parser.add_argument('provider', type=str, required=True, nullable=False, location='json') - parser.add_argument('credential', type=dict, required=True, nullable=False, location='json') + parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') args = parser.parse_args() - data_source_api_key_binding = ApiKeyAuthService.create_provider_auth(current_user.tenant_id, args) + ApiKeyAuthService.validate_api_key_auth_args(args) + ApiKeyAuthService.create_provider_auth(current_user.current_tenant_id, args) + return {'result': 'success'}, 200 api.add_resource(ApiKeyAuthDataSource, '/api-key-auth/data-source') diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 30dc6ac8459047..9cfe98b67ebcd5 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -512,6 +512,7 @@ def get(self, vector_type): else: raise ValueError("Unsupported vector db type.") + class DatasetErrorDocs(Resource): @setup_required @login_required diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 9dedcefe0f7ba4..edec06576c7b59 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -250,7 +250,7 @@ def post(self, dataset_id): DocumentService.document_create_args_validate(args) try: - documents, batch = DocumentService.save_document_with_dataset_id(dataset, args, current_user) + documents, batch = DocumentService. save_document_with_dataset_id(dataset, args, current_user) except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) except QuotaExceededError: diff --git a/api/controllers/console/datasets/website.py b/api/controllers/console/datasets/website.py new file mode 100644 index 00000000000000..6caf715aa31958 --- /dev/null +++ b/api/controllers/console/datasets/website.py @@ -0,0 +1,42 @@ +from flask_restful import Resource, reqparse + +from controllers.console import api +from controllers.console.setup import setup_required +from controllers.console.wraps import account_initialization_required +from libs.login import login_required +from services.website_service import WebsiteService + + +class WebsiteCrawlApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def post(self): + parser = reqparse.RequestParser() + parser.add_argument('provider', type=str, choices=['firecrawl'], + required=True, nullable=True, location='json') + parser.add_argument('url', type=str, required=True, nullable=True, location='json') + parser.add_argument('options', type=dict, required=True, nullable=True, location='json') + args = parser.parse_args() + WebsiteService.document_create_args_validate(args) + # crawl url + result = WebsiteService.crawl_url(args) + return result, 200 + + +class WebsiteCrawlStatusApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, job_id: str): + parser = reqparse.RequestParser() + parser.add_argument('provider', type=str, choices=['firecrawl'], required=True, location='args') + args = parser.parse_args() + # get crawl status + result = WebsiteService.get_crawl_status(job_id, args['provider']) + return result, 200 + + +api.add_resource(WebsiteCrawlApi, '/website/crawl') +api.add_resource(WebsiteCrawlStatusApi, '/website/crawl/status/') diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 51e77393d660e8..1de1033cc1c74e 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -376,6 +376,20 @@ def _extract(self, index_processor: BaseIndexProcessor, dataset_document: Datase document_model=dataset_document.doc_form ) text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule['mode']) + elif dataset_document.data_source_type == 'website': + if (not data_source_info or 'provider' not in data_source_info + or 'url' not in data_source_info or 'job_id' not in data_source_info): + raise ValueError("no website import info found") + extract_setting = ExtractSetting( + datasource_type="website", + website_info={ + "provider": data_source_info['provider'], + "job_id": data_source_info['job_id'], + "url": data_source_info['url'] + }, + document_model=dataset_document.doc_form + ) + text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule['mode']) # update document status to splitting self._update_document_index_status( document_id=dataset_document.id, diff --git a/api/core/rag/extractor/entity/datasource_type.py b/api/core/rag/extractor/entity/datasource_type.py index 902835571d94f4..4ce379e96bda71 100644 --- a/api/core/rag/extractor/entity/datasource_type.py +++ b/api/core/rag/extractor/entity/datasource_type.py @@ -4,4 +4,4 @@ class DatasourceType(Enum): FILE = "upload_file" NOTION = "notion_import" - URL = "url" + WEBSITE = "website" diff --git a/api/core/rag/extractor/entity/extract_setting.py b/api/core/rag/extractor/entity/extract_setting.py index 69b17039fbf200..096252f32f5047 100644 --- a/api/core/rag/extractor/entity/extract_setting.py +++ b/api/core/rag/extractor/entity/extract_setting.py @@ -20,18 +20,14 @@ class Config: def __init__(self, **data) -> None: super().__init__(**data) -class FirecrawlInfo(BaseModel): + +class WebsiteInfo(BaseModel): """ - Firecrawl import info. + website import info. """ + provider: str + job_id: str url: str - mode: str - ## [Review] Not sure if these belong here - firecrawl_api_key: str - firecrawl_base_url: str - ## --- - document: Document = None - tenant_id: str class Config: arbitrary_types_allowed = True @@ -47,7 +43,7 @@ class ExtractSetting(BaseModel): datasource_type: str upload_file: UploadFile = None notion_info: NotionInfo = None - firecrawl_info: FirecrawlInfo = None + website_info: WebsiteInfo = None document_model: str = None class Config: diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index dc416d5d376b3e..b18855db45c54d 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -142,12 +142,14 @@ def extract(cls, extract_setting: ExtractSetting, is_automatic: bool = False, tenant_id=extract_setting.notion_info.tenant_id, ) return extractor.extract() - elif extract_setting.datasource_type == DatasourceType.URL.value: - # [Review] Not sure if api key and base url belong here. - extractor = FirecrawlWebExtractor( - api_key=extract_setting.firecrawl_info.firecrawl_api_key, - base_url=extract_setting.firecrawl_info.firecrawl_base_url, - url=extract_setting.firecrawl_info.url, mode=extract_setting.firecrawl_info.mode) - return extractor.extract() + elif extract_setting.datasource_type == DatasourceType.WEBSITE.value: + if extract_setting.website_info.provider == 'firecrawl': + extractor = FirecrawlWebExtractor( + url=extract_setting.website_info.url, + job_id=extract_setting.website_info.job_id + ) + return extractor.extract() + else: + raise ValueError(f"Unsupported website provider: {extract_setting.website_info.provider}") else: raise ValueError(f"Unsupported datasource type: {extract_setting.datasource_type}") diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index bdb7c8e247aa7d..559ee3ba59ba72 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -1,13 +1,12 @@ -import os import time import requests class FirecrawlApp: - def __init__(self, api_key=None, base_url='https://api.firecrawl.dev'): - self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') - self.base_url = base_url or os.getenv('FIRECRAWL_BASE_URL') + def __init__(self, api_key=None, base_url=None): + self.api_key = api_key + self.base_url = base_url or 'https://api.firecrawl.dev' if self.api_key is None and self.base_url == 'https://api.firecrawl.dev': raise ValueError('No API key provided') @@ -36,34 +35,8 @@ def scrape_url(self, url, params=None): raise Exception(f'Failed to scrape URL. Status code: {response.status_code}. Error: {error_message}') else: raise Exception(f'Failed to scrape URL. Status code: {response.status_code}') - - def search(self, query, params=None): - headers = { - 'Content-Type': 'application/json', - 'Authorization': f'Bearer {self.api_key}' - } - json_data = {'query': query} - if params: - json_data.update(params) - response = requests.post( - f'{self.base_url}/v0/search', - headers=headers, - json=json_data - ) - if response.status_code == 200: - response = response.json() - if response['success'] == True: - return response['data'] - else: - raise Exception(f'Failed to search. Error: {response["error"]}') - - elif response.status_code in [402, 409, 500]: - error_message = response.json().get('error', 'Unknown error occurred') - raise Exception(f'Failed to search. Status code: {response.status_code}. Error: {error_message}') - else: - raise Exception(f'Failed to search. Status code: {response.status_code}') - def crawl_url(self, url, params=None, wait_until_done=True, polling_interval=2, timeout=500): + def crawl_url(self, url, params=None) -> str: start_time = time.time() headers = self._prepare_headers() json_data = {'url': url} @@ -72,21 +45,12 @@ def crawl_url(self, url, params=None, wait_until_done=True, polling_interval=2, response = self._post_request(f'{self.base_url}/v0/crawl', json_data, headers) if response.status_code == 200: job_id = response.json().get('jobId') - if wait_until_done: - while True: - elapsed_time = time.time() - start_time - if elapsed_time > timeout: - raise Exception('Firecrawl: Crawl job timed out.') - result = self._monitor_job_status(job_id, headers, polling_interval) - if result is not None: - return result - else: - return {'jobId': job_id} + return job_id else: self._handle_error(response, 'start crawl job') - def check_crawl_status(self, job_id): + def check_crawl_status(self, job_id) -> dict: headers = self._prepare_headers() response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) if response.status_code == 200: @@ -117,22 +81,6 @@ def _get_request(self, url, headers, retries=3, backoff_factor=0.5): else: return response return response - - def _monitor_job_status(self, job_id, headers, polling_interval): - status_response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) - if status_response.status_code == 200: - status_data = status_response.json() - if status_data['status'] == 'completed': - if 'data' in status_data: - return status_data['data'] - else: - raise Exception('Crawl job completed but no data was returned') - elif status_data['status'] in ['active', 'paused', 'pending', 'queued']: - time.sleep(max(polling_interval, 2)) # Wait for the specified polling_interval before checking again - else: - raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}') - else: - self._handle_error(status_response, 'check crawl status') def _handle_error(self, response, action): if response.status_code in [402, 409, 500]: diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index f0d03956e7461c..5ff88e269d007f 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -1,6 +1,6 @@ from core.rag.extractor.extractor_base import BaseExtractor -from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp from core.rag.models.document import Document +from services.website_service import WebsiteService class FirecrawlWebExtractor(BaseExtractor): @@ -20,42 +20,16 @@ class FirecrawlWebExtractor(BaseExtractor): def __init__( self, url: str, - api_key: str, - base_url: str = 'https://api.firecrawl.dev', - mode: str = 'crawl', + job_id: str ): """Initialize with url, api_key, base_url and mode.""" self._url = url - self._api_key = api_key - self._base_url = base_url - self._mode = mode - self._firecrawl_app = FirecrawlApp(api_key=self._api_key, base_url=self._base_url) + self.job_id = job_id def extract(self) -> list[Document]: + """Extract content from the URL.""" documents = [] - if self._mode == 'scrape': - content = self._scrape_url() - if content: - documents.append(self._create_document(content)) - elif self._mode in ['crawl', 'crawl_return_urls']: - items = self._crawl_url(return_only_urls=(self._mode == 'crawl_return_urls')) - for item in items: - if item: - documents.append(self._create_document(item, is_url=(self._mode == 'crawl_return_urls'))) - return documents - - def _create_document(self, content, is_url=False): - if is_url: - return Document(page_content=content.get('url', '')) - else: - return Document(page_content=content.get('markdown', '')) - - def _scrape_url(self): - return self._firecrawl_app.scrape_url(self._url) - - def _crawl_url(self, return_only_urls=False): - return self._firecrawl_app.crawl_url(self._url, { - "crawlerOptions": { - "returnOnlyUrls": return_only_urls - } - }) + document = WebsiteService.get_crawl_url_data(self.job_id, 'firecrawl', self._url) + if document: + documents.append(document) + return [] \ No newline at end of file diff --git a/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py b/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py new file mode 100644 index 00000000000000..1591e87f7ee592 --- /dev/null +++ b/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py @@ -0,0 +1,67 @@ +"""add-api-key-auth-binding + +Revision ID: 7b45942e39bb +Revises: 47cc7df8c4f3 +Create Date: 2024-05-14 07:31:29.702766 + +""" +from alembic import op +import models as models +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '7b45942e39bb' +down_revision = '47cc7df8c4f3' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('data_source_api_key_auth_bindings', + sa.Column('id', models.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('tenant_id', models.StringUUID(), nullable=False), + sa.Column('category', sa.String(length=255), nullable=False), + sa.Column('provider', sa.String(length=255), nullable=False), + sa.Column('credentials', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('disabled', sa.Boolean(), server_default=sa.text('false'), nullable=True), + sa.PrimaryKeyConstraint('id', name='data_source_api_key_auth_binding_pkey') + ) + with op.batch_alter_table('data_source_api_key_auth_bindings', schema=None) as batch_op: + batch_op.create_index('data_source_api_key_auth_binding_provider_idx', ['provider'], unique=False) + batch_op.create_index('data_source_api_key_auth_binding_tenant_id_idx', ['tenant_id'], unique=False) + + with op.batch_alter_table('data_source_bindings', schema=None) as batch_op: + batch_op.drop_index('source_binding_tenant_id_idx') + batch_op.drop_index('source_info_idx') + + op.rename_table('data_source_bindings', 'data_source_oauth_bindings') + + with op.batch_alter_table('data_source_oauth_bindings', schema=None) as batch_op: + batch_op.create_index('source_binding_tenant_id_idx', ['tenant_id'], unique=False) + batch_op.create_index('source_info_idx', ['source_info'], unique=False, postgresql_using='gin') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + + with op.batch_alter_table('data_source_oauth_bindings', schema=None) as batch_op: + batch_op.drop_index('source_info_idx', postgresql_using='gin') + batch_op.drop_index('source_binding_tenant_id_idx') + + op.rename_table('data_source_oauth_bindings', 'data_source_bindings') + + with op.batch_alter_table('data_source_bindings', schema=None) as batch_op: + batch_op.create_index('source_info_idx', ['source_info'], unique=False) + batch_op.create_index('source_binding_tenant_id_idx', ['tenant_id'], unique=False) + + with op.batch_alter_table('data_source_api_key_auth_bindings', schema=None) as batch_op: + batch_op.drop_index('data_source_api_key_auth_binding_tenant_id_idx') + batch_op.drop_index('data_source_api_key_auth_binding_provider_idx') + + op.drop_table('data_source_api_key_auth_bindings') + # ### end Alembic commands ### diff --git a/api/services/auth/api_key_auth_base.py b/api/services/auth/api_key_auth_base.py new file mode 100644 index 00000000000000..8484e9c410e2a6 --- /dev/null +++ b/api/services/auth/api_key_auth_base.py @@ -0,0 +1,9 @@ +from abc import ABC + + +class ApiKeyAuthBase(ABC): + def __init__(self, credentials: dict): + self.credentials = credentials + + def validate_credentials(self): + raise NotImplementedError diff --git a/api/services/auth/api_key_auth_factory.py b/api/services/auth/api_key_auth_factory.py new file mode 100644 index 00000000000000..ccd0023c44d84d --- /dev/null +++ b/api/services/auth/api_key_auth_factory.py @@ -0,0 +1,14 @@ + +from services.auth.firecrawl import FirecrawlAuth + + +class ApiKeyAuthFactory: + + def __init__(self, provider: str, credentials: dict): + if provider == 'firecrawl': + self.auth = FirecrawlAuth(credentials) + else: + raise ValueError('Invalid provider') + + def validate_credentials(self): + return self.auth.validate_credentials() diff --git a/api/services/auth/api_key_auth_service.py b/api/services/auth/api_key_auth_service.py index 8630eb6615ff3f..6fe572ebcd779b 100644 --- a/api/services/auth/api_key_auth_service.py +++ b/api/services/auth/api_key_auth_service.py @@ -1,14 +1,8 @@ -import uuid -from typing import List - -from flask_login import current_user -from sqlalchemy import func -from werkzeug.exceptions import NotFound +import json from extensions.ext_database import db -from models.dataset import Dataset -from models.model import App, Tag, TagBinding from models.source import DataSourceApiKeyAuthBinding +from services.auth.api_key_auth_factory import ApiKeyAuthFactory class ApiKeyAuthService: @@ -22,13 +16,27 @@ def get_provider_auth_list(tenant_id: str) -> list: return data_source_api_key_bindings @staticmethod - def create_provider_auth(tenant_id: str, args: dict) -> DataSourceApiKeyAuthBinding: - data_source_api_key_binding = DataSourceApiKeyAuthBinding() - data_source_api_key_binding.tenant_id = tenant_id - data_source_api_key_binding.category = args['category'] - data_source_api_key_binding.provider = args['provider'] - data_source_api_key_binding.credentials = args['credential'] - db.session.add(data_source_api_key_binding) - db.session.commit() - return data_source_api_key_binding - pass + def create_provider_auth(tenant_id: str, args: dict): + auth_result = ApiKeyAuthFactory(args['provider'], args['credentials']).validate_credentials() + if auth_result: + data_source_api_key_binding = DataSourceApiKeyAuthBinding() + data_source_api_key_binding.tenant_id = tenant_id + data_source_api_key_binding.category = args['category'] + data_source_api_key_binding.provider = args['provider'] + data_source_api_key_binding.credentials = json.dumps(args['credentials'], ensure_ascii=False) + db.session.add(data_source_api_key_binding) + db.session.commit() + + @classmethod + def validate_api_key_auth_args(cls, args): + if 'category' not in args or not args['category']: + raise ValueError('category is required') + if 'provider' not in args or not args['provider']: + raise ValueError('provider is required') + if 'credentials' not in args or not args['credentials']: + raise ValueError('credentials is required') + if not isinstance(args['credentials'], dict): + raise ValueError('credentials must be a dictionary') + if 'auth_type' not in args['credentials'] or not args['credentials']['auth_type']: + raise ValueError('auth_type is required') + diff --git a/api/services/auth/firecrawl.py b/api/services/auth/firecrawl.py index f76b976739c226..ee219a1bcb445b 100644 --- a/api/services/auth/firecrawl.py +++ b/api/services/auth/firecrawl.py @@ -1,14 +1,21 @@ -import os import requests +from services.auth.api_key_auth_base import ApiKeyAuthBase -class FirecrawlAuth: - def __init__(self, api_key=None, endpoint=None): - self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY') - if self.api_key is None: + +class FirecrawlAuth(ApiKeyAuthBase): + def __init__(self, credentials: dict): + super().__init__(credentials) + auth_type = credentials.get('auth_type') + if auth_type != 'bearer': + raise ValueError('Invalid auth type, Firecrawl auth type must be Bearer') + self.api_key = credentials.get('config').get('api_key', None) + self.base_url = credentials.get('config').get('base_url', 'https://api.firecrawl.dev') + + if not self.api_key: raise ValueError('No API key provided') - def _validate_credentials(self): + def validate_credentials(self): headers = self._prepare_headers() options = { 'url': 'https://example.com', @@ -21,19 +28,11 @@ def _validate_credentials(self): 'onlyMainContent': True } } - response = self._post_request('https://api.firecrawl.dev/v0/crawl', options, headers) + response = self._post_request(f'{self.base_url}/v0/crawl', options, headers) if response.status_code == 200: return True else: - self._handle_error(response, 'start crawl job') - - def check_crawl_status(self, job_id): - headers = self._prepare_headers() - response = self._get_request(f'https://api.firecrawl.dev/v0/crawl/status/{job_id}', headers) - if response.status_code == 200: - return response.json() - else: - self._handle_error(response, 'check crawl status') + self._handle_error(response) def _prepare_headers(self): return { @@ -44,32 +43,9 @@ def _prepare_headers(self): def _post_request(self, url, data, headers): return requests.post(url, headers=headers, json=data) - def _get_request(self, url, headers): - return requests.get(url, headers=headers) - - def _monitor_job_status(self, job_id, headers, timeout): - import time - while True: - status_response = self._get_request(f'https://api.firecrawl.dev/v0/crawl/status/{job_id}', headers) - if status_response.status_code == 200: - status_data = status_response.json() - if status_data['status'] == 'completed': - if 'data' in status_data: - return status_data['data'] - else: - raise Exception('Crawl job completed but no data was returned') - elif status_data['status'] in ['active', 'paused', 'pending', 'queued']: - if timeout < 2: - timeout = 2 - time.sleep(timeout) # Wait for the specified timeout before checking again - else: - raise Exception(f'Crawl job failed or was stopped. Status: {status_data["status"]}') - else: - self._handle_error(status_response, 'check crawl status') - - def _handle_error(self, response, action): + def _handle_error(self, response): if response.status_code in [402, 409, 500]: error_message = response.json().get('error', 'Unknown error occurred') - raise Exception(f'Failed to {action}. Status code: {response.status_code}. Error: {error_message}') + raise Exception(f'Failed to authorize. Status code: {response.status_code}. Error: {error_message}') else: - raise Exception(f'Unexpected error occurred while trying to {action}. Status code: {response.status_code}') + raise Exception(f'Unexpected error occurred while trying to authorize. Status code: {response.status_code}') diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 0ceef2f800be62..86ad29e3164055 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -523,6 +523,9 @@ def save_document_with_dataset_id(dataset: Dataset, document_data: dict, notion_info_list = document_data["data_source"]['info_list']['notion_info_list'] for notion_info in notion_info_list: count = count + len(notion_info['pages']) + elif document_data["data_source"]["type"] == "website": + website_info = document_data["data_source"]['info_list']['website_info'] + count = len(website_info['urls']) batch_upload_limit = int(current_app.config['BATCH_UPLOAD_LIMIT']) if count > batch_upload_limit: raise ValueError(f"You have reached the batch upload limit of {batch_upload_limit}.") @@ -695,6 +698,26 @@ def save_document_with_dataset_id(dataset: Dataset, document_data: dict, # delete not selected documents if len(exist_document) > 0: clean_notion_document_task.delay(list(exist_document.values()), dataset.id) + elif document_data["data_source"]["type"] == "website": + website_info = document_data["data_source"]['info_list']['website_info'] + urls = website_info['urls'] + for url in urls: + data_source_info = { + 'url': url, + 'provider': website_info['provider'], + 'job_id': website_info['job_id'] + } + document = DocumentService.build_document(dataset, dataset_process_rule.id, + document_data["data_source"]["type"], + document_data["doc_form"], + document_data["doc_language"], + data_source_info, created_from, position, + account, website_info['url'], batch) + db.session.add(document) + db.session.flush() + document_ids.append(document.id) + documents.append(document) + position += 1 db.session.commit() # trigger async task @@ -813,6 +836,15 @@ def update_document_with_dataset_id(dataset: Dataset, document_data: dict, "notion_page_icon": page['page_icon'], "type": page['type'] } + elif document_data["data_source"]["type"] == "website": + website_info = document_data["data_source"]['info_list']['website_info'] + urls = website_info['urls'] + for url in urls: + data_source_info = { + 'url': url, + 'provider': website_info['provider'], + 'job_id': website_info['job_id'] + } document.data_source_type = document_data["data_source"]["type"] document.data_source_info = json.dumps(data_source_info) document.name = file_name @@ -851,6 +883,9 @@ def save_document_without_dataset_id(tenant_id: str, document_data: dict, accoun notion_info_list = document_data["data_source"]['info_list']['notion_info_list'] for notion_info in notion_info_list: count = count + len(notion_info['pages']) + elif document_data["data_source"]["type"] == "website": + website_info = document_data["data_source"]['info_list']['website_info'] + count = len(website_info['urls']) batch_upload_limit = int(current_app.config['BATCH_UPLOAD_LIMIT']) if count > batch_upload_limit: raise ValueError(f"You have reached the batch upload limit of {batch_upload_limit}.") @@ -951,6 +986,10 @@ def data_source_args_validate(cls, args: dict): if 'notion_info_list' not in args['data_source']['info_list'] or not args['data_source']['info_list'][ 'notion_info_list']: raise ValueError("Notion source info is required") + if args['data_source']['type'] == 'website': + if 'website_info' not in args['data_source']['info_list'] or not args['data_source']['info_list'][ + 'website_info']: + raise ValueError("Website source info is required") @classmethod def process_rule_args_validate(cls, args: dict): diff --git a/api/services/website_service.py b/api/services/website_service.py new file mode 100644 index 00000000000000..11d1b0d0070dc0 --- /dev/null +++ b/api/services/website_service.py @@ -0,0 +1,128 @@ +import json + +from flask_login import current_user +from werkzeug.exceptions import NotFound + +from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp +from extensions.ext_database import db +from models.source import DataSourceApiKeyAuthBinding + + +class WebsiteService: + + @classmethod + def document_create_args_validate(cls, args: dict): + if 'url' not in args or not args['url']: + raise ValueError('url is required') + if 'options' not in args or not args['options']: + raise ValueError('options is required') + if 'limit' not in args['options'] or not args['options']['limit']: + raise ValueError('limit is required') + + @classmethod + def crawl_url(cls, args: dict) -> dict: + provider = args.get('provider') + url = args.get('url') + options = args.get('options') + if provider == 'firecrawl': + data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( + DataSourceApiKeyAuthBinding.tenant_id == current_user.current_tenant_id, + DataSourceApiKeyAuthBinding.category == 'website', + DataSourceApiKeyAuthBinding.provider == 'firecrawl', + DataSourceApiKeyAuthBinding.disabled.is_(False) + ).first() + if not data_source_api_key_bindings: + raise NotFound('Firecrawl API key not found') + credentials = json.loads(data_source_api_key_bindings.credentials) + firecrawl_app = FirecrawlApp(api_key=credentials.get('config').get('api_key'), + base_url=credentials.get('config').get('base_url', None)) + crawl_sub_pages = options.get('crawl_sub_pages', False) + only_main_content = options.get('only_main_content', False) + if not crawl_sub_pages: + params = { + 'crawlerOptions': { + "includes": [], + "excludes": [], + "generateImgAltText": True, + "maxDepth": 1, + "limit": 1, + 'returnOnlyUrls': False, + 'pageOptions': { + 'onlyMainContent': only_main_content, + "includeHtml": False + } + } + } + else: + includes = ','.join(options.get('includes')) if options.get('includes') else [] + excludes = ','.join(options.get('excludes')) if options.get('excludes') else [] + params = { + 'crawlerOptions': { + "includes": includes if includes else [], + "excludes": excludes if excludes else [], + "generateImgAltText": True, + "maxDepth": options.get('max_depth', 1), + "limit": options.get('limit', 1), + 'returnOnlyUrls': False, + 'pageOptions': { + 'onlyMainContent': only_main_content, + "includeHtml": False + } + } + } + job_id = firecrawl_app.crawl_url(url, params) + return { + 'status': 'active', + 'job_id': job_id + } + else: + raise ValueError('Invalid provider') + + @classmethod + def get_crawl_status(cls, job_id: str, provider: str) -> dict: + if provider == 'firecrawl': + data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( + DataSourceApiKeyAuthBinding.tenant_id == current_user.current_tenant_id, + DataSourceApiKeyAuthBinding.category == 'website', + DataSourceApiKeyAuthBinding.provider == 'firecrawl', + DataSourceApiKeyAuthBinding.disabled.is_(False) + ).first() + if not data_source_api_key_bindings: + raise NotFound('Firecrawl API key not found') + credentials = json.loads(data_source_api_key_bindings.credentials) + firecrawl_app = FirecrawlApp(api_key=credentials.get('config').get('api_key'), + base_url=credentials.get('config').get('base_url', None)) + result = firecrawl_app.check_crawl_status(job_id) + crawl_status_data = { + 'status': result.get('status', 'active'), + 'job_id': job_id, + 'data': result.get('data', []) + } + else: + raise ValueError('Invalid provider') + return crawl_status_data + + @classmethod + def get_crawl_url_data(cls, job_id: str, provider: str, url: str) -> dict: + if provider == 'firecrawl': + data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( + DataSourceApiKeyAuthBinding.tenant_id == current_user.current_tenant_id, + DataSourceApiKeyAuthBinding.category == 'website', + DataSourceApiKeyAuthBinding.provider == 'firecrawl', + DataSourceApiKeyAuthBinding.disabled.is_(False) + ).first() + if not data_source_api_key_bindings: + raise NotFound('Firecrawl API key not found') + credentials = json.loads(data_source_api_key_bindings.credentials) + firecrawl_app = FirecrawlApp(api_key=credentials.get('config').get('api_key'), + base_url=credentials.get('config').get('base_url', None)) + result = firecrawl_app.check_crawl_status(job_id) + if result.get('status') != 'completed': + raise ValueError('Crawl job is not completed') + data = result.get('data') + for item in data: + if item.get('data'): + if item.get('data').get('metadata').get('sourceURL') == url: + return item.get('data').get('markdown') + else: + raise ValueError('Invalid provider') \ No newline at end of file From 0628cefc40ab7c7206bd8536878c7b91ed1f0d80 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Wed, 15 May 2024 11:00:03 +0800 Subject: [PATCH 018/273] add document rename function --- .../console/datasets/datasets_document.py | 24 +++++++++++++++++++ api/services/dataset_service.py | 21 ++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 9dedcefe0f7ba4..2c1b5c6a4c47d8 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -924,6 +924,28 @@ def post(self, dataset_id): return {'result': 'success'}, 204 +class DocumentRenameApi(DocumentResource): + @setup_required + @login_required + @account_initialization_required + @marshal_with(document_fields) + def post(self, dataset_id, document_id): + # The role of the current user in the ta table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + + parser = reqparse.RequestParser() + parser.add_argument('name', type=str, required=True, nullable=False, location='json') + args = parser.parse_args() + + try: + document = DocumentService.rename_document(dataset_id, document_id, args['name']) + except services.errors.document.DocumentIndexingError: + raise DocumentIndexingError('Cannot delete document during indexing.') + + return document + + api.add_resource(GetProcessRuleApi, '/datasets/process-rule') api.add_resource(DatasetDocumentListApi, '/datasets//documents') @@ -950,3 +972,5 @@ def post(self, dataset_id): api.add_resource(DocumentPauseApi, '/datasets//documents//processing/pause') api.add_resource(DocumentRecoverApi, '/datasets//documents//processing/resume') api.add_resource(DocumentRetryApi, '/datasets//retry') +api.add_resource(DocumentRenameApi, + '/datasets//documents//rename') diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index ab8b8487af8ae3..bed60400ef88bf 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -451,6 +451,27 @@ def delete_document(document): db.session.delete(document) db.session.commit() + @staticmethod + def rename_document(dataset_id: str, document_id: str, name: str) -> Document: + dataset = DatasetService.get_dataset(dataset_id) + if not dataset: + raise ValueError('Dataset not found.') + + document = DocumentService.get_document(dataset_id, document_id) + + if not document: + raise ValueError('Document not found.') + + if document.tenant_id != current_user.current_tenant_id: + raise ValueError('No permission.') + + document.name = name + + db.session.add(document) + db.session.commit() + + return document + @staticmethod def pause_document(document): if document.indexing_status not in ["waiting", "parsing", "cleaning", "splitting", "indexing"]: From 47c4b73ded4a384e1519b407dd1cebee2b9f5ce4 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Wed, 15 May 2024 14:50:03 +0800 Subject: [PATCH 019/273] support variable input in question classifier node's instruction --- .../question_classifier_node.py | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/api/core/workflow/nodes/question_classifier/question_classifier_node.py b/api/core/workflow/nodes/question_classifier/question_classifier_node.py index af1e68b92a6c74..e538edb9146c30 100644 --- a/api/core/workflow/nodes/question_classifier/question_classifier_node.py +++ b/api/core/workflow/nodes/question_classifier/question_classifier_node.py @@ -12,6 +12,7 @@ from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate from core.prompt.simple_prompt_transform import ModelMode from core.prompt.utils.prompt_message_util import PromptMessageUtil +from core.prompt.utils.prompt_template_parser import PromptTemplateParser from core.workflow.entities.base_node_data_entities import BaseNodeData from core.workflow.entities.node_entities import NodeRunMetadataKey, NodeRunResult, NodeType from core.workflow.entities.variable_pool import VariablePool @@ -26,6 +27,7 @@ QUESTION_CLASSIFIER_USER_PROMPT_2, QUESTION_CLASSIFIER_USER_PROMPT_3, ) +from core.workflow.utils.variable_template_parser import VariableTemplateParser from libs.json_in_md_parser import parse_and_check_json_markdown from models.workflow import WorkflowNodeExecutionStatus @@ -47,6 +49,9 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: model_instance, model_config = self._fetch_model_config(node_data.model) # fetch memory memory = self._fetch_memory(node_data.memory, variable_pool, model_instance) + # fetch instruction + instruction = self._format_instruction(node_data.instruction, variable_pool) + node_data.instruction = instruction # fetch prompt messages prompt_messages, stop = self._fetch_prompt( node_data=node_data, @@ -269,8 +274,30 @@ def _get_prompt_template(self, node_data: QuestionClassifierNodeData, query: str text=QUESTION_CLASSIFIER_COMPLETION_PROMPT.format(histories=memory_str, input_text=input_text, categories=json.dumps(categories), - classification_instructions=instruction, ensure_ascii=False) + classification_instructions=instruction, + ensure_ascii=False) ) else: raise ValueError(f"Model mode {model_mode} not support.") + + def _format_instruction(self, instruction: str, variable_pool: VariablePool) -> str: + inputs = {} + + variable_selectors = [] + variable_template_parser = VariableTemplateParser(template=instruction) + variable_selectors.extend(variable_template_parser.extract_variable_selectors()) + for variable_selector in variable_selectors: + variable_value = variable_pool.get_variable_value(variable_selector.value_selector) + if variable_value is None: + raise ValueError(f'Variable {variable_selector.variable} not found') + + inputs[variable_selector.variable] = variable_value + + prompt_template = PromptTemplateParser(template=instruction, with_variable_tmpl=True) + prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs} + + instruction = prompt_template.format( + prompt_inputs + ) + return instruction From 1a03d3057146f68865612a28eb45ee95ac2d12e6 Mon Sep 17 00:00:00 2001 From: takatost Date: Fri, 10 May 2024 17:37:09 +0800 Subject: [PATCH 020/273] add load balancing models --- api/models/provider.py | 53 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/api/models/provider.py b/api/models/provider.py index eb6ec4beb46ee8..4c14c33f095cee 100644 --- a/api/models/provider.py +++ b/api/models/provider.py @@ -47,7 +47,7 @@ class Provider(db.Model): id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) - provider_name = db.Column(db.String(40), nullable=False) + provider_name = db.Column(db.String(255), nullable=False) provider_type = db.Column(db.String(40), nullable=False, server_default=db.text("'custom'::character varying")) encrypted_config = db.Column(db.Text, nullable=True) is_valid = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) @@ -94,7 +94,7 @@ class ProviderModel(db.Model): id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) - provider_name = db.Column(db.String(40), nullable=False) + provider_name = db.Column(db.String(255), nullable=False) model_name = db.Column(db.String(255), nullable=False) model_type = db.Column(db.String(40), nullable=False) encrypted_config = db.Column(db.Text, nullable=True) @@ -112,7 +112,7 @@ class TenantDefaultModel(db.Model): id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) - provider_name = db.Column(db.String(40), nullable=False) + provider_name = db.Column(db.String(255), nullable=False) model_name = db.Column(db.String(255), nullable=False) model_type = db.Column(db.String(40), nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) @@ -128,7 +128,7 @@ class TenantPreferredModelProvider(db.Model): id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) - provider_name = db.Column(db.String(40), nullable=False) + provider_name = db.Column(db.String(255), nullable=False) preferred_provider_type = db.Column(db.String(40), nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) @@ -143,7 +143,7 @@ class ProviderOrder(db.Model): id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) tenant_id = db.Column(StringUUID, nullable=False) - provider_name = db.Column(db.String(40), nullable=False) + provider_name = db.Column(db.String(255), nullable=False) account_id = db.Column(StringUUID, nullable=False) payment_product_id = db.Column(db.String(191), nullable=False) payment_id = db.Column(db.String(191)) @@ -157,3 +157,46 @@ class ProviderOrder(db.Model): refunded_at = db.Column(db.DateTime) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + + +class ProviderModelSetting(db.Model): + """ + Provider model settings for record the model enabled status and load balancing status. + """ + __tablename__ = 'provider_model_settings' + __table_args__ = ( + db.PrimaryKeyConstraint('id', name='provider_model_setting_pkey'), + db.Index('provider_model_setting_tenant_provider_model_idx', 'tenant_id', 'provider_name', 'model_type'), + ) + + id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) + tenant_id = db.Column(StringUUID, nullable=False) + provider_name = db.Column(db.String(255), nullable=False) + model_name = db.Column(db.String(255), nullable=False) + model_type = db.Column(db.String(40), nullable=False) + enabled = db.Column(db.Boolean, nullable=False, server_default=db.text('true')) + load_balancing_enabled = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) + created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + + +class LoadBalancingModelConfig(db.Model): + """ + Configurations for load balancing models. + """ + __tablename__ = 'load_balancing_model_configs' + __table_args__ = ( + db.PrimaryKeyConstraint('id', name='load_balancing_model_config_pkey'), + db.Index('load_balancing_model_config_tenant_provider_model_idx', 'tenant_id', 'provider_name', 'model_type'), + ) + + id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) + tenant_id = db.Column(StringUUID, nullable=False) + provider_name = db.Column(db.String(255), nullable=False) + model_name = db.Column(db.String(255), nullable=False) + model_type = db.Column(db.String(40), nullable=False) + name = db.Column(db.String(255), nullable=False) + encrypted_config = db.Column(db.Text, nullable=True) + enabled = db.Column(db.Boolean, nullable=False, server_default=db.text('true')) + created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) From bbc4b461161530afdaca7e2c4c2faca93485d62c Mon Sep 17 00:00:00 2001 From: takatost Date: Fri, 10 May 2024 22:39:44 +0800 Subject: [PATCH 021/273] add lb configs fetch --- api/config.py | 36 ++-- api/controllers/console/workspace/models.py | 4 + api/core/entities/model_entities.py | 12 +- api/core/entities/provider_configuration.py | 91 ++++++++-- api/core/entities/provider_entities.py | 18 ++ api/core/helper/model_provider_cache.py | 1 + api/core/provider_manager.py | 160 +++++++++++++++++- .../4e99a8df00ff_add_load_balancing.py | 126 ++++++++++++++ .../entities/model_provider_entities.py | 13 +- api/services/feature_service.py | 36 ++-- api/services/model_provider_service.py | 16 +- 11 files changed, 453 insertions(+), 60 deletions(-) create mode 100644 api/migrations/versions/4e99a8df00ff_add_load_balancing.py diff --git a/api/config.py b/api/config.py index eba27c7c708146..7ad13f1a4938b1 100644 --- a/api/config.py +++ b/api/config.py @@ -68,6 +68,7 @@ 'INVITE_EXPIRY_HOURS': 72, 'BILLING_ENABLED': 'False', 'CAN_REPLACE_LOGO': 'False', + 'MODEL_LB_ENABLED': 'False', 'ETL_TYPE': 'dify', 'KEYWORD_STORE': 'jieba', 'BATCH_UPLOAD_LIMIT': 20, @@ -117,6 +118,7 @@ def __init__(self): self.LOG_FILE = get_env('LOG_FILE') self.LOG_FORMAT = get_env('LOG_FORMAT') self.LOG_DATEFORMAT = get_env('LOG_DATEFORMAT') + self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED') # The backend URL prefix of the console API. # used to concatenate the login authorization callback or notion integration callback. @@ -198,6 +200,12 @@ def __init__(self): if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') + # ------------------------ + # Code Execution Sandbox Configurations. + # ------------------------ + self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT') + self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY') + # ------------------------ # File Storage Configurations. # ------------------------ @@ -310,6 +318,15 @@ def __init__(self): self.UPLOAD_FILE_SIZE_LIMIT = int(get_env('UPLOAD_FILE_SIZE_LIMIT')) self.UPLOAD_FILE_BATCH_LIMIT = int(get_env('UPLOAD_FILE_BATCH_LIMIT')) self.UPLOAD_IMAGE_FILE_SIZE_LIMIT = int(get_env('UPLOAD_IMAGE_FILE_SIZE_LIMIT')) + self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT') + + # RAG ETL Configurations. + self.ETL_TYPE = get_env('ETL_TYPE') + self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL') + self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE') + + # Tool Configurations. + self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE') # Moderation in app Configurations. self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE')) @@ -361,18 +378,13 @@ def __init__(self): self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE') self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN') - self.ETL_TYPE = get_env('ETL_TYPE') - self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL') + # Platform Billing Configurations. self.BILLING_ENABLED = get_bool_env('BILLING_ENABLED') - self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO') - - self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT') - - self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT') - self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY') - self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED') - self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE') - - self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE') + # ------------------------ + # Enterprise feature Configurations. + # **Before using, please contact business@dify.ai by email to inquire about licensing matters.** + # ------------------------ self.ENTERPRISE_ENABLED = get_bool_env('ENTERPRISE_ENABLED') + self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO') + self.MODEL_LB_ENABLED = get_bool_env('MODEL_LB_ENABLED') diff --git a/api/controllers/console/workspace/models.py b/api/controllers/console/workspace/models.py index 23239b1902539b..e9c51dfc7536ea 100644 --- a/api/controllers/console/workspace/models.py +++ b/api/controllers/console/workspace/models.py @@ -259,6 +259,10 @@ def get(self, model_type): api.add_resource(ModelProviderModelApi, '/workspaces/current/model-providers//models') +api.add_resource(ModelProviderModelApi, '/workspaces/current/model-providers//models/enable', + endpoint='model-provider-model-enable') +api.add_resource(ModelProviderModelApi, '/workspaces/current/model-providers//models/disable', + endpoint='model-provider-model-disable') api.add_resource(ModelProviderModelCredentialApi, '/workspaces/current/model-providers//models/credentials') api.add_resource(ModelProviderModelValidateApi, diff --git a/api/core/entities/model_entities.py b/api/core/entities/model_entities.py index 05719e5b8d7590..9a797c1c95363d 100644 --- a/api/core/entities/model_entities.py +++ b/api/core/entities/model_entities.py @@ -16,6 +16,7 @@ class ModelStatus(Enum): NO_CONFIGURE = "no-configure" QUOTA_EXCEEDED = "quota-exceeded" NO_PERMISSION = "no-permission" + DISABLED = "disabled" class SimpleModelProviderEntity(BaseModel): @@ -43,12 +44,19 @@ def __init__(self, provider_entity: ProviderEntity) -> None: ) -class ModelWithProviderEntity(ProviderModel): +class ProviderModelWithStatusEntity(ProviderModel): + """ + Model class for model response. + """ + status: ModelStatus + load_balancing_enabled: bool = False + + +class ModelWithProviderEntity(ProviderModelWithStatusEntity): """ Model with provider entity. """ provider: SimpleModelProviderEntity - status: ModelStatus class DefaultModelProviderEntity(BaseModel): diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index 303034693deb1c..4777f916ab1bb7 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -1,6 +1,7 @@ import datetime import json import logging +from collections import defaultdict from collections.abc import Iterator from json import JSONDecodeError from typing import Optional @@ -8,7 +9,12 @@ from pydantic import BaseModel from core.entities.model_entities import ModelStatus, ModelWithProviderEntity, SimpleModelProviderEntity -from core.entities.provider_entities import CustomConfiguration, SystemConfiguration, SystemConfigurationStatus +from core.entities.provider_entities import ( + CustomConfiguration, + ModelSettings, + SystemConfiguration, + SystemConfigurationStatus, +) from core.helper import encrypter from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType from core.model_runtime.entities.model_entities import FetchFrom, ModelType @@ -39,6 +45,7 @@ class ProviderConfiguration(BaseModel): using_provider_type: ProviderType system_configuration: SystemConfiguration custom_configuration: CustomConfiguration + model_settings: list[ModelSettings] def __init__(self, **data): super().__init__(**data) @@ -522,15 +529,22 @@ def get_provider_models(self, model_type: Optional[ModelType] = None, else: model_types = provider_instance.get_provider_schema().supported_model_types + # Group model settings by model type and model + model_setting_map = defaultdict(dict) + for model_setting in self.model_settings: + model_setting_map[model_setting.model_type][model_setting.model] = model_setting + if self.using_provider_type == ProviderType.SYSTEM: provider_models = self._get_system_provider_models( model_types=model_types, - provider_instance=provider_instance + provider_instance=provider_instance, + model_setting_map=model_setting_map ) else: provider_models = self._get_custom_provider_models( model_types=model_types, - provider_instance=provider_instance + provider_instance=provider_instance, + model_setting_map=model_setting_map ) if only_active: @@ -541,18 +555,27 @@ def get_provider_models(self, model_type: Optional[ModelType] = None, def _get_system_provider_models(self, model_types: list[ModelType], - provider_instance: ModelProvider) -> list[ModelWithProviderEntity]: + provider_instance: ModelProvider, + model_setting_map: dict[ModelType, dict[str, ModelSettings]]) \ + -> list[ModelWithProviderEntity]: """ Get system provider models. :param model_types: model types :param provider_instance: provider instance + :param model_setting_map: model setting map :return: """ provider_models = [] for model_type in model_types: - provider_models.extend( - [ + for m in provider_instance.models(model_type): + status = ModelStatus.ACTIVE + if m.model_type in model_setting_map and m.model in model_setting_map[m.model_type]: + model_setting = model_setting_map[m.model_type][m.model] + if model_setting.enabled is False: + status = ModelStatus.DISABLED + + provider_models.append( ModelWithProviderEntity( model=m.model, label=m.label, @@ -562,11 +585,9 @@ def _get_system_provider_models(self, model_properties=m.model_properties, deprecated=m.deprecated, provider=SimpleModelProviderEntity(self.provider), - status=ModelStatus.ACTIVE + status=status ) - for m in provider_instance.models(model_type) - ] - ) + ) if self.provider.provider not in original_provider_configurate_methods: original_provider_configurate_methods[self.provider.provider] = [] @@ -611,6 +632,13 @@ def _get_system_provider_models(self, if custom_model_schema.model_type not in model_types: continue + status = ModelStatus.ACTIVE + if (custom_model_schema.model_type in model_setting_map + and custom_model_schema.model in model_setting_map[custom_model_schema.model_type]): + model_setting = model_setting_map[custom_model_schema.model_type][custom_model_schema.model] + if model_setting.enabled is False: + status = ModelStatus.DISABLED + provider_models.append( ModelWithProviderEntity( model=custom_model_schema.model, @@ -621,7 +649,7 @@ def _get_system_provider_models(self, model_properties=custom_model_schema.model_properties, deprecated=custom_model_schema.deprecated, provider=SimpleModelProviderEntity(self.provider), - status=ModelStatus.ACTIVE + status=status ) ) @@ -632,16 +660,20 @@ def _get_system_provider_models(self, m.status = ModelStatus.NO_PERMISSION elif not quota_configuration.is_valid: m.status = ModelStatus.QUOTA_EXCEEDED + return provider_models def _get_custom_provider_models(self, model_types: list[ModelType], - provider_instance: ModelProvider) -> list[ModelWithProviderEntity]: + provider_instance: ModelProvider, + model_setting_map: dict[ModelType, dict[str, ModelSettings]]) \ + -> list[ModelWithProviderEntity]: """ Get custom provider models. :param model_types: model types :param provider_instance: provider instance + :param model_setting_map: model setting map :return: """ provider_models = [] @@ -656,6 +688,16 @@ def _get_custom_provider_models(self, models = provider_instance.models(model_type) for m in models: + status = ModelStatus.ACTIVE if credentials else ModelStatus.NO_CONFIGURE + load_balancing_enabled = False + if m.model_type in model_setting_map and m.model in model_setting_map[m.model_type]: + model_setting = model_setting_map[m.model_type][m.model] + if model_setting.enabled is False: + status = ModelStatus.DISABLED + + if len(model_setting.load_balancing_configs) > 1: + load_balancing_enabled = True + provider_models.append( ModelWithProviderEntity( model=m.model, @@ -666,7 +708,8 @@ def _get_custom_provider_models(self, model_properties=m.model_properties, deprecated=m.deprecated, provider=SimpleModelProviderEntity(self.provider), - status=ModelStatus.ACTIVE if credentials else ModelStatus.NO_CONFIGURE + status=status, + load_balancing_enabled=load_balancing_enabled ) ) @@ -690,6 +733,17 @@ def _get_custom_provider_models(self, if not custom_model_schema: continue + status = ModelStatus.ACTIVE + load_balancing_enabled = False + if (custom_model_schema.model_type in model_setting_map + and custom_model_schema.model in model_setting_map[custom_model_schema.model_type]): + model_setting = model_setting_map[custom_model_schema.model_type][custom_model_schema.model] + if model_setting.enabled is False: + status = ModelStatus.DISABLED + + if len(model_setting.load_balancing_configs) > 1: + load_balancing_enabled = True + provider_models.append( ModelWithProviderEntity( model=custom_model_schema.model, @@ -700,12 +754,21 @@ def _get_custom_provider_models(self, model_properties=custom_model_schema.model_properties, deprecated=custom_model_schema.deprecated, provider=SimpleModelProviderEntity(self.provider), - status=ModelStatus.ACTIVE + status=status, + load_balancing_enabled=load_balancing_enabled ) ) return provider_models + def _get_load_balancing_configs(self, model: str) -> dict: + """ + Get load balancing configs. + :param model: model name + :return: + """ + return {} + class ProviderConfigurations(BaseModel): """ diff --git a/api/core/entities/provider_entities.py b/api/core/entities/provider_entities.py index 114dfaf9111ab8..7f36e6c15d19f5 100644 --- a/api/core/entities/provider_entities.py +++ b/api/core/entities/provider_entities.py @@ -72,3 +72,21 @@ class CustomConfiguration(BaseModel): """ provider: Optional[CustomProviderConfiguration] = None models: list[CustomModelConfiguration] = [] + + +class ModelLoadBalancingConfiguration(BaseModel): + """ + Class for model load balancing configuration. + """ + name: str + credentials: dict + + +class ModelSettings(BaseModel): + """ + Model class for model settings. + """ + model: str + model_type: ModelType + enabled: bool = True + load_balancing_configs: list[ModelLoadBalancingConfiguration] = [] diff --git a/api/core/helper/model_provider_cache.py b/api/core/helper/model_provider_cache.py index 81e589f65bf1a6..29cb4acc7d03c2 100644 --- a/api/core/helper/model_provider_cache.py +++ b/api/core/helper/model_provider_cache.py @@ -9,6 +9,7 @@ class ProviderCredentialsCacheType(Enum): PROVIDER = "provider" MODEL = "provider_model" + LOAD_BALANCING_MODEL = "load_balancing_provider_model" class ProviderCredentialsCache: diff --git a/api/core/provider_manager.py b/api/core/provider_manager.py index 0db84d3b6959a6..379902c79f87fc 100644 --- a/api/core/provider_manager.py +++ b/api/core/provider_manager.py @@ -11,6 +11,8 @@ CustomConfiguration, CustomModelConfiguration, CustomProviderConfiguration, + ModelLoadBalancingConfiguration, + ModelSettings, QuotaConfiguration, SystemConfiguration, ) @@ -26,13 +28,16 @@ from extensions import ext_hosting_provider from extensions.ext_database import db from models.provider import ( + LoadBalancingModelConfig, Provider, ProviderModel, + ProviderModelSetting, ProviderQuotaType, ProviderType, TenantDefaultModel, TenantPreferredModelProvider, ) +from services.feature_service import FeatureService class ProviderManager: @@ -98,6 +103,13 @@ def get_configurations(self, tenant_id: str) -> ProviderConfigurations: # Get All preferred provider types of the workspace provider_name_to_preferred_model_provider_records_dict = self._get_all_preferred_model_providers(tenant_id) + # Get All provider model settings + provider_name_to_provider_model_settings_dict = self._get_all_provider_model_settings(tenant_id) + + # Get All load balancing configs + provider_name_to_provider_load_balancing_model_configs_dict \ + = self._get_all_provider_load_balancing_configs(tenant_id) + provider_configurations = ProviderConfigurations( tenant_id=tenant_id ) @@ -167,13 +179,28 @@ def get_configurations(self, tenant_id: str) -> ProviderConfigurations: if has_valid_quota: using_provider_type = ProviderType.SYSTEM + # Get provider load balancing configs + provider_model_settings = provider_name_to_provider_model_settings_dict.get(provider_name) + + # Get provider load balancing configs + provider_load_balancing_configs \ + = provider_name_to_provider_load_balancing_model_configs_dict.get(provider_name) + + # Convert to model settings + model_settings = self._to_model_settings( + provider_entity=provider_entity, + provider_model_settings=provider_model_settings, + load_balancing_model_configs=provider_load_balancing_configs + ) + provider_configuration = ProviderConfiguration( tenant_id=tenant_id, provider=provider_entity, preferred_provider_type=preferred_provider_type, using_provider_type=using_provider_type, system_configuration=system_configuration, - custom_configuration=custom_configuration + custom_configuration=custom_configuration, + model_settings=model_settings ) provider_configurations[provider_name] = provider_configuration @@ -371,7 +398,7 @@ def _get_all_preferred_model_providers(self, tenant_id: str) -> dict[str, Tenant """ Get All preferred provider types of the workspace. - :param tenant_id: + :param tenant_id: workspace id :return: """ preferred_provider_types = db.session.query(TenantPreferredModelProvider) \ @@ -386,6 +413,48 @@ def _get_all_preferred_model_providers(self, tenant_id: str) -> dict[str, Tenant return provider_name_to_preferred_provider_type_records_dict + def _get_all_provider_model_settings(self, tenant_id: str) -> dict[str, list[ProviderModelSetting]]: + """ + Get All provider model settings of the workspace. + + :param tenant_id: workspace id + :return: + """ + provider_model_settings = db.session.query(ProviderModelSetting) \ + .filter( + ProviderModelSetting.tenant_id == tenant_id + ).all() + + provider_name_to_provider_model_settings_dict = defaultdict(list) + for provider_model_setting in provider_model_settings: + (provider_name_to_provider_model_settings_dict[provider_model_setting.provider_name] + .append(provider_model_setting)) + + return provider_name_to_provider_model_settings_dict + + def _get_all_provider_load_balancing_configs(self, tenant_id: str) -> dict[str, list[LoadBalancingModelConfig]]: + """ + Get All provider load balancing configs of the workspace. + + :param tenant_id: workspace id + :return: + """ + model_load_balancing_enabled = FeatureService.get_features(tenant_id).model_load_balancing_enabled + if not model_load_balancing_enabled: + return dict() + + provider_load_balancing_configs = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id + ).all() + + provider_name_to_provider_load_balancing_model_configs_dict = defaultdict(list) + for provider_load_balancing_config in provider_load_balancing_configs: + (provider_name_to_provider_load_balancing_model_configs_dict[provider_load_balancing_config.provider_name] + .append(provider_load_balancing_config)) + + return provider_name_to_provider_load_balancing_model_configs_dict + def _init_trial_provider_records(self, tenant_id: str, provider_name_to_provider_records_dict: dict[str, list]) -> dict[str, list]: """ @@ -759,3 +828,90 @@ def _extract_secret_variables(self, credential_form_schemas: list[CredentialForm secret_input_form_variables.append(credential_form_schema.variable) return secret_input_form_variables + + def _to_model_settings(self, provider_entity: ProviderEntity, + provider_model_settings: Optional[list[ProviderModelSetting]] = None, + load_balancing_model_configs: Optional[list[LoadBalancingModelConfig]] = None) \ + -> list[ModelSettings]: + """ + Convert to model settings. + + :param provider_model_settings: provider model settings include enabled, load balancing enabled + :param load_balancing_model_configs: load balancing model configs + :return: + """ + # Get provider model credential secret variables + model_credential_secret_variables = self._extract_secret_variables( + provider_entity.model_credential_schema.credential_form_schemas + if provider_entity.model_credential_schema else [] + ) + + model_settings = [] + if not provider_model_settings: + return model_settings + + for provider_model_setting in provider_model_settings: + load_balancing_configs = [] + if provider_model_setting.load_balancing_enabled and load_balancing_model_configs: + for load_balancing_model_config in load_balancing_model_configs: + if (load_balancing_model_config.model_name == provider_model_setting.model_name + and load_balancing_model_config.model_type == provider_model_setting.model_type): + if not load_balancing_model_config.enabled: + continue + + if not load_balancing_model_config.encrypted_config: + continue + + provider_model_credentials_cache = ProviderCredentialsCache( + tenant_id=load_balancing_model_config.tenant_id, + identity_id=load_balancing_model_config.id, + cache_type=ProviderCredentialsCacheType.LOAD_BALANCING_MODEL + ) + + # Get cached provider model credentials + cached_provider_model_credentials = provider_model_credentials_cache.get() + + if not cached_provider_model_credentials: + try: + provider_model_credentials = json.loads(load_balancing_model_config.encrypted_config) + except JSONDecodeError: + continue + + # Get decoding rsa key and cipher for decrypting credentials + if self.decoding_rsa_key is None or self.decoding_cipher_rsa is None: + self.decoding_rsa_key, self.decoding_cipher_rsa = encrypter.get_decrypt_decoding( + load_balancing_model_config.tenant_id) + + for variable in model_credential_secret_variables: + if variable in provider_model_credentials: + try: + provider_model_credentials[variable] = encrypter.decrypt_token_with_decoding( + provider_model_credentials.get(variable), + self.decoding_rsa_key, + self.decoding_cipher_rsa + ) + except ValueError: + pass + + # cache provider model credentials + provider_model_credentials_cache.set( + credentials=provider_model_credentials + ) + else: + provider_model_credentials = cached_provider_model_credentials + + load_balancing_configs.append(ModelLoadBalancingConfiguration( + name=load_balancing_model_config.name, + credentials=provider_model_credentials + )) + + model_settings.append( + ModelSettings( + model=provider_model_setting.model_name, + model_type=ModelType.value_of(provider_model_setting.model_type), + enabled=provider_model_setting.enabled, + load_balancing_configs=load_balancing_configs + ) + ) + + return model_settings diff --git a/api/migrations/versions/4e99a8df00ff_add_load_balancing.py b/api/migrations/versions/4e99a8df00ff_add_load_balancing.py new file mode 100644 index 00000000000000..5c14d0b0678b65 --- /dev/null +++ b/api/migrations/versions/4e99a8df00ff_add_load_balancing.py @@ -0,0 +1,126 @@ +"""add load balancing + +Revision ID: 4e99a8df00ff +Revises: 47cc7df8c4f3 +Create Date: 2024-05-10 12:08:09.812736 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '4e99a8df00ff' +down_revision = '47cc7df8c4f3' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('load_balancing_model_configs', + sa.Column('id', models.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('tenant_id', models.StringUUID(), nullable=False), + sa.Column('provider_name', sa.String(length=255), nullable=False), + sa.Column('model_name', sa.String(length=255), nullable=False), + sa.Column('model_type', sa.String(length=40), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('encrypted_config', sa.Text(), nullable=True), + sa.Column('enabled', sa.Boolean(), server_default=sa.text('true'), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='load_balancing_model_config_pkey') + ) + with op.batch_alter_table('load_balancing_model_configs', schema=None) as batch_op: + batch_op.create_index('load_balancing_model_config_tenant_provider_model_idx', ['tenant_id', 'provider_name', 'model_type'], unique=False) + + op.create_table('provider_model_settings', + sa.Column('id', models.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('tenant_id', models.StringUUID(), nullable=False), + sa.Column('provider_name', sa.String(length=255), nullable=False), + sa.Column('model_name', sa.String(length=255), nullable=False), + sa.Column('model_type', sa.String(length=40), nullable=False), + sa.Column('enabled', sa.Boolean(), server_default=sa.text('true'), nullable=False), + sa.Column('load_balancing_enabled', sa.Boolean(), server_default=sa.text('false'), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='provider_model_setting_pkey') + ) + with op.batch_alter_table('provider_model_settings', schema=None) as batch_op: + batch_op.create_index('provider_model_setting_tenant_provider_model_idx', ['tenant_id', 'provider_name', 'model_type'], unique=False) + + with op.batch_alter_table('provider_models', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False) + + with op.batch_alter_table('provider_orders', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False) + + with op.batch_alter_table('providers', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False) + + with op.batch_alter_table('tenant_default_models', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False) + + with op.batch_alter_table('tenant_preferred_model_providers', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.VARCHAR(length=40), + type_=sa.String(length=255), + existing_nullable=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('tenant_preferred_model_providers', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False) + + with op.batch_alter_table('tenant_default_models', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False) + + with op.batch_alter_table('providers', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False) + + with op.batch_alter_table('provider_orders', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False) + + with op.batch_alter_table('provider_models', schema=None) as batch_op: + batch_op.alter_column('provider_name', + existing_type=sa.String(length=255), + type_=sa.VARCHAR(length=40), + existing_nullable=False) + + with op.batch_alter_table('provider_model_settings', schema=None) as batch_op: + batch_op.drop_index('provider_model_setting_tenant_provider_model_idx') + + op.drop_table('provider_model_settings') + with op.batch_alter_table('load_balancing_model_configs', schema=None) as batch_op: + batch_op.drop_index('load_balancing_model_config_tenant_provider_model_idx') + + op.drop_table('load_balancing_model_configs') + # ### end Alembic commands ### diff --git a/api/services/entities/model_provider_entities.py b/api/services/entities/model_provider_entities.py index 6cdd5090ae8751..77bb5e08c32839 100644 --- a/api/services/entities/model_provider_entities.py +++ b/api/services/entities/model_provider_entities.py @@ -4,10 +4,10 @@ from flask import current_app from pydantic import BaseModel -from core.entities.model_entities import ModelStatus, ModelWithProviderEntity +from core.entities.model_entities import ModelWithProviderEntity, ProviderModelWithStatusEntity from core.entities.provider_entities import QuotaConfiguration from core.model_runtime.entities.common_entities import I18nObject -from core.model_runtime.entities.model_entities import ModelType, ProviderModel +from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.entities.provider_entities import ( ConfigurateMethod, ModelCredentialSchema, @@ -79,13 +79,6 @@ def __init__(self, **data) -> None: ) -class ModelResponse(ProviderModel): - """ - Model class for model response. - """ - status: ModelStatus - - class ProviderWithModelsResponse(BaseModel): """ Model class for provider with models response. @@ -95,7 +88,7 @@ class ProviderWithModelsResponse(BaseModel): icon_small: Optional[I18nObject] = None icon_large: Optional[I18nObject] = None status: CustomConfigurationStatus - models: list[ModelResponse] + models: list[ProviderModelWithStatusEntity] def __init__(self, **data) -> None: super().__init__(**data) diff --git a/api/services/feature_service.py b/api/services/feature_service.py index 3cf51d11a012e5..9615793e94f7f0 100644 --- a/api/services/feature_service.py +++ b/api/services/feature_service.py @@ -28,6 +28,7 @@ class FeatureModel(BaseModel): documents_upload_quota: LimitationModel = LimitationModel(size=0, limit=50) docs_processing: str = 'standard' can_replace_logo: bool = False + model_load_balancing_enabled: bool = False class FeatureService: @@ -46,6 +47,7 @@ def get_features(cls, tenant_id: str) -> FeatureModel: @classmethod def _fulfill_params_from_env(cls, features: FeatureModel): features.can_replace_logo = current_app.config['CAN_REPLACE_LOGO'] + features.model_load_balancing_enabled = current_app.config['MODEL_LB_ENABLED'] @classmethod def _fulfill_params_from_billing_api(cls, features: FeatureModel, tenant_id: str): @@ -55,21 +57,31 @@ def _fulfill_params_from_billing_api(cls, features: FeatureModel, tenant_id: str features.billing.subscription.plan = billing_info['subscription']['plan'] features.billing.subscription.interval = billing_info['subscription']['interval'] - features.members.size = billing_info['members']['size'] - features.members.limit = billing_info['members']['limit'] + if 'members' in billing_info: + features.members.size = billing_info['members']['size'] + features.members.limit = billing_info['members']['limit'] - features.apps.size = billing_info['apps']['size'] - features.apps.limit = billing_info['apps']['limit'] + if 'apps' in billing_info: + features.apps.size = billing_info['apps']['size'] + features.apps.limit = billing_info['apps']['limit'] - features.vector_space.size = billing_info['vector_space']['size'] - features.vector_space.limit = billing_info['vector_space']['limit'] + if 'vector_space' in billing_info: + features.vector_space.size = billing_info['vector_space']['size'] + features.vector_space.limit = billing_info['vector_space']['limit'] - features.documents_upload_quota.size = billing_info['documents_upload_quota']['size'] - features.documents_upload_quota.limit = billing_info['documents_upload_quota']['limit'] + if 'documents_upload_quota' in billing_info: + features.documents_upload_quota.size = billing_info['documents_upload_quota']['size'] + features.documents_upload_quota.limit = billing_info['documents_upload_quota']['limit'] - features.annotation_quota_limit.size = billing_info['annotation_quota_limit']['size'] - features.annotation_quota_limit.limit = billing_info['annotation_quota_limit']['limit'] + if 'annotation_quota_limit' in billing_info: + features.annotation_quota_limit.size = billing_info['annotation_quota_limit']['size'] + features.annotation_quota_limit.limit = billing_info['annotation_quota_limit']['limit'] - features.docs_processing = billing_info['docs_processing'] - features.can_replace_logo = billing_info['can_replace_logo'] + if 'docs_processing' in billing_info: + features.docs_processing = billing_info['docs_processing'] + if 'can_replace_logo' in billing_info: + features.can_replace_logo = billing_info['can_replace_logo'] + + if 'model_load_balancing_enabled' in billing_info: + features.model_load_balancing_enabled = billing_info['model_load_balancing_enabled'] diff --git a/api/services/model_provider_service.py b/api/services/model_provider_service.py index 5a4342ae0386db..c30aae829c98d1 100644 --- a/api/services/model_provider_service.py +++ b/api/services/model_provider_service.py @@ -6,7 +6,7 @@ import requests from flask import current_app -from core.entities.model_entities import ModelStatus +from core.entities.model_entities import ModelStatus, ProviderModelWithStatusEntity from core.model_runtime.entities.model_entities import ModelType, ParameterRule from core.model_runtime.model_providers import model_provider_factory from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel @@ -16,7 +16,6 @@ CustomConfigurationResponse, CustomConfigurationStatus, DefaultModelResponse, - ModelResponse, ModelWithProviderEntityResponse, ProviderResponse, ProviderWithModelsResponse, @@ -303,6 +302,9 @@ def get_models_by_model_type(self, tenant_id: str, model_type: str) -> list[Prov if model.deprecated: continue + if model.status != ModelStatus.ACTIVE: + continue + provider_models[model.provider.provider].append(model) # convert to ProviderWithModelsResponse list @@ -313,24 +315,22 @@ def get_models_by_model_type(self, tenant_id: str, model_type: str) -> list[Prov first_model = models[0] - has_active_models = any([model.status == ModelStatus.ACTIVE for model in models]) - providers_with_models.append( ProviderWithModelsResponse( provider=provider, label=first_model.provider.label, icon_small=first_model.provider.icon_small, icon_large=first_model.provider.icon_large, - status=CustomConfigurationStatus.ACTIVE - if has_active_models else CustomConfigurationStatus.NO_CONFIGURE, - models=[ModelResponse( + status=CustomConfigurationStatus.ACTIVE, + models=[ProviderModelWithStatusEntity( model=model.model, label=model.label, model_type=model.model_type, features=model.features, fetch_from=model.fetch_from, model_properties=model.model_properties, - status=model.status + status=model.status, + load_balancing_enabled=model.load_balancing_enabled ) for model in models] ) ) From 7c02770cb7d15495b03192a27b58b373634c82d9 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 13:18:11 +0800 Subject: [PATCH 022/273] refactor version --- api/controllers/console/version.py | 37 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/api/controllers/console/version.py b/api/controllers/console/version.py index a50e4c41a88b41..faf36c4f40020f 100644 --- a/api/controllers/console/version.py +++ b/api/controllers/console/version.py @@ -17,13 +17,19 @@ def get(self): args = parser.parse_args() check_update_url = current_app.config['CHECK_UPDATE_URL'] - if not check_update_url: - return { - 'version': '0.0.0', - 'release_date': '', - 'release_notes': '', - 'can_auto_update': False + result = { + 'version': current_app.config['CURRENT_VERSION'], + 'release_date': '', + 'release_notes': '', + 'can_auto_update': False, + 'features': { + 'can_replace_logo': current_app.config['CAN_REPLACE_LOGO'], + 'model_load_balancing_enabled': current_app.config['MODEL_LB_ENABLED'] } + } + + if not check_update_url: + return result try: response = requests.get(check_update_url, { @@ -31,20 +37,15 @@ def get(self): }) except Exception as error: logging.warning("Check update version error: {}.".format(str(error))) - return { - 'version': args.get('current_version'), - 'release_date': '', - 'release_notes': '', - 'can_auto_update': False - } + result['version'] = args.get('current_version') + return result content = json.loads(response.content) - return { - 'version': content['version'], - 'release_date': content['releaseDate'], - 'release_notes': content['releaseNotes'], - 'can_auto_update': content['canAutoUpdate'] - } + result['version'] = content['version'] + result['release_date'] = content['releaseDate'] + result['release_notes'] = content['releaseNotes'] + result['can_auto_update'] = content['canAutoUpdate'] + return result api.add_resource(VersionApi, '/version') From 87c0df815debf7b2458e08abedac5b72950aeaaf Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 13:48:11 +0800 Subject: [PATCH 023/273] add model enable / disable api --- api/controllers/console/workspace/models.py | 54 ++++++++++++++++- api/core/entities/provider_configuration.py | 64 ++++++++++++++++++++- api/services/model_provider_service.py | 48 ++++++++++++++++ 3 files changed, 163 insertions(+), 3 deletions(-) diff --git a/api/controllers/console/workspace/models.py b/api/controllers/console/workspace/models.py index e9c51dfc7536ea..719d69da957fa2 100644 --- a/api/controllers/console/workspace/models.py +++ b/api/controllers/console/workspace/models.py @@ -175,6 +175,56 @@ def get(self, provider: str): } +class ModelProviderModelEnableApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def patch(self, provider: str): + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + args = parser.parse_args() + + model_provider_service = ModelProviderService() + model_provider_service.enable_model( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'] + ) + + return {'result': 'success'} + + +class ModelProviderModelDisableApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def patch(self, provider: str): + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + args = parser.parse_args() + + model_provider_service = ModelProviderService() + model_provider_service.disable_model( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'] + ) + + return {'result': 'success'} + + class ModelProviderModelValidateApi(Resource): @setup_required @@ -259,9 +309,9 @@ def get(self, model_type): api.add_resource(ModelProviderModelApi, '/workspaces/current/model-providers//models') -api.add_resource(ModelProviderModelApi, '/workspaces/current/model-providers//models/enable', +api.add_resource(ModelProviderModelEnableApi, '/workspaces/current/model-providers//models/enable', endpoint='model-provider-model-enable') -api.add_resource(ModelProviderModelApi, '/workspaces/current/model-providers//models/disable', +api.add_resource(ModelProviderModelDisableApi, '/workspaces/current/model-providers//models/disable', endpoint='model-provider-model-disable') api.add_resource(ModelProviderModelCredentialApi, '/workspaces/current/model-providers//models/credentials') diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index 4777f916ab1bb7..23bf0316e90746 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -28,7 +28,7 @@ from core.model_runtime.model_providers.__base.ai_model import AIModel from core.model_runtime.model_providers.__base.model_provider import ModelProvider from extensions.ext_database import db -from models.provider import Provider, ProviderModel, ProviderType, TenantPreferredModelProvider +from models.provider import Provider, ProviderModel, ProviderModelSetting, ProviderType, TenantPreferredModelProvider logger = logging.getLogger(__name__) @@ -409,6 +409,68 @@ def delete_custom_model_credentials(self, model_type: ModelType, model: str) -> provider_model_credentials_cache.delete() + def enable_model(self, model_type: ModelType, model: str) -> ProviderModelSetting: + """ + Enable model. + :param model_type: model type + :param model: model name + :return: + """ + model_setting = db.session.query(ProviderModelSetting) \ + .filter( + ProviderModelSetting.tenant_id == self.tenant_id, + ProviderModelSetting.provider_name == self.provider.provider, + ProviderModelSetting.model_type == model_type.to_origin_model_type(), + ProviderModelSetting.model_name == model + ).first() + + if model_setting: + model_setting.enabled = True + db.session.commit() + else: + model_setting = ProviderModelSetting( + tenant_id=self.tenant_id, + provider_name=self.provider.provider, + model_type=model_type.to_origin_model_type(), + model_name=model, + enabled=True + ) + db.session.add(model_setting) + db.session.commit() + + return model_setting + + def disable_model(self, model_type: ModelType, model: str) -> ProviderModelSetting: + """ + Disable model. + :param model_type: model type + :param model: model name + :return: + """ + model_setting = db.session.query(ProviderModelSetting) \ + .filter( + ProviderModelSetting.tenant_id == self.tenant_id, + ProviderModelSetting.provider_name == self.provider.provider, + ProviderModelSetting.model_type == model_type.to_origin_model_type(), + ProviderModelSetting.model_name == model + ).first() + + if model_setting: + model_setting.enabled = False + db.session.commit() + else: + model_setting = ProviderModelSetting( + tenant_id=self.tenant_id, + provider_name=self.provider.provider, + model_type=model_type.to_origin_model_type(), + model_name=model, + enabled=False + ) + db.session.add(model_setting) + db.session.commit() + + return model_setting + def get_provider_instance(self) -> ModelProvider: """ Get provider instance. diff --git a/api/services/model_provider_service.py b/api/services/model_provider_service.py index c30aae829c98d1..385af685f98f44 100644 --- a/api/services/model_provider_service.py +++ b/api/services/model_provider_service.py @@ -486,6 +486,54 @@ def switch_preferred_provider(self, tenant_id: str, provider: str, preferred_pro # Switch preferred provider type provider_configuration.switch_preferred_provider_type(preferred_provider_type_enum) + def enable_model(self, tenant_id: str, provider: str, model: str, model_type: str) -> None: + """ + enable model. + + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Enable model + provider_configuration.enable_model( + model=model, + model_type=ModelType.value_of(model_type) + ) + + def disable_model(self, tenant_id: str, provider: str, model: str, model_type: str) -> None: + """ + disable model. + + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Enable model + provider_configuration.disable_model( + model=model, + model_type=ModelType.value_of(model_type) + ) + def free_quota_submit(self, tenant_id: str, provider: str): api_key = os.environ.get("FREE_QUOTA_APPLY_API_KEY") api_base_url = os.environ.get("FREE_QUOTA_APPLY_BASE_URL") From 6a73f5c1454111d8f553821b7d5a4672decb2077 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 18:06:04 +0800 Subject: [PATCH 024/273] add model lb enable / disable api --- api/controllers/console/__init__.py | 2 +- api/controllers/console/feature.py | 7 +- .../workspace/load_balancing_config.py | 203 ++++++++++++++++++ api/core/entities/provider_configuration.py | 82 ++++++- api/services/model_load_balancing_service.py | 61 ++++++ 5 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 api/controllers/console/workspace/load_balancing_config.py create mode 100644 api/services/model_load_balancing_service.py diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 498557cd512e6f..42ca3345e4af46 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -57,4 +57,4 @@ from .tag import tags # Import workspace controllers -from .workspace import account, members, model_providers, models, tool_providers, workspace +from .workspace import account, load_balancing_config, members, model_providers, models, tool_providers, workspace diff --git a/api/controllers/console/feature.py b/api/controllers/console/feature.py index 325652a44794cc..11105b31445096 100644 --- a/api/controllers/console/feature.py +++ b/api/controllers/console/feature.py @@ -1,15 +1,20 @@ from flask_login import current_user from flask_restful import Resource +from libs.login import login_required from services.enterprise.enterprise_feature_service import EnterpriseFeatureService from services.feature_service import FeatureService from . import api -from .wraps import cloud_utm_record +from .setup import setup_required +from .wraps import account_initialization_required, cloud_utm_record class FeatureApi(Resource): + @setup_required + @login_required + @account_initialization_required @cloud_utm_record def get(self): return FeatureService.get_features(current_user.current_tenant_id).dict() diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py new file mode 100644 index 00000000000000..d320b6c60db8d7 --- /dev/null +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -0,0 +1,203 @@ +from flask_restful import Resource, reqparse +from werkzeug.exceptions import Forbidden + +from controllers.console import api +from controllers.console.setup import setup_required +from controllers.console.wraps import account_initialization_required +from core.model_runtime.entities.model_entities import ModelType +from libs.login import current_user, login_required +from models.account import TenantAccountRole +from services.model_load_balancing_service import ModelLoadBalancingService + + +class ModelLoadBalancingEnableApi(Resource): + @setup_required + @login_required + @account_initialization_required + def patch(self, provider: str): + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + args = parser.parse_args() + + # enable model load balancing + model_load_balancing_service = ModelLoadBalancingService() + model_load_balancing_service.enable_model_load_balancing( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'] + ) + + return {'result': 'success'} + + +class ModelLoadBalancingDisableApi(Resource): + @setup_required + @login_required + @account_initialization_required + def patch(self, provider: str): + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + args = parser.parse_args() + + # disable model load balancing + model_load_balancing_service = ModelLoadBalancingService() + model_load_balancing_service.disable_model_load_balancing( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'] + ) + + return {'result': 'success'} + + +class LoadBalancingConfigListApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def get(self, provider: str): + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='args') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='args') + args = parser.parse_args() + + # TODO + + return {} + + @setup_required + @login_required + @account_initialization_required + def post(self, provider: str): + if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + raise Forbidden() + + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + parser.add_argument('name', type=str, required=True, nullable=False, location='json') + parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') + args = parser.parse_args() + + # TODO + + return {} + + +class LoadBalancingConfigApi(Resource): + @setup_required + @login_required + @account_initialization_required + def post(self, provider: str, config_id: str): + if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + raise Forbidden() + + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + parser.add_argument('name', type=str, required=True, nullable=False, location='json') + parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') + args = parser.parse_args() + + # TODO + + return {} + + @setup_required + @login_required + @account_initialization_required + def delete(self, provider: str, config_id: str): + if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + raise Forbidden() + + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='args') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='args') + args = parser.parse_args() + + # TODO + + return {} + + +class LoadBalancingConfigEnableApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def patch(self, provider: str, config_id: str): + if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + raise Forbidden() + + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + args = parser.parse_args() + + # TODO + + return {} + + +class LoadBalancingConfigDisableApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def patch(self, provider: str, config_id: str): + if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + raise Forbidden() + + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='json') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='json') + args = parser.parse_args() + + # TODO + + return {} + + +# Model Load Balancing Feature +api.add_resource(ModelLoadBalancingEnableApi, + '/workspaces/current/model-providers//models/load-balancing-configs/enable') +api.add_resource(ModelLoadBalancingDisableApi, + '/workspaces/current/model-providers//models/load-balancing-configs/disable') + +# Load Balancing Config +api.add_resource(LoadBalancingConfigListApi, + '/workspaces/current/model-providers//models/load-balancing-configs') +api.add_resource(LoadBalancingConfigApi, + '/workspaces/current/model-providers//models/load-balancing-configs/') +api.add_resource(LoadBalancingConfigEnableApi, + '/workspaces/current/model-providers//models/load-balancing-configs//enable') +api.add_resource(LoadBalancingConfigDisableApi, + '/workspaces/current/model-providers//models/load-balancing-configs//disable') diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index 23bf0316e90746..b1f0c6ee76cf74 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -28,7 +28,14 @@ from core.model_runtime.model_providers.__base.ai_model import AIModel from core.model_runtime.model_providers.__base.model_provider import ModelProvider from extensions.ext_database import db -from models.provider import Provider, ProviderModel, ProviderModelSetting, ProviderType, TenantPreferredModelProvider +from models.provider import ( + LoadBalancingModelConfig, + Provider, + ProviderModel, + ProviderModelSetting, + ProviderType, + TenantPreferredModelProvider, +) logger = logging.getLogger(__name__) @@ -471,6 +478,79 @@ def disable_model(self, model_type: ModelType, model: str) -> ProviderModelSetti return model_setting + def enable_model_load_balancing(self, model_type: ModelType, model: str) -> ProviderModelSetting: + """ + Enable model load balancing. + :param model_type: model type + :param model: model name + :return: + """ + load_balancing_config_count = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == self.tenant_id, + LoadBalancingModelConfig.provider_name == self.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model + ).count() + + if load_balancing_config_count <= 1: + raise ValueError('Model load balancing configuration must be more than 1.') + + model_setting = db.session.query(ProviderModelSetting) \ + .filter( + ProviderModelSetting.tenant_id == self.tenant_id, + ProviderModelSetting.provider_name == self.provider.provider, + ProviderModelSetting.model_type == model_type.to_origin_model_type(), + ProviderModelSetting.model_name == model + ).first() + + if model_setting: + model_setting.load_balancing_enabled = True + db.session.commit() + else: + model_setting = ProviderModelSetting( + tenant_id=self.tenant_id, + provider_name=self.provider.provider, + model_type=model_type.to_origin_model_type(), + model_name=model, + load_balancing_enabled=True + ) + db.session.add(model_setting) + db.session.commit() + + return model_setting + + def disable_model_load_balancing(self, model_type: ModelType, model: str) -> ProviderModelSetting: + """ + Disable model load balancing. + :param model_type: model type + :param model: model name + :return: + """ + model_setting = db.session.query(ProviderModelSetting) \ + .filter( + ProviderModelSetting.tenant_id == self.tenant_id, + ProviderModelSetting.provider_name == self.provider.provider, + ProviderModelSetting.model_type == model_type.to_origin_model_type(), + ProviderModelSetting.model_name == model + ).first() + + if model_setting: + model_setting.load_balancing_enabled = False + db.session.commit() + else: + model_setting = ProviderModelSetting( + tenant_id=self.tenant_id, + provider_name=self.provider.provider, + model_type=model_type.to_origin_model_type(), + model_name=model, + load_balancing_enabled=False + ) + db.session.add(model_setting) + db.session.commit() + + return model_setting + def get_provider_instance(self) -> ModelProvider: """ Get provider instance. diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py new file mode 100644 index 00000000000000..aeb9d78da9f12f --- /dev/null +++ b/api/services/model_load_balancing_service.py @@ -0,0 +1,61 @@ +import logging + +from core.model_runtime.entities.model_entities import ModelType +from core.provider_manager import ProviderManager + +logger = logging.getLogger(__name__) + + +class ModelLoadBalancingService: + + def __init__(self) -> None: + self.provider_manager = ProviderManager() + + def enable_model_load_balancing(self, tenant_id: str, provider: str, model: str, model_type: str) -> None: + """ + enable model load balancing. + + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Enable model load balancing + provider_configuration.enable_model_load_balancing( + model=model, + model_type=ModelType.value_of(model_type) + ) + + def disable_model_load_balancing(self, tenant_id: str, provider: str, model: str, model_type: str) -> None: + """ + disable model load balancing. + + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # disable model load balancing + provider_configuration.disable_model_load_balancing( + model=model, + model_type=ModelType.value_of(model_type) + ) + From 76c837b739a08199682844f86bc0dd7c00e9d391 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 19:42:48 +0800 Subject: [PATCH 025/273] add lb config list --- .../workspace/load_balancing_config.py | 16 ++++- api/fields/load_balancing_config.py | 12 ++++ api/services/model_load_balancing_service.py | 70 +++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 api/fields/load_balancing_config.py diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py index d320b6c60db8d7..912972a70e4ec2 100644 --- a/api/controllers/console/workspace/load_balancing_config.py +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -1,10 +1,11 @@ -from flask_restful import Resource, reqparse +from flask_restful import Resource, marshal_with, reqparse from werkzeug.exceptions import Forbidden from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required from core.model_runtime.entities.model_entities import ModelType +from fields.load_balancing_config import load_balancing_config_list_fields from libs.login import current_user, login_required from models.account import TenantAccountRole from services.model_load_balancing_service import ModelLoadBalancingService @@ -65,6 +66,7 @@ class LoadBalancingConfigListApi(Resource): @setup_required @login_required @account_initialization_required + @marshal_with(load_balancing_config_list_fields) def get(self, provider: str): tenant_id = current_user.current_tenant_id @@ -74,9 +76,17 @@ def get(self, provider: str): choices=[mt.value for mt in ModelType], location='args') args = parser.parse_args() - # TODO + model_load_balancing_service = ModelLoadBalancingService() + load_balancing_configs = model_load_balancing_service.get_load_balancing_configs( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'] + ) - return {} + return { + 'data': load_balancing_configs + } @setup_required @login_required diff --git a/api/fields/load_balancing_config.py b/api/fields/load_balancing_config.py new file mode 100644 index 00000000000000..93806a10b89856 --- /dev/null +++ b/api/fields/load_balancing_config.py @@ -0,0 +1,12 @@ +from flask_restful import fields + +load_balancing_config_fields = { + 'id': fields.String, + 'name': fields.String, + 'enabled': fields.Boolean +} + + +load_balancing_config_list_fields = { + 'data': fields.List(fields.Nested(load_balancing_config_fields)) +} diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index aeb9d78da9f12f..8445a454ec45fd 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -2,6 +2,8 @@ from core.model_runtime.entities.model_entities import ModelType from core.provider_manager import ProviderManager +from extensions.ext_database import db +from models.provider import LoadBalancingModelConfig logger = logging.getLogger(__name__) @@ -59,3 +61,71 @@ def disable_model_load_balancing(self, tenant_id: str, provider: str, model: str model_type=ModelType.value_of(model_type) ) + def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, model_type: str) \ + -> list[LoadBalancingModelConfig]: + """ + Get load balancing configurations. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Convert model type to ModelType + model_type = ModelType.value_of(model_type) + + # Get load balancing configurations + load_balancing_configs = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model + ).order_by(LoadBalancingModelConfig.created_at).all() + + # check if the inherit configuration exists, inherit is represented for the provider or model custom credentials + inherit_config_exists = False + for load_balancing_config in load_balancing_configs: + if load_balancing_config.name == '__inherit__': + inherit_config_exists = True + break + + if not inherit_config_exists: + # Initialize the inherit configuration + inherit_config = self._init_inherit_config(tenant_id, provider, model, model_type) + + # prepend the inherit configuration + load_balancing_configs.insert(0, inherit_config) + + return load_balancing_configs + + def _init_inherit_config(self, tenant_id: str, provider: str, model: str, model_type: ModelType) \ + -> LoadBalancingModelConfig: + """ + Initialize the inherit configuration. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :return: + """ + # Initialize the inherit configuration + inherit_config = LoadBalancingModelConfig( + tenant_id=tenant_id, + provider_name=provider, + model_type=model_type.to_origin_model_type(), + model_name=model, + name='__inherit__' + ) + db.session.add(inherit_config) + db.session.commit() + + return inherit_config From 5af820a34b203c4b905859faa77335dea8647939 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 21:06:32 +0800 Subject: [PATCH 026/273] add lb config detail --- .../workspace/load_balancing_config.py | 32 ++++++++- api/core/entities/provider_configuration.py | 6 +- api/services/model_load_balancing_service.py | 69 +++++++++++++++++++ 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py index 912972a70e4ec2..5f70874ada89a1 100644 --- a/api/controllers/console/workspace/load_balancing_config.py +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -1,5 +1,5 @@ from flask_restful import Resource, marshal_with, reqparse -from werkzeug.exceptions import Forbidden +from werkzeug.exceptions import Forbidden, NotFound from controllers.console import api from controllers.console.setup import setup_required @@ -111,6 +111,36 @@ def post(self, provider: str): class LoadBalancingConfigApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, provider: str, config_id: str): + if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + raise Forbidden() + + tenant_id = current_user.current_tenant_id + + parser = reqparse.RequestParser() + parser.add_argument('model', type=str, required=True, nullable=False, location='args') + parser.add_argument('model_type', type=str, required=True, nullable=False, + choices=[mt.value for mt in ModelType], location='args') + args = parser.parse_args() + + # get model load balancing config detail + model_load_balancing_service = ModelLoadBalancingService() + result = model_load_balancing_service.get_load_balancing_config( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'], + config_id=config_id + ) + + if not result: + raise NotFound() + + return result + @setup_required @login_required @account_initialization_required diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index b1f0c6ee76cf74..254f03736ec84d 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -144,7 +144,7 @@ def get_custom_credentials(self, obfuscated: bool = False) -> Optional[dict]: return credentials # Obfuscate credentials - return self._obfuscated_credentials( + return self.obfuscated_credentials( credentials=credentials, credential_form_schemas=self.provider.provider_credential_schema.credential_form_schemas if self.provider.provider_credential_schema else [] @@ -288,7 +288,7 @@ def get_custom_model_credentials(self, model_type: ModelType, model: str, obfusc return credentials # Obfuscate credentials - return self._obfuscated_credentials( + return self.obfuscated_credentials( credentials=credentials, credential_form_schemas=self.provider.model_credential_schema.credential_form_schemas if self.provider.model_credential_schema else [] @@ -616,7 +616,7 @@ def _extract_secret_variables(self, credential_form_schemas: list[CredentialForm return secret_input_form_variables - def _obfuscated_credentials(self, credentials: dict, credential_form_schemas: list[CredentialFormSchema]) -> dict: + def obfuscated_credentials(self, credentials: dict, credential_form_schemas: list[CredentialFormSchema]) -> dict: """ Obfuscated credentials. diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index 8445a454ec45fd..dfe59f5449efc2 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -1,4 +1,5 @@ import logging +from typing import Optional from core.model_runtime.entities.model_entities import ModelType from core.provider_manager import ProviderManager @@ -107,6 +108,74 @@ def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, return load_balancing_configs + def get_load_balancing_config(self, tenant_id: str, provider: str, model: str, model_type: str, config_id: str) \ + -> Optional[dict]: + """ + Get load balancing configuration. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :param config_id: load balancing config id + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Convert model type to ModelType + model_type = ModelType.value_of(model_type) + + # Get load balancing configurations + load_balancing_model_config = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model, + LoadBalancingModelConfig.id == config_id + ).first() + + if not load_balancing_model_config: + return None + + credentials = {} + for model_setting in provider_configuration.model_settings: + if model_setting.model == model and model_setting.model_type == model_type: + load_balancing_configs = model_setting.load_balancing_configs + for load_balancing_config in load_balancing_configs: + if load_balancing_config.name == load_balancing_config.name: + credentials = load_balancing_config.credentials + + # Get credential form schemas from model credential schema or provider credential schema + if provider_configuration.provider.model_credential_schema: + credential_form_schemas = (provider_configuration.provider.model_credential_schema + .credential_form_schemas) + else: + credential_form_schemas = (provider_configuration.provider.provider_credential_schema + .credential_form_schemas) + + if not credential_form_schemas: + credential_form_schemas = [] + + # Obfuscate credentials + credentials = provider_configuration.obfuscated_credentials( + credentials=credentials, + credential_form_schemas=credential_form_schemas + ) + break + + return { + 'id': load_balancing_model_config.id, + 'name': load_balancing_model_config.name, + 'credentials': credentials, + 'enabled': load_balancing_model_config.enabled + } + def _init_inherit_config(self, tenant_id: str, provider: str, model: str, model_type: ModelType) \ -> LoadBalancingModelConfig: """ From da9295baf025343cc60ec80f477e1f9237a05409 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 21:13:24 +0800 Subject: [PATCH 027/273] optimize --- api/core/entities/provider_entities.py | 1 + api/core/provider_manager.py | 1 + api/services/model_load_balancing_service.py | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/core/entities/provider_entities.py b/api/core/entities/provider_entities.py index 7f36e6c15d19f5..1eaa6ea02ca296 100644 --- a/api/core/entities/provider_entities.py +++ b/api/core/entities/provider_entities.py @@ -78,6 +78,7 @@ class ModelLoadBalancingConfiguration(BaseModel): """ Class for model load balancing configuration. """ + id: str name: str credentials: dict diff --git a/api/core/provider_manager.py b/api/core/provider_manager.py index 379902c79f87fc..2abdf046a3ceb6 100644 --- a/api/core/provider_manager.py +++ b/api/core/provider_manager.py @@ -901,6 +901,7 @@ def _to_model_settings(self, provider_entity: ProviderEntity, provider_model_credentials = cached_provider_model_credentials load_balancing_configs.append(ModelLoadBalancingConfiguration( + id=load_balancing_model_config.id, name=load_balancing_model_config.name, credentials=provider_model_credentials )) diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index dfe59f5449efc2..ac1977959de4e6 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -147,9 +147,9 @@ def get_load_balancing_config(self, tenant_id: str, provider: str, model: str, m for model_setting in provider_configuration.model_settings: if model_setting.model == model and model_setting.model_type == model_type: load_balancing_configs = model_setting.load_balancing_configs - for load_balancing_config in load_balancing_configs: - if load_balancing_config.name == load_balancing_config.name: - credentials = load_balancing_config.credentials + for lb_config in load_balancing_configs: + if lb_config.id == load_balancing_model_config.id: + credentials = lb_config.credentials # Get credential form schemas from model credential schema or provider credential schema if provider_configuration.provider.model_credential_schema: From d98d9c5a847cc018343dc05bdd0122e96ab034db Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 22:36:07 +0800 Subject: [PATCH 028/273] fix bugs --- .../workspace/load_balancing_config.py | 36 ++- api/core/entities/provider_configuration.py | 20 +- api/services/model_load_balancing_service.py | 270 ++++++++++++++++-- api/services/workflow_service.py | 2 +- 4 files changed, 286 insertions(+), 42 deletions(-) diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py index 5f70874ada89a1..f7b6acbd508936 100644 --- a/api/controllers/console/workspace/load_balancing_config.py +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -5,6 +5,7 @@ from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError from fields.load_balancing_config import load_balancing_config_list_fields from libs.login import current_user, login_required from models.account import TenantAccountRole @@ -105,9 +106,22 @@ def post(self, provider: str): parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') args = parser.parse_args() - # TODO + # create model load balancing config + model_load_balancing_service = ModelLoadBalancingService() - return {} + try: + model_load_balancing_service.create_load_balancing_config( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'], + name=args['name'], + credentials=args['credentials'] + ) + except CredentialsValidateFailedError as ex: + raise ValueError(str(ex)) + + return {'result': 'success'}, 201 class LoadBalancingConfigApi(Resource): @@ -158,9 +172,23 @@ def post(self, provider: str, config_id: str): parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') args = parser.parse_args() - # TODO + # create model load balancing config + model_load_balancing_service = ModelLoadBalancingService() - return {} + try: + model_load_balancing_service.update_load_balancing_config( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'], + config_id=config_id, + name=args['name'], + credentials=args['credentials'] + ) + except CredentialsValidateFailedError as ex: + raise ValueError(str(ex)) + + return {'result': 'success'} @setup_required @login_required diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index 254f03736ec84d..5f74eeb2e90da2 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -165,7 +165,7 @@ def custom_credentials_validate(self, credentials: dict) -> tuple[Provider, dict ).first() # Get provider credential secret variables - provider_credential_secret_variables = self._extract_secret_variables( + provider_credential_secret_variables = self.extract_secret_variables( self.provider.provider_credential_schema.credential_form_schemas if self.provider.provider_credential_schema else [] ) @@ -316,7 +316,7 @@ def custom_model_credentials_validate(self, model_type: ModelType, model: str, c ).first() # Get provider credential secret variables - provider_credential_secret_variables = self._extract_secret_variables( + provider_credential_secret_variables = self.extract_secret_variables( self.provider.model_credential_schema.credential_form_schemas if self.provider.model_credential_schema else [] ) @@ -433,6 +433,7 @@ def enable_model(self, model_type: ModelType, model: str) -> ProviderModelSettin if model_setting: model_setting.enabled = True + model_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() else: model_setting = ProviderModelSetting( @@ -464,6 +465,7 @@ def disable_model(self, model_type: ModelType, model: str) -> ProviderModelSetti if model_setting: model_setting.enabled = False + model_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() else: model_setting = ProviderModelSetting( @@ -506,6 +508,7 @@ def enable_model_load_balancing(self, model_type: ModelType, model: str) -> Prov if model_setting: model_setting.load_balancing_enabled = True + model_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() else: model_setting = ProviderModelSetting( @@ -537,6 +540,7 @@ def disable_model_load_balancing(self, model_type: ModelType, model: str) -> Pro if model_setting: model_setting.load_balancing_enabled = False + model_setting.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() else: model_setting = ProviderModelSetting( @@ -602,7 +606,7 @@ def switch_preferred_provider_type(self, provider_type: ProviderType) -> None: db.session.commit() - def _extract_secret_variables(self, credential_form_schemas: list[CredentialFormSchema]) -> list[str]: + def extract_secret_variables(self, credential_form_schemas: list[CredentialFormSchema]) -> list[str]: """ Extract secret input form variables. @@ -625,7 +629,7 @@ def obfuscated_credentials(self, credentials: dict, credential_form_schemas: lis :return: """ # Get provider credential secret variables - credential_secret_variables = self._extract_secret_variables( + credential_secret_variables = self.extract_secret_variables( credential_form_schemas ) @@ -903,14 +907,6 @@ def _get_custom_provider_models(self, return provider_models - def _get_load_balancing_configs(self, model: str) -> dict: - """ - Get load balancing configs. - :param model: model name - :return: - """ - return {} - class ProviderConfigurations(BaseModel): """ diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index ac1977959de4e6..4f67c66fa30d8c 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -1,7 +1,17 @@ +import datetime +import json import logging +from json import JSONDecodeError from typing import Optional +from core.entities.provider_configuration import ProviderConfiguration +from core.helper import encrypter from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.entities.provider_entities import ( + ModelCredentialSchema, + ProviderCredentialSchema, +) +from core.model_runtime.model_providers import model_provider_factory from core.provider_manager import ProviderManager from extensions.ext_database import db from models.provider import LoadBalancingModelConfig @@ -143,31 +153,22 @@ def get_load_balancing_config(self, tenant_id: str, provider: str, model: str, m if not load_balancing_model_config: return None - credentials = {} - for model_setting in provider_configuration.model_settings: - if model_setting.model == model and model_setting.model_type == model_type: - load_balancing_configs = model_setting.load_balancing_configs - for lb_config in load_balancing_configs: - if lb_config.id == load_balancing_model_config.id: - credentials = lb_config.credentials - - # Get credential form schemas from model credential schema or provider credential schema - if provider_configuration.provider.model_credential_schema: - credential_form_schemas = (provider_configuration.provider.model_credential_schema - .credential_form_schemas) - else: - credential_form_schemas = (provider_configuration.provider.provider_credential_schema - .credential_form_schemas) - - if not credential_form_schemas: - credential_form_schemas = [] - - # Obfuscate credentials - credentials = provider_configuration.obfuscated_credentials( - credentials=credentials, - credential_form_schemas=credential_form_schemas - ) - break + try: + if load_balancing_model_config.encrypted_config: + credentials = json.loads(load_balancing_model_config.encrypted_config) + else: + credentials = {} + except JSONDecodeError: + credentials = {} + + # Get credential form schemas from model credential schema or provider credential schema + credential_schemas = self._get_credential_schema(provider_configuration) + + # Obfuscate credentials + credentials = provider_configuration.obfuscated_credentials( + credentials=credentials, + credential_form_schemas=credential_schemas.credential_form_schemas + ) return { 'id': load_balancing_model_config.id, @@ -198,3 +199,222 @@ def _init_inherit_config(self, tenant_id: str, provider: str, model: str, model_ db.session.commit() return inherit_config + + def create_load_balancing_config(self, tenant_id: str, + provider: str, + model: str, + model_type: str, + name: str, + credentials: dict) -> None: + """ + Create load balancing configuration. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :param name: load balancing config name + :param credentials: load balancing config credentials + :raises ValueError: if provider does not exist + :raises CredentialsValidateFailedError: if credentials validation failed + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Convert model type to ModelType + model_type = ModelType.value_of(model_type) + + if name == '__inherit__': + raise ValueError('Invalid load balancing config name') + + # Get load balancing configurations + load_balancing_model_config = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model, + LoadBalancingModelConfig.name == name + ).first() + + if load_balancing_model_config: + raise ValueError('Load balancing config name already exists') + + # validate custom provider config + credentials = self._custom_credentials_validate( + tenant_id=tenant_id, + provider_configuration=provider_configuration, + model_type=model_type, + model=model, + credentials=credentials + ) + + # create load balancing config + load_balancing_model_config = LoadBalancingModelConfig( + tenant_id=tenant_id, + provider_name=provider_configuration.provider.provider, + model_type=model_type.to_origin_model_type(), + model_name=model, + name=name, + encrypted_config=json.dumps(credentials) + ) + + db.session.add(load_balancing_model_config) + db.session.commit() + + def update_load_balancing_config(self, tenant_id: str, + provider: str, + model: str, + model_type: str, + config_id: str, + name: str, + credentials: dict) -> None: + """ + Update load balancing configuration. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :param config_id: load balancing config id + :param name: load balancing config name + :param credentials: load balancing config credentials + :raises ValueError: if provider does not exist + :raises CredentialsValidateFailedError: if credentials validation failed + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Convert model type to ModelType + model_type = ModelType.value_of(model_type) + + if name == '__inherit__': + raise ValueError('Invalid load balancing config name') + + # Get load balancing configurations + load_balancing_model_config = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model, + LoadBalancingModelConfig.id == config_id + ).first() + + if not load_balancing_model_config: + raise ValueError('Load balancing config does not exist') + + # check duplicate name + duplicate_name_load_balancing_model_config = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model, + LoadBalancingModelConfig.id != config_id, + LoadBalancingModelConfig.name == name + ).first() + + if duplicate_name_load_balancing_model_config: + raise ValueError('Load balancing config name already exists') + + # validate custom provider config + credentials = self._custom_credentials_validate( + tenant_id=tenant_id, + provider_configuration=provider_configuration, + model_type=model_type, + model=model, + credentials=credentials, + load_balancing_model_config=load_balancing_model_config + ) + + # update load balancing config + load_balancing_model_config.name = name + load_balancing_model_config.encrypted_config = json.dumps(credentials) + load_balancing_model_config.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) + db.session.commit() + + def _custom_credentials_validate(self, tenant_id: str, + provider_configuration: ProviderConfiguration, + model_type: ModelType, + model: str, + credentials: dict, + load_balancing_model_config: Optional[LoadBalancingModelConfig] = None) -> dict: + """ + Validate custom credentials. + :param tenant_id: workspace id + :param provider_configuration: provider configuration + :param model_type: model type + :param model: model name + :param credentials: credentials + :param load_balancing_model_config: load balancing model config + :return: + """ + # Get credential form schemas from model credential schema or provider credential schema + credential_schemas = self._get_credential_schema(provider_configuration) + + # Get provider credential secret variables + provider_credential_secret_variables = provider_configuration.extract_secret_variables( + credential_schemas.credential_form_schemas + ) + + if load_balancing_model_config: + try: + # fix origin data + if load_balancing_model_config.encrypted_config: + original_credentials = json.loads(load_balancing_model_config.encrypted_config) + else: + original_credentials = {} + except JSONDecodeError: + original_credentials = {} + + # encrypt credentials + for key, value in credentials.items(): + if key in provider_credential_secret_variables: + # if send [__HIDDEN__] in secret input, it will be same as original value + if value == '[__HIDDEN__]' and key in original_credentials: + credentials[key] = encrypter.decrypt_token(tenant_id, original_credentials[key]) + + if isinstance(credential_schemas, ModelCredentialSchema): + credentials = model_provider_factory.model_credentials_validate( + provider=provider_configuration.provider.provider, + model_type=model_type, + model=model, + credentials=credentials + ) + else: + credentials = model_provider_factory.provider_credentials_validate( + provider=provider_configuration.provider.provider, + credentials=credentials + ) + + for key, value in credentials.items(): + if key in provider_credential_secret_variables: + credentials[key] = encrypter.encrypt_token(tenant_id, value) + + return credentials + + def _get_credential_schema(self, provider_configuration: ProviderConfiguration) \ + -> ModelCredentialSchema | ProviderCredentialSchema: + """ + Get form schemas. + :param provider_configuration: provider configuration + :return: + """ + # Get credential form schemas from model credential schema or provider credential schema + if provider_configuration.provider.model_credential_schema: + credential_schema = provider_configuration.provider.model_credential_schema + else: + credential_schema = provider_configuration.provider.provider_credential_schema + + return credential_schema diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 456ab0dcb02045..6235ecf0a36543 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -68,7 +68,7 @@ def sync_draft_workflow(self, app_model: App, account: Account) -> Workflow: """ Sync draft workflow - @throws WorkflowHashNotEqualError + :raises WorkflowHashNotEqualError """ # fetch draft workflow by app_model workflow = self.get_draft_workflow(app_model=app_model) From 0b5a5ae119912c8d7e9d8c47778323cd277a6832 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 22:44:06 +0800 Subject: [PATCH 029/273] add delete config --- .../workspace/load_balancing_config.py | 12 +++- api/services/model_load_balancing_service.py | 59 +++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py index f7b6acbd508936..de34348db64be6 100644 --- a/api/controllers/console/workspace/load_balancing_config.py +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -205,9 +205,17 @@ def delete(self, provider: str, config_id: str): choices=[mt.value for mt in ModelType], location='args') args = parser.parse_args() - # TODO + # delete model load balancing config + model_load_balancing_service = ModelLoadBalancingService() + model_load_balancing_service.delete_load_balancing_config( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'], + config_id=config_id + ) - return {} + return {'result': 'success'}, 204 class LoadBalancingConfigEnableApi(Resource): diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index 4f67c66fa30d8c..1a9dea61024e91 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -6,6 +6,7 @@ from core.entities.provider_configuration import ProviderConfiguration from core.helper import encrypter +from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.entities.provider_entities import ( ModelCredentialSchema, @@ -265,6 +266,7 @@ def create_load_balancing_config(self, tenant_id: str, ) db.session.add(load_balancing_model_config) + db.session.flush() db.session.commit() def update_load_balancing_config(self, tenant_id: str, @@ -344,6 +346,8 @@ def update_load_balancing_config(self, tenant_id: str, load_balancing_model_config.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) db.session.commit() + self._clear_credentials_cache(tenant_id, config_id) + def _custom_credentials_validate(self, tenant_id: str, provider_configuration: ProviderConfiguration, model_type: ModelType, @@ -418,3 +422,58 @@ def _get_credential_schema(self, provider_configuration: ProviderConfiguration) credential_schema = provider_configuration.provider.provider_credential_schema return credential_schema + + def delete_load_balancing_config(self, tenant_id: str, provider: str, model: str, model_type: str, config_id: str) \ + -> None: + """ + Delete load balancing configuration. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :param config_id: load balancing config id + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Convert model type to ModelType + model_type = ModelType.value_of(model_type) + + # Get load balancing configurations + load_balancing_model_config = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model, + LoadBalancingModelConfig.id == config_id + ).first() + + if not load_balancing_model_config: + raise ValueError('Load balancing config does not exist') + + db.session.delete(load_balancing_model_config) + db.session.commit() + + self._clear_credentials_cache(tenant_id, config_id) + + def _clear_credentials_cache(self, tenant_id: str, config_id: str) -> None: + """ + Clear credentials cache. + :param tenant_id: workspace id + :param config_id: load balancing config id + :return: + """ + provider_model_credentials_cache = ProviderCredentialsCache( + tenant_id=tenant_id, + identity_id=config_id, + cache_type=ProviderCredentialsCacheType.LOAD_BALANCING_MODEL + ) + + provider_model_credentials_cache.delete() From 020e2bcd09cff56e594f5fbf43e978313fbbc594 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 22:49:17 +0800 Subject: [PATCH 030/273] add config enable / disable --- .../workspace/load_balancing_config.py | 24 +++++- api/services/model_load_balancing_service.py | 79 +++++++++++++++++++ 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py index de34348db64be6..31a2ad24185a54 100644 --- a/api/controllers/console/workspace/load_balancing_config.py +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -235,9 +235,17 @@ def patch(self, provider: str, config_id: str): choices=[mt.value for mt in ModelType], location='json') args = parser.parse_args() - # TODO + # enable model load balancing config + model_load_balancing_service = ModelLoadBalancingService() + model_load_balancing_service.enable_load_balancing_config( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'], + config_id=config_id + ) - return {} + return {"result": "success"} class LoadBalancingConfigDisableApi(Resource): @@ -257,9 +265,17 @@ def patch(self, provider: str, config_id: str): choices=[mt.value for mt in ModelType], location='json') args = parser.parse_args() - # TODO + # disable model load balancing config + model_load_balancing_service = ModelLoadBalancingService() + model_load_balancing_service.disable_load_balancing_config( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'], + config_id=config_id + ) - return {} + return {"result": "success"} # Model Load Balancing Feature diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index 1a9dea61024e91..79ee74bb0c9c4f 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -477,3 +477,82 @@ def _clear_credentials_cache(self, tenant_id: str, config_id: str) -> None: ) provider_model_credentials_cache.delete() + + def enable_load_balancing_config(self, tenant_id: str, provider: str, model: str, model_type: str, config_id: str) \ + -> None: + """ + Enable load balancing configuration. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :param config_id: load balancing config id + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Convert model type to ModelType + model_type = ModelType.value_of(model_type) + + # Get load balancing configurations + load_balancing_model_config = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model, + LoadBalancingModelConfig.id == config_id + ).first() + + if not load_balancing_model_config: + raise ValueError('Load balancing config does not exist') + + load_balancing_model_config.enabled = True + load_balancing_model_config.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) + db.session.commit() + + def disable_load_balancing_config(self, tenant_id: str, provider: str, model: str, model_type: str, config_id: str) \ + -> None: + """ + Disable load balancing configuration. + :param tenant_id: workspace id + :param provider: provider name + :param model: model name + :param model_type: model type + :param config_id: load balancing config id + :return: + """ + # Get all provider configurations of the current workspace + provider_configurations = self.provider_manager.get_configurations(tenant_id) + + # Get provider configuration + provider_configuration = provider_configurations.get(provider) + + if not provider_configuration: + raise ValueError(f"Provider {provider} does not exist.") + + # Convert model type to ModelType + model_type = ModelType.value_of(model_type) + + # Get load balancing configurations + load_balancing_model_config = db.session.query(LoadBalancingModelConfig) \ + .filter( + LoadBalancingModelConfig.tenant_id == tenant_id, + LoadBalancingModelConfig.provider_name == provider_configuration.provider.provider, + LoadBalancingModelConfig.model_type == model_type.to_origin_model_type(), + LoadBalancingModelConfig.model_name == model, + LoadBalancingModelConfig.id == config_id + ).first() + + if not load_balancing_model_config: + raise ValueError('Load balancing config does not exist') + + load_balancing_model_config.enabled = False + load_balancing_model_config.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) + db.session.commit() From 03848948301bc1114c82e96649d6746f69be3974 Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 23:35:03 +0800 Subject: [PATCH 031/273] add model disable raise when model invoke --- api/core/entities/provider_configuration.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index 5f74eeb2e90da2..42de012e8a251f 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -76,6 +76,14 @@ def get_current_credentials(self, model_type: ModelType, model: str) -> Optional :param model: model name :return: """ + if self.model_settings: + # check if model is disabled by admin + for model_setting in self.model_settings: + if (model_setting.model_type == model_type + and model_setting.model == model + and not model_setting.enabled): + raise ValueError(f'Model {model} is disabled.') + if self.using_provider_type == ProviderType.SYSTEM: restrict_models = [] for quota_configuration in self.system_configuration.quota_configurations: @@ -753,7 +761,8 @@ def _get_system_provider_models(self, break if should_use_custom_model: - if original_provider_configurate_methods[self.provider.provider] == [ConfigurateMethod.CUSTOMIZABLE_MODEL]: + if original_provider_configurate_methods[self.provider.provider] == [ + ConfigurateMethod.CUSTOMIZABLE_MODEL]: # only customizable model for restrict_model in restrict_models: copy_credentials = self.system_configuration.credentials.copy() From be8481fa59b3c72f6d4128424ca6fb2673ea0ebf Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 11 May 2024 23:45:10 +0800 Subject: [PATCH 032/273] add comment --- api/core/entities/provider_configuration.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index 42de012e8a251f..551d013f00178c 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -76,13 +76,15 @@ def get_current_credentials(self, model_type: ModelType, model: str) -> Optional :param model: model name :return: """ + current_model_setting = None if self.model_settings: # check if model is disabled by admin for model_setting in self.model_settings: if (model_setting.model_type == model_type - and model_setting.model == model - and not model_setting.enabled): - raise ValueError(f'Model {model} is disabled.') + and model_setting.model == model): + current_model_setting = model_setting + if not model_setting.enabled: + raise ValueError(f'Model {model} is disabled.') if self.using_provider_type == ProviderType.SYSTEM: restrict_models = [] @@ -102,6 +104,11 @@ def get_current_credentials(self, model_type: ModelType, model: str) -> Optional return copy_credentials else: + # check if load balancing is enabled + if current_model_setting.load_balancing_configs: + # TODO use load balancing proxy to choose credentials + pass + if self.custom_configuration.models: for model_configuration in self.custom_configuration.models: if model_configuration.model_type == model_type and model_configuration.model == model: From d60eb3b61e9fe3cbd6dc6368f43e1d34b8767053 Mon Sep 17 00:00:00 2001 From: takatost Date: Mon, 13 May 2024 22:00:55 +0800 Subject: [PATCH 033/273] add round-robin lb invoke --- api/core/application_manager.py | 0 api/core/entities/provider_configuration.py | 17 +- api/core/extension/extensible.py | 2 +- .../{utils => helper}/module_import_helper.py | 0 api/core/{utils => helper}/position_helper.py | 0 api/core/model_manager.py | 215 +++++++++++++++++- .../model_providers/__base/ai_model.py | 2 +- .../model_providers/__base/model_provider.py | 2 +- .../model_providers/model_provider_factory.py | 4 +- api/core/provider_manager.py | 6 + api/core/tools/provider/builtin/_positions.py | 2 +- .../tools/provider/builtin_tool_provider.py | 2 +- api/core/tools/tool_manager.py | 2 +- .../utils/test_module_import_helper.py | 2 +- 14 files changed, 227 insertions(+), 29 deletions(-) delete mode 100644 api/core/application_manager.py rename api/core/{utils => helper}/module_import_helper.py (100%) rename api/core/{utils => helper}/position_helper.py (100%) diff --git a/api/core/application_manager.py b/api/core/application_manager.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/api/core/entities/provider_configuration.py b/api/core/entities/provider_configuration.py index 551d013f00178c..e685b96d009bb2 100644 --- a/api/core/entities/provider_configuration.py +++ b/api/core/entities/provider_configuration.py @@ -76,13 +76,11 @@ def get_current_credentials(self, model_type: ModelType, model: str) -> Optional :param model: model name :return: """ - current_model_setting = None if self.model_settings: # check if model is disabled by admin for model_setting in self.model_settings: if (model_setting.model_type == model_type and model_setting.model == model): - current_model_setting = model_setting if not model_setting.enabled: raise ValueError(f'Model {model} is disabled.') @@ -104,20 +102,17 @@ def get_current_credentials(self, model_type: ModelType, model: str) -> Optional return copy_credentials else: - # check if load balancing is enabled - if current_model_setting.load_balancing_configs: - # TODO use load balancing proxy to choose credentials - pass - + credentials = None if self.custom_configuration.models: for model_configuration in self.custom_configuration.models: if model_configuration.model_type == model_type and model_configuration.model == model: - return model_configuration.credentials + credentials = model_configuration.credentials + break if self.custom_configuration.provider: - return self.custom_configuration.provider.credentials - else: - return None + credentials = self.custom_configuration.provider.credentials + + return credentials def get_system_configuration_status(self) -> SystemConfigurationStatus: """ diff --git a/api/core/extension/extensible.py b/api/core/extension/extensible.py index d2ec555d6c30f8..3a37c6492e149b 100644 --- a/api/core/extension/extensible.py +++ b/api/core/extension/extensible.py @@ -7,7 +7,7 @@ from pydantic import BaseModel -from core.utils.position_helper import sort_to_dict_by_position_map +from core.helper.position_helper import sort_to_dict_by_position_map class ExtensionModule(enum.Enum): diff --git a/api/core/utils/module_import_helper.py b/api/core/helper/module_import_helper.py similarity index 100% rename from api/core/utils/module_import_helper.py rename to api/core/helper/module_import_helper.py diff --git a/api/core/utils/position_helper.py b/api/core/helper/position_helper.py similarity index 100% rename from api/core/utils/position_helper.py rename to api/core/helper/position_helper.py diff --git a/api/core/model_manager.py b/api/core/model_manager.py index 8c0633992767dc..e09a669cf23070 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -1,7 +1,10 @@ +import logging +import os from collections.abc import Generator from typing import IO, Optional, Union, cast -from core.entities.provider_configuration import ProviderModelBundle +from core.entities.provider_configuration import ProviderConfiguration, ProviderModelBundle +from core.entities.provider_entities import ModelLoadBalancingConfiguration from core.errors.error import ProviderTokenNotInitError from core.model_runtime.callbacks.base_callback import Callback from core.model_runtime.entities.llm_entities import LLMResult @@ -9,6 +12,7 @@ from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.entities.rerank_entities import RerankResult from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult +from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeConnectionError, InvokeRateLimitError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.model_providers.__base.moderation_model import ModerationModel from core.model_runtime.model_providers.__base.rerank_model import RerankModel @@ -16,6 +20,10 @@ from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.provider_manager import ProviderManager +from extensions.ext_redis import redis_client +from models.provider import ProviderType + +logger = logging.getLogger(__name__) class ModelInstance: @@ -29,6 +37,12 @@ def __init__(self, provider_model_bundle: ProviderModelBundle, model: str) -> No self.provider = provider_model_bundle.configuration.provider.provider self.credentials = self._fetch_credentials_from_bundle(provider_model_bundle, model) self.model_type_instance = self.provider_model_bundle.model_type_instance + self.load_balancing_manager = self._get_load_balancing_manager( + configuration=provider_model_bundle.configuration, + model_type=provider_model_bundle.model_type_instance.model_type, + model=model, + credentials=self.credentials + ) def _fetch_credentials_from_bundle(self, provider_model_bundle: ProviderModelBundle, model: str) -> dict: """ @@ -37,8 +51,10 @@ def _fetch_credentials_from_bundle(self, provider_model_bundle: ProviderModelBun :param model: model name :return: """ - credentials = provider_model_bundle.configuration.get_current_credentials( - model_type=provider_model_bundle.model_type_instance.model_type, + configuration = provider_model_bundle.configuration + model_type = provider_model_bundle.model_type_instance.model_type + credentials = configuration.get_current_credentials( + model_type=model_type, model=model ) @@ -47,6 +63,43 @@ def _fetch_credentials_from_bundle(self, provider_model_bundle: ProviderModelBun return credentials + def _get_load_balancing_manager(self, configuration: ProviderConfiguration, + model_type: ModelType, + model: str, + credentials: dict) -> Optional["LBModelManager"]: + """ + Get load balancing model credentials + :param configuration: provider configuration + :param model_type: model type + :param model: model name + :param credentials: model credentials + :return: + """ + if configuration.model_settings and configuration.using_provider_type == ProviderType.CUSTOM: + current_model_setting = None + # check if model is disabled by admin + for model_setting in configuration.model_settings: + if (model_setting.model_type == model_type + and model_setting.model == model): + current_model_setting = model_setting + break + + # check if load balancing is enabled + if current_model_setting and current_model_setting.load_balancing_configs: + # use load balancing proxy to choose credentials + lb_model_manager = LBModelManager( + tenant_id=configuration.tenant_id, + provider=configuration.provider.provider, + model_type=model_type, + model=model, + load_balancing_configs=current_model_setting.load_balancing_configs, + managed_credentials=credentials + ) + + return lb_model_manager + + return None + def invoke_llm(self, prompt_messages: list[PromptMessage], model_parameters: Optional[dict] = None, tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, stream: bool = True, user: Optional[str] = None, callbacks: list[Callback] = None) \ @@ -67,7 +120,8 @@ def invoke_llm(self, prompt_messages: list[PromptMessage], model_parameters: Opt raise Exception("Model type instance is not LargeLanguageModel") self.model_type_instance = cast(LargeLanguageModel, self.model_type_instance) - return self.model_type_instance.invoke( + return self._round_robin_invoke( + function=self.model_type_instance.invoke, model=self.model, credentials=self.credentials, prompt_messages=prompt_messages, @@ -92,7 +146,8 @@ def invoke_text_embedding(self, texts: list[str], user: Optional[str] = None) \ raise Exception("Model type instance is not TextEmbeddingModel") self.model_type_instance = cast(TextEmbeddingModel, self.model_type_instance) - return self.model_type_instance.invoke( + return self._round_robin_invoke( + function=self.model_type_instance.invoke, model=self.model, credentials=self.credentials, texts=texts, @@ -117,7 +172,8 @@ def invoke_rerank(self, query: str, docs: list[str], score_threshold: Optional[f raise Exception("Model type instance is not RerankModel") self.model_type_instance = cast(RerankModel, self.model_type_instance) - return self.model_type_instance.invoke( + return self._round_robin_invoke( + function=self.model_type_instance.invoke, model=self.model, credentials=self.credentials, query=query, @@ -140,7 +196,8 @@ def invoke_moderation(self, text: str, user: Optional[str] = None) \ raise Exception("Model type instance is not ModerationModel") self.model_type_instance = cast(ModerationModel, self.model_type_instance) - return self.model_type_instance.invoke( + return self._round_robin_invoke( + function=self.model_type_instance.invoke, model=self.model, credentials=self.credentials, text=text, @@ -160,7 +217,8 @@ def invoke_speech2text(self, file: IO[bytes], user: Optional[str] = None) \ raise Exception("Model type instance is not Speech2TextModel") self.model_type_instance = cast(Speech2TextModel, self.model_type_instance) - return self.model_type_instance.invoke( + return self._round_robin_invoke( + function=self.model_type_instance.invoke, model=self.model, credentials=self.credentials, file=file, @@ -183,7 +241,8 @@ def invoke_tts(self, content_text: str, tenant_id: str, voice: str, streaming: b raise Exception("Model type instance is not TTSModel") self.model_type_instance = cast(TTSModel, self.model_type_instance) - return self.model_type_instance.invoke( + return self._round_robin_invoke( + function=self.model_type_instance.invoke, model=self.model, credentials=self.credentials, content_text=content_text, @@ -193,6 +252,36 @@ def invoke_tts(self, content_text: str, tenant_id: str, voice: str, streaming: b streaming=streaming ) + def _round_robin_invoke(self, function: callable, *args, **kwargs): + """ + Round-robin invoke + :param function: function to invoke + :param args: function args + :param kwargs: function kwargs + :return: + """ + if not self.load_balancing_manager: + return function(*args, **kwargs) + + while True: + lb_config = self.load_balancing_manager.fetch_next() + if not lb_config: + raise ProviderTokenNotInitError("Model credentials is not initialized.") + + try: + del kwargs['credentials'] + return function(*args, **kwargs, credentials=lb_config.credentials) + except InvokeRateLimitError: + # expire in 60 seconds + self.load_balancing_manager.cooldown(lb_config, expire=60) + continue + except InvokeAuthorizationError | InvokeConnectionError: + # expire in 10 seconds + self.load_balancing_manager.cooldown(lb_config, expire=10) + continue + except Exception as e: + raise e + def get_tts_voices(self, language: str) -> list: """ Invoke large language tts model voices @@ -226,6 +315,7 @@ def get_model_instance(self, tenant_id: str, provider: str, model_type: ModelTyp """ if not provider: return self.get_default_model_instance(tenant_id, model_type) + provider_model_bundle = self._provider_manager.get_provider_model_bundle( tenant_id=tenant_id, provider=provider, @@ -255,3 +345,110 @@ def get_default_model_instance(self, tenant_id: str, model_type: ModelType) -> M model_type=model_type, model=default_model_entity.model ) + + +class LBModelManager: + def __init__(self, tenant_id: str, + provider: str, + model_type: ModelType, + model: str, + load_balancing_configs: list[ModelLoadBalancingConfiguration], + managed_credentials: Optional[dict] = None) -> None: + """ + Load balancing model manager + :param load_balancing_configs: all load balancing configurations + :param managed_credentials: credentials if load balancing configuration name is __inherit__ + """ + self._tenant_id = tenant_id + self._provider = provider + self._model_type = model_type + self._model = model + self._load_balancing_configs = load_balancing_configs + + for load_balancing_config in self._load_balancing_configs: + if load_balancing_config.name == "__inherit__": + if not managed_credentials: + # remove __inherit__ if managed credentials is not provided + self._load_balancing_configs.remove(load_balancing_config) + else: + load_balancing_config.credentials = managed_credentials + + def fetch_next(self) -> Optional[ModelLoadBalancingConfiguration]: + """ + Get next model load balancing config + Strategy: Round Robin + :return: + """ + cache_key = "model_lb_index:{}:{}:{}:{}".format( + self._tenant_id, + self._provider, + self._model_type.value, + self._model + ) + + cooldown_load_balancing_configs = [] + max_index = len(self._load_balancing_configs) + + while True: + current_index = redis_client.incr(cache_key) + if current_index > max_index: + current_index = current_index % max_index + redis_client.set(cache_key, current_index) + + redis_client.expire(cache_key, 3600) + + real_index = current_index - 1 + if real_index > max_index: + real_index = 0 + + config = self._load_balancing_configs[real_index] + + if self.is_cooldown(config): + cooldown_load_balancing_configs.append(config) + if len(cooldown_load_balancing_configs) >= len(self._load_balancing_configs): + # all configs are in cooldown + return None + + continue + + if bool(os.environ.get("DEBUG", 'False').lower() == 'true'): + logger.info(f"Model LB\nid: {config.id}\nname:{config.name}\n" + f"tenant_id: {self._tenant_id}\nprovider: {self._provider}\n" + f"model_type: {self._model_type.value}\nmodel: {self._model}") + + return config + + return None + + def cooldown(self, config: ModelLoadBalancingConfiguration, expire: int = 60): + """ + Cooldown model load balancing config + :param config: model load balancing config + :param expire: cooldown time + :return: + """ + cooldown_cache_key = "model_lb_index:cooldown:{}:{}:{}:{}:{}".format( + self._tenant_id, + self._provider, + self._model_type.value, + self._model, + config.id + ) + + redis_client.setex(cooldown_cache_key, expire, 'true') + + def is_cooldown(self, config: ModelLoadBalancingConfiguration) -> bool: + """ + Check if model load balancing config is in cooldown + :param config: model load balancing config + :return: + """ + cooldown_cache_key = "model_lb_index:cooldown:{}:{}:{}:{}:{}".format( + self._tenant_id, + self._provider, + self._model_type.value, + self._model, + config.id + ) + + return redis_client.exists(cooldown_cache_key) diff --git a/api/core/model_runtime/model_providers/__base/ai_model.py b/api/core/model_runtime/model_providers/__base/ai_model.py index 34a737549381de..ff369c7baead2c 100644 --- a/api/core/model_runtime/model_providers/__base/ai_model.py +++ b/api/core/model_runtime/model_providers/__base/ai_model.py @@ -5,6 +5,7 @@ import yaml +from core.helper.position_helper import get_position_map, sort_by_position_map from core.model_runtime.entities.common_entities import I18nObject from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE from core.model_runtime.entities.model_entities import ( @@ -18,7 +19,6 @@ ) from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer -from core.utils.position_helper import get_position_map, sort_by_position_map class AIModel(ABC): diff --git a/api/core/model_runtime/model_providers/__base/model_provider.py b/api/core/model_runtime/model_providers/__base/model_provider.py index 7c839a9672e335..e7be2733154e85 100644 --- a/api/core/model_runtime/model_providers/__base/model_provider.py +++ b/api/core/model_runtime/model_providers/__base/model_provider.py @@ -3,10 +3,10 @@ import yaml +from core.helper.module_import_helper import get_subclasses_from_module, import_module_from_source from core.model_runtime.entities.model_entities import AIModelEntity, ModelType from core.model_runtime.entities.provider_entities import ProviderEntity from core.model_runtime.model_providers.__base.ai_model import AIModel -from core.utils.module_import_helper import get_subclasses_from_module, import_module_from_source class ModelProvider(ABC): diff --git a/api/core/model_runtime/model_providers/model_provider_factory.py b/api/core/model_runtime/model_providers/model_provider_factory.py index 44a1cf2e84318a..26c4199d169384 100644 --- a/api/core/model_runtime/model_providers/model_provider_factory.py +++ b/api/core/model_runtime/model_providers/model_provider_factory.py @@ -4,13 +4,13 @@ from pydantic import BaseModel +from core.helper.module_import_helper import load_single_subclass_from_source +from core.helper.position_helper import get_position_map, sort_to_dict_by_position_map from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.entities.provider_entities import ProviderConfig, ProviderEntity, SimpleProviderEntity from core.model_runtime.model_providers.__base.model_provider import ModelProvider from core.model_runtime.schema_validators.model_credential_schema_validator import ModelCredentialSchemaValidator from core.model_runtime.schema_validators.provider_credential_schema_validator import ProviderCredentialSchemaValidator -from core.utils.module_import_helper import load_single_subclass_from_source -from core.utils.position_helper import get_position_map, sort_to_dict_by_position_map logger = logging.getLogger(__name__) diff --git a/api/core/provider_manager.py b/api/core/provider_manager.py index 2abdf046a3ceb6..996ad7b9fd87da 100644 --- a/api/core/provider_manager.py +++ b/api/core/provider_manager.py @@ -860,6 +860,12 @@ def _to_model_settings(self, provider_entity: ProviderEntity, continue if not load_balancing_model_config.encrypted_config: + if load_balancing_model_config.name == "__inherit__": + load_balancing_configs.append(ModelLoadBalancingConfiguration( + id=load_balancing_model_config.id, + name=load_balancing_model_config.name, + credentials={} + )) continue provider_model_credentials_cache = ProviderCredentialsCache( diff --git a/api/core/tools/provider/builtin/_positions.py b/api/core/tools/provider/builtin/_positions.py index d0826ddcf0067a..e1722e98f96213 100644 --- a/api/core/tools/provider/builtin/_positions.py +++ b/api/core/tools/provider/builtin/_positions.py @@ -1,7 +1,7 @@ import os.path +from core.helper.position_helper import get_position_map, sort_by_position_map from core.tools.entities.user_entities import UserToolProvider -from core.utils.position_helper import get_position_map, sort_by_position_map class BuiltinToolProviderSort: diff --git a/api/core/tools/provider/builtin_tool_provider.py b/api/core/tools/provider/builtin_tool_provider.py index c2178cdd407901..4aaf09eb58f2d0 100644 --- a/api/core/tools/provider/builtin_tool_provider.py +++ b/api/core/tools/provider/builtin_tool_provider.py @@ -4,6 +4,7 @@ from yaml import FullLoader, load +from core.helper.module_import_helper import load_single_subclass_from_source from core.tools.entities.tool_entities import ToolParameter, ToolProviderCredentials, ToolProviderType from core.tools.entities.user_entities import UserToolProviderCredentials from core.tools.errors import ( @@ -15,7 +16,6 @@ from core.tools.provider.tool_provider import ToolProviderController from core.tools.tool.builtin_tool import BuiltinTool from core.tools.tool.tool import Tool -from core.utils.module_import_helper import load_single_subclass_from_source class BuiltinToolProviderController(ToolProviderController): diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py index d46f1f22a574d2..84892dd518881b 100644 --- a/api/core/tools/tool_manager.py +++ b/api/core/tools/tool_manager.py @@ -9,6 +9,7 @@ from flask import current_app from core.agent.entities import AgentToolEntity +from core.helper.module_import_helper import load_single_subclass_from_source from core.model_runtime.utils.encoders import jsonable_encoder from core.tools import * from core.tools.entities.common_entities import I18nObject @@ -28,7 +29,6 @@ ToolConfigurationManager, ToolParameterConfigurationManager, ) -from core.utils.module_import_helper import load_single_subclass_from_source from core.workflow.nodes.tool.entities import ToolEntity from extensions.ext_database import db from models.tools import ApiToolProvider, BuiltinToolProvider diff --git a/api/tests/integration_tests/utils/test_module_import_helper.py b/api/tests/integration_tests/utils/test_module_import_helper.py index 39ac41b648459d..256c9a911f104f 100644 --- a/api/tests/integration_tests/utils/test_module_import_helper.py +++ b/api/tests/integration_tests/utils/test_module_import_helper.py @@ -1,6 +1,6 @@ import os -from core.utils.module_import_helper import import_module_from_source, load_single_subclass_from_source +from core.helper.module_import_helper import import_module_from_source, load_single_subclass_from_source from tests.integration_tests.utils.parent_class import ParentClass From bcbfe92cd81e209e9d0a021f3f5a13e002ae0c2f Mon Sep 17 00:00:00 2001 From: takatost Date: Mon, 13 May 2024 22:47:36 +0800 Subject: [PATCH 034/273] fix bugs --- api/core/indexing_runner.py | 19 +++----------- api/core/model_manager.py | 33 +++++++++++++++++++++--- api/core/splitter/fixed_text_splitter.py | 9 ++----- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 51e77393d660e8..637180dda7a710 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -283,11 +283,7 @@ def indexing_estimate(self, tenant_id: str, extract_settings: list[ExtractSettin if len(preview_texts) < 5: preview_texts.append(document.page_content) if indexing_technique == 'high_quality' or embedding_model_instance: - embedding_model_type_instance = embedding_model_instance.model_type_instance - embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_type_instance) - tokens += embedding_model_type_instance.get_num_tokens( - model=embedding_model_instance.model, - credentials=embedding_model_instance.credentials, + tokens += embedding_model_instance.get_text_embedding_num_tokens( texts=[self.filter_string(document.page_content)] ) @@ -654,10 +650,6 @@ def _load(self, index_processor: BaseIndexProcessor, dataset: Dataset, tokens = 0 chunk_size = 10 - embedding_model_type_instance = None - if embedding_model_instance: - embedding_model_type_instance = embedding_model_instance.model_type_instance - embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_type_instance) # create keyword index create_keyword_thread = threading.Thread(target=self._process_keyword_index, args=(current_app._get_current_object(), @@ -670,8 +662,7 @@ def _load(self, index_processor: BaseIndexProcessor, dataset: Dataset, chunk_documents = documents[i:i + chunk_size] futures.append(executor.submit(self._process_chunk, current_app._get_current_object(), index_processor, chunk_documents, dataset, - dataset_document, embedding_model_instance, - embedding_model_type_instance)) + dataset_document, embedding_model_instance)) for future in futures: tokens += future.result() @@ -712,7 +703,7 @@ def _process_keyword_index(self, flask_app, dataset_id, document_id, documents): db.session.commit() def _process_chunk(self, flask_app, index_processor, chunk_documents, dataset, dataset_document, - embedding_model_instance, embedding_model_type_instance): + embedding_model_instance): with flask_app.app_context(): # check document is paused self._check_document_paused_status(dataset_document.id) @@ -720,9 +711,7 @@ def _process_chunk(self, flask_app, index_processor, chunk_documents, dataset, d tokens = 0 if dataset.indexing_technique == 'high_quality' or embedding_model_type_instance: tokens += sum( - embedding_model_type_instance.get_num_tokens( - embedding_model_instance.model, - embedding_model_instance.credentials, + embedding_model_instance.get_text_embedding_num_tokens( [document.page_content] ) for document in chunk_documents diff --git a/api/core/model_manager.py b/api/core/model_manager.py index e09a669cf23070..53a0a5770ac3c6 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -154,6 +154,24 @@ def invoke_text_embedding(self, texts: list[str], user: Optional[str] = None) \ user=user ) + def get_text_embedding_num_tokens(self, texts: list[str]) -> int: + """ + Get number of tokens for text embedding + + :param texts: texts to embed + :return: + """ + if not isinstance(self.model_type_instance, TextEmbeddingModel): + raise Exception("Model type instance is not TextEmbeddingModel") + + self.model_type_instance = cast(TextEmbeddingModel, self.model_type_instance) + return self._round_robin_invoke( + function=self.model_type_instance.get_num_tokens, + model=self.model, + credentials=self.credentials, + texts=texts + ) + def invoke_rerank(self, query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None) \ @@ -263,21 +281,28 @@ def _round_robin_invoke(self, function: callable, *args, **kwargs): if not self.load_balancing_manager: return function(*args, **kwargs) + last_exception = None while True: lb_config = self.load_balancing_manager.fetch_next() if not lb_config: - raise ProviderTokenNotInitError("Model credentials is not initialized.") + if not last_exception: + raise ProviderTokenNotInitError("Model credentials is not initialized.") + else: + raise last_exception try: - del kwargs['credentials'] + if 'credentials' in kwargs: + del kwargs['credentials'] return function(*args, **kwargs, credentials=lb_config.credentials) - except InvokeRateLimitError: + except InvokeRateLimitError as e: # expire in 60 seconds self.load_balancing_manager.cooldown(lb_config, expire=60) + last_exception = e continue - except InvokeAuthorizationError | InvokeConnectionError: + except (InvokeAuthorizationError, InvokeConnectionError) as e: # expire in 10 seconds self.load_balancing_manager.cooldown(lb_config, expire=10) + last_exception = e continue except Exception as e: raise e diff --git a/api/core/splitter/fixed_text_splitter.py b/api/core/splitter/fixed_text_splitter.py index a1510259ac17b1..d0d6f69179d505 100644 --- a/api/core/splitter/fixed_text_splitter.py +++ b/api/core/splitter/fixed_text_splitter.py @@ -1,10 +1,9 @@ """Functionality for splitting text.""" from __future__ import annotations -from typing import Any, Optional, cast +from typing import Any, Optional from core.model_manager import ModelInstance -from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer from core.splitter.text_splitter import ( TS, @@ -35,11 +34,7 @@ def _token_encoder(text: str) -> int: return 0 if embedding_model_instance: - embedding_model_type_instance = embedding_model_instance.model_type_instance - embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_type_instance) - return embedding_model_type_instance.get_num_tokens( - model=embedding_model_instance.model, - credentials=embedding_model_instance.credentials, + return embedding_model_instance.get_text_embedding_num_tokens( texts=[text] ) else: From 80d0cd8610001b238f7a6ae2e80c197516169bea Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 14 May 2024 19:05:23 +0800 Subject: [PATCH 035/273] fix get num token func use --- api/core/app/apps/base_app_runner.py | 25 ++++++++++--------- .../easy_ui_based_generate_task_pipeline.py | 18 +++++++------ api/core/docstore/dataset_docstore.py | 9 ++----- api/core/memory/token_buffer_memory.py | 13 ++-------- api/core/model_manager.py | 21 ++++++++++++++++ api/core/prompt/prompt_transform.py | 14 +++++------ .../question_classifier_node.py | 12 ++++----- api/services/dataset_service.py | 18 +++---------- .../batch_create_segment_to_index_task.py | 8 +----- 9 files changed, 66 insertions(+), 72 deletions(-) diff --git a/api/core/app/apps/base_app_runner.py b/api/core/app/apps/base_app_runner.py index f1f426b27efacc..545463c8bd69cd 100644 --- a/api/core/app/apps/base_app_runner.py +++ b/api/core/app/apps/base_app_runner.py @@ -1,6 +1,6 @@ import time from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Optional, Union from core.app.app_config.entities import ExternalDataVariableEntity, PromptTemplateEntity from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom @@ -16,11 +16,11 @@ from core.external_data_tool.external_data_fetch import ExternalDataFetch from core.file.file_obj import FileVar from core.memory.token_buffer_memory import TokenBufferMemory +from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage from core.model_runtime.entities.model_entities import ModelPropertyKey from core.model_runtime.errors.invoke import InvokeBadRequestError -from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.moderation.input_moderation import InputModeration from core.prompt.advanced_prompt_transform import AdvancedPromptTransform from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate, MemoryConfig @@ -45,8 +45,11 @@ def get_pre_calculate_rest_tokens(self, app_record: App, :param query: query :return: """ - model_type_instance = model_config.provider_model_bundle.model_type_instance - model_type_instance = cast(LargeLanguageModel, model_type_instance) + # Invoke model + model_instance = ModelInstance( + provider_model_bundle=model_config.provider_model_bundle, + model=model_config.model + ) model_context_tokens = model_config.model_schema.model_properties.get(ModelPropertyKey.CONTEXT_SIZE) @@ -73,9 +76,7 @@ def get_pre_calculate_rest_tokens(self, app_record: App, query=query ) - prompt_tokens = model_type_instance.get_num_tokens( - model_config.model, - model_config.credentials, + prompt_tokens = model_instance.get_llm_num_tokens( prompt_messages ) @@ -89,8 +90,10 @@ def get_pre_calculate_rest_tokens(self, app_record: App, def recalc_llm_max_tokens(self, model_config: ModelConfigWithCredentialsEntity, prompt_messages: list[PromptMessage]): # recalc max_tokens if sum(prompt_token + max_tokens) over model token limit - model_type_instance = model_config.provider_model_bundle.model_type_instance - model_type_instance = cast(LargeLanguageModel, model_type_instance) + model_instance = ModelInstance( + provider_model_bundle=model_config.provider_model_bundle, + model=model_config.model + ) model_context_tokens = model_config.model_schema.model_properties.get(ModelPropertyKey.CONTEXT_SIZE) @@ -107,9 +110,7 @@ def recalc_llm_max_tokens(self, model_config: ModelConfigWithCredentialsEntity, if max_tokens is None: max_tokens = 0 - prompt_tokens = model_type_instance.get_num_tokens( - model_config.model, - model_config.credentials, + prompt_tokens = model_instance.get_llm_num_tokens( prompt_messages ) diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py index a7dbb4754c38da..f71470edb242df 100644 --- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py @@ -37,6 +37,7 @@ ) from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline from core.app.task_pipeline.message_cycle_manage import MessageCycleManage +from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage from core.model_runtime.entities.message_entities import ( AssistantPromptMessage, @@ -317,29 +318,30 @@ def _handle_stop(self, event: QueueStopEvent) -> None: """ model_config = self._model_config model = model_config.model - model_type_instance = model_config.provider_model_bundle.model_type_instance - model_type_instance = cast(LargeLanguageModel, model_type_instance) + + model_instance = ModelInstance( + provider_model_bundle=model_config.provider_model_bundle, + model=model_config.model + ) # calculate num tokens prompt_tokens = 0 if event.stopped_by != QueueStopEvent.StopBy.ANNOTATION_REPLY: - prompt_tokens = model_type_instance.get_num_tokens( - model, - model_config.credentials, + prompt_tokens = model_instance.get_llm_num_tokens( self._task_state.llm_result.prompt_messages ) completion_tokens = 0 if event.stopped_by == QueueStopEvent.StopBy.USER_MANUAL: - completion_tokens = model_type_instance.get_num_tokens( - model, - model_config.credentials, + completion_tokens = model_instance.get_llm_num_tokens( [self._task_state.llm_result.message] ) credentials = model_config.credentials # transform usage + model_type_instance = model_config.provider_model_bundle.model_type_instance + model_type_instance = cast(LargeLanguageModel, model_type_instance) self._task_state.llm_result.usage = model_type_instance._calc_response_usage( model, credentials, diff --git a/api/core/docstore/dataset_docstore.py b/api/core/docstore/dataset_docstore.py index 7567493b9f91ed..e7ec78acc0aaad 100644 --- a/api/core/docstore/dataset_docstore.py +++ b/api/core/docstore/dataset_docstore.py @@ -1,11 +1,10 @@ from collections.abc import Sequence -from typing import Any, Optional, cast +from typing import Any, Optional from sqlalchemy import func from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType -from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel from core.rag.models.document import Document from extensions.ext_database import db from models.dataset import Dataset, DocumentSegment @@ -95,11 +94,7 @@ def add_documents( # calc embedding use tokens if embedding_model: - model_type_instance = embedding_model.model_type_instance - model_type_instance = cast(TextEmbeddingModel, model_type_instance) - tokens = model_type_instance.get_num_tokens( - model=embedding_model.model, - credentials=embedding_model.credentials, + tokens = embedding_model.get_text_embedding_num_tokens( texts=[doc.page_content] ) else: diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index cd0b2508d407f1..6b53104c709d23 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -9,8 +9,6 @@ TextPromptMessageContent, UserPromptMessage, ) -from core.model_runtime.entities.model_entities import ModelType -from core.model_runtime.model_providers import model_provider_factory from extensions.ext_database import db from models.model import AppMode, Conversation, Message @@ -78,12 +76,7 @@ def get_history_prompt_messages(self, max_token_limit: int = 2000, return [] # prune the chat message if it exceeds the max token limit - provider_instance = model_provider_factory.get_provider_instance(self.model_instance.provider) - model_type_instance = provider_instance.get_model_instance(ModelType.LLM) - - curr_message_tokens = model_type_instance.get_num_tokens( - self.model_instance.model, - self.model_instance.credentials, + curr_message_tokens = self.model_instance.get_llm_num_tokens( prompt_messages ) @@ -91,9 +84,7 @@ def get_history_prompt_messages(self, max_token_limit: int = 2000, pruned_memory = [] while curr_message_tokens > max_token_limit and prompt_messages: pruned_memory.append(prompt_messages.pop(0)) - curr_message_tokens = model_type_instance.get_num_tokens( - self.model_instance.model, - self.model_instance.credentials, + curr_message_tokens = self.model_instance.get_llm_num_tokens( prompt_messages ) diff --git a/api/core/model_manager.py b/api/core/model_manager.py index 53a0a5770ac3c6..c86f610eea63ba 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -133,6 +133,27 @@ def invoke_llm(self, prompt_messages: list[PromptMessage], model_parameters: Opt callbacks=callbacks ) + def get_llm_num_tokens(self, prompt_messages: list[PromptMessage], + tools: Optional[list[PromptMessageTool]] = None) -> int: + """ + Get number of tokens for llm + + :param prompt_messages: prompt messages + :param tools: tools for tool calling + :return: + """ + if not isinstance(self.model_type_instance, LargeLanguageModel): + raise Exception("Model type instance is not LargeLanguageModel") + + self.model_type_instance = cast(LargeLanguageModel, self.model_type_instance) + return self._round_robin_invoke( + function=self.model_type_instance.get_num_tokens, + model=self.model, + credentials=self.credentials, + prompt_messages=prompt_messages, + tools=tools + ) + def invoke_text_embedding(self, texts: list[str], user: Optional[str] = None) \ -> TextEmbeddingResult: """ diff --git a/api/core/prompt/prompt_transform.py b/api/core/prompt/prompt_transform.py index 9bf2ae090f7686..d8e2d2f76d2201 100644 --- a/api/core/prompt/prompt_transform.py +++ b/api/core/prompt/prompt_transform.py @@ -1,10 +1,10 @@ -from typing import Optional, cast +from typing import Optional from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity from core.memory.token_buffer_memory import TokenBufferMemory +from core.model_manager import ModelInstance from core.model_runtime.entities.message_entities import PromptMessage from core.model_runtime.entities.model_entities import ModelPropertyKey -from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.prompt.entities.advanced_prompt_entities import MemoryConfig @@ -25,12 +25,12 @@ def _calculate_rest_token(self, prompt_messages: list[PromptMessage], model_context_tokens = model_config.model_schema.model_properties.get(ModelPropertyKey.CONTEXT_SIZE) if model_context_tokens: - model_type_instance = model_config.provider_model_bundle.model_type_instance - model_type_instance = cast(LargeLanguageModel, model_type_instance) + model_instance = ModelInstance( + provider_model_bundle=model_config.provider_model_bundle, + model=model_config.model + ) - curr_message_tokens = model_type_instance.get_num_tokens( - model_config.model, - model_config.credentials, + curr_message_tokens = model_instance.get_llm_num_tokens( prompt_messages ) diff --git a/api/core/workflow/nodes/question_classifier/question_classifier_node.py b/api/core/workflow/nodes/question_classifier/question_classifier_node.py index af1e68b92a6c74..e62f92054b440c 100644 --- a/api/core/workflow/nodes/question_classifier/question_classifier_node.py +++ b/api/core/workflow/nodes/question_classifier/question_classifier_node.py @@ -4,9 +4,9 @@ from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity from core.memory.token_buffer_memory import TokenBufferMemory +from core.model_manager import ModelInstance from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageRole from core.model_runtime.entities.model_entities import ModelPropertyKey -from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.utils.encoders import jsonable_encoder from core.prompt.advanced_prompt_transform import AdvancedPromptTransform from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate @@ -189,12 +189,12 @@ def _calculate_rest_token(self, node_data: QuestionClassifierNodeData, query: st model_context_tokens = model_config.model_schema.model_properties.get(ModelPropertyKey.CONTEXT_SIZE) if model_context_tokens: - model_type_instance = model_config.provider_model_bundle.model_type_instance - model_type_instance = cast(LargeLanguageModel, model_type_instance) + model_instance = ModelInstance( + provider_model_bundle=model_config.provider_model_bundle, + model=model_config.model + ) - curr_message_tokens = model_type_instance.get_num_tokens( - model_config.model, - model_config.credentials, + curr_message_tokens = model_instance.get_llm_num_tokens( prompt_messages ) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index ab8b8487af8ae3..9bfa4a01b790dc 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -4,7 +4,7 @@ import random import time import uuid -from typing import Optional, cast +from typing import Optional from flask import current_app from flask_login import current_user @@ -13,7 +13,6 @@ from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType -from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel from core.rag.datasource.keyword.keyword_factory import Keyword from core.rag.models.document import Document as RAGDocument from events.dataset_event import dataset_was_deleted @@ -1123,10 +1122,7 @@ def create_segment(cls, args: dict, document: Document, dataset: Dataset): model=dataset.embedding_model ) # calc embedding use tokens - model_type_instance = cast(TextEmbeddingModel, embedding_model.model_type_instance) - tokens = model_type_instance.get_num_tokens( - model=embedding_model.model, - credentials=embedding_model.credentials, + tokens = embedding_model.get_text_embedding_num_tokens( texts=[content] ) lock_name = 'add_segment_lock_document_id_{}'.format(document.id) @@ -1194,10 +1190,7 @@ def multi_create_segment(cls, segments: list, document: Document, dataset: Datas tokens = 0 if dataset.indexing_technique == 'high_quality' and embedding_model: # calc embedding use tokens - model_type_instance = cast(TextEmbeddingModel, embedding_model.model_type_instance) - tokens = model_type_instance.get_num_tokens( - model=embedding_model.model, - credentials=embedding_model.credentials, + tokens = embedding_model.get_text_embedding_num_tokens( texts=[content] ) segment_document = DocumentSegment( @@ -1300,10 +1293,7 @@ def update_segment(cls, args: dict, segment: DocumentSegment, document: Document ) # calc embedding use tokens - model_type_instance = cast(TextEmbeddingModel, embedding_model.model_type_instance) - tokens = model_type_instance.get_num_tokens( - model=embedding_model.model, - credentials=embedding_model.credentials, + tokens = embedding_model.get_text_embedding_num_tokens( texts=[content] ) segment.content = content diff --git a/api/tasks/batch_create_segment_to_index_task.py b/api/tasks/batch_create_segment_to_index_task.py index d6dc970477cf8c..67cc03bdebee54 100644 --- a/api/tasks/batch_create_segment_to_index_task.py +++ b/api/tasks/batch_create_segment_to_index_task.py @@ -2,7 +2,6 @@ import logging import time import uuid -from typing import cast import click from celery import shared_task @@ -11,7 +10,6 @@ from core.indexing_runner import IndexingRunner from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType -from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel from extensions.ext_database import db from extensions.ext_redis import redis_client from libs import helper @@ -59,16 +57,12 @@ def batch_create_segment_to_index_task(job_id: str, content: list, dataset_id: s model=dataset.embedding_model ) - model_type_instance = embedding_model.model_type_instance - model_type_instance = cast(TextEmbeddingModel, model_type_instance) for segment in content: content = segment['content'] doc_id = str(uuid.uuid4()) segment_hash = helper.generate_text_hash(content) # calc embedding use tokens - tokens = model_type_instance.get_num_tokens( - model=embedding_model.model, - credentials=embedding_model.credentials, + tokens = embedding_model.get_text_embedding_num_tokens( texts=[content] ) if embedding_model else 0 max_position = db.session.query(func.max(DocumentSegment.position)).filter( From 03a60465deaa9565684ebcf7d47e437215036adc Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 14 May 2024 19:55:17 +0800 Subject: [PATCH 036/273] remove llm invoke --- api/core/tools/model/tool_model_manager.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/api/core/tools/model/tool_model_manager.py b/api/core/tools/model/tool_model_manager.py index e97d78d6998030..fbf7b69f8a2e61 100644 --- a/api/core/tools/model/tool_model_manager.py +++ b/api/core/tools/model/tool_model_manager.py @@ -71,10 +71,8 @@ def calculate_tokens( if not model_instance: raise InvokeModelError('Model not found') - llm_model = cast(LargeLanguageModel, model_instance.model_type_instance) - # get tokens - tokens = llm_model.get_num_tokens(model_instance.model, model_instance.credentials, prompt_messages) + tokens = model_instance.get_llm_num_tokens(prompt_messages) return tokens @@ -106,13 +104,8 @@ def invoke( tenant_id=tenant_id, model_type=ModelType.LLM, ) - llm_model = cast(LargeLanguageModel, model_instance.model_type_instance) - - # get model credentials - model_credentials = model_instance.credentials - # get prompt tokens - prompt_tokens = llm_model.get_num_tokens(model_instance.model, model_credentials, prompt_messages) + prompt_tokens = model_instance.get_llm_num_tokens(prompt_messages) model_parameters = { 'temperature': 0.8, @@ -142,9 +135,7 @@ def invoke( db.session.commit() try: - response: LLMResult = llm_model.invoke( - model=model_instance.model, - credentials=model_credentials, + response: LLMResult = model_instance.invoke_llm( prompt_messages=prompt_messages, model_parameters=model_parameters, tools=[], stop=[], stream=False, user=user_id, callbacks=[] @@ -174,4 +165,4 @@ def invoke( db.session.commit() - return response \ No newline at end of file + return response From afaf5c19552f87bc7f2e6c57ec988ac6afb94887 Mon Sep 17 00:00:00 2001 From: takatost Date: Wed, 15 May 2024 11:49:19 +0800 Subject: [PATCH 037/273] optimize round-robin --- api/app.py | 33 +++++++++++++++++++++++++++++++++ api/core/model_manager.py | 6 ++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/api/app.py b/api/app.py index a36750410e156d..10b0bd63fd3106 100644 --- a/api/app.py +++ b/api/app.py @@ -1,5 +1,7 @@ import os +from extensions.ext_redis import redis_client + if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true': from gevent import monkey @@ -267,5 +269,36 @@ def pool_stat(): } +@app.route('/test') +def test(): + max_index = 7 + cache_key = "model_lb_index:{}:{}:{}:{}".format( + 'a', + 'b', + 'c', + 'd' + ) + result = redis_client.incr(cache_key) + if result >= 20: + current_index = 1 + redis_client.set(cache_key, current_index) + else: + current_index = result + + redis_client.expire(cache_key, 3600) + if current_index >= max_index: + current_index = current_index % max_index + 1 + + real_index = current_index - 1 + if real_index > max_index: + real_index = 0 + + return { + 'original_index': result, + 'current_index': current_index, + 'real_index': real_index + } + + if __name__ == '__main__': app.run(host='0.0.0.0', port=5001) diff --git a/api/core/model_manager.py b/api/core/model_manager.py index c86f610eea63ba..5138b8d5b76244 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -437,11 +437,13 @@ def fetch_next(self) -> Optional[ModelLoadBalancingConfiguration]: while True: current_index = redis_client.incr(cache_key) - if current_index > max_index: - current_index = current_index % max_index + if current_index >= 10000000: + current_index = 1 redis_client.set(cache_key, current_index) redis_client.expire(cache_key, 3600) + if current_index >= max_index: + current_index = current_index % max_index + 1 real_index = current_index - 1 if real_index > max_index: From bc9bdc1fe2b0133026af84db43d4e692f512ccb3 Mon Sep 17 00:00:00 2001 From: takatost Date: Wed, 15 May 2024 11:49:45 +0800 Subject: [PATCH 038/273] remove test code --- api/app.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/api/app.py b/api/app.py index 10b0bd63fd3106..a36750410e156d 100644 --- a/api/app.py +++ b/api/app.py @@ -1,7 +1,5 @@ import os -from extensions.ext_redis import redis_client - if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true': from gevent import monkey @@ -269,36 +267,5 @@ def pool_stat(): } -@app.route('/test') -def test(): - max_index = 7 - cache_key = "model_lb_index:{}:{}:{}:{}".format( - 'a', - 'b', - 'c', - 'd' - ) - result = redis_client.incr(cache_key) - if result >= 20: - current_index = 1 - redis_client.set(cache_key, current_index) - else: - current_index = result - - redis_client.expire(cache_key, 3600) - if current_index >= max_index: - current_index = current_index % max_index + 1 - - real_index = current_index - 1 - if real_index > max_index: - real_index = 0 - - return { - 'original_index': result, - 'current_index': current_index, - 'real_index': real_index - } - - if __name__ == '__main__': app.run(host='0.0.0.0', port=5001) From acae13acb93d08437fb30b69a414ec52b6da4d1e Mon Sep 17 00:00:00 2001 From: takatost Date: Wed, 15 May 2024 14:58:13 +0800 Subject: [PATCH 039/273] optimize config comments --- api/config.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/api/config.py b/api/config.py index 7ad13f1a4938b1..9e4e6951737481 100644 --- a/api/config.py +++ b/api/config.py @@ -211,22 +211,30 @@ def __init__(self): # ------------------------ self.STORAGE_TYPE = get_env('STORAGE_TYPE') self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH') + + # S3 Storage settings self.S3_ENDPOINT = get_env('S3_ENDPOINT') self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME') self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY') self.S3_SECRET_KEY = get_env('S3_SECRET_KEY') self.S3_REGION = get_env('S3_REGION') self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE') + + # Azure Blob Storage settings self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME') self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY') self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME') self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL') + + # Google Cloud Storage settings self.ALIYUN_OSS_BUCKET_NAME=get_env('ALIYUN_OSS_BUCKET_NAME') self.ALIYUN_OSS_ACCESS_KEY=get_env('ALIYUN_OSS_ACCESS_KEY') self.ALIYUN_OSS_SECRET_KEY=get_env('ALIYUN_OSS_SECRET_KEY') self.ALIYUN_OSS_ENDPOINT=get_env('ALIYUN_OSS_ENDPOINT') self.ALIYUN_OSS_REGION=get_env('ALIYUN_OSS_REGION') self.ALIYUN_OSS_AUTH_VERSION=get_env('ALIYUN_OSS_AUTH_VERSION') + + # Google Cloud Storage settings self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME') self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64') @@ -236,6 +244,7 @@ def __init__(self): # ------------------------ self.VECTOR_STORE = get_env('VECTOR_STORE') self.KEYWORD_STORE = get_env('KEYWORD_STORE') + # qdrant settings self.QDRANT_URL = get_env('QDRANT_URL') self.QDRANT_API_KEY = get_env('QDRANT_API_KEY') @@ -378,6 +387,9 @@ def __init__(self): self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE') self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN') + # Model Load Balancing Configurations. + self.MODEL_LB_ENABLED = get_bool_env('MODEL_LB_ENABLED') + # Platform Billing Configurations. self.BILLING_ENABLED = get_bool_env('BILLING_ENABLED') @@ -387,4 +399,3 @@ def __init__(self): # ------------------------ self.ENTERPRISE_ENABLED = get_bool_env('ENTERPRISE_ENABLED') self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO') - self.MODEL_LB_ENABLED = get_bool_env('MODEL_LB_ENABLED') From 844e4800aa64b9791f045f02dab3ad23b6f94abf Mon Sep 17 00:00:00 2001 From: takatost Date: Wed, 15 May 2024 18:10:50 +0800 Subject: [PATCH 040/273] add _to_model_settings tests --- api/core/provider_manager.py | 2 +- .../unit_tests/core/test_provider_manager.py | 183 ++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 api/tests/unit_tests/core/test_provider_manager.py diff --git a/api/core/provider_manager.py b/api/core/provider_manager.py index 996ad7b9fd87da..1bca4a64e99f89 100644 --- a/api/core/provider_manager.py +++ b/api/core/provider_manager.py @@ -917,7 +917,7 @@ def _to_model_settings(self, provider_entity: ProviderEntity, model=provider_model_setting.model_name, model_type=ModelType.value_of(provider_model_setting.model_type), enabled=provider_model_setting.enabled, - load_balancing_configs=load_balancing_configs + load_balancing_configs=load_balancing_configs if len(load_balancing_configs) > 1 else [] ) ) diff --git a/api/tests/unit_tests/core/test_provider_manager.py b/api/tests/unit_tests/core/test_provider_manager.py new file mode 100644 index 00000000000000..072b6f100f3a16 --- /dev/null +++ b/api/tests/unit_tests/core/test_provider_manager.py @@ -0,0 +1,183 @@ +from core.entities.provider_entities import ModelSettings +from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.model_providers import model_provider_factory +from core.provider_manager import ProviderManager +from models.provider import LoadBalancingModelConfig, ProviderModelSetting + + +def test__to_model_settings(mocker): + # Get all provider entities + provider_entities = model_provider_factory.get_providers() + + provider_entity = None + for provider in provider_entities: + if provider.provider == 'openai': + provider_entity = provider + + # Mocking the inputs + provider_model_settings = [ProviderModelSetting( + id='id', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + enabled=True, + load_balancing_enabled=True + )] + load_balancing_model_configs = [ + LoadBalancingModelConfig( + id='id1', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + name='__inherit__', + encrypted_config=None, + enabled=True + ), + LoadBalancingModelConfig( + id='id2', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + name='first', + encrypted_config='{"openai_api_key": "fake_key"}', + enabled=True + ) + ] + + mocker.patch('core.helper.model_provider_cache.ProviderCredentialsCache.get', return_value={"openai_api_key": "fake_key"}) + + provider_manager = ProviderManager() + + # Running the method + result = provider_manager._to_model_settings( + provider_entity, + provider_model_settings, + load_balancing_model_configs + ) + + # Asserting that the result is as expected + assert len(result) == 1 + assert isinstance(result[0], ModelSettings) + assert result[0].model == 'gpt-4' + assert result[0].model_type == ModelType.LLM + assert result[0].enabled is True + assert len(result[0].load_balancing_configs) == 2 + assert result[0].load_balancing_configs[0].name == '__inherit__' + assert result[0].load_balancing_configs[1].name == 'first' + + +def test__to_model_settings_only_one_lb(mocker): + # Get all provider entities + provider_entities = model_provider_factory.get_providers() + + provider_entity = None + for provider in provider_entities: + if provider.provider == 'openai': + provider_entity = provider + + # Mocking the inputs + provider_model_settings = [ProviderModelSetting( + id='id', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + enabled=True, + load_balancing_enabled=True + )] + load_balancing_model_configs = [ + LoadBalancingModelConfig( + id='id1', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + name='__inherit__', + encrypted_config=None, + enabled=True + ) + ] + + mocker.patch('core.helper.model_provider_cache.ProviderCredentialsCache.get', return_value={"openai_api_key": "fake_key"}) + + provider_manager = ProviderManager() + + # Running the method + result = provider_manager._to_model_settings( + provider_entity, + provider_model_settings, + load_balancing_model_configs + ) + + # Asserting that the result is as expected + assert len(result) == 1 + assert isinstance(result[0], ModelSettings) + assert result[0].model == 'gpt-4' + assert result[0].model_type == ModelType.LLM + assert result[0].enabled is True + assert len(result[0].load_balancing_configs) == 0 + + +def test__to_model_settings_lb_disabled(mocker): + # Get all provider entities + provider_entities = model_provider_factory.get_providers() + + provider_entity = None + for provider in provider_entities: + if provider.provider == 'openai': + provider_entity = provider + + # Mocking the inputs + provider_model_settings = [ProviderModelSetting( + id='id', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + enabled=True, + load_balancing_enabled=False + )] + load_balancing_model_configs = [ + LoadBalancingModelConfig( + id='id1', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + name='__inherit__', + encrypted_config=None, + enabled=True + ), + LoadBalancingModelConfig( + id='id2', + tenant_id='tenant_id', + provider_name='openai', + model_name='gpt-4', + model_type='text-generation', + name='first', + encrypted_config='{"openai_api_key": "fake_key"}', + enabled=True + ) + ] + + mocker.patch('core.helper.model_provider_cache.ProviderCredentialsCache.get', return_value={"openai_api_key": "fake_key"}) + + provider_manager = ProviderManager() + + # Running the method + result = provider_manager._to_model_settings( + provider_entity, + provider_model_settings, + load_balancing_model_configs + ) + + # Asserting that the result is as expected + assert len(result) == 1 + assert isinstance(result[0], ModelSettings) + assert result[0].model == 'gpt-4' + assert result[0].model_type == ModelType.LLM + assert result[0].enabled is True + assert len(result[0].load_balancing_configs) == 0 From 95c66c92b3f6f8b11eac97a83290421cac7ab28b Mon Sep 17 00:00:00 2001 From: takatost Date: Thu, 16 May 2024 00:14:25 +0800 Subject: [PATCH 041/273] add lb_model_manager test --- api/core/model_manager.py | 6 +- .../unit_tests/core/test_model_manager.py | 77 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 api/tests/unit_tests/core/test_model_manager.py diff --git a/api/core/model_manager.py b/api/core/model_manager.py index 5138b8d5b76244..2ae6e0bbed7b34 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -442,8 +442,8 @@ def fetch_next(self) -> Optional[ModelLoadBalancingConfiguration]: redis_client.set(cache_key, current_index) redis_client.expire(cache_key, 3600) - if current_index >= max_index: - current_index = current_index % max_index + 1 + if current_index > max_index: + current_index = current_index % max_index real_index = current_index - 1 if real_index > max_index: @@ -468,7 +468,7 @@ def fetch_next(self) -> Optional[ModelLoadBalancingConfiguration]: return None - def cooldown(self, config: ModelLoadBalancingConfiguration, expire: int = 60): + def cooldown(self, config: ModelLoadBalancingConfiguration, expire: int = 60) -> None: """ Cooldown model load balancing config :param config: model load balancing config diff --git a/api/tests/unit_tests/core/test_model_manager.py b/api/tests/unit_tests/core/test_model_manager.py new file mode 100644 index 00000000000000..495dd4bdfeab1d --- /dev/null +++ b/api/tests/unit_tests/core/test_model_manager.py @@ -0,0 +1,77 @@ +from unittest.mock import MagicMock + +import pytest + +from core.entities.provider_entities import ModelLoadBalancingConfiguration +from core.model_manager import LBModelManager +from core.model_runtime.entities.model_entities import ModelType + + +@pytest.fixture +def lb_model_manager(): + load_balancing_configs = [ + ModelLoadBalancingConfiguration( + id='id1', + name='__inherit__', + credentials={} + ), + ModelLoadBalancingConfiguration( + id='id2', + name='first', + credentials={"openai_api_key": "fake_key"} + ), + ModelLoadBalancingConfiguration( + id='id3', + name='second', + credentials={"openai_api_key": "fake_key"} + ) + ] + + lb_model_manager = LBModelManager( + tenant_id='tenant_id', + provider='openai', + model_type=ModelType.LLM, + model='gpt-4', + load_balancing_configs=load_balancing_configs, + managed_credentials={"openai_api_key": "fake_key"} + ) + + lb_model_manager.cooldown = MagicMock(return_value=None) + + def is_cooldown(config: ModelLoadBalancingConfiguration): + if config.id == 'id1': + return True + + return False + + lb_model_manager.is_cooldown = MagicMock(side_effect=is_cooldown) + + return lb_model_manager + + +def test_lb_model_manager_fetch_next(mocker, lb_model_manager): + assert len(lb_model_manager._load_balancing_configs) == 3 + + config1 = lb_model_manager._load_balancing_configs[0] + config2 = lb_model_manager._load_balancing_configs[1] + config3 = lb_model_manager._load_balancing_configs[2] + + assert lb_model_manager.is_cooldown(config1) is True + assert lb_model_manager.is_cooldown(config2) is False + assert lb_model_manager.is_cooldown(config3) is False + + start_index = 0 + def incr(key): + nonlocal start_index + start_index += 1 + return start_index + + mocker.patch('redis.Redis.incr', side_effect=incr) + mocker.patch('redis.Redis.set', return_value=None) + mocker.patch('redis.Redis.expire', return_value=None) + + config = lb_model_manager.fetch_next() + assert config == config2 + + config = lb_model_manager.fetch_next() + assert config == config3 From 503ed7ff768600ee6687242dfc8b521698721b98 Mon Sep 17 00:00:00 2001 From: takatost Date: Thu, 16 May 2024 00:32:18 +0800 Subject: [PATCH 042/273] add in cooldown and ttl return --- .../workspace/load_balancing_config.py | 4 +-- api/core/model_manager.py | 33 +++++++++++++++++-- api/fields/load_balancing_config.py | 12 ------- api/services/model_load_balancing_service.py | 24 ++++++++++++-- .../unit_tests/core/test_model_manager.py | 8 ++--- 5 files changed, 58 insertions(+), 23 deletions(-) delete mode 100644 api/fields/load_balancing_config.py diff --git a/api/controllers/console/workspace/load_balancing_config.py b/api/controllers/console/workspace/load_balancing_config.py index 31a2ad24185a54..8d0ba2a75fc2d9 100644 --- a/api/controllers/console/workspace/load_balancing_config.py +++ b/api/controllers/console/workspace/load_balancing_config.py @@ -1,4 +1,4 @@ -from flask_restful import Resource, marshal_with, reqparse +from flask_restful import Resource, reqparse from werkzeug.exceptions import Forbidden, NotFound from controllers.console import api @@ -6,7 +6,6 @@ from controllers.console.wraps import account_initialization_required from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.errors.validate import CredentialsValidateFailedError -from fields.load_balancing_config import load_balancing_config_list_fields from libs.login import current_user, login_required from models.account import TenantAccountRole from services.model_load_balancing_service import ModelLoadBalancingService @@ -67,7 +66,6 @@ class LoadBalancingConfigListApi(Resource): @setup_required @login_required @account_initialization_required - @marshal_with(load_balancing_config_list_fields) def get(self, provider: str): tenant_id = current_user.current_tenant_id diff --git a/api/core/model_manager.py b/api/core/model_manager.py index 2ae6e0bbed7b34..a4aaac48309e6d 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -451,7 +451,7 @@ def fetch_next(self) -> Optional[ModelLoadBalancingConfiguration]: config = self._load_balancing_configs[real_index] - if self.is_cooldown(config): + if self.in_cooldown(config): cooldown_load_balancing_configs.append(config) if len(cooldown_load_balancing_configs) >= len(self._load_balancing_configs): # all configs are in cooldown @@ -485,7 +485,7 @@ def cooldown(self, config: ModelLoadBalancingConfiguration, expire: int = 60) -> redis_client.setex(cooldown_cache_key, expire, 'true') - def is_cooldown(self, config: ModelLoadBalancingConfiguration) -> bool: + def in_cooldown(self, config: ModelLoadBalancingConfiguration) -> bool: """ Check if model load balancing config is in cooldown :param config: model load balancing config @@ -500,3 +500,32 @@ def is_cooldown(self, config: ModelLoadBalancingConfiguration) -> bool: ) return redis_client.exists(cooldown_cache_key) + + @classmethod + def get_config_in_cooldown_and_ttl(cls, tenant_id: str, + provider: str, + model_type: ModelType, + model: str, + config_id: str) -> tuple[bool, int]: + """ + Get model load balancing config is in cooldown and ttl + :param tenant_id: workspace id + :param provider: provider name + :param model_type: model type + :param model: model name + :param config_id: model load balancing config id + :return: + """ + cooldown_cache_key = "model_lb_index:cooldown:{}:{}:{}:{}:{}".format( + tenant_id, + provider, + model_type.value, + model, + config_id + ) + + ttl = redis_client.ttl(cooldown_cache_key) + if ttl == -2: + return False, 0 + + return True, ttl diff --git a/api/fields/load_balancing_config.py b/api/fields/load_balancing_config.py deleted file mode 100644 index 93806a10b89856..00000000000000 --- a/api/fields/load_balancing_config.py +++ /dev/null @@ -1,12 +0,0 @@ -from flask_restful import fields - -load_balancing_config_fields = { - 'id': fields.String, - 'name': fields.String, - 'enabled': fields.Boolean -} - - -load_balancing_config_list_fields = { - 'data': fields.List(fields.Nested(load_balancing_config_fields)) -} diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index 79ee74bb0c9c4f..4de697b92153e9 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -7,6 +7,7 @@ from core.entities.provider_configuration import ProviderConfiguration from core.helper import encrypter from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType +from core.model_manager import LBModelManager from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.entities.provider_entities import ( ModelCredentialSchema, @@ -74,7 +75,7 @@ def disable_model_load_balancing(self, tenant_id: str, provider: str, model: str ) def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, model_type: str) \ - -> list[LoadBalancingModelConfig]: + -> list[dict]: """ Get load balancing configurations. :param tenant_id: workspace id @@ -117,7 +118,26 @@ def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, # prepend the inherit configuration load_balancing_configs.insert(0, inherit_config) - return load_balancing_configs + # fetch status and ttl for each config + datas = [] + for load_balancing_config in load_balancing_configs: + in_cooldown, ttl = LBModelManager.get_config_in_cooldown_and_ttl( + tenant_id=tenant_id, + provider=provider, + model=model, + model_type=model_type, + config_id=load_balancing_config.id + ) + + datas.append({ + 'id': load_balancing_config.id, + 'name': load_balancing_config.name, + 'enabled': load_balancing_config.enabled, + 'in_cooldown': in_cooldown, + 'ttl': ttl + }) + + return datas def get_load_balancing_config(self, tenant_id: str, provider: str, model: str, model_type: str, config_id: str) \ -> Optional[dict]: diff --git a/api/tests/unit_tests/core/test_model_manager.py b/api/tests/unit_tests/core/test_model_manager.py index 495dd4bdfeab1d..3024a54a4d325c 100644 --- a/api/tests/unit_tests/core/test_model_manager.py +++ b/api/tests/unit_tests/core/test_model_manager.py @@ -44,7 +44,7 @@ def is_cooldown(config: ModelLoadBalancingConfiguration): return False - lb_model_manager.is_cooldown = MagicMock(side_effect=is_cooldown) + lb_model_manager.in_cooldown = MagicMock(side_effect=is_cooldown) return lb_model_manager @@ -56,9 +56,9 @@ def test_lb_model_manager_fetch_next(mocker, lb_model_manager): config2 = lb_model_manager._load_balancing_configs[1] config3 = lb_model_manager._load_balancing_configs[2] - assert lb_model_manager.is_cooldown(config1) is True - assert lb_model_manager.is_cooldown(config2) is False - assert lb_model_manager.is_cooldown(config3) is False + assert lb_model_manager.in_cooldown(config1) is True + assert lb_model_manager.in_cooldown(config2) is False + assert lb_model_manager.in_cooldown(config3) is False start_index = 0 def incr(key): From e217ad9f5fc5e29ac0446b739c3e9b240b319dd3 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 16 May 2024 16:38:59 +0800 Subject: [PATCH 043/273] add website sync --- .../console/datasets/datasets_document.py | 27 ++++++ api/core/indexing_runner.py | 10 +- .../rag/extractor/entity/datasource_type.py | 2 +- .../rag/extractor/entity/extract_setting.py | 2 + api/core/rag/extractor/extract_processor.py | 4 +- .../rag/extractor/firecrawl/firecrawl_app.py | 39 ++++++-- .../firecrawl/firecrawl_web_extractor.py | 38 ++++++-- api/models/source.py | 6 +- api/services/auth/api_key_auth_service.py | 18 ++++ api/services/dataset_service.py | 32 ++++-- api/services/website_service.py | 97 ++++++++++++------- .../sync_website_document_indexing_task.py | 90 +++++++++++++++++ .../rag/extractor/firecrawl/test_firecrawl.py | 51 +++++----- 13 files changed, 327 insertions(+), 89 deletions(-) create mode 100644 api/tasks/sync_website_document_indexing_task.py diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index edec06576c7b59..0e7ac9f8305641 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -924,6 +924,32 @@ def post(self, dataset_id): return {'result': 'success'}, 204 +class WebsiteDocumentSyncApi(DocumentResource): + @setup_required + @login_required + @account_initialization_required + def post(self, dataset_id, document_id): + """sync website document.""" + dataset_id = str(dataset_id) + dataset = DatasetService.get_dataset(dataset_id) + if not dataset: + raise NotFound('Dataset not found.') + document_id = str(document_id) + document = DocumentService.get_document(dataset.id, document_id) + if not document: + raise NotFound('Document not found.') + if document.tenant_id != current_user.current_tenant_id: + raise Forbidden('No permission.') + if document.data_source_type != 'website_crawl': + raise ValueError('Document is not a website document.') + # 403 if document is archived + if DocumentService.check_archived(document): + raise ArchivedDocumentImmutableError() + # sync document + DocumentService.sync_website_document(dataset_id, document) + + return {'result': 'success'}, 204 + api.add_resource(GetProcessRuleApi, '/datasets/process-rule') api.add_resource(DatasetDocumentListApi, '/datasets//documents') @@ -950,3 +976,4 @@ def post(self, dataset_id): api.add_resource(DocumentPauseApi, '/datasets//documents//processing/pause') api.add_resource(DocumentRecoverApi, '/datasets//documents//processing/resume') api.add_resource(DocumentRetryApi, '/datasets//retry') +api.add_resource(WebsiteDocumentSyncApi, '/datasets///website-sync') \ No newline at end of file diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 1de1033cc1c74e..5578033e00c418 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -340,7 +340,7 @@ def indexing_estimate(self, tenant_id: str, extract_settings: list[ExtractSettin def _extract(self, index_processor: BaseIndexProcessor, dataset_document: DatasetDocument, process_rule: dict) \ -> list[Document]: # load file - if dataset_document.data_source_type not in ["upload_file", "notion_import"]: + if dataset_document.data_source_type not in ["upload_file", "notion_import", "website_crawl"]: return [] data_source_info = dataset_document.data_source_info_dict @@ -376,16 +376,18 @@ def _extract(self, index_processor: BaseIndexProcessor, dataset_document: Datase document_model=dataset_document.doc_form ) text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule['mode']) - elif dataset_document.data_source_type == 'website': + elif dataset_document.data_source_type == 'website_crawl': if (not data_source_info or 'provider' not in data_source_info or 'url' not in data_source_info or 'job_id' not in data_source_info): raise ValueError("no website import info found") extract_setting = ExtractSetting( - datasource_type="website", + datasource_type="website_crawl", website_info={ "provider": data_source_info['provider'], "job_id": data_source_info['job_id'], - "url": data_source_info['url'] + "url": data_source_info['url'], + "mode": data_source_info['mode'], + "only_main_content": data_source_info['only_main_content'] }, document_model=dataset_document.doc_form ) diff --git a/api/core/rag/extractor/entity/datasource_type.py b/api/core/rag/extractor/entity/datasource_type.py index 4ce379e96bda71..19ad300d110fe6 100644 --- a/api/core/rag/extractor/entity/datasource_type.py +++ b/api/core/rag/extractor/entity/datasource_type.py @@ -4,4 +4,4 @@ class DatasourceType(Enum): FILE = "upload_file" NOTION = "notion_import" - WEBSITE = "website" + WEBSITE = "website_crawl" diff --git a/api/core/rag/extractor/entity/extract_setting.py b/api/core/rag/extractor/entity/extract_setting.py index 096252f32f5047..9bf99d06af75ef 100644 --- a/api/core/rag/extractor/entity/extract_setting.py +++ b/api/core/rag/extractor/entity/extract_setting.py @@ -28,6 +28,8 @@ class WebsiteInfo(BaseModel): provider: str job_id: str url: str + mode: str + only_main_content: bool = False class Config: arbitrary_types_allowed = True diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index b18855db45c54d..88b3058554a046 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -146,7 +146,9 @@ def extract(cls, extract_setting: ExtractSetting, is_automatic: bool = False, if extract_setting.website_info.provider == 'firecrawl': extractor = FirecrawlWebExtractor( url=extract_setting.website_info.url, - job_id=extract_setting.website_info.job_id + job_id=extract_setting.website_info.job_id, + mode=extract_setting.website_info.mode, + only_main_content=extract_setting.website_info.only_main_content ) return extractor.extract() else: diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 559ee3ba59ba72..e8fe59e44b8deb 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -9,8 +9,8 @@ def __init__(self, api_key=None, base_url=None): self.base_url = base_url or 'https://api.firecrawl.dev' if self.api_key is None and self.base_url == 'https://api.firecrawl.dev': raise ValueError('No API key provided') - - def scrape_url(self, url, params=None): + + def scrape_url(self, url, params=None) -> dict: headers = { 'Content-Type': 'application/json', 'Authorization': f'Bearer {self.api_key}' @@ -26,10 +26,16 @@ def scrape_url(self, url, params=None): if response.status_code == 200: response = response.json() if response['success'] == True: - return response['data'] + data = response['data'] + return { + 'title': data.get('metadata').get('title'), + 'description': data.get('metadata').get('description'), + 'source_url': data.get('metadata').get('sourceURL'), + 'markdown': data.get('markdown') + } else: raise Exception(f'Failed to scrape URL. Error: {response["error"]}') - + elif response.status_code in [402, 409, 500]: error_message = response.json().get('error', 'Unknown error occurred') raise Exception(f'Failed to scrape URL. Status code: {response.status_code}. Error: {error_message}') @@ -49,12 +55,33 @@ def crawl_url(self, url, params=None) -> str: else: self._handle_error(response, 'start crawl job') - def check_crawl_status(self, job_id) -> dict: headers = self._prepare_headers() response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) if response.status_code == 200: - return response.json() + crawl_status_response = response.json() + if crawl_status_response.get('status') != 'completed': + return { + 'status': crawl_status_response.get('status'), + 'data': [] + } + else: + data = crawl_status_response.get('data', []) + url_data_list = [] + for item in data: + if item.get('success', False): + if item.get('data'): + url_data = { + 'title': item.get('data').get('metadata').get('title'), + 'description': item.get('data').get('metadata').get('description'), + 'source_url': item.get('data').get('metadata').get('sourceURL'), + 'markdown': item.get('data').get('markdown') + } + url_data_list.append(url_data) + return { + 'status': 'completed', + 'data': url_data_list + } else: self._handle_error(response, 'check crawl status') diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index 5ff88e269d007f..870ae8d272458e 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -4,7 +4,6 @@ class FirecrawlWebExtractor(BaseExtractor): - """ Crawl and scrape websites and return content in clean llm-ready markdown. @@ -16,20 +15,43 @@ class FirecrawlWebExtractor(BaseExtractor): mode: The mode of operation. Defaults to 'scrape'. Options are 'crawl', 'scrape' and 'crawl_return_urls'. """ - def __init__( - self, - url: str, - job_id: str + self, + url: str, + job_id: str, + mode: str = 'crawl', + only_main_content: bool = False ): """Initialize with url, api_key, base_url and mode.""" self._url = url self.job_id = job_id + self.mode = mode + self.only_main_content = only_main_content def extract(self) -> list[Document]: """Extract content from the URL.""" documents = [] - document = WebsiteService.get_crawl_url_data(self.job_id, 'firecrawl', self._url) - if document: + if self.mode == 'crawl': + crawl_data = WebsiteService.get_crawl_url_data(self.job_id, 'firecrawl', self._url) + if crawl_data is None: + return [] + document = Document(page_content=crawl_data.get('markdown', ''), + metadata={ + 'source_url': crawl_data.get('source_url'), + 'description': crawl_data.get('description'), + 'title': crawl_data.get('title') + } + ) + documents.append(document) + elif self.mode == 'scrape': + scrape_data = WebsiteService.get_scrape_url_data('firecrawl', self._url, self.only_main_content) + + document = Document(page_content=scrape_data.get('markdown', ''), + metadata={ + 'source_url': scrape_data.get('source_url'), + 'description': scrape_data.get('description'), + 'title': scrape_data.get('title') + } + ) documents.append(document) - return [] \ No newline at end of file + return documents diff --git a/api/models/source.py b/api/models/source.py index b870f1fa37db74..265e68f014c6c2 100644 --- a/api/models/source.py +++ b/api/models/source.py @@ -47,8 +47,8 @@ def to_dict(self): 'tenant_id': self.tenant_id, 'category': self.category, 'provider': self.provider, - 'credentials': json.dumps(self.credentials, ensure_ascii=False), - 'created_at': self.created_at, - 'updated_at': self.updated_at, + 'credentials': json.loads(self.credentials), + 'created_at': self.created_at.timestamp(), + 'updated_at': self.updated_at.timestamp(), 'disabled': self.disabled } diff --git a/api/services/auth/api_key_auth_service.py b/api/services/auth/api_key_auth_service.py index 6fe572ebcd779b..6cab79f6924202 100644 --- a/api/services/auth/api_key_auth_service.py +++ b/api/services/auth/api_key_auth_service.py @@ -1,5 +1,6 @@ import json +from core.helper import encrypter from extensions.ext_database import db from models.source import DataSourceApiKeyAuthBinding from services.auth.api_key_auth_factory import ApiKeyAuthFactory @@ -19,6 +20,10 @@ def get_provider_auth_list(tenant_id: str) -> list: def create_provider_auth(tenant_id: str, args: dict): auth_result = ApiKeyAuthFactory(args['provider'], args['credentials']).validate_credentials() if auth_result: + # Encrypt the api key + api_key = encrypter.encrypt_token(tenant_id, args['credentials']['config']['api_key']) + args['credentials']['config']['api_key'] = api_key + data_source_api_key_binding = DataSourceApiKeyAuthBinding() data_source_api_key_binding.tenant_id = tenant_id data_source_api_key_binding.category = args['category'] @@ -27,6 +32,19 @@ def create_provider_auth(tenant_id: str, args: dict): db.session.add(data_source_api_key_binding) db.session.commit() + @staticmethod + def get_auth_credentials(tenant_id: str, category: str, provider: str): + data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( + DataSourceApiKeyAuthBinding.tenant_id == tenant_id, + DataSourceApiKeyAuthBinding.category == category, + DataSourceApiKeyAuthBinding.provider == provider, + DataSourceApiKeyAuthBinding.disabled.is_(False) + ).first() + if not data_source_api_key_bindings: + return None + credentials = json.loads(data_source_api_key_bindings.credentials) + return credentials + @classmethod def validate_api_key_auth_args(cls, args): if 'category' not in args or not args['category']: diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 86ad29e3164055..0966083314106c 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -49,6 +49,8 @@ from tasks.duplicate_document_indexing_task import duplicate_document_indexing_task from tasks.recover_document_indexing_task import recover_document_indexing_task from tasks.retry_document_indexing_task import retry_document_indexing_task +from tasks.sync_website_document_indexing_task import sync_website_document_indexing_task + class DatasetService: @@ -498,6 +500,20 @@ def retry_document(dataset_id: str, documents: list[Document]): retry_document_indexing_task.delay(dataset_id, document_ids) @staticmethod + def sync_website_document(dataset_id: str, document: Document): + # sync document indexing + document.indexing_status = 'waiting' + data_source_info = document.data_source_info_dict + data_source_info['mode'] = 'scrape' + document.data_source_info = json.dumps(data_source_info, ensure_ascii=False) + db.session.add(document) + db.session.commit() + # add sync flag + sync_indexing_cache_key = 'document_{}_is_sync'.format(document.id) + redis_client.setex(sync_indexing_cache_key, 600, 1) + + sync_website_document_indexing_task.delay(dataset_id, document.id) + @staticmethod def get_documents_position(dataset_id): document = Document.query.filter_by(dataset_id=dataset_id).order_by(Document.position.desc()).first() if document: @@ -523,8 +539,8 @@ def save_document_with_dataset_id(dataset: Dataset, document_data: dict, notion_info_list = document_data["data_source"]['info_list']['notion_info_list'] for notion_info in notion_info_list: count = count + len(notion_info['pages']) - elif document_data["data_source"]["type"] == "website": - website_info = document_data["data_source"]['info_list']['website_info'] + elif document_data["data_source"]["type"] == "website_crawl": + website_info = document_data["data_source"]['info_list']['website_info_list'] count = len(website_info['urls']) batch_upload_limit = int(current_app.config['BATCH_UPLOAD_LIMIT']) if count > batch_upload_limit: @@ -698,8 +714,8 @@ def save_document_with_dataset_id(dataset: Dataset, document_data: dict, # delete not selected documents if len(exist_document) > 0: clean_notion_document_task.delay(list(exist_document.values()), dataset.id) - elif document_data["data_source"]["type"] == "website": - website_info = document_data["data_source"]['info_list']['website_info'] + elif document_data["data_source"]["type"] == "website_crawl": + website_info = document_data["data_source"]['info_list']['website_info_list'] urls = website_info['urls'] for url in urls: data_source_info = { @@ -836,8 +852,8 @@ def update_document_with_dataset_id(dataset: Dataset, document_data: dict, "notion_page_icon": page['page_icon'], "type": page['type'] } - elif document_data["data_source"]["type"] == "website": - website_info = document_data["data_source"]['info_list']['website_info'] + elif document_data["data_source"]["type"] == "website_crawl": + website_info = document_data["data_source"]['info_list']['website_info_list'] urls = website_info['urls'] for url in urls: data_source_info = { @@ -883,8 +899,8 @@ def save_document_without_dataset_id(tenant_id: str, document_data: dict, accoun notion_info_list = document_data["data_source"]['info_list']['notion_info_list'] for notion_info in notion_info_list: count = count + len(notion_info['pages']) - elif document_data["data_source"]["type"] == "website": - website_info = document_data["data_source"]['info_list']['website_info'] + elif document_data["data_source"]["type"] == "website_crawl": + website_info = document_data["data_source"]['info_list']['website_info_list'] count = len(website_info['urls']) batch_upload_limit = int(current_app.config['BATCH_UPLOAD_LIMIT']) if count > batch_upload_limit: diff --git a/api/services/website_service.py b/api/services/website_service.py index 11d1b0d0070dc0..6d46a1626e4c7a 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -1,11 +1,14 @@ import json +from typing import Any from flask_login import current_user from werkzeug.exceptions import NotFound +from core.helper import encrypter from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp from extensions.ext_database import db from models.source import DataSourceApiKeyAuthBinding +from services.auth.api_key_auth_service import ApiKeyAuthService class WebsiteService: @@ -24,17 +27,16 @@ def crawl_url(cls, args: dict) -> dict: provider = args.get('provider') url = args.get('url') options = args.get('options') + credentials = ApiKeyAuthService.get_auth_credentials(current_user.current_tenant_id, + 'website', + provider) if provider == 'firecrawl': - data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( - DataSourceApiKeyAuthBinding.tenant_id == current_user.current_tenant_id, - DataSourceApiKeyAuthBinding.category == 'website', - DataSourceApiKeyAuthBinding.provider == 'firecrawl', - DataSourceApiKeyAuthBinding.disabled.is_(False) - ).first() - if not data_source_api_key_bindings: - raise NotFound('Firecrawl API key not found') - credentials = json.loads(data_source_api_key_bindings.credentials) - firecrawl_app = FirecrawlApp(api_key=credentials.get('config').get('api_key'), + # decrypt api_key + api_key = encrypter.decrypt_token( + tenant_id=current_user.current_tenant_id, + token=credentials.get('config').get('api_key') + ) + firecrawl_app = FirecrawlApp(api_key=api_key, base_url=credentials.get('config').get('base_url', None)) crawl_sub_pages = options.get('crawl_sub_pages', False) only_main_content = options.get('only_main_content', False) @@ -80,17 +82,16 @@ def crawl_url(cls, args: dict) -> dict: @classmethod def get_crawl_status(cls, job_id: str, provider: str) -> dict: + credentials = ApiKeyAuthService.get_auth_credentials(current_user.current_tenant_id, + 'website', + provider) if provider == 'firecrawl': - data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( - DataSourceApiKeyAuthBinding.tenant_id == current_user.current_tenant_id, - DataSourceApiKeyAuthBinding.category == 'website', - DataSourceApiKeyAuthBinding.provider == 'firecrawl', - DataSourceApiKeyAuthBinding.disabled.is_(False) - ).first() - if not data_source_api_key_bindings: - raise NotFound('Firecrawl API key not found') - credentials = json.loads(data_source_api_key_bindings.credentials) - firecrawl_app = FirecrawlApp(api_key=credentials.get('config').get('api_key'), + # decrypt api_key + api_key = encrypter.decrypt_token( + tenant_id=current_user.current_tenant_id, + token=credentials.get('config').get('api_key') + ) + firecrawl_app = FirecrawlApp(api_key=api_key, base_url=credentials.get('config').get('base_url', None)) result = firecrawl_app.check_crawl_status(job_id) crawl_status_data = { @@ -103,26 +104,50 @@ def get_crawl_status(cls, job_id: str, provider: str) -> dict: return crawl_status_data @classmethod - def get_crawl_url_data(cls, job_id: str, provider: str, url: str) -> dict: + def get_crawl_url_data(cls, job_id: str, provider: str, url: str) -> dict | None: + credentials = ApiKeyAuthService.get_auth_credentials(current_user.current_tenant_id, + 'website', + provider) if provider == 'firecrawl': - data_source_api_key_bindings = db.session.query(DataSourceApiKeyAuthBinding).filter( - DataSourceApiKeyAuthBinding.tenant_id == current_user.current_tenant_id, - DataSourceApiKeyAuthBinding.category == 'website', - DataSourceApiKeyAuthBinding.provider == 'firecrawl', - DataSourceApiKeyAuthBinding.disabled.is_(False) - ).first() - if not data_source_api_key_bindings: - raise NotFound('Firecrawl API key not found') - credentials = json.loads(data_source_api_key_bindings.credentials) - firecrawl_app = FirecrawlApp(api_key=credentials.get('config').get('api_key'), + # decrypt api_key + api_key = encrypter.decrypt_token( + tenant_id=current_user.current_tenant_id, + token=credentials.get('config').get('api_key') + ) + firecrawl_app = FirecrawlApp(api_key=api_key, base_url=credentials.get('config').get('base_url', None)) result = firecrawl_app.check_crawl_status(job_id) if result.get('status') != 'completed': raise ValueError('Crawl job is not completed') data = result.get('data') - for item in data: - if item.get('data'): - if item.get('data').get('metadata').get('sourceURL') == url: - return item.get('data').get('markdown') + if data: + for item in data: + if item.get('data').get('source_url') == url: + return item + return None else: - raise ValueError('Invalid provider') \ No newline at end of file + raise ValueError('Invalid provider') + + @classmethod + def get_scrape_url_data(cls, provider: str, url: str, only_main_content: bool) -> dict | None: + credentials = ApiKeyAuthService.get_auth_credentials(current_user.current_tenant_id, + 'website', + provider) + if provider == 'firecrawl': + # decrypt api_key + api_key = encrypter.decrypt_token( + tenant_id=current_user.current_tenant_id, + token=credentials.get('config').get('api_key') + ) + firecrawl_app = FirecrawlApp(api_key=api_key, + base_url=credentials.get('config').get('base_url', None)) + params = { + 'pageOptions': { + 'onlyMainContent': only_main_content, + "includeHtml": False + } + } + result = firecrawl_app.scrape_url(url, params) + return result + else: + raise ValueError('Invalid provider') diff --git a/api/tasks/sync_website_document_indexing_task.py b/api/tasks/sync_website_document_indexing_task.py new file mode 100644 index 00000000000000..f96d68c90e4095 --- /dev/null +++ b/api/tasks/sync_website_document_indexing_task.py @@ -0,0 +1,90 @@ +import datetime +import logging +import time + +import click +from celery import shared_task + +from core.indexing_runner import IndexingRunner +from core.rag.index_processor.index_processor_factory import IndexProcessorFactory +from extensions.ext_database import db +from extensions.ext_redis import redis_client +from models.dataset import Dataset, Document, DocumentSegment +from services.feature_service import FeatureService + + +@shared_task(queue='dataset') +def sync_website_document_indexing_task(dataset_id: str, document_id: str): + """ + Async process document + :param dataset_id: + :param document_id: + + Usage: sunc_website_document_indexing_task.delay(dataset_id, document_id) + """ + start_at = time.perf_counter() + + dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first() + + sync_indexing_cache_key = 'document_{}_is_sync'.format(document_id) + # check document limit + features = FeatureService.get_features(dataset.tenant_id) + try: + if features.billing.enabled: + vector_space = features.vector_space + if 0 < vector_space.limit <= vector_space.size: + raise ValueError("Your total number of documents plus the number of uploads have over the limit of " + "your subscription.") + except Exception as e: + document = db.session.query(Document).filter( + Document.id == document_id, + Document.dataset_id == dataset_id + ).first() + if document: + document.indexing_status = 'error' + document.error = str(e) + document.stopped_at = datetime.datetime.utcnow() + db.session.add(document) + db.session.commit() + redis_client.delete(sync_indexing_cache_key) + return + + logging.info(click.style('Start sync website document: {}'.format(document_id), fg='green')) + document = db.session.query(Document).filter( + Document.id == document_id, + Document.dataset_id == dataset_id + ).first() + try: + if document: + # clean old data + index_processor = IndexProcessorFactory(document.doc_form).init_index_processor() + + segments = db.session.query(DocumentSegment).filter(DocumentSegment.document_id == document_id).all() + if segments: + index_node_ids = [segment.index_node_id for segment in segments] + # delete from vector index + index_processor.clean(dataset, index_node_ids) + + for segment in segments: + db.session.delete(segment) + db.session.commit() + + document.indexing_status = 'parsing' + document.processing_started_at = datetime.datetime.utcnow() + db.session.add(document) + db.session.commit() + + indexing_runner = IndexingRunner() + indexing_runner.run([document]) + redis_client.delete(sync_indexing_cache_key) + except Exception as ex: + document.indexing_status = 'error' + document.error = str(ex) + document.stopped_at = datetime.datetime.utcnow() + db.session.add(document) + db.session.commit() + logging.info(click.style(str(ex), fg='yellow')) + redis_client.delete(sync_indexing_cache_key) + pass + end_at = time.perf_counter() + logging.info(click.style('Sync dataset: {} latency: {}'.format(dataset_id, end_at - start_at), fg='green')) diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py index 19b372649911b1..7463ee38bf218d 100644 --- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py +++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py @@ -1,37 +1,44 @@ import os +from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp from core.rag.extractor.firecrawl.firecrawl_web_extractor import FirecrawlWebExtractor from core.rag.models.document import Document def test_firecrawl_web_extractor_scrape_mode(): url = "https://dify.ai" - api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-' + api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-' base_url = 'https://api.firecrawl.dev' - mode = 'scrape' - firecrawl_web_extractor = FirecrawlWebExtractor(url, api_key, base_url, mode) - documents = firecrawl_web_extractor.extract() - print(documents) - assert isinstance(documents, list) - assert all(isinstance(doc, Document) for doc in documents) + firecrawl_app = FirecrawlApp(api_key=api_key, + base_url=base_url) + params = { + 'pageOptions': { + 'onlyMainContent': True, + "includeHtml": False + } + } + data = firecrawl_app.scrape_url(url, params) + print(data) + assert isinstance(data, dict) + def test_firecrawl_web_extractor_crawl_mode(): url = "https://firecrawl.dev" api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-' base_url = 'https://api.firecrawl.dev' - mode = 'crawl' - firecrawl_web_extractor = FirecrawlWebExtractor(url, api_key, base_url, mode) - documents = firecrawl_web_extractor.extract() - print(documents) - assert isinstance(documents, list) - assert all(isinstance(doc, Document) for doc in documents) + firecrawl_app = FirecrawlApp(api_key=api_key, + base_url=base_url) + params = { + 'crawlerOptions': { + "includes": [], + "excludes": [], + "generateImgAltText": True, + "maxDepth": 1, + "limit": 1, + 'returnOnlyUrls': False, -def test_firecrawl_web_extractor_crawl_return_urls_mode(): - url = "https://mendable.ai" - api_key = os.getenv('FIRECRAWL_API_KEY') or 'fc-' - base_url = 'https://api.firecrawl.dev' - mode = 'crawl_return_urls' - firecrawl_web_extractor = FirecrawlWebExtractor(url, api_key, base_url, mode) - documents = firecrawl_web_extractor.extract() - assert isinstance(documents, list) - assert all(isinstance(doc, Document) for doc in documents) + } + } + job_id = firecrawl_app.crawl_url(url, params) + print(job_id) + assert isinstance(job_id, str) From 677ce6caaab30f9a970a91d208976e643e3dd263 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 16 May 2024 16:46:55 +0800 Subject: [PATCH 044/273] add website sync --- api/services/dataset_service.py | 16 ++++++++++++---- api/tasks/sync_website_document_indexing_task.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 0966083314106c..5db6e80fb7c2bc 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -488,12 +488,16 @@ def recover_document(document): @staticmethod def retry_document(dataset_id: str, documents: list[Document]): for document in documents: + # add retry flag + retry_indexing_cache_key = 'document_{}_is_retried'.format(document.id) + cache_result = redis_client.get(retry_indexing_cache_key) + if cache_result is not None: + raise ValueError("Document is being retried, please try again later") # retry document indexing document.indexing_status = 'waiting' db.session.add(document) db.session.commit() - # add retry flag - retry_indexing_cache_key = 'document_{}_is_retried'.format(document.id) + redis_client.setex(retry_indexing_cache_key, 600, 1) # trigger async task document_ids = [document.id for document in documents] @@ -501,6 +505,11 @@ def retry_document(dataset_id: str, documents: list[Document]): @staticmethod def sync_website_document(dataset_id: str, document: Document): + # add sync flag + sync_indexing_cache_key = 'document_{}_is_sync'.format(document.id) + cache_result = redis_client.get(sync_indexing_cache_key) + if cache_result is not None: + raise ValueError("Document is being synced, please try again later") # sync document indexing document.indexing_status = 'waiting' data_source_info = document.data_source_info_dict @@ -508,8 +517,7 @@ def sync_website_document(dataset_id: str, document: Document): document.data_source_info = json.dumps(data_source_info, ensure_ascii=False) db.session.add(document) db.session.commit() - # add sync flag - sync_indexing_cache_key = 'document_{}_is_sync'.format(document.id) + redis_client.setex(sync_indexing_cache_key, 600, 1) sync_website_document_indexing_task.delay(dataset_id, document.id) diff --git a/api/tasks/sync_website_document_indexing_task.py b/api/tasks/sync_website_document_indexing_task.py index f96d68c90e4095..320da8718a12cb 100644 --- a/api/tasks/sync_website_document_indexing_task.py +++ b/api/tasks/sync_website_document_indexing_task.py @@ -87,4 +87,4 @@ def sync_website_document_indexing_task(dataset_id: str, document_id: str): redis_client.delete(sync_indexing_cache_key) pass end_at = time.perf_counter() - logging.info(click.style('Sync dataset: {} latency: {}'.format(dataset_id, end_at - start_at), fg='green')) + logging.info(click.style('Sync document: {} latency: {}'.format(document_id, end_at - start_at), fg='green')) From 328a0f4b6016d4d99ba452e2370472ff3d0a94f9 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 16 May 2024 16:52:52 +0800 Subject: [PATCH 045/273] Update to put method --- api/controllers/console/datasets/datasets_document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 2c1b5c6a4c47d8..5597a88ef3938d 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -929,7 +929,7 @@ class DocumentRenameApi(DocumentResource): @login_required @account_initialization_required @marshal_with(document_fields) - def post(self, dataset_id, document_id): + def put(self, dataset_id, document_id): # The role of the current user in the ta table must be admin or owner if not current_user.is_admin_or_owner: raise Forbidden() From 8a4396a20974c8d116609a515409c364ec7a38b0 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 16 May 2024 17:43:48 +0800 Subject: [PATCH 046/273] firecrawl Data format --- .../rag/extractor/firecrawl/firecrawl_app.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index e8fe59e44b8deb..d30eb4b9a538e9 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -69,15 +69,13 @@ def check_crawl_status(self, job_id) -> dict: data = crawl_status_response.get('data', []) url_data_list = [] for item in data: - if item.get('success', False): - if item.get('data'): - url_data = { - 'title': item.get('data').get('metadata').get('title'), - 'description': item.get('data').get('metadata').get('description'), - 'source_url': item.get('data').get('metadata').get('sourceURL'), - 'markdown': item.get('data').get('markdown') - } - url_data_list.append(url_data) + url_data = { + 'title': item.get('metadata').get('title'), + 'description': item.get('metadata').get('description'), + 'source_url': item.get('metadata').get('sourceURL'), + 'markdown': item.get('markdown') + } + url_data_list.append(url_data) return { 'status': 'completed', 'data': url_data_list From 4fee326fe648d0a7ad9f7561847721e42d04c5c4 Mon Sep 17 00:00:00 2001 From: nite-knite Date: Fri, 17 May 2024 18:39:35 +0800 Subject: [PATCH 047/273] feat: load balancing config modal styling --- .../config/assistant-type-picker/index.tsx | 2 +- web/app/components/app/overview/appCard.tsx | 8 +- web/app/components/base/button/index.css | 6 +- web/app/components/base/button/index.tsx | 22 +-- .../chat/chat/answer/workflow-process.tsx | 2 +- .../base/image-uploader/image-list.tsx | 5 +- .../base/image-uploader/image-preview.tsx | 2 +- web/app/components/base/modal/index.css | 7 + web/app/components/base/modal/index.tsx | 15 +- .../base/simple-pie-chart/index.module.css | 4 + .../base/simple-pie-chart/index.tsx | 66 +++++++ web/app/components/base/switch/index.tsx | 4 +- .../custom/custom-app-header-brand/index.tsx | 6 +- .../custom/custom-web-app-brand/index.tsx | 6 +- .../model-provider-page/declarations.ts | 8 +- .../model-provider-page/hooks.ts | 10 +- .../model-provider-page/index.tsx | 12 +- .../model-provider-page/model-badge/index.tsx | 2 +- .../model-provider-page/model-modal/index.tsx | 8 +- .../model-selector/popup-item.tsx | 4 +- .../provider-added-card/credential-panel.tsx | 4 +- .../provider-added-card/index.tsx | 18 +- .../model-balancing-modal.tsx | 163 ++++++++++++++++++ .../provider-added-card/model-list-item.tsx | 100 +++++++++++ .../provider-added-card/model-list.tsx | 104 ++++------- .../provider-card/index.tsx | 8 +- web/app/components/tools/tool-list/index.tsx | 4 +- web/app/components/workflow/block-icon.tsx | 4 +- .../components/workflow/header/checklist.tsx | 4 +- .../components/workflow/operator/index.tsx | 2 +- .../workflow/panel/chat-record/index.tsx | 2 +- .../panel/debug-and-preview/index.tsx | 4 +- web/app/styles/globals.css | 3 +- web/context/modal-context.tsx | 6 +- web/i18n/en-US/common.ts | 10 +- web/i18n/zh-Hans/common.ts | 8 + web/tailwind.config.js | 7 + 37 files changed, 487 insertions(+), 163 deletions(-) create mode 100644 web/app/components/base/modal/index.css create mode 100644 web/app/components/base/simple-pie-chart/index.module.css create mode 100644 web/app/components/base/simple-pie-chart/index.tsx create mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx create mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx diff --git a/web/app/components/app/configuration/config/assistant-type-picker/index.tsx b/web/app/components/app/configuration/config/assistant-type-picker/index.tsx index e19e7fffa3b1c8..3bdf0cbcf71bd3 100644 --- a/web/app/components/app/configuration/config/assistant-type-picker/index.tsx +++ b/web/app/components/app/configuration/config/assistant-type-picker/index.tsx @@ -123,7 +123,7 @@ const AssistantTypePicker: FC = ({
-
+
{t('appDebug.assistantType.name')}
@@ -162,7 +161,7 @@ function AppCard({ ? t('appOverview.overview.appInfo.accessibleAddress') : t('appOverview.overview.apiInfo.accessibleAddress')}
-
+
{isApp ? appUrl : apiUrl} @@ -186,8 +185,7 @@ function AppCard({ onClick={onGenCode} >
diff --git a/web/app/components/base/button/index.css b/web/app/components/base/button/index.css index 81fb5fda4522cf..94cd71817ec854 100644 --- a/web/app/components/base/button/index.css +++ b/web/app/components/base/button/index.css @@ -3,10 +3,10 @@ @layer components { .btn { @apply inline-flex justify-center items-center content-center h-9 leading-5 rounded-lg px-4 py-2 text-base cursor-pointer whitespace-nowrap; - } + }; .btn-default { - @apply border-solid border border-gray-200 cursor-pointer text-gray-500 hover:bg-white hover:shadow-sm hover:border-gray-300; + @apply border-solid border border-gray-200 cursor-pointer text-gray-700 hover:bg-white hover:shadow-sm hover:border-gray-300; } .btn-default-disabled { @@ -28,4 +28,4 @@ .btn-warning-disabled { @apply bg-red-600/75 cursor-not-allowed text-white; } -} \ No newline at end of file +} diff --git a/web/app/components/base/button/index.tsx b/web/app/components/base/button/index.tsx index e617a5d12d4f0a..34f16ad556f772 100644 --- a/web/app/components/base/button/index.tsx +++ b/web/app/components/base/button/index.tsx @@ -1,16 +1,16 @@ -import type { FC, MouseEventHandler } from 'react' -import React from 'react' +import type { FC, MouseEventHandler, PropsWithChildren } from 'react' +import React, { memo } from 'react' +import classNames from 'classnames' import Spinner from '../spinner' -export type IButtonProps = { +export type IButtonProps = PropsWithChildren<{ type?: string className?: string disabled?: boolean loading?: boolean tabIndex?: number - children: React.ReactNode onClick?: MouseEventHandler -} +}> const Button: FC = ({ type, @@ -21,22 +21,22 @@ const Button: FC = ({ loading = false, tabIndex, }) => { - let style = 'cursor-pointer' + let typeClassNames = 'cursor-pointer' switch (type) { case 'primary': - style = (disabled || loading) ? 'btn-primary-disabled' : 'btn-primary' + typeClassNames = (disabled || loading) ? 'btn-primary-disabled' : 'btn-primary' break case 'warning': - style = (disabled || loading) ? 'btn-warning-disabled' : 'btn-warning' + typeClassNames = (disabled || loading) ? 'btn-warning-disabled' : 'btn-warning' break default: - style = disabled ? 'btn-default-disabled' : 'btn-default' + typeClassNames = disabled ? 'btn-default-disabled' : 'btn-default' break } return (
@@ -47,4 +47,4 @@ const Button: FC = ({ ) } -export default React.memo(Button) +export default memo(Button) diff --git a/web/app/components/base/chat/chat/answer/workflow-process.tsx b/web/app/components/base/chat/chat/answer/workflow-process.tsx index 4eef0b938fbc5d..c9ce144f0a3081 100644 --- a/web/app/components/base/chat/chat/answer/workflow-process.tsx +++ b/web/app/components/base/chat/chat/answer/workflow-process.tsx @@ -51,7 +51,7 @@ const WorkflowProcessItem = ({ return (
= ({
= ({ type="button" className={cn( 'absolute z-10 -top-[9px] -right-[9px] items-center justify-center w-[18px] h-[18px]', - 'bg-white hover:bg-gray-50 border-[0.5px] border-black/[0.02] rounded-2xl shadow-lg', + 'bg-white hover:bg-gray-50 border-[0.5px] border-black/2 rounded-2xl shadow-lg', item.progress === -1 ? 'flex' : 'hidden group-hover:flex', )} onClick={() => onRemove && onRemove(item._id)} diff --git a/web/app/components/base/image-uploader/image-preview.tsx b/web/app/components/base/image-uploader/image-preview.tsx index 8670e103540c1d..4fd071fd73a371 100644 --- a/web/app/components/base/image-uploader/image-preview.tsx +++ b/web/app/components/base/image-uploader/image-preview.tsx @@ -18,7 +18,7 @@ const ImagePreview: FC = ({ className='max-w-full max-h-full' />
diff --git a/web/app/components/base/modal/index.css b/web/app/components/base/modal/index.css new file mode 100644 index 00000000000000..b1b4648be62357 --- /dev/null +++ b/web/app/components/base/modal/index.css @@ -0,0 +1,7 @@ +.modal-dialog { + @apply relative z-10; +} + +.modal-panel { + @apply w-full max-w-md transform rounded-2xl bg-white p-6 text-left align-middle shadow-xl transition-all; +} diff --git a/web/app/components/base/modal/index.tsx b/web/app/components/base/modal/index.tsx index 403cb319ba5717..4e1f184dabc484 100644 --- a/web/app/components/base/modal/index.tsx +++ b/web/app/components/base/modal/index.tsx @@ -1,16 +1,17 @@ import { Dialog, Transition } from '@headlessui/react' import { Fragment } from 'react' import { XMarkIcon } from '@heroicons/react/24/outline' +import classNames from 'classnames' // https://headlessui.com/react/dialog type IModal = { className?: string wrapperClassName?: string isShow: boolean - onClose: () => void + onClose?: () => void title?: React.ReactNode description?: React.ReactNode - children: React.ReactNode + children?: React.ReactNode closable?: boolean overflowVisible?: boolean } @@ -19,7 +20,7 @@ export default function Modal({ className, wrapperClassName, isShow, - onClose, + onClose = () => { }, title, description, children, @@ -28,7 +29,7 @@ export default function Modal({ }: IModal) { return ( - + - + {title && { + const option: EChartsOption = useMemo(() => ({ + series: [ + { + type: 'pie', + radius: ['83%', '100%'], + animation: false, + data: [ + { value: 100, itemStyle: { color: stroke } }, + ], + emphasis: { + disabled: true, + }, + labelLine: { + show: false, + }, + cursor: 'default', + }, + { + type: 'pie', + radius: '83%', + animationDuration: 600, + data: [ + { value: percentage, itemStyle: { color: fill } }, + { value: 100 - percentage, itemStyle: { color: '#fff' } }, + ], + emphasis: { + disabled: true, + }, + labelLine: { + show: false, + }, + cursor: 'default', + }, + ], + }), [stroke, fill, percentage]) + + return ( + + ) +} + +export default memo(SimplePieChart) diff --git a/web/app/components/base/switch/index.tsx b/web/app/components/base/switch/index.tsx index 8b7d1216992e2d..7d13b0cb9ac4b7 100644 --- a/web/app/components/base/switch/index.tsx +++ b/web/app/components/base/switch/index.tsx @@ -8,9 +8,10 @@ type SwitchProps = { size?: 'sm' | 'md' | 'lg' | 'l' defaultValue?: boolean disabled?: boolean + className?: string } -const Switch = ({ onChange, size = 'lg', defaultValue = false, disabled = false }: SwitchProps) => { +const Switch = ({ onChange, size = 'lg', defaultValue = false, disabled = false, className }: SwitchProps) => { const [enabled, setEnabled] = useState(defaultValue) useEffect(() => { setEnabled(defaultValue) @@ -49,6 +50,7 @@ const Switch = ({ onChange, size = 'lg', defaultValue = false, disabled = false enabled ? 'bg-blue-600' : 'bg-gray-200', 'relative inline-flex flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out', disabled ? '!opacity-50 !cursor-not-allowed' : '', + className, )} > { return (
{t('custom.app.title')}
-
+
@@ -43,7 +43,7 @@ const CustomAppHeaderBrand = () => {
+ } + > +
+
+
+
+ {Boolean(model) && ( + + )} +
+
+
{t('common.modelProvider.providerManaged')}
+
Todo
+
+
+
+ +
+
+
+ +
+
+
{t('common.modelProvider.loadBalancing')}
+
Todo
+
+
+ +
+
+
+
+ + + +
+
{t('common.modelProvider.defaultConfig')}
+ {t('common.modelProvider.providerManaged')} +
+
+
+ + + + + + + +
+ value} + /> +
+
+ +
+
+
+ + + +
+
Another
+ {t('common.modelProvider.providerManaged')} +
+
+
+ + + + + + + +
+ value} + /> +
+
+ +
+
+ {t('common.modelProvider.addConfig')} +
+
+
+ +
+ + {t('common.modelProvider.loadBalancingLeastKeyWarning')} +
+
+
+ +
+ + +
+ + ) +} + +export default memo(ModelBalancingModal) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx new file mode 100644 index 00000000000000..64ae35d0e73f47 --- /dev/null +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx @@ -0,0 +1,100 @@ +import { memo } from 'react' +import { useTranslation } from 'react-i18next' +import classNames from 'classnames' +import type { CustomConfigurationModelFixedFields, ModelItem, ModelProvider } from '../declarations' +import { ConfigurationMethodEnum, ModelStatusEnum } from '../declarations' +import ModelBadge from '../model-badge' +import ModelIcon from '../model-icon' +import ModelName from '../model-name' +import Button from '@/app/components/base/button' +import { Balance } from '@/app/components/base/icons/src/vender/line/financeAndECommerce' +import { Settings01 } from '@/app/components/base/icons/src/vender/line/general' +import Switch from '@/app/components/base/switch' +import TooltipPlus from '@/app/components/base/tooltip-plus' + +export type ModelListItemProps = { + model: ModelItem + provider: ModelProvider + isConfigurable: boolean + onConfig: (currentCustomConfigrationModelFixedFields?: CustomConfigurationModelFixedFields) => void + onModifyLoadBalancing?: (model: ModelItem) => void +} + +const ModelListItem = ({ model, provider, isConfigurable, onConfig, onModifyLoadBalancing }: ModelListItemProps) => { + const { t } = useTranslation() + + return ( +
+ + + {/* TODO: check feature switch */} + + + {t('common.modelProvider.loadBalancingHeadline')} + + +
+ { + model.fetch_from === ConfigurationMethodEnum.customizableModel && ( + + ) + } + {/* */} + {/* TODO: check feature switch */} + {!model.deprecated && [ModelStatusEnum.active, ModelStatusEnum.disabled].includes(model.status) && ( + + )} + { + model.deprecated + ? ( + {t('common.modelProvider.modelHasBeenDeprecated')}} offset={{ mainAxis: 4 }}> + + + ) + : ( + value} + /> + ) + } +
+
+ ) +} + +export default memo(ModelListItem) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx index 7ff0a9642799d6..93392c8866a2f0 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx @@ -1,31 +1,25 @@ import type { FC } from 'react' +import { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import type { - CustomConfigrationModelFixedFields, + CustomConfigurationModelFixedFields, ModelItem, ModelProvider, } from '../declarations' import { - ConfigurateMethodEnum, - ModelStatusEnum, + ConfigurationMethodEnum, } from '../declarations' -import { useLanguage } from '../hooks' -import ModelIcon from '../model-icon' -import ModelName from '../model-name' // import Tab from './tab' -import ModelBadge from '../model-badge' import AddModelButton from './add-model-button' -import { Settings01 } from '@/app/components/base/icons/src/vender/line/general' +import ModelListItem from './model-list-item' +import ModelBalancingModal from './model-balancing-modal' import { ChevronDownDouble } from '@/app/components/base/icons/src/vender/line/arrows' -import Button from '@/app/components/base/button' -import Switch from '@/app/components/base/switch' -import { Balance } from '@/app/components/base/icons/src/vender/line/financeAndECommerce' type ModelListProps = { provider: ModelProvider models: ModelItem[] onCollapse: () => void - onConfig: (currentCustomConfigrationModelFixedFields?: CustomConfigrationModelFixedFields) => void + onConfig: (currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields) => void } const ModelList: FC = ({ provider, @@ -34,10 +28,13 @@ const ModelList: FC = ({ onConfig, }) => { const { t } = useTranslation() - const language = useLanguage() - const configurateMethods = provider.configurate_methods.filter(method => method !== ConfigurateMethodEnum.fetchFromRemote) - const canCustomConfig = configurateMethods.includes(ConfigurateMethodEnum.customizableModel) - // const canSystemConfig = configurateMethods.includes(ConfigurateMethodEnum.predefinedModel) + const configurativeMethods = provider.configurate_methods.filter(method => method !== ConfigurationMethodEnum.fetchFromRemote) + const isConfigurable = configurativeMethods.includes(ConfigurationMethodEnum.customizableModel) + + const [balancingModel, setBalancingModel] = useState() + const onModifyLoadBalancing = useCallback((model: ModelItem) => { + setBalancingModel(model) + }, []) return (
@@ -56,14 +53,14 @@ const ModelList: FC = ({ {/* { - canCustomConfig && canSystemConfig && ( + isConfigurable && canSystemConfig && ( {}} /> ) } */} { - canCustomConfig && ( + isConfigurable && (
onConfig()} />
@@ -72,66 +69,25 @@ const ModelList: FC = ({
{ models.map(model => ( -
- - - {/* TODO: check feature switch */} - - - {t('common.modelProvider.loadBalancing')} - - -
- { - model.fetch_from === ConfigurateMethodEnum.customizableModel && ( - - ) - } - {/* */} - {/* TODO: check feature switch */} - {!model.deprecated && [ModelStatusEnum.active, ModelStatusEnum.disabled].includes(model.status) && ( - - )} - value} - /> -
-
+ {...{ + model, + provider, + isConfigurable, + onConfig, + onModifyLoadBalancing, + }} + /> )) }
+ setBalancingModel(undefined), + }} />
) } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-card/index.tsx b/web/app/components/header/account-setting/model-provider-page/provider-card/index.tsx index 0e5eb7a236c08e..54dc78150fd973 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-card/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-card/index.tsx @@ -4,7 +4,7 @@ import type { ModelProvider, TypeWithI18N, } from '../declarations' -import { ConfigurateMethodEnum } from '../declarations' +import { ConfigurationMethodEnum } from '../declarations' import { DEFAULT_BACKGROUND_COLOR, MODEL_PROVIDER_QUOTA_GET_FREE, @@ -25,7 +25,7 @@ import { IS_CE_EDITION } from '@/config' type ProviderCardProps = { provider: ModelProvider - onOpenModal: (configurateMethod: ConfigurateMethodEnum) => void + onOpenModal: (configurateMethod: ConfigurationMethodEnum) => void } const TIP_MAP: { [k: string]: TypeWithI18N } = { @@ -54,7 +54,7 @@ const ProviderCard: FC = ({ updateModelProviders() } const handleFreeQuota = useFreeQuota(handleFreeQuotaSuccess) - const configurateMethods = provider.configurate_methods.filter(method => method !== ConfigurateMethodEnum.fetchFromRemote) + const configurateMethods = provider.configurate_methods.filter(method => method !== ConfigurationMethodEnum.fetchFromRemote) const canGetFreeQuota = MODEL_PROVIDER_QUOTA_GET_FREE.includes(provider.provider) && !IS_CE_EDITION && provider.system_configuration.enabled return ( @@ -111,7 +111,7 @@ const ProviderCard: FC = ({
{ configurateMethods.map((method) => { - if (method === ConfigurateMethodEnum.predefinedModel) { + if (method === ConfigurationMethodEnum.predefinedModel) { return ( - ) + model.fetch_from === ConfigurationMethodEnum.customizableModel + ? ( + + ) + : (modelLoadBalancingEnabled && !model.deprecated && [ModelStatusEnum.active, ModelStatusEnum.disabled].includes(model.status)) + ? ( + + ) + : null } - {/* */} - {/* TODO: check feature switch */} - {!model.deprecated && [ModelStatusEnum.active, ModelStatusEnum.disabled].includes(model.status) && ( - - )} { model.deprecated ? ( @@ -85,10 +101,11 @@ const ModelListItem = ({ model, provider, isConfigurable, onConfig, onModifyLoad ) : ( value} + onChange={onEnablingStateChange} /> ) } diff --git a/web/context/provider-context.tsx b/web/context/provider-context.tsx index 4d7fe393abd4f5..04e3f19bfed682 100644 --- a/web/context/provider-context.tsx +++ b/web/context/provider-context.tsx @@ -1,6 +1,6 @@ 'use client' -import { createContext, useContext } from 'use-context-selector' +import { createContext, useContext, useContextSelector } from 'use-context-selector' import useSWR from 'swr' import { useEffect, useState } from 'react' import { @@ -19,7 +19,7 @@ import { fetchCurrentPlanInfo } from '@/service/billing' import { parseCurrentPlan } from '@/app/components/billing/utils' import { defaultPlan } from '@/app/components/billing/config' -const ProviderContext = createContext<{ +type ProviderContextState = { modelProviders: ModelProvider[] textGenerationModelList: Model[] supportRetrievalMethods: RETRIEVE_METHOD[] @@ -33,36 +33,43 @@ const ProviderContext = createContext<{ enableBilling: boolean onPlanInfoChanged: () => void enableReplaceWebAppLogo: boolean - enableModelLoadBalancing: boolean -}>({ - modelProviders: [], - textGenerationModelList: [], - supportRetrievalMethods: [], - isAPIKeySet: true, - plan: { - type: Plan.sandbox, - usage: { - vectorSpace: 32, - buildApps: 12, - teamMembers: 1, - annotatedResponse: 1, - }, - total: { - vectorSpace: 200, - buildApps: 50, - teamMembers: 1, - annotatedResponse: 10, - }, - }, - isFetchedPlan: false, - enableBilling: false, - onPlanInfoChanged: () => { }, - enableReplaceWebAppLogo: false, - enableModelLoadBalancing: false, - }) + modelLoadBalancingEnabled: boolean +} +const ProviderContext = createContext({ + modelProviders: [], + textGenerationModelList: [], + supportRetrievalMethods: [], + isAPIKeySet: true, + plan: { + type: Plan.sandbox, + usage: { + vectorSpace: 32, + buildApps: 12, + teamMembers: 1, + annotatedResponse: 1, + }, + total: { + vectorSpace: 200, + buildApps: 50, + teamMembers: 1, + annotatedResponse: 10, + }, + }, + isFetchedPlan: false, + enableBilling: false, + onPlanInfoChanged: () => { }, + enableReplaceWebAppLogo: false, + modelLoadBalancingEnabled: false, +}) export const useProviderContext = () => useContext(ProviderContext) +// Adding a dangling comma to avoid the generic parsing issue in tsx, see: +// https://github.com/microsoft/TypeScript/issues/15713 +// eslint-disable-next-line @typescript-eslint/comma-dangle +export const useProviderContextSelector = (selector: (state: ProviderContextState) => T): T => + useContextSelector(ProviderContext, selector) + type ProviderContextProviderProps = { children: React.ReactNode } @@ -78,7 +85,7 @@ export const ProviderContextProvider = ({ const [isFetchedPlan, setIsFetchedPlan] = useState(false) const [enableBilling, setEnableBilling] = useState(true) const [enableReplaceWebAppLogo, setEnableReplaceWebAppLogo] = useState(false) - const [enableModelLoadBalancing, setEnableModelLoadBalancing] = useState(false) + const [modelLoadBalancingEnabled, setModelLoadBalancingEnabled] = useState(false) const fetchPlan = async () => { const data = await fetchCurrentPlanInfo() @@ -90,7 +97,7 @@ export const ProviderContextProvider = ({ setIsFetchedPlan(true) } if (data.model_load_balancing_enabled) - setEnableModelLoadBalancing(true) + setModelLoadBalancingEnabled(true) } useEffect(() => { fetchPlan() @@ -107,7 +114,7 @@ export const ProviderContextProvider = ({ enableBilling, onPlanInfoChanged: fetchPlan, enableReplaceWebAppLogo, - enableModelLoadBalancing, + modelLoadBalancingEnabled, }}> {children} diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index f25152aa321f11..12a434ef4db41c 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -344,6 +344,7 @@ const translation = { apiKeyRateLimit: 'Rate limit was reached, available after {{seconds}}s', addConfig: 'Add Config', loadBalancingLeastKeyWarning: 'To enable load balancing at least 2 keys must be enabled.', + loadBalancingInfo: 'By default, load balancing uses the Round-robin strategy. If rate limiting is triggered, a 1-minute cooldown period will be applied.', }, dataSource: { add: 'Add a data source', diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index dd49f79b4ed078..84ae3ae081a699 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -344,6 +344,7 @@ const translation = { apiKeyRateLimit: '已达频率上限，{{seconds}}秒后恢复', addConfig: '增加配置', loadBalancingLeastKeyWarning: '至少启用 2 个 Key 以使用负载均衡', + loadBalancingInfo: '默认情况下，负载平衡使用 Round-robin 策略。如果触发速率限制，将应用 1 分钟的冷却时间', }, dataSource: { add: '添加数据源', diff --git a/web/service/common.ts b/web/service/common.ts index 3a7d97af14c160..de22627889ab6c 100644 --- a/web/service/common.ts +++ b/web/service/common.ts @@ -32,6 +32,7 @@ import type { ModelItem, ModelParameterRule, ModelProvider, + ModelTypeEnum, } from '@/app/components/header/account-setting/model-provider-page/declarations' import type { RETRIEVE_METHOD } from '@/types/app' @@ -271,3 +272,9 @@ type RetrievalMethodsRes = { export const fetchSupportRetrievalMethods: Fetcher = (url) => { return get(url) } + +export const enableModel = (url: string, body: { model: string; model_type: ModelTypeEnum }) => + patch(url, { body }) + +export const disableModel = (url: string, body: { model: string; model_type: ModelTypeEnum }) => + patch(url, { body }) From 71644024a27445c02002a01c26e24b8babff21d5 Mon Sep 17 00:00:00 2001 From: nite-knite Date: Sun, 26 May 2024 16:24:00 +0800 Subject: [PATCH 051/273] feat: fetch model load balancing config --- .../model-provider-page/declarations.ts | 16 + .../model-provider-page/hooks.ts | 2 +- .../model-provider-page/model-modal/index.tsx | 4 +- .../model-balancing-config-entry-modal.tsx | 32 ++ .../model-balancing-modal.tsx | 295 +++++++++++------- .../provider-added-card/model-list.tsx | 14 +- .../provider-added-card/priority-selector.tsx | 2 +- web/i18n/en-US/common.ts | 1 + web/service/common.ts | 11 + 9 files changed, 262 insertions(+), 115 deletions(-) create mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx diff --git a/web/app/components/header/account-setting/model-provider-page/declarations.ts b/web/app/components/header/account-setting/model-provider-page/declarations.ts index a19d52e36ed333..c1f24d4e283daf 100644 --- a/web/app/components/header/account-setting/model-provider-page/declarations.ts +++ b/web/app/components/header/account-setting/model-provider-page/declarations.ts @@ -224,3 +224,19 @@ export type ModelParameterRule = { options?: string[] tagPlaceholder?: TypeWithI18N } + +export type ModelLoadBalancingConfigEntry = { + /** model balancing config entry id */ + id: string + /** is model balancing enabled */ + enabled: boolean + /** config entry name */ + name: string + /** model balancing credential */ + credentials: Record +} + +export type ModelLoadBalancingConfig = { + enabled: boolean + configs: ModelLoadBalancingConfigEntry[] +} diff --git a/web/app/components/header/account-setting/model-provider-page/hooks.ts b/web/app/components/header/account-setting/model-provider-page/hooks.ts index 6ee58df0c20e21..9fd887f96203d9 100644 --- a/web/app/components/header/account-setting/model-provider-page/hooks.ts +++ b/web/app/components/header/account-setting/model-provider-page/hooks.ts @@ -62,7 +62,7 @@ export const useLanguage = () => { return locale.replace('-', '_') } -export const useProviderCrenditialsFormSchemasValue = ( +export const useProviderCredentialsFormSchemasValue = ( provider: string, configurateMethod: ConfigurationMethodEnum, configured?: boolean, diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx index 1c46180cd7e49e..60ebf329e98b14 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx @@ -28,7 +28,7 @@ import { } from '../utils' import { useLanguage, - useProviderCrenditialsFormSchemasValue, + useProviderCredentialsFormSchemasValue, } from '../hooks' import ProviderIcon from '../provider-icon' import { useValidate } from '../../key-validator/hooks' @@ -61,7 +61,7 @@ const ModelModal: FC = ({ onSave, }) => { const providerFormSchemaPredefined = configurateMethod === ConfigurationMethodEnum.predefinedModel - const formSchemasValue = useProviderCrenditialsFormSchemasValue( + const formSchemasValue = useProviderCredentialsFormSchemasValue( provider.provider, configurateMethod, providerFormSchemaPredefined && provider.custom_configuration.status === CustomConfigurationStatusEnum.active, diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx new file mode 100644 index 00000000000000..9f65196658e257 --- /dev/null +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx @@ -0,0 +1,32 @@ +import { memo } from 'react' +import { useTranslation } from 'react-i18next' +import Modal from '@/app/components/base/modal' +import Form from '@/app/components/header/account-setting/model-provider-page/model-modal/Form' + +// create and export model balancing config modal +const ModelBalancingConfigEntryModal = () => { + const { t } = useTranslation() + + return ( + +
{ }} + formSchemas={[]} + validating={false} + showOnVariableMap={{}} + isEditMode={false} + readonly={false} + isShowDefaultValue={false} + /> + + ) +} + +export default memo(ModelBalancingConfigEntryModal) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx index 181b3343f8b924..175e0b3a9689c1 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx @@ -1,7 +1,8 @@ -import { memo } from 'react' +import { memo, useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import classNames from 'classnames' -import type { ModelItem, ModelProvider } from '../declarations' +import useSWR from 'swr' +import type { ModelItem, ModelLoadBalancingConfig, ModelLoadBalancingConfigEntry, ModelProvider } from '../declarations' import ModelIcon from '../model-icon' import ModelName from '../model-name' import Modal from '@/app/components/base/modal' @@ -13,10 +14,12 @@ import { AlertTriangle } from '@/app/components/base/icons/src/vender/line/alert import SimplePieChart from '@/app/components/base/simple-pie-chart' import TooltipPlus from '@/app/components/base/tooltip-plus' import Button from '@/app/components/base/button' +import { fetchModelLoadBalancingConfig } from '@/service/common' +import Loading from '@/app/components/base/loading' export type ModelBalancingModalProps = { provider: ModelProvider - model?: ModelItem + model: ModelItem open?: boolean onClose?: () => void } @@ -25,7 +28,53 @@ export type ModelBalancingModalProps = { const ModelBalancingModal = ({ provider, model, open = false, onClose }: ModelBalancingModalProps) => { const { t } = useTranslation() - // const { data: providersData } = useSWR(`/workspaces/current/model-providers/${provider.provider}/models/load-balancing-configs/:config_id`, fetchModelProviders) + // useProviderCredentialsFormSchemasValue(provider.provider, model.fetch_from) + const { data } = useSWR( + `/workspaces/current/model-providers/${provider.provider}/models/credentials?model=${model.model}&model_type=${model.model_type}`, + fetchModelLoadBalancingConfig, + ) + + const originalConfig = data?.load_balancing + const [draftConfig, setDraftConfig] = useState() + useEffect(() => { + if (originalConfig && !draftConfig) + setDraftConfig(originalConfig) + }, [draftConfig, originalConfig]) + + const toggleModalBalancing = useCallback((enabled: boolean) => { + if (draftConfig) { + setDraftConfig({ + ...draftConfig, + enabled, + }) + } + }, [draftConfig]) + + const updateConfigEntry = useCallback( + ( + index: number, + modifier: (entry: ModelLoadBalancingConfigEntry) => ModelLoadBalancingConfigEntry, + ) => { + setDraftConfig((prev) => { + if (!prev) + return prev + const newConfigs = [...prev.configs] + newConfigs[index] = modifier(newConfigs[index]) + return { + ...prev, + configs: newConfigs, + } + }) + }, + [], + ) + + const toggleConfigEntryEnabled = useCallback((index: number, state?: boolean) => { + updateConfigEntry(index, entry => ({ + ...entry, + enabled: typeof state === 'boolean' ? state : !entry.enabled, + })) + }, [updateConfigEntry]) return ( } > -
-
-
-
- {Boolean(model) && ( - - )} -
-
-
{t('common.modelProvider.providerManaged')}
-
Todo
-
-
-
- -
-
-
- -
-
-
- {t('common.modelProvider.loadBalancing')} - - - -
-
Todo
-
-
- -
-
-
-
- - - -
-
{t('common.modelProvider.defaultConfig')}
- {t('common.modelProvider.providerManaged')} -
-
-
- - - - - - - + {!draftConfig + ? + : ( + <> +
+
toggleModalBalancing(false) : undefined} + > +
+
+ {Boolean(model) && ( + + )} +
+
+
{t('common.modelProvider.providerManaged')}
+
Todo
+
- value} - />
-
-
-
-
- - - +
toggleModalBalancing(true) : undefined} + > +
+
+ +
+
+
+ {t('common.modelProvider.loadBalancing')} + + + +
+
Todo
+
-
Another
- {t('common.modelProvider.providerManaged')} -
-
-
- - - - - - - -
- value} - /> -
-
+ {draftConfig.enabled && ( +
+ {draftConfig.configs.map((config, index) => { + const isProviderManaged = config.name === '__inherit__' + return ( +
+
+
+ + + +
+
+ {isProviderManaged ? t('common.modelProvider.defaultConfig') : config.name} +
+ {isProviderManaged && ( + {t('common.modelProvider.providerManaged')} + )} +
+
+ {!isProviderManaged && ( + <> +
+ + + + + + + +
+ + )} + toggleConfigEntryEnabled(index, value)} + /> +
+
+ ) + })} + +
+
+
+ + + +
+
Another
+ {t('common.modelProvider.providerManaged')} +
+
+
+ + + + + + + +
+ value} + /> +
+
-
-
- {t('common.modelProvider.addConfig')} +
+
+ {t('common.modelProvider.addConfig')} +
+
+
+ )} + { + draftConfig.enabled && draftConfig.configs.length < 2 && ( +
+ + {t('common.modelProvider.loadBalancingLeastKeyWarning')} +
+ ) + }
-
- -
- - {t('common.modelProvider.loadBalancingLeastKeyWarning')} -
-
-
-
- - -
- +
+ + +
+ + ) + } + ) } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx index 93392c8866a2f0..b3a712334e74bb 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx @@ -82,12 +82,14 @@ const ModelList: FC = ({ )) }
- setBalancingModel(undefined), - }} /> + {Boolean(balancingModel) && ( + setBalancingModel(undefined), + }} /> + )}
) } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/priority-selector.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/priority-selector.tsx index 3b9f73c55c0f68..ee8b5c2acab5c6 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/priority-selector.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/priority-selector.tsx @@ -18,7 +18,7 @@ const Selector: FC = ({ const options = [ { key: PreferredProviderTypeEnum.custom, - text: 'API', + text: t('common.modelProvider.apiKey'), }, { key: PreferredProviderTypeEnum.system, diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index 12a434ef4db41c..e7baebe9ce9971 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -278,6 +278,7 @@ const translation = { key: 'Rerank Model', tip: 'Rerank model will reorder the candidate document list based on the semantic match with user query, improving the results of semantic ranking', }, + apiKey: 'API-KEY', quota: 'Quota', searchModel: 'Search model', noModelFound: 'No model found for {{model}}', diff --git a/web/service/common.ts b/web/service/common.ts index de22627889ab6c..24ee032b7260b4 100644 --- a/web/service/common.ts +++ b/web/service/common.ts @@ -30,6 +30,7 @@ import type { DefaultModelResponse, Model, ModelItem, + ModelLoadBalancingConfig, ModelParameterRule, ModelProvider, ModelTypeEnum, @@ -170,6 +171,16 @@ export const fetchModelProviderCredentials: Fetcher<{ credentials?: Record }>(url) } +export const fetchModelLoadBalancingConfig: Fetcher<{ + credentials?: Record + load_balancing: ModelLoadBalancingConfig +}, string> = (url) => { + return get<{ + credentials?: Record + load_balancing: ModelLoadBalancingConfig + }>(url) +} + export const fetchModelProviderModelList: Fetcher<{ data: ModelItem[] }, string> = (url) => { return get<{ data: ModelItem[] }>(url) } From a4c9f4af224f4d9bc22f7ef0acec0ea7485461a3 Mon Sep 17 00:00:00 2001 From: nite-knite Date: Sun, 26 May 2024 16:35:04 +0800 Subject: [PATCH 052/273] feat: model balancing config entry removal --- .../provider-added-card/model-balancing-modal.tsx | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx index 175e0b3a9689c1..5ba7d698a09465 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx @@ -53,13 +53,17 @@ const ModelBalancingModal = ({ provider, model, open = false, onClose }: ModelBa const updateConfigEntry = useCallback( ( index: number, - modifier: (entry: ModelLoadBalancingConfigEntry) => ModelLoadBalancingConfigEntry, + modifier: (entry: ModelLoadBalancingConfigEntry) => ModelLoadBalancingConfigEntry | undefined, ) => { setDraftConfig((prev) => { if (!prev) return prev const newConfigs = [...prev.configs] - newConfigs[index] = modifier(newConfigs[index]) + const modifiedConfig = modifier(newConfigs[index]) + if (modifiedConfig) + newConfigs[index] = modifiedConfig + else + newConfigs.splice(index, 1) return { ...prev, configs: newConfigs, @@ -176,7 +180,10 @@ const ModelBalancingModal = ({ provider, model, open = false, onClose }: ModelBa - + updateConfigEntry(index, () => undefined)} + > From 23a2e5269a2e0b5a2592d030deb375cce797244c Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 11:09:35 +0800 Subject: [PATCH 053/273] feat: choose prompt editor --- .../components/advanced-setting.tsx | 46 +++++++++++++------ .../nodes/question-classifier/panel.tsx | 9 ++++ .../nodes/question-classifier/use-config.ts | 23 ++++++++++ 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx b/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx index f95dfce31ab80f..12fcee37595eb1 100644 --- a/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx +++ b/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx @@ -2,9 +2,9 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' -import TextEditor from '../../_base/components/editor/text-editor' import MemoryConfig from '../../_base/components/memory-config' -import type { Memory } from '@/app/components/workflow/types' +import Editor from '@/app/components/workflow/nodes/_base/components/prompt/editor' +import type { Memory, Node, NodeOutPutVar } from '@/app/components/workflow/types' import TooltipPlus from '@/app/components/base/tooltip-plus' import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general' const i18nPrefix = 'workflow.nodes.questionClassifiers' @@ -16,6 +16,15 @@ type Props = { memory?: Memory onMemoryChange: (memory?: Memory) => void readonly?: boolean + isChatModel: boolean + isChatApp: boolean + hasSetBlockStatus?: { + context: boolean + history: boolean + query: boolean + } + nodesOutputVars: NodeOutPutVar[] + availableNodes: Node[] } const AdvancedSetting: FC = ({ @@ -25,13 +34,18 @@ const AdvancedSetting: FC = ({ memory, onMemoryChange, readonly, + isChatModel, + isChatApp, + hasSetBlockStatus, + nodesOutputVars, + availableNodes, }) => { const { t } = useTranslation() return ( <> - {t(`${i18nPrefix}.instruction`)} @@ -45,15 +59,21 @@ const AdvancedSetting: FC = ({ } value={instruction} onChange={onInstructionChange} - minHeight={160} - placeholder={t(`${i18nPrefix}.instructionPlaceholder`)!} - headerRight={( -
-
{instruction?.length || 0}
-
-
- )} - readonly={readonly} + // minHeight={160} + // placeholder={t(`${i18nPrefix}.instructionPlaceholder`)!} + // headerRight={( + //
+ //
{instruction?.length || 0}
+ //
+ //
+ // )} + readOnly={readonly} + isChatModel={isChatModel} + isChatApp={isChatApp} + isShowContext={false} + hasSetBlockStatus={hasSetBlockStatus} + nodesOutputVars={nodesOutputVars} + availableNodes={availableNodes} /> {!hideMemorySetting && ( > = ({ inputs, handleModelChanged, isChatMode, + isChatModel, handleCompletionParamsChange, handleQueryVarChange, handleTopicsChange, + hasSetBlockStatus, + availableVars, + availableNodesWithParent, handleInstructionChange, handleMemoryChange, isShowSingleRun, @@ -99,6 +103,11 @@ const Panel: FC> = ({ memory={inputs.memory} onMemoryChange={handleMemoryChange} readonly={readOnly} + isChatApp={isChatMode} + isChatModel={isChatModel} + hasSetBlockStatus={hasSetBlockStatus} + nodesOutputVars={availableVars} + availableNodes={availableNodesWithParent} />
diff --git a/web/app/components/workflow/nodes/question-classifier/use-config.ts b/web/app/components/workflow/nodes/question-classifier/use-config.ts index d5bc4ce0bd2466..540b78a79f4ce9 100644 --- a/web/app/components/workflow/nodes/question-classifier/use-config.ts +++ b/web/app/components/workflow/nodes/question-classifier/use-config.ts @@ -7,11 +7,13 @@ import { useWorkflow, } from '../../hooks' import { useStore } from '../../store' +import useAvailableVarList from '../_base/hooks/use-available-var-list' import type { QuestionClassifierNodeType } from './types' import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud' import useOneStepRun from '@/app/components/workflow/nodes/_base/hooks/use-one-step-run' import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { checkHasQueryBlock } from '@/app/components/base/prompt-editor/constants' const useConfig = (id: string, payload: QuestionClassifierNodeType) => { const { nodesReadOnly: readOnly } = useNodesReadOnly() @@ -93,6 +95,24 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { setInputs(newInputs) }, [inputs, setInputs]) + const filterInputVar = useCallback((varPayload: Var) => { + return [VarType.number, VarType.string].includes(varPayload.type) + }, []) + + const { + availableVars, + availableNodesWithParent, + } = useAvailableVarList(id, { + onlyLeafNodeVar: false, + filterVar: filterInputVar, + }) + + const hasSetBlockStatus = { + history: false, + query: isChatMode ? checkHasQueryBlock(inputs.instruction) : false, + context: false, + } + const handleInstructionChange = useCallback((instruction: string) => { const newInputs = produce(inputs, (draft) => { draft.instruction = instruction @@ -147,6 +167,9 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { handleQueryVarChange, filterVar, handleTopicsChange: handleClassesChange, + hasSetBlockStatus, + availableVars, + availableNodesWithParent, handleInstructionChange, handleMemoryChange, isShowSingleRun, From 78f1d8723ea4ef991c2c2a62e95a0d73d8be713b Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 11:42:31 +0800 Subject: [PATCH 054/273] feat: set var in single run --- .../nodes/question-classifier/panel.tsx | 9 ++++++--- .../nodes/question-classifier/use-config.ts | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/web/app/components/workflow/nodes/question-classifier/panel.tsx b/web/app/components/workflow/nodes/question-classifier/panel.tsx index 0b155945a57a6f..c82490edfa0a11 100644 --- a/web/app/components/workflow/nodes/question-classifier/panel.tsx +++ b/web/app/components/workflow/nodes/question-classifier/panel.tsx @@ -35,6 +35,9 @@ const Panel: FC> = ({ availableVars, availableNodesWithParent, handleInstructionChange, + inputVarValues, + varInputs, + setInputVarValues, handleMemoryChange, isShowSingleRun, hideSingleRun, @@ -134,9 +137,9 @@ const Panel: FC> = ({ variable: 'query', type: InputVarType.paragraph, required: true, - }], - values: { query }, - onChange: keyValue => setQuery((keyValue as any).query), + }, ...varInputs], + values: inputVarValues, + onChange: setInputVarValues, }, ]} runningStatus={runningStatus} diff --git a/web/app/components/workflow/nodes/question-classifier/use-config.ts b/web/app/components/workflow/nodes/question-classifier/use-config.ts index 540b78a79f4ce9..7d84a653e35751 100644 --- a/web/app/components/workflow/nodes/question-classifier/use-config.ts +++ b/web/app/components/workflow/nodes/question-classifier/use-config.ts @@ -131,6 +131,7 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { const { isShowSingleRun, hideSingleRun, + getInputVars, runningStatus, handleRun, handleStop, @@ -153,6 +154,22 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { }) }, [runInputData, setRunInputData]) + const varInputs = getInputVars([inputs.instruction]) + const inputVarValues = (() => { + const vars: Record = { + query, + } + Object.keys(runInputData) + .forEach((key) => { + vars[key] = runInputData[key] + }) + return vars + })() + + const setInputVarValues = useCallback((newPayload: Record) => { + setRunInputData(newPayload) + }, [setRunInputData]) + const filterVar = useCallback((varPayload: Var) => { return varPayload.type === VarType.string }, []) @@ -171,6 +188,9 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { availableVars, availableNodesWithParent, handleInstructionChange, + varInputs, + inputVarValues, + setInputVarValues, handleMemoryChange, isShowSingleRun, hideSingleRun, From c4c9c2bffca2dfc1aa8a6821b9ca02de95ac83ea Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 13:47:28 +0800 Subject: [PATCH 055/273] feat: rename and remove --- .../workflow/nodes/_base/components/variable/utils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index 6545358ee41872..e4b181b18dcc4b 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -510,7 +510,10 @@ export const getNodeUsedVars = (node: Node): ValueSelector[] => { break } case BlockEnum.QuestionClassifier: { - res = [(data as QuestionClassifierNodeType).query_variable_selector] + const payload = (data as QuestionClassifierNodeType) + res = [payload.query_variable_selector] + const varInInstructions = matchNotSystemVars([payload.instruction || '']) + res.push(...varInInstructions) break } case BlockEnum.HttpRequest: { @@ -725,6 +728,7 @@ export const updateNodeVars = (oldNode: Node, oldVarSelector: ValueSelector, new const payload = data as QuestionClassifierNodeType if (payload.query_variable_selector.join('.') === oldVarSelector.join('.')) payload.query_variable_selector = newVarSelector + payload.instruction = replaceOldVarInText(payload.instruction, oldVarSelector, newVarSelector) break } case BlockEnum.HttpRequest: { From 4d8a25d1daf04ee188b275a9bf018252379080a8 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 14:00:59 +0800 Subject: [PATCH 056/273] fix: remove useless code --- .../question-classifier/components/advanced-setting.tsx | 9 --------- .../workflow/nodes/question-classifier/panel.tsx | 2 -- .../workflow/nodes/question-classifier/use-config.ts | 1 - 3 files changed, 12 deletions(-) diff --git a/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx b/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx index 12fcee37595eb1..2742c1df56d1ee 100644 --- a/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx +++ b/web/app/components/workflow/nodes/question-classifier/components/advanced-setting.tsx @@ -45,7 +45,6 @@ const AdvancedSetting: FC = ({ return ( <> {t(`${i18nPrefix}.instruction`)} @@ -59,14 +58,6 @@ const AdvancedSetting: FC = ({ } value={instruction} onChange={onInstructionChange} - // minHeight={160} - // placeholder={t(`${i18nPrefix}.instructionPlaceholder`)!} - // headerRight={( - //
- //
{instruction?.length || 0}
- //
- //
- // )} readOnly={readonly} isChatModel={isChatModel} isChatApp={isChatApp} diff --git a/web/app/components/workflow/nodes/question-classifier/panel.tsx b/web/app/components/workflow/nodes/question-classifier/panel.tsx index c82490edfa0a11..59c40c8f5a1932 100644 --- a/web/app/components/workflow/nodes/question-classifier/panel.tsx +++ b/web/app/components/workflow/nodes/question-classifier/panel.tsx @@ -44,8 +44,6 @@ const Panel: FC> = ({ runningStatus, handleRun, handleStop, - query, - setQuery, runResult, filterVar, } = useConfig(id, data) diff --git a/web/app/components/workflow/nodes/question-classifier/use-config.ts b/web/app/components/workflow/nodes/question-classifier/use-config.ts index 7d84a653e35751..883ce85f438e12 100644 --- a/web/app/components/workflow/nodes/question-classifier/use-config.ts +++ b/web/app/components/workflow/nodes/question-classifier/use-config.ts @@ -69,7 +69,6 @@ const useConfig = (id: string, payload: QuestionClassifierNodeType) => { draft.query_variable_selector = newVar as ValueSelector }) setInputs(newInputs) - // console.log(newInputs.query_variable_selector) }, [inputs, setInputs]) useEffect(() => { From 7e4c117ebb4d04ade0c060c579bbccb22fda2be4 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 17:21:32 +0800 Subject: [PATCH 057/273] feat: support rename in list --- .../components/datasets/documents/list.tsx | 78 +++++++++++++++---- .../datasets/documents/rename-modal.tsx | 75 ++++++++++++++++++ web/i18n/en-US/dataset-documents.ts | 2 + web/i18n/zh-Hans/dataset-documents.ts | 2 + web/service/datasets.ts | 6 ++ 5 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 web/app/components/datasets/documents/rename-modal.tsx diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 01618b6e6b84e2..2b612d4382f291 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -1,8 +1,8 @@ /* eslint-disable no-mixed-operators */ 'use client' import type { FC, SVGProps } from 'react' -import React, { useEffect, useState } from 'react' -import { useDebounceFn } from 'ahooks' +import React, { useCallback, useEffect, useState } from 'react' +import { useBoolean, useDebounceFn } from 'ahooks' import { ArrowDownIcon, TrashIcon } from '@heroicons/react/24/outline' import { ExclamationCircleIcon } from '@heroicons/react/24/solid' import { pick } from 'lodash-es' @@ -11,7 +11,10 @@ import { useRouter } from 'next/navigation' import { useTranslation } from 'react-i18next' import cn from 'classnames' import dayjs from 'dayjs' +import { Edit03 } from '../../base/icons/src/vender/solid/general' +import TooltipPlus from '../../base/tooltip-plus' import s from './style.module.css' +import RenameModal from './rename-modal' import Switch from '@/app/components/base/switch' import Divider from '@/app/components/base/divider' import Popover from '@/app/components/base/popover' @@ -39,7 +42,7 @@ export const SettingsIcon = ({ className }: SVGProps) => { export const SyncIcon = () => { return - + } @@ -326,13 +329,30 @@ const DocumentList: FC = ({ embeddingAvailable, documents = } } + const [currDocument, setCurrDocument] = useState(null) + const [isShowRenameModal, { + setTrue: setShowRenameModalTrue, + setFalse: setShowRenameModalFalse, + }] = useBoolean(false) + const handleShowRenameModal = useCallback((doc: LocalDoc) => { + setCurrDocument(doc) + setShowRenameModalTrue() + }, [setShowRenameModalTrue]) + const handleRenamed = useCallback(() => { + onUpdate() + }, [onUpdate]) + return (
- + - @@ -391,6 +429,16 @@ const DocumentList: FC = ({ embeddingAvailable, documents = })}
#{t('datasetDocuments.list.table.header.fileName')} +
+ {t('datasetDocuments.list.table.header.fileName')} +
+ 		{t('datasetDocuments.list.table.header.words')} {t('datasetDocuments.list.table.header.hitCount')} @@ -355,17 +375,35 @@ const DocumentList: FC = ({ embeddingAvailable, documents = router.push(`/datasets/${datasetId}/documents/${doc.id}`) }}> {doc.position} - { - doc?.data_source_type === DataSourceType.NOTION - ? - :
- } - { - doc.data_source_type === DataSourceType.NOTION - ? {doc.name} - : {doc?.name?.replace(/\.[^/.]+$/, '')}.{suffix} - } + 		+
+ + { + doc?.data_source_type === DataSourceType.NOTION + ? + :
+ } + { + doc.data_source_type === DataSourceType.NOTION + ? {doc.name} + : {doc?.name?.replace(/\.[^/.]+$/, '')}.{suffix} + } + +
+ +
{ + e.stopPropagation() + handleShowRenameModal(doc) + }} + > + +
+ +
+
+ 		{renderCount(doc.word_count)} {renderCount(doc.hit_count)}
+ + {isShowRenameModal && currDocument && ( + + )}
) } diff --git a/web/app/components/datasets/documents/rename-modal.tsx b/web/app/components/datasets/documents/rename-modal.tsx new file mode 100644 index 00000000000000..a47d47e4b3f1da --- /dev/null +++ b/web/app/components/datasets/documents/rename-modal.tsx @@ -0,0 +1,75 @@ +'use client' +import type { FC } from 'react' +import React, { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useBoolean } from 'ahooks' +import Toast from '../../base/toast' +import Modal from '@/app/components/base/modal' +import Button from '@/app/components/base/button' +import { renameDocumentName } from '@/service/datasets' + +type Props = { + datasetId: string + documentId: string + name: string + onClose: () => void + onSaved: () => void +} + +const RenameModal: FC = ({ + documentId, + datasetId, + name, + onClose, + onSaved, +}) => { + const { t } = useTranslation() + + const [newName, setNewName] = useState(name) + const [saveLoading, { + setTrue: setSaveLoadingTrue, + setFalse: setSaveLoadingFalse, + }] = useBoolean(false) + + const handleSave = async () => { + setSaveLoadingTrue() + try { + await renameDocumentName({ + datasetId, + documentId, + name: newName, + }) + Toast.notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + onSaved() + onClose() + } + catch (error) { + if (error) + Toast.notify({ type: 'error', message: error.toString() }) + } + finally { + setSaveLoadingFalse() + } + } + + return ( + +
{t('datasetDocuments.list.table.name')}
+ setNewName(e.target.value)} + /> + +
+ + +
+ + ) +} +export default React.memo(RenameModal) diff --git a/web/i18n/en-US/dataset-documents.ts b/web/i18n/en-US/dataset-documents.ts index 6a8cf6c5cae430..b431965323cd27 100644 --- a/web/i18n/en-US/dataset-documents.ts +++ b/web/i18n/en-US/dataset-documents.ts @@ -13,6 +13,8 @@ const translation = { status: 'STATUS', action: 'ACTION', }, + rename: 'Rename', + name: 'Name', }, action: { uploadFile: 'Upload new file', diff --git a/web/i18n/zh-Hans/dataset-documents.ts b/web/i18n/zh-Hans/dataset-documents.ts index 903f4b916e4b12..9ea5e7aa0f58d6 100644 --- a/web/i18n/zh-Hans/dataset-documents.ts +++ b/web/i18n/zh-Hans/dataset-documents.ts @@ -13,6 +13,8 @@ const translation = { status: '状态', action: '操作', }, + rename: '重命名', + name: '名称', }, action: { uploadFile: '上传新文件', diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 628aa4df6a218c..65e7b28ff811e8 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -114,6 +114,12 @@ export const fetchDocumentDetail: Fetcher(`/datasets/${datasetId}/documents/${documentId}`, { params }) } +export const renameDocumentName: Fetcher = ({ datasetId, documentId, name }) => { + return put(`/datasets/${datasetId}/documents/${documentId}/rename`, { + body: { name }, + }) +} + export const pauseDocIndexing: Fetcher = ({ datasetId, documentId }) => { return patch(`/datasets/${datasetId}/documents/${documentId}/processing/pause`) } From 0a12e93b27e11c98cf0cf860077741d5abd37576 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 17:35:39 +0800 Subject: [PATCH 058/273] feat: rename in dropdown action --- .../datasets/documents/detail/index.tsx | 1 + .../components/datasets/documents/list.tsx | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/web/app/components/datasets/documents/detail/index.tsx b/web/app/components/datasets/documents/detail/index.tsx index cda6e98a26ae4a..8e2bac69268656 100644 --- a/web/app/components/datasets/documents/detail/index.tsx +++ b/web/app/components/datasets/documents/detail/index.tsx @@ -150,6 +150,7 @@ const DocumentDetail: FC = ({ datasetId, documentId }) => { scene='detail' embeddingAvailable={embeddingAvailable} detail={{ + name: documentDetail?.name || '', enabled: documentDetail?.enabled || false, archived: documentDetail?.archived || false, id: documentId, diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 2b612d4382f291..ef97b9d82bbbe6 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -110,6 +110,7 @@ type OperationName = 'delete' | 'archive' | 'enable' | 'disable' | 'sync' | 'un_ export const OperationAction: FC<{ embeddingAvailable: boolean detail: { + name: string enabled: boolean archived: boolean id: string @@ -167,6 +168,25 @@ export const OperationAction: FC<{ onOperate(operationName) }, { wait: 500 }) + const [currDocument, setCurrDocument] = useState<{ + id: string + name: string + } | null>(null) + const [isShowRenameModal, { + setTrue: setShowRenameModalTrue, + setFalse: setShowRenameModalFalse, + }] = useBoolean(false) + const handleShowRenameModal = useCallback((doc: { + id: string + name: string + }) => { + setCurrDocument(doc) + setShowRenameModalTrue() + }, [setShowRenameModalTrue]) + const handleRenamed = useCallback(() => { + onUpdate() + }, [onUpdate]) + return
e.stopPropagation()}> {isListScene && !embeddingAvailable && ( { }} disabled={true} size='md' /> @@ -216,6 +236,15 @@ export const OperationAction: FC<{ } {!archived && ( <> +
{ + handleShowRenameModal({ + id: detail.id, + name: detail.name, + }) + }}> + + {t('datasetDocuments.list.table.rename')} +
router.push(`/datasets/${datasetId}/documents/${detail.id}/settings`)}> {t('datasetDocuments.list.action.settings')} @@ -275,6 +304,16 @@ export const OperationAction: FC<{
} + + {isShowRenameModal && currDocument && ( + + )}
} @@ -421,7 +460,7 @@ const DocumentList: FC = ({ embeddingAvailable, documents = From e4df4f62ef18e53fbf1d0991310c5d50e9316139 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 18:12:00 +0800 Subject: [PATCH 059/273] fix: url --- web/service/datasets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 65e7b28ff811e8..302b16f6f55567 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -115,7 +115,7 @@ export const fetchDocumentDetail: Fetcher = ({ datasetId, documentId, name }) => { - return put(`/datasets/${datasetId}/documents/${documentId}/rename`, { + return post(`/datasets/${datasetId}/documents/${documentId}/rename`, { body: { name }, }) } From a3d3417fe567b2528173d8f7c99b7a8651b0a60e Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 28 May 2024 18:17:24 +0800 Subject: [PATCH 060/273] fix: get document name from doc name --- web/app/components/datasets/documents/list.tsx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index ef97b9d82bbbe6..4254b98088da75 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -406,7 +406,7 @@ const DocumentList: FC = ({ embeddingAvailable, documents = {localDocs.map((doc) => { - const suffix = doc.name.split('.').pop() || 'txt' + const documentType = 'txt'// TODO: wait for api. doc.name.split('.').pop() || 'txt' return = ({ embeddingAvailable, documents = { doc?.data_source_type === DataSourceType.NOTION ? - :
+ :
} { - doc.data_source_type === DataSourceType.NOTION - ? {doc.name} - : {doc?.name?.replace(/\.[^/.]+$/, '')}.{suffix} + doc.name }
From b97872415045d45c7429fe709df9bc40522bbccc Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 28 May 2024 20:38:18 +0800 Subject: [PATCH 061/273] update migration down revision --- api/migrations/versions/4e99a8df00ff_add_load_balancing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/migrations/versions/4e99a8df00ff_add_load_balancing.py b/api/migrations/versions/4e99a8df00ff_add_load_balancing.py index 5c14d0b0678b65..67d7b9fbf54875 100644 --- a/api/migrations/versions/4e99a8df00ff_add_load_balancing.py +++ b/api/migrations/versions/4e99a8df00ff_add_load_balancing.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = '4e99a8df00ff' -down_revision = '47cc7df8c4f3' +down_revision = '64a70a7aab8b' branch_labels = None depends_on = None From ed32dfecb7e2f48ca7a337bc23e3944db5ac10e5 Mon Sep 17 00:00:00 2001 From: nite-knite Date: Wed, 29 May 2024 00:00:21 +0800 Subject: [PATCH 062/273] feat: add model load balancing configs and config entry modal --- .../model-provider-page/declarations.ts | 6 +- .../model-provider-page/hooks.ts | 12 +- .../model-provider-page/index.tsx | 14 +- .../model-provider-page/model-modal/index.tsx | 31 +- .../model-load-balancing-entry-modal.tsx | 325 ++++++++++++++++++ .../model-selector/popup-item.tsx | 2 +- .../model-balancing-config-entry-modal.tsx | 32 -- .../model-balancing-modal.tsx | 262 -------------- .../provider-added-card/model-list-item.tsx | 2 +- .../provider-added-card/model-list.tsx | 4 +- .../model-load-balancing-configs.tsx | 219 ++++++++++++ .../model-load-balancing-modal.tsx | 119 +++++++ web/app/components/tools/tool-list/index.tsx | 4 +- web/context/modal-context.tsx | 76 ++-- 14 files changed, 764 insertions(+), 344 deletions(-) create mode 100644 web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx delete mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx delete mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx create mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx create mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx diff --git a/web/app/components/header/account-setting/model-provider-page/declarations.ts b/web/app/components/header/account-setting/model-provider-page/declarations.ts index c1f24d4e283daf..d2312bb317205e 100644 --- a/web/app/components/header/account-setting/model-provider-page/declarations.ts +++ b/web/app/components/header/account-setting/model-provider-page/declarations.ts @@ -228,12 +228,16 @@ export type ModelParameterRule = { export type ModelLoadBalancingConfigEntry = { /** model balancing config entry id */ id: string - /** is model balancing enabled */ + /** is config entry enabled */ enabled: boolean /** config entry name */ name: string /** model balancing credential */ credentials: Record + /** is config entry currently removed from Round-robin queue */ + in_cooldown: boolean + /** cooldown time (in seconds) */ + ttl: number } export type ModelLoadBalancingConfig = { diff --git a/web/app/components/header/account-setting/model-provider-page/hooks.ts b/web/app/components/header/account-setting/model-provider-page/hooks.ts index 9fd887f96203d9..340f86769f1606 100644 --- a/web/app/components/header/account-setting/model-provider-page/hooks.ts +++ b/web/app/components/header/account-setting/model-provider-page/hooks.ts @@ -7,7 +7,7 @@ import { import useSWR, { useSWRConfig } from 'swr' import { useContext } from 'use-context-selector' import type { - CustomConfigrationModelFixedFields, + CustomConfigurationModelFixedFields, DefaultModel, DefaultModelResponse, Model, @@ -66,7 +66,7 @@ export const useProviderCredentialsFormSchemasValue = ( provider: string, configurateMethod: ConfigurationMethodEnum, configured?: boolean, - currentCustomConfigrationModelFixedFields?: CustomConfigrationModelFixedFields, + currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields, ) => { const { data: predefinedFormSchemasValue } = useSWR( (configurateMethod === ConfigurationMethodEnum.predefinedModel && configured) @@ -75,8 +75,8 @@ export const useProviderCredentialsFormSchemasValue = ( fetchModelProviderCredentials, ) const { data: customFormSchemasValue } = useSWR( - (configurateMethod === ConfigurationMethodEnum.customizableModel && currentCustomConfigrationModelFixedFields) - ? `/workspaces/current/model-providers/${provider}/models/credentials?model=${currentCustomConfigrationModelFixedFields?.__model_name}&model_type=${currentCustomConfigrationModelFixedFields?.__model_type}` + (configurateMethod === ConfigurationMethodEnum.customizableModel && currentCustomConfigurationModelFixedFields) + ? `/workspaces/current/model-providers/${provider}/models/credentials?model=${currentCustomConfigurationModelFixedFields?.__model_name}&model_type=${currentCustomConfigurationModelFixedFields?.__model_type}` : null, fetchModelProviderCredentials, ) @@ -87,12 +87,12 @@ export const useProviderCredentialsFormSchemasValue = ( : customFormSchemasValue?.credentials ? { ...customFormSchemasValue?.credentials, - ...currentCustomConfigrationModelFixedFields, + ...currentCustomConfigurationModelFixedFields, } : undefined }, [ configurateMethod, - currentCustomConfigrationModelFixedFields, + currentCustomConfigurationModelFixedFields, customFormSchemasValue?.credentials, predefinedFormSchemasValue?.credentials, ]) diff --git a/web/app/components/header/account-setting/model-provider-page/index.tsx b/web/app/components/header/account-setting/model-provider-page/index.tsx index 53a78dd43bd34a..b8f0caf93e6ae7 100644 --- a/web/app/components/header/account-setting/model-provider-page/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/index.tsx @@ -4,7 +4,7 @@ import SystemModelSelector from './system-model-selector' import ProviderAddedCard, { UPDATE_MODEL_PROVIDER_CUSTOM_MODEL_LIST } from './provider-added-card' import ProviderCard from './provider-card' import type { - CustomConfigrationModelFixedFields, + CustomConfigurationModelFixedFields, ModelProvider, } from './declarations' import { @@ -58,13 +58,13 @@ const ModelProviderPage = () => { const handleOpenModal = ( provider: ModelProvider, configurateMethod: ConfigurationMethodEnum, - customConfigrationModelFixedFields?: CustomConfigrationModelFixedFields, + CustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields, ) => { setShowModelModal({ payload: { currentProvider: provider, - currentConfigurateMethod: configurateMethod, - currentCustomConfigrationModelFixedFields: customConfigrationModelFixedFields, + currentConfigurationMethod: configurateMethod, + currentCustomConfigurationModelFixedFields: CustomConfigurationModelFixedFields, }, onSaveCallback: () => { updateModelProviders() @@ -81,8 +81,8 @@ const ModelProviderPage = () => { payload: provider.provider, } as any) - if (customConfigrationModelFixedFields?.__model_type) - updateModelList(customConfigrationModelFixedFields?.__model_type) + if (CustomConfigurationModelFixedFields?.__model_type) + updateModelList(CustomConfigurationModelFixedFields?.__model_type) } }, }) @@ -117,7 +117,7 @@ const ModelProviderPage = () => { handleOpenModal(provider, configurateMethod, currentCustomConfigrationModelFixedFields)} + onOpenModal={(configurateMethod: ConfigurationMethodEnum, currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields) => handleOpenModal(provider, configurateMethod, currentCustomConfigurationModelFixedFields)} /> )) } diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx index 60ebf329e98b14..1e598af0d975cb 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx @@ -11,8 +11,9 @@ import type { CredentialFormSchema, CredentialFormSchemaRadio, CredentialFormSchemaSelect, - CustomConfigrationModelFixedFields, + CustomConfigurationModelFixedFields, FormValue, + ModelLoadBalancingConfig, ModelProvider, } from '../declarations' import { @@ -33,6 +34,7 @@ import { import ProviderIcon from '../provider-icon' import { useValidate } from '../../key-validator/hooks' import { ValidatedStatus } from '../../key-validator/declarations' +import ModelLoadBalancingConfigs from '../provider-added-card/model-load-balancing-configs' import Form from './Form' import Button from '@/app/components/base/button' import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security' @@ -48,7 +50,7 @@ import ConfirmCommon from '@/app/components/base/confirm/common' type ModelModalProps = { provider: ModelProvider configurateMethod: ConfigurationMethodEnum - currentCustomConfigrationModelFixedFields?: CustomConfigrationModelFixedFields + currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields onCancel: () => void onSave: () => void } @@ -56,7 +58,7 @@ type ModelModalProps = { const ModelModal: FC = ({ provider, configurateMethod, - currentCustomConfigrationModelFixedFields, + currentCustomConfigurationModelFixedFields, onCancel, onSave, }) => { @@ -65,7 +67,7 @@ const ModelModal: FC = ({ provider.provider, configurateMethod, providerFormSchemaPredefined && provider.custom_configuration.status === CustomConfigurationStatusEnum.active, - currentCustomConfigrationModelFixedFields, + currentCustomConfigurationModelFixedFields, ) const isEditMode = !!formSchemasValue const { t } = useTranslation() @@ -73,6 +75,17 @@ const ModelModal: FC = ({ const language = useLanguage() const [loading, setLoading] = useState(false) const [showConfirm, setShowConfirm] = useState(false) + + const originalConfig: ModelLoadBalancingConfig = useMemo(() => ({ + enabled: false, + configs: [], + }), []) + const [draftConfig, setDraftConfig] = useState() + useEffect(() => { + if (originalConfig && !draftConfig) + setDraftConfig(originalConfig) + }, [draftConfig, originalConfig]) + const formSchemas = useMemo(() => { return providerFormSchemaPredefined ? provider.provider_credential_schema.credential_form_schemas @@ -151,7 +164,7 @@ const ModelModal: FC = ({ useEffect(() => { setValue(initialFormSchemasValue) }, [initialFormSchemasValue]) - const [validate, validating, validatedStatusState] = useValidate(value) + const [_, validating, validatedStatusState] = useValidate(value) const filteredRequiredFormSchemas = requiredFormSchemas.filter((requiredFormSchema) => { if (requiredFormSchema.show_on.length && requiredFormSchema.show_on.every(showOnItem => value[showOnItem.variable] === showOnItem.value)) return true @@ -241,6 +254,14 @@ const ModelModal: FC = ({ showOnVariableMap={showOnVariableMap} isEditMode={isEditMode} /> + + +
{ (provider.help && (provider.help.title || provider.help.url)) diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx new file mode 100644 index 00000000000000..ea4c07d5e4df82 --- /dev/null +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx @@ -0,0 +1,325 @@ +import type { FC } from 'react' +import { + memo, + useCallback, + useEffect, + useMemo, + useState, +} from 'react' +import { useTranslation } from 'react-i18next' +import type { + CredentialFormSchema, + CredentialFormSchemaRadio, + CredentialFormSchemaSelect, + CustomConfigurationModelFixedFields, + FormValue, + ModelProvider, +} from '../declarations' +import { + ConfigurationMethodEnum, + CustomConfigurationStatusEnum, + FormTypeEnum, +} from '../declarations' +import { + genModelNameFormSchema, + genModelTypeFormSchema, + removeCredentials, + saveCredentials, +} from '../utils' +import { + useLanguage, + useProviderCredentialsFormSchemasValue, +} from '../hooks' +import { useValidate } from '../../key-validator/hooks' +import { ValidatedStatus } from '../../key-validator/declarations' +import Form from './Form' +import Button from '@/app/components/base/button' +import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security' +import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general' +import { AlertCircle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' +import { + PortalToFollowElem, + PortalToFollowElemContent, +} from '@/app/components/base/portal-to-follow-elem' +import { useToastContext } from '@/app/components/base/toast' +import ConfirmCommon from '@/app/components/base/confirm/common' + +type ModelModalProps = { + provider: ModelProvider + configurateMethod: ConfigurationMethodEnum + currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields + credentials?: Record + onCancel: () => void + onSave: () => void +} + +const ModelLoadBalancingEntryModal: FC = ({ + provider, + configurateMethod, + currentCustomConfigurationModelFixedFields, + credentials, + onCancel, + onSave, +}) => { + const providerFormSchemaPredefined = configurateMethod === ConfigurationMethodEnum.predefinedModel + const formSchemasValue = useProviderCredentialsFormSchemasValue( + provider.provider, + configurateMethod, + providerFormSchemaPredefined && provider.custom_configuration.status === CustomConfigurationStatusEnum.active, + currentCustomConfigurationModelFixedFields, + ) + const isEditMode = !!credentials + const { t } = useTranslation() + const { notify } = useToastContext() + const language = useLanguage() + const [loading, setLoading] = useState(false) + const [showConfirm, setShowConfirm] = useState(false) + const formSchemas = useMemo(() => { + return providerFormSchemaPredefined + ? provider.provider_credential_schema.credential_form_schemas + : [ + genModelTypeFormSchema(provider.supported_model_types), + genModelNameFormSchema(provider.model_credential_schema?.model), + ...provider.model_credential_schema.credential_form_schemas, + ] + }, [ + providerFormSchemaPredefined, + provider.provider_credential_schema?.credential_form_schemas, + provider.supported_model_types, + provider.model_credential_schema?.credential_form_schemas, + provider.model_credential_schema?.model, + ]) + const [ + requiredFormSchemas, + secretFormSchemas, + defaultFormSchemaValue, + showOnVariableMap, + ] = useMemo(() => { + const requiredFormSchemas: CredentialFormSchema[] = [] + const secretFormSchemas: CredentialFormSchema[] = [] + const defaultFormSchemaValue: Record = {} + const showOnVariableMap: Record = {} + + formSchemas.forEach((formSchema) => { + if (formSchema.required) + requiredFormSchemas.push(formSchema) + + if (formSchema.type === FormTypeEnum.secretInput) + secretFormSchemas.push(formSchema) + + if (formSchema.default) + defaultFormSchemaValue[formSchema.variable] = formSchema.default + + if (formSchema.show_on.length) { + formSchema.show_on.forEach((showOnItem) => { + if (!showOnVariableMap[showOnItem.variable]) + showOnVariableMap[showOnItem.variable] = [] + + if (!showOnVariableMap[showOnItem.variable].includes(formSchema.variable)) + showOnVariableMap[showOnItem.variable].push(formSchema.variable) + }) + } + + if (formSchema.type === FormTypeEnum.select || formSchema.type === FormTypeEnum.radio) { + (formSchema as (CredentialFormSchemaRadio | CredentialFormSchemaSelect)).options.forEach((option) => { + if (option.show_on.length) { + option.show_on.forEach((showOnItem) => { + if (!showOnVariableMap[showOnItem.variable]) + showOnVariableMap[showOnItem.variable] = [] + + if (!showOnVariableMap[showOnItem.variable].includes(formSchema.variable)) + showOnVariableMap[showOnItem.variable].push(formSchema.variable) + }) + } + }) + } + }) + + return [ + requiredFormSchemas, + secretFormSchemas, + defaultFormSchemaValue, + showOnVariableMap, + ] + }, [formSchemas]) + const initialFormSchemasValue = useMemo(() => { + return { + ...defaultFormSchemaValue, + ...formSchemasValue, + } + }, [formSchemasValue, defaultFormSchemaValue]) + const [value, setValue] = useState(initialFormSchemasValue) + useEffect(() => { + setValue(initialFormSchemasValue) + }, [initialFormSchemasValue]) + const [_, validating, validatedStatusState] = useValidate(value) + const filteredRequiredFormSchemas = requiredFormSchemas.filter((requiredFormSchema) => { + if (requiredFormSchema.show_on.length && requiredFormSchema.show_on.every(showOnItem => value[showOnItem.variable] === showOnItem.value)) + return true + + if (!requiredFormSchema.show_on.length) + return true + + return false + }) + const getSecretValues = useCallback((v: FormValue) => { + return secretFormSchemas.reduce((prev, next) => { + if (v[next.variable] === initialFormSchemasValue[next.variable]) + prev[next.variable] = '[__HIDDEN__]' + + return prev + }, {} as Record) + }, [initialFormSchemasValue, secretFormSchemas]) + + const handleValueChange = (v: FormValue) => { + setValue(v) + } + const handleSave = async () => { + try { + setLoading(true) + + const res = await saveCredentials( + providerFormSchemaPredefined, + provider.provider, + { + ...value, + ...getSecretValues(value), + }, + ) + if (res.result === 'success') { + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + onSave() + onCancel() + } + } + finally { + setLoading(false) + } + } + + const handleRemove = async () => { + try { + setLoading(true) + + const res = await removeCredentials( + providerFormSchemaPredefined, + provider.provider, + value, + ) + if (res.result === 'success') { + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + onSave() + onCancel() + } + } + finally { + setLoading(false) + } + } + + return ( + + +
+
+
+
+
{t('common.modelProvider.addConfig')}
+
+ +
+ { + (provider.help && (provider.help.title || provider.help.url)) + ? ( + !provider.help.url && e.preventDefault()} + > + {provider.help.title?.[language] || provider.help.url[language] || provider.help.title?.en_US || provider.help.url.en_US} + + + ) + :
+ } +
+ { + isEditMode && ( + + ) + } + + +
+
+
+
+ { + (validatedStatusState.status === ValidatedStatus.Error && validatedStatusState.message) + ? ( +
+ + {validatedStatusState.message} +
+ ) + : ( +
+ + {t('common.modelProvider.encrypted.front')} + + PKCS1_OAEP + + {t('common.modelProvider.encrypted.back')} +
+ ) + } +
+
+ { + showConfirm && ( + setShowConfirm(false)} + onConfirm={handleRemove} + confirmWrapperClassName='z-[70]' + /> + ) + } +
+ + + ) +} + +export default memo(ModelLoadBalancingEntryModal) diff --git a/web/app/components/header/account-setting/model-provider-page/model-selector/popup-item.tsx b/web/app/components/header/account-setting/model-provider-page/model-selector/popup-item.tsx index 27219e579da193..82672158a9cd7b 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-selector/popup-item.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-selector/popup-item.tsx @@ -49,7 +49,7 @@ const PopupItem: FC = ({ setShowModelModal({ payload: { currentProvider, - currentConfigurateMethod: ConfigurationMethodEnum.predefinedModel, + currentConfigurationMethod: ConfigurationMethodEnum.predefinedModel, }, onSaveCallback: () => { updateModelProviders() diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx deleted file mode 100644 index 9f65196658e257..00000000000000 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-config-entry-modal.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import { memo } from 'react' -import { useTranslation } from 'react-i18next' -import Modal from '@/app/components/base/modal' -import Form from '@/app/components/header/account-setting/model-provider-page/model-modal/Form' - -// create and export model balancing config modal -const ModelBalancingConfigEntryModal = () => { - const { t } = useTranslation() - - return ( - - { }} - formSchemas={[]} - validating={false} - showOnVariableMap={{}} - isEditMode={false} - readonly={false} - isShowDefaultValue={false} - /> - - ) -} - -export default memo(ModelBalancingConfigEntryModal) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx deleted file mode 100644 index 5ba7d698a09465..00000000000000 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-balancing-modal.tsx +++ /dev/null @@ -1,262 +0,0 @@ -import { memo, useCallback, useEffect, useState } from 'react' -import { useTranslation } from 'react-i18next' -import classNames from 'classnames' -import useSWR from 'swr' -import type { ModelItem, ModelLoadBalancingConfig, ModelLoadBalancingConfigEntry, ModelProvider } from '../declarations' -import ModelIcon from '../model-icon' -import ModelName from '../model-name' -import Modal from '@/app/components/base/modal' -import { Balance } from '@/app/components/base/icons/src/vender/line/financeAndECommerce' -import Switch from '@/app/components/base/switch' -import Indicator from '@/app/components/header/indicator' -import { Edit02, HelpCircle, Plus02, Trash03 } from '@/app/components/base/icons/src/vender/line/general' -import { AlertTriangle } from '@/app/components/base/icons/src/vender/line/alertsAndFeedback' -import SimplePieChart from '@/app/components/base/simple-pie-chart' -import TooltipPlus from '@/app/components/base/tooltip-plus' -import Button from '@/app/components/base/button' -import { fetchModelLoadBalancingConfig } from '@/service/common' -import Loading from '@/app/components/base/loading' - -export type ModelBalancingModalProps = { - provider: ModelProvider - model: ModelItem - open?: boolean - onClose?: () => void -} - -// model balancing config modal -const ModelBalancingModal = ({ provider, model, open = false, onClose }: ModelBalancingModalProps) => { - const { t } = useTranslation() - - // useProviderCredentialsFormSchemasValue(provider.provider, model.fetch_from) - const { data } = useSWR( - `/workspaces/current/model-providers/${provider.provider}/models/credentials?model=${model.model}&model_type=${model.model_type}`, - fetchModelLoadBalancingConfig, - ) - - const originalConfig = data?.load_balancing - const [draftConfig, setDraftConfig] = useState() - useEffect(() => { - if (originalConfig && !draftConfig) - setDraftConfig(originalConfig) - }, [draftConfig, originalConfig]) - - const toggleModalBalancing = useCallback((enabled: boolean) => { - if (draftConfig) { - setDraftConfig({ - ...draftConfig, - enabled, - }) - } - }, [draftConfig]) - - const updateConfigEntry = useCallback( - ( - index: number, - modifier: (entry: ModelLoadBalancingConfigEntry) => ModelLoadBalancingConfigEntry | undefined, - ) => { - setDraftConfig((prev) => { - if (!prev) - return prev - const newConfigs = [...prev.configs] - const modifiedConfig = modifier(newConfigs[index]) - if (modifiedConfig) - newConfigs[index] = modifiedConfig - else - newConfigs.splice(index, 1) - return { - ...prev, - configs: newConfigs, - } - }) - }, - [], - ) - - const toggleConfigEntryEnabled = useCallback((index: number, state?: boolean) => { - updateConfigEntry(index, entry => ({ - ...entry, - enabled: typeof state === 'boolean' ? state : !entry.enabled, - })) - }, [updateConfigEntry]) - - return ( - -
{t('common.modelProvider.configLoadBalancing')}
- {Boolean(model) && ( -
- - -
- )} -
- } - > - {!draftConfig - ? - : ( - <> -
-
toggleModalBalancing(false) : undefined} - > -
-
- {Boolean(model) && ( - - )} -
-
-
{t('common.modelProvider.providerManaged')}
-
Todo
-
-
-
- -
toggleModalBalancing(true) : undefined} - > -
-
- -
-
-
- {t('common.modelProvider.loadBalancing')} - - - -
-
Todo
-
-
- {draftConfig.enabled && ( -
- {draftConfig.configs.map((config, index) => { - const isProviderManaged = config.name === '__inherit__' - return ( -
-
-
- - - -
-
- {isProviderManaged ? t('common.modelProvider.defaultConfig') : config.name} -
- {isProviderManaged && ( - {t('common.modelProvider.providerManaged')} - )} -
-
- {!isProviderManaged && ( - <> -
- - - - updateConfigEntry(index, () => undefined)} - > - - - -
- - )} - toggleConfigEntryEnabled(index, value)} - /> -
-
- ) - })} - -
-
-
- - - -
-
Another
- {t('common.modelProvider.providerManaged')} -
-
-
- - - - - - - -
- value} - /> -
-
- -
-
- {t('common.modelProvider.addConfig')} -
-
-
- )} - { - draftConfig.enabled && draftConfig.configs.length < 2 && ( -
- - {t('common.modelProvider.loadBalancingLeastKeyWarning')} -
- ) - } -
-
- -
- - -
- - ) - } - - ) -} - -export default memo(ModelBalancingModal) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx index 2cc1348265653e..e5093cc0f4bd72 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx @@ -19,7 +19,7 @@ export type ModelListItemProps = { model: ModelItem provider: ModelProvider isConfigurable: boolean - onConfig: (currentCustomConfigrationModelFixedFields?: CustomConfigurationModelFixedFields) => void + onConfig: (currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields) => void onModifyLoadBalancing?: (model: ModelItem) => void } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx index b3a712334e74bb..443e054ddc987c 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx @@ -12,7 +12,7 @@ import { // import Tab from './tab' import AddModelButton from './add-model-button' import ModelListItem from './model-list-item' -import ModelBalancingModal from './model-balancing-modal' +import ModelLoadBalancingModal from './model-load-balancing-modal' import { ChevronDownDouble } from '@/app/components/base/icons/src/vender/line/arrows' type ModelListProps = { @@ -83,7 +83,7 @@ const ModelList: FC = ({ }
{Boolean(balancingModel) && ( - > + provider: ModelProvider + configurationMethod: ConfigurationMethodEnum + withSwitch?: boolean +} + +const ModelLoadBalancingConfigs = ({ draftConfig, setDraftConfig, provider, configurationMethod, withSwitch = false }: ModelLoadBalancingConfigsProps) => { + const { t } = useTranslation() + + const updateConfigEntry = useCallback( + ( + index: number, + modifier: (entry: ModelLoadBalancingConfigEntry) => ModelLoadBalancingConfigEntry | undefined, + ) => { + setDraftConfig((prev) => { + if (!prev) + return prev + const newConfigs = [...prev.configs] + const modifiedConfig = modifier(newConfigs[index]) + if (modifiedConfig) + newConfigs[index] = modifiedConfig + else + newConfigs.splice(index, 1) + return { + ...prev, + configs: newConfigs, + } + }) + }, + [setDraftConfig], + ) + + const toggleModalBalancing = useCallback((enabled: boolean) => { + if (draftConfig) { + setDraftConfig({ + ...draftConfig, + enabled, + }) + } + }, [draftConfig, setDraftConfig]) + + const toggleConfigEntryEnabled = useCallback((index: number, state?: boolean) => { + updateConfigEntry(index, entry => ({ + ...entry, + enabled: typeof state === 'boolean' ? state : !entry.enabled, + })) + }, [updateConfigEntry]) + + const setShowModelLoadBalancingEntryModal = useModalContextSelector(state => state.setShowModelLoadBalancingEntryModal) + + const toggleAddEntryModel = useCallback(() => { + setShowModelLoadBalancingEntryModal({ + payload: { + currentProvider: provider, + currentConfigurationMethod: configurationMethod, + currentCustomConfigurationModelFixedFields: undefined, + }, + onSaveCallback: () => { + // onRefreshData() + }, + }) + }, [configurationMethod, provider, setShowModelLoadBalancingEntryModal]) + + if (!draftConfig) + return null + + return ( +
toggleModalBalancing(true) : undefined} + > +
+
+ +
+
+
+ {t('common.modelProvider.loadBalancing')} + + + +
+
Todo
+
+ { + withSwitch && ( + toggleModalBalancing(value)} + /> + ) + } +
+ {draftConfig.enabled && ( +
+ {draftConfig.configs.map((config, index) => { + const isProviderManaged = config.name === '__inherit__' + return ( +
+
+
+ {(config.in_cooldown && Boolean(config.ttl)) + ? ( + + + + ) + : ( + + + + )} +
+
+ {isProviderManaged ? t('common.modelProvider.defaultConfig') : config.name} +
+ {isProviderManaged && ( + {t('common.modelProvider.providerManaged')} + )} +
+
+ {!isProviderManaged && ( + <> +
+ + + + updateConfigEntry(index, () => undefined)} + > + + + +
+ + )} + toggleConfigEntryEnabled(index, value)} + /> +
+
+ ) + })} + +
+
+
+ + + +
+
Another
+ {t('common.modelProvider.providerManaged')} +
+
+
+ + + + + + + +
+ value} + /> +
+
+ +
+
+ {t('common.modelProvider.addConfig')} +
+
+
+ )} + { + draftConfig.enabled && draftConfig.configs.length < 2 && ( +
+ + {t('common.modelProvider.loadBalancingLeastKeyWarning')} +
+ ) + } +
+ ) +} + +export default ModelLoadBalancingConfigs diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx new file mode 100644 index 00000000000000..51e894cb84df01 --- /dev/null +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx @@ -0,0 +1,119 @@ +import { memo, useCallback, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' +import classNames from 'classnames' +import useSWR from 'swr' +import type { ModelItem, ModelLoadBalancingConfig, ModelProvider } from '../declarations' +import ModelIcon from '../model-icon' +import ModelName from '../model-name' +import ModelLoadBalancingConfigs from './model-load-balancing-configs' +import Modal from '@/app/components/base/modal' +import Button from '@/app/components/base/button' +import { fetchModelLoadBalancingConfig } from '@/service/common' +import Loading from '@/app/components/base/loading' + +export type ModelLoadBalancingModalProps = { + provider: ModelProvider + model: ModelItem + open?: boolean + onClose?: () => void +} + +// model balancing config modal +const ModelLoadBalancingModal = ({ provider, model, open = false, onClose }: ModelLoadBalancingModalProps) => { + const { t } = useTranslation() + + // useProviderCredentialsFormSchemasValue(provider.provider, model.fetch_from) + const { data } = useSWR( + `/workspaces/current/model-providers/${provider.provider}/models/credentials?model=${model.model}&model_type=${model.model_type}`, + fetchModelLoadBalancingConfig, + ) + + const originalConfig = data?.load_balancing + const [draftConfig, setDraftConfig] = useState() + useEffect(() => { + if (originalConfig && !draftConfig) + setDraftConfig(originalConfig) + }, [draftConfig, originalConfig]) + + const toggleModalBalancing = useCallback((enabled: boolean) => { + if (draftConfig) { + setDraftConfig({ + ...draftConfig, + enabled, + }) + } + }, [draftConfig]) + + return ( + +
{t('common.modelProvider.configLoadBalancing')}
+ {Boolean(model) && ( +
+ + +
+ )} +
+ } + > + {!draftConfig + ? + : ( + <> +
+
toggleModalBalancing(false) : undefined} + > +
+
+ {Boolean(model) && ( + + )} +
+
+
{t('common.modelProvider.providerManaged')}
+
Todo
+
+
+
+ + +
+ +
+ + +
+ + ) + } + + ) +} + +export default memo(ModelLoadBalancingModal) diff --git a/web/app/components/tools/tool-list/index.tsx b/web/app/components/tools/tool-list/index.tsx index 78eb5c6ce299a6..b10469d62ca1fe 100644 --- a/web/app/components/tools/tool-list/index.tsx +++ b/web/app/components/tools/tool-list/index.tsx @@ -58,8 +58,8 @@ const ToolList: FC = ({ setShowModelModal({ payload: { currentProvider: provider, - currentConfigurateMethod: ConfigurationMethodEnum.predefinedModel, - currentCustomConfigrationModelFixedFields: undefined, + currentConfigurationMethod: ConfigurationMethodEnum.predefinedModel, + currentCustomConfigurationModelFixedFields: undefined, }, onSaveCallback: () => { onRefreshData() diff --git a/web/context/modal-context.tsx b/web/context/modal-context.tsx index ae49f30636a158..adf830ea941300 100644 --- a/web/context/modal-context.tsx +++ b/web/context/modal-context.tsx @@ -2,7 +2,7 @@ import type { Dispatch, SetStateAction } from 'react' import { useCallback, useState } from 'react' -import { createContext, useContext } from 'use-context-selector' +import { createContext, useContext, useContextSelector } from 'use-context-selector' import { useRouter, useSearchParams } from 'next/navigation' import AccountSetting from '@/app/components/header/account-setting' import ApiBasedExtensionModal from '@/app/components/header/account-setting/api-based-extension-page/modal' @@ -12,7 +12,7 @@ import AnnotationFullModal from '@/app/components/billing/annotation-full/modal' import ModelModal from '@/app/components/header/account-setting/model-provider-page/model-modal' import type { ConfigurationMethodEnum, - CustomConfigrationModelFixedFields, + CustomConfigurationModelFixedFields, ModelProvider, } from '@/app/components/header/account-setting/model-provider-page/declarations' @@ -22,6 +22,7 @@ import type { ApiBasedExtension, ExternalDataTool, } from '@/models/common' +import ModelLoadBalancingEntryModal from '@/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal' export type ModalState = { payload: T @@ -32,10 +33,10 @@ export type ModalState = { export type ModelModalType = { currentProvider: ModelProvider - currentConfigurateMethod: ConfigurationMethodEnum - currentCustomConfigrationModelFixedFields?: CustomConfigrationModelFixedFields + currentConfigurationMethod: ConfigurationMethodEnum + currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields } -const ModalContext = createContext<{ +export type ModalContextState = { setShowAccountSettingModal: Dispatch | null>> setShowApiBasedExtensionModal: Dispatch | null>> setShowModerationSettingModal: Dispatch | null>> @@ -43,18 +44,27 @@ const ModalContext = createContext<{ setShowPricingModal: Dispatch> setShowAnnotationFullModal: () => void setShowModelModal: Dispatch | null>> -}>({ - setShowAccountSettingModal: () => { }, - setShowApiBasedExtensionModal: () => { }, - setShowModerationSettingModal: () => { }, - setShowExternalDataToolModal: () => { }, - setShowPricingModal: () => { }, - setShowAnnotationFullModal: () => { }, - setShowModelModal: () => { }, - }) + setShowModelLoadBalancingEntryModal: Dispatch | null>> +} +const ModalContext = createContext({ + setShowAccountSettingModal: () => { }, + setShowApiBasedExtensionModal: () => { }, + setShowModerationSettingModal: () => { }, + setShowExternalDataToolModal: () => { }, + setShowPricingModal: () => { }, + setShowAnnotationFullModal: () => { }, + setShowModelModal: () => { }, + setShowModelLoadBalancingEntryModal: () => { }, +}) export const useModalContext = () => useContext(ModalContext) +// Adding a dangling comma to avoid the generic parsing issue in tsx, see: +// https://github.com/microsoft/TypeScript/issues/15713 +// eslint-disable-next-line @typescript-eslint/comma-dangle +export const useModalContextSelector = (selector: (state: ModalContextState) => T): T => + useContextSelector(ModalContext, selector) + type ModalContextProviderProps = { children: React.ReactNode } @@ -66,34 +76,31 @@ export const ModalContextProvider = ({ const [showModerationSettingModal, setShowModerationSettingModal] = useState | null>(null) const [showExternalDataToolModal, setShowExternalDataToolModal] = useState | null>(null) const [showModelModal, setShowModelModal] = useState | null>(null) + const [showModelLoadBalancingEntryModal, setShowModelLoadBalancingEntryModal] = useState | null>(null) const searchParams = useSearchParams() const router = useRouter() const [showPricingModal, setShowPricingModal] = useState(searchParams.get('show-pricing') === '1') const [showAnnotationFullModal, setShowAnnotationFullModal] = useState(false) const handleCancelAccountSettingModal = () => { setShowAccountSettingModal(null) - if (showAccountSettingModal?.onCancelCallback) showAccountSettingModal?.onCancelCallback() } const handleCancelModerationSettingModal = () => { setShowModerationSettingModal(null) - if (showModerationSettingModal?.onCancelCallback) showModerationSettingModal.onCancelCallback() } const handleCancelExternalDataToolModal = () => { setShowExternalDataToolModal(null) - if (showExternalDataToolModal?.onCancelCallback) showExternalDataToolModal.onCancelCallback() } const handleCancelModelModal = useCallback(() => { setShowModelModal(null) - if (showModelModal?.onCancelCallback) showModelModal.onCancelCallback() }, [showModelModal]) @@ -101,35 +108,42 @@ export const ModalContextProvider = ({ const handleSaveModelModal = useCallback(() => { if (showModelModal?.onSaveCallback) showModelModal.onSaveCallback(showModelModal.payload) - setShowModelModal(null) }, [showModelModal]) + const handleCancelModelLoadBalancingEntryModal = useCallback(() => { + setShowModelLoadBalancingEntryModal(null) + if (showModelLoadBalancingEntryModal?.onCancelCallback) + showModelLoadBalancingEntryModal.onCancelCallback() + }, [showModelLoadBalancingEntryModal]) + + const handleSaveModelLoadBalancingEntryModal = useCallback(() => { + if (showModelLoadBalancingEntryModal?.onSaveCallback) + showModelLoadBalancingEntryModal.onSaveCallback(showModelLoadBalancingEntryModal.payload) + setShowModelLoadBalancingEntryModal(null) + }, [showModelLoadBalancingEntryModal]) + const handleSaveApiBasedExtension = (newApiBasedExtension: ApiBasedExtension) => { if (showApiBasedExtensionModal?.onSaveCallback) showApiBasedExtensionModal.onSaveCallback(newApiBasedExtension) - setShowApiBasedExtensionModal(null) } const handleSaveModeration = (newModerationConfig: ModerationConfig) => { if (showModerationSettingModal?.onSaveCallback) showModerationSettingModal.onSaveCallback(newModerationConfig) - setShowModerationSettingModal(null) } const handleSaveExternalDataTool = (newExternalDataTool: ExternalDataTool) => { if (showExternalDataToolModal?.onSaveCallback) showExternalDataToolModal.onSaveCallback(newExternalDataTool) - setShowExternalDataToolModal(null) } const handleValidateBeforeSaveExternalDataTool = (newExternalDataTool: ExternalDataTool) => { if (showExternalDataToolModal?.onValidateBeforeSaveCallback) return showExternalDataToolModal?.onValidateBeforeSaveCallback(newExternalDataTool) - return true } @@ -142,6 +156,7 @@ export const ModalContextProvider = ({ setShowPricingModal: () => setShowPricingModal(true), setShowAnnotationFullModal: () => setShowAnnotationFullModal(true), setShowModelModal, + setShowModelLoadBalancingEntryModal, }}> <> {children} @@ -205,13 +220,24 @@ export const ModalContextProvider = ({ !!showModelModal && ( ) } + { + !!showModelLoadBalancingEntryModal && ( + + ) + } ) From 87c6d8842abb4b6c94ba36766bdfc87d8df83fad Mon Sep 17 00:00:00 2001 From: nite-knite Date: Thu, 30 May 2024 16:56:21 +0800 Subject: [PATCH 063/273] feat: load provider load balancing config, add upgrade tip --- .../model-provider-page/hooks.ts | 23 +- .../model-provider-page/model-modal/index.tsx | 20 +- .../model-load-balancing-entry-modal.tsx | 53 ++-- .../provider-added-card/model-list-item.tsx | 1 + .../model-load-balancing-configs.tsx | 240 +++++++++--------- .../model-load-balancing-modal.tsx | 2 +- web/context/modal-context.tsx | 2 +- web/i18n/en-US/common.ts | 1 + web/i18n/zh-Hans/common.ts | 1 + web/service/common.ts | 8 +- 10 files changed, 191 insertions(+), 160 deletions(-) diff --git a/web/app/components/header/account-setting/model-provider-page/hooks.ts b/web/app/components/header/account-setting/model-provider-page/hooks.ts index 340f86769f1606..a227f88f46e427 100644 --- a/web/app/components/header/account-setting/model-provider-page/hooks.ts +++ b/web/app/components/header/account-setting/model-provider-page/hooks.ts @@ -62,27 +62,27 @@ export const useLanguage = () => { return locale.replace('-', '_') } -export const useProviderCredentialsFormSchemasValue = ( +export const useProviderCredentialsAndLoadBalancing = ( provider: string, - configurateMethod: ConfigurationMethodEnum, + configurationMethod: ConfigurationMethodEnum, configured?: boolean, currentCustomConfigurationModelFixedFields?: CustomConfigurationModelFixedFields, ) => { const { data: predefinedFormSchemasValue } = useSWR( - (configurateMethod === ConfigurationMethodEnum.predefinedModel && configured) + (configurationMethod === ConfigurationMethodEnum.predefinedModel && configured) ? `/workspaces/current/model-providers/${provider}/credentials` : null, fetchModelProviderCredentials, ) const { data: customFormSchemasValue } = useSWR( - (configurateMethod === ConfigurationMethodEnum.customizableModel && currentCustomConfigurationModelFixedFields) + (configurationMethod === ConfigurationMethodEnum.customizableModel && currentCustomConfigurationModelFixedFields) ? `/workspaces/current/model-providers/${provider}/models/credentials?model=${currentCustomConfigurationModelFixedFields?.__model_name}&model_type=${currentCustomConfigurationModelFixedFields?.__model_type}` : null, fetchModelProviderCredentials, ) - const value = useMemo(() => { - return configurateMethod === ConfigurationMethodEnum.predefinedModel + const credentials = useMemo(() => { + return configurationMethod === ConfigurationMethodEnum.predefinedModel ? predefinedFormSchemasValue?.credentials : customFormSchemasValue?.credentials ? { @@ -91,13 +91,20 @@ export const useProviderCredentialsFormSchemasValue = ( } : undefined }, [ - configurateMethod, + configurationMethod, currentCustomConfigurationModelFixedFields, customFormSchemasValue?.credentials, predefinedFormSchemasValue?.credentials, ]) - return value + return { + credentials, + loadBalancing: (configurationMethod === ConfigurationMethodEnum.predefinedModel + ? predefinedFormSchemasValue + : customFormSchemasValue + )?.load_balancing, + } + // as ([Record | undefined, ModelLoadBalancingConfig | undefined]) } export const useModelList = (type: ModelTypeEnum) => { diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx index 1e598af0d975cb..5e8ed3423c0842 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/index.tsx @@ -29,7 +29,7 @@ import { } from '../utils' import { useLanguage, - useProviderCredentialsFormSchemasValue, + useProviderCredentialsAndLoadBalancing, } from '../hooks' import ProviderIcon from '../provider-icon' import { useValidate } from '../../key-validator/hooks' @@ -63,7 +63,10 @@ const ModelModal: FC = ({ onSave, }) => { const providerFormSchemaPredefined = configurateMethod === ConfigurationMethodEnum.predefinedModel - const formSchemasValue = useProviderCredentialsFormSchemasValue( + const { + credentials: formSchemasValue, + loadBalancing: originalConfig, + } = useProviderCredentialsAndLoadBalancing( provider.provider, configurateMethod, providerFormSchemaPredefined && provider.custom_configuration.status === CustomConfigurationStatusEnum.active, @@ -76,10 +79,6 @@ const ModelModal: FC = ({ const [loading, setLoading] = useState(false) const [showConfirm, setShowConfirm] = useState(false) - const originalConfig: ModelLoadBalancingConfig = useMemo(() => ({ - enabled: false, - configs: [], - }), []) const [draftConfig, setDraftConfig] = useState() useEffect(() => { if (originalConfig && !draftConfig) @@ -92,7 +91,7 @@ const ModelModal: FC = ({ : [ genModelTypeFormSchema(provider.supported_model_types), genModelNameFormSchema(provider.model_credential_schema?.model), - ...provider.model_credential_schema.credential_form_schemas, + ...(draftConfig?.enabled ? [] : provider.model_credential_schema.credential_form_schemas), ] }, [ providerFormSchemaPredefined, @@ -100,6 +99,7 @@ const ModelModal: FC = ({ provider.supported_model_types, provider.model_credential_schema?.credential_form_schemas, provider.model_credential_schema?.model, + draftConfig?.enabled, ]) const [ requiredFormSchemas, @@ -154,11 +154,11 @@ const ModelModal: FC = ({ showOnVariableMap, ] }, [formSchemas]) - const initialFormSchemasValue = useMemo(() => { + const initialFormSchemasValue: Record = useMemo(() => { return { ...defaultFormSchemaValue, ...formSchemasValue, - } + } as Record }, [formSchemasValue, defaultFormSchemaValue]) const [value, setValue] = useState(initialFormSchemasValue) useEffect(() => { @@ -245,6 +245,7 @@ const ModelModal: FC = ({
{renderTitlePrefix()}
+ = ({ isEditMode={isEditMode} /> +
onCancel: () => void @@ -55,16 +54,16 @@ type ModelModalProps = { const ModelLoadBalancingEntryModal: FC = ({ provider, - configurateMethod, + configurationMethod, currentCustomConfigurationModelFixedFields, credentials, onCancel, onSave, }) => { - const providerFormSchemaPredefined = configurateMethod === ConfigurationMethodEnum.predefinedModel - const formSchemasValue = useProviderCredentialsFormSchemasValue( + const providerFormSchemaPredefined = configurationMethod === ConfigurationMethodEnum.predefinedModel + const { credentials: formSchemasValue } = useProviderCredentialsAndLoadBalancing( provider.provider, - configurateMethod, + configurationMethod, providerFormSchemaPredefined && provider.custom_configuration.status === CustomConfigurationStatusEnum.active, currentCustomConfigurationModelFixedFields, ) @@ -75,19 +74,35 @@ const ModelLoadBalancingEntryModal: FC = ({ const [loading, setLoading] = useState(false) const [showConfirm, setShowConfirm] = useState(false) const formSchemas = useMemo(() => { - return providerFormSchemaPredefined - ? provider.provider_credential_schema.credential_form_schemas - : [ - genModelTypeFormSchema(provider.supported_model_types), - genModelNameFormSchema(provider.model_credential_schema?.model), - ...provider.model_credential_schema.credential_form_schemas, - ] + return [ + { + type: FormTypeEnum.textInput, + label: { + en_US: 'Config Name', + zh_Hans: '配置名称', + }, + variable: '__model_name', + required: true, + show_on: [], + placeholder: { + en_US: 'Enter your Config Name here', + zh_Hans: '输入配置名称', + }, + } as CredentialFormSchemaTextInput, + ...( + providerFormSchemaPredefined + ? provider.provider_credential_schema.credential_form_schemas + : [ + // genModelTypeFormSchema(provider.supported_model_types), + // genModelNameFormSchema(provider.model_credential_schema?.model), + ...provider.model_credential_schema.credential_form_schemas, + ] + ), + ] }, [ providerFormSchemaPredefined, provider.provider_credential_schema?.credential_form_schemas, - provider.supported_model_types, provider.model_credential_schema?.credential_form_schemas, - provider.model_credential_schema?.model, ]) const [ requiredFormSchemas, @@ -142,11 +157,11 @@ const ModelLoadBalancingEntryModal: FC = ({ showOnVariableMap, ] }, [formSchemas]) - const initialFormSchemasValue = useMemo(() => { + const initialFormSchemasValue: Record = useMemo(() => { return { ...defaultFormSchemaValue, ...formSchemasValue, - } + } as Record }, [formSchemasValue, defaultFormSchemaValue]) const [value, setValue] = useState(initialFormSchemasValue) useEffect(() => { diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx index e5093cc0f4bd72..20071aa30d9c25 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx @@ -80,6 +80,7 @@ const ModelListItem = ({ model, provider, isConfigurable, onConfig, onModifyLoad {t('common.modelProvider.config')} ) + // TODO: show config for sandbox mode but not CE : (modelLoadBalancingEnabled && !model.deprecated && [ModelStatusEnum.active, ModelStatusEnum.disabled].includes(model.status)) ? ( diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx index d98605a1916f5f..51019f7bb13135 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx @@ -27,6 +27,7 @@ import { } from '../hooks' import { useValidate } from '../../key-validator/hooks' import { ValidatedStatus } from '../../key-validator/declarations' +import { validateLoadBalancingCredentials } from '../utils' import Form from './Form' import Button from '@/app/components/base/button' import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security' @@ -46,6 +47,7 @@ type ModelModalProps = { entry?: ModelLoadBalancingConfigEntry onCancel: () => void onSave: (entry: ModelLoadBalancingConfigEntry) => void + onRemove: () => void } const ModelLoadBalancingEntryModal: FC = ({ @@ -55,6 +57,7 @@ const ModelLoadBalancingEntryModal: FC = ({ entry, onCancel, onSave, + onRemove, }) => { const providerFormSchemaPredefined = configurationMethod === ConfigurationMethodEnum.predefinedModel // const { credentials: formSchemasValue } = useProviderCredentialsAndLoadBalancing( @@ -156,6 +159,7 @@ const ModelLoadBalancingEntryModal: FC = ({ setResult({ ...defaultFormSchemaValue, ...entry.credentials, + id: entry.id, } as Record) } }, [entry, defaultFormSchemaValue, result]) @@ -192,57 +196,42 @@ const ModelLoadBalancingEntryModal: FC = ({ }, {} as Record) }, [initialFormSchemasValue, secretFormSchemas]) - const handleValueChange = ({ __model_type, __model_name, ...v }: FormValue) => { + // const handleValueChange = ({ __model_type, __model_name, ...v }: FormValue) => { + const handleValueChange = (v: FormValue) => { setValue(v) - setResult(v) } const handleSave = async () => { try { setLoading(true) - // const res = await saveCredentials( - // providerFormSchemaPredefined, - // provider.provider, - // { - // ...value, - // ...getSecretValues(value), - // }, - // ) - // if (res.result === 'success') { - // notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) - getSecretValues(value) - onSave({ - ...(entry || {}), - name: result!.name as string, - credentials: result!, - }) - // onCancel() - // } + const res = await validateLoadBalancingCredentials( + providerFormSchemaPredefined, + provider.provider, + { + ...value, + ...getSecretValues(value), + }, + ) + if (res.status === ValidatedStatus.Success) { + // notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + onSave({ + ...(entry || {}), + name: value.name as string, + credentials: value as Record, + }) + // onCancel() + } + else { + notify({ type: 'error', message: res.message || '' }) + } } finally { setLoading(false) } } - const handleRemove = async () => { - try { - setLoading(true) - - // const res = await removeCredentials( - // providerFormSchemaPredefined, - // provider.provider, - // value, - // ) - // if (res.result === 'success') { - // notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) - // // TODO - // onSave() - // onCancel() - // } - } - finally { - setLoading(false) - } + const handleRemove = () => { + onRemove?.() } return ( diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/index.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/index.tsx index aa178d586ba7ff..ff0a665ce359ed 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/index.tsx @@ -154,6 +154,7 @@ const ProviderAddedCard: FC = ({ models={modelList} onCollapse={() => setCollapsed(true)} onConfig={currentCustomConfigurationModelFixedFields => onOpenModal(ConfigurationMethodEnum.customizableModel, currentCustomConfigurationModelFixedFields)} + onChange={(provider: string) => getModelList(provider)} /> ) } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx index 20071aa30d9c25..3bde1af392076a 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx @@ -12,8 +12,9 @@ import { Balance } from '@/app/components/base/icons/src/vender/line/financeAndE import { Settings01 } from '@/app/components/base/icons/src/vender/line/general' import Switch from '@/app/components/base/switch' import TooltipPlus from '@/app/components/base/tooltip-plus' -import { useProviderContextSelector } from '@/context/provider-context' +import { useProviderContext, useProviderContextSelector } from '@/context/provider-context' import { disableModel, enableModel } from '@/service/common' +import { Plan } from '@/app/components/billing/type' export type ModelListItemProps = { model: ModelItem @@ -25,6 +26,7 @@ export type ModelListItemProps = { const ModelListItem = ({ model, provider, isConfigurable, onConfig, onModifyLoadBalancing }: ModelListItemProps) => { const { t } = useTranslation() + const { plan } = useProviderContext() const modelLoadBalancingEnabled = useProviderContextSelector(state => state.modelLoadBalancingEnabled) const toggleModelEnablingStatus = useCallback(async (enabled: boolean) => { @@ -80,8 +82,7 @@ const ModelListItem = ({ model, provider, isConfigurable, onConfig, onModifyLoad {t('common.modelProvider.config')} ) - // TODO: show config for sandbox mode but not CE - : (modelLoadBalancingEnabled && !model.deprecated && [ModelStatusEnum.active, ModelStatusEnum.disabled].includes(model.status)) + : ((modelLoadBalancingEnabled || plan.type === Plan.sandbox) && !model.deprecated && [ModelStatusEnum.active, ModelStatusEnum.disabled].includes(model.status)) ? (
diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx index 32ff43aefde352..01005890bd95c5 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx @@ -15,6 +15,7 @@ import UpgradeBtn from '@/app/components/billing/upgrade-btn' import s from '@/app/components/custom/style.module.css' import GridMask from '@/app/components/base/grid-mask' import { useProviderContextSelector } from '@/context/provider-context' +import { IS_CE_EDITION } from '@/config' export type ModelLoadBalancingConfigsProps = { draftConfig?: ModelLoadBalancingConfig @@ -62,13 +63,13 @@ const ModelLoadBalancingConfigs = ({ ) const toggleModalBalancing = useCallback((enabled: boolean) => { - if (draftConfig) { + if ((modelLoadBalancingEnabled || !enabled) && draftConfig) { setDraftConfig({ ...draftConfig, enabled, }) } - }, [draftConfig, setDraftConfig]) + }, [draftConfig, modelLoadBalancingEnabled, setDraftConfig]) const toggleConfigEntryEnabled = useCallback((index: number, state?: boolean) => { updateConfigEntry(index, entry => ({ @@ -86,6 +87,7 @@ const ModelLoadBalancingConfigs = ({ currentConfigurationMethod: configurationMethod, currentCustomConfigurationModelFixedFields, entry, + index, }, onSaveCallback: ({ entry: result }) => { if (entry) { @@ -100,13 +102,29 @@ const ModelLoadBalancingConfigs = ({ setDraftConfig(prev => ({ ...prev, enabled: !!prev?.enabled, - configs: (prev?.configs || []).concat([result!]), + configs: (prev?.configs || []).concat([{ ...result!, enabled: true }]), })) } // onRefreshData() }, + onRemoveCallback: ({ index }) => { + if (index !== undefined && (draftConfig?.configs?.length ?? 0) > index) { + setDraftConfig(prev => ({ + ...prev, + enabled: !!prev?.enabled, + configs: prev?.configs.filter((_, i) => i !== index) || [], + })) + } + }, }) - }, [configurationMethod, currentCustomConfigurationModelFixedFields, provider, setDraftConfig, setShowModelLoadBalancingEntryModal]) + }, [ + configurationMethod, + currentCustomConfigurationModelFixedFields, + draftConfig?.configs?.length, + provider, + setDraftConfig, + setShowModelLoadBalancingEntryModal, + ]) if (!draftConfig) return null @@ -139,15 +157,16 @@ const ModelLoadBalancingConfigs = ({ withSwitch && ( toggleModalBalancing(value)} /> ) }
{draftConfig.enabled && ( -
+
{draftConfig.configs.map((config, index) => { const isProviderManaged = config.name === '__inherit__' return ( @@ -224,7 +243,7 @@ const ModelLoadBalancingConfigs = ({ }
- {!modelLoadBalancingEnabled && ( + {!modelLoadBalancingEnabled && !IS_CE_EDITION && (
void + onSave?: (provider: string) => void } // model balancing config modal -const ModelLoadBalancingModal = ({ provider, model, open = false, onClose }: ModelLoadBalancingModalProps) => { +const ModelLoadBalancingModal = ({ provider, model, open = false, onClose, onSave }: ModelLoadBalancingModalProps) => { const { t } = useTranslation() + const { notify } = useToastContext() - const { data } = useSWR( + const [loading, setLoading] = useState(false) + + const { data, mutate } = useSWR( `/workspaces/current/model-providers/${provider.provider}/models/credentials?model=${model.model}&model_type=${model.model_type}`, fetchModelLoadBalancingConfig, ) const originalConfig = data?.load_balancing const [draftConfig, setDraftConfig] = useState() + const originalConfigMap = useMemo(() => { + if (!originalConfig) + return {} + return originalConfig?.configs.reduce((prev, config) => { + if (config.id) + prev[config.id] = config + return prev + }, {} as Record) + }, [originalConfig]) useEffect(() => { if (originalConfig && !draftConfig) setDraftConfig(originalConfig) @@ -43,6 +59,50 @@ const ModelLoadBalancingModal = ({ provider, model, open = false, onClose }: Mod } }, [draftConfig]) + const extendedSecretFormSchemas = useMemo( + () => provider.provider_credential_schema.credential_form_schemas.filter( + ({ type }) => type === FormTypeEnum.secretInput, + ), + [provider.provider_credential_schema.credential_form_schemas], + ) + + const encodeConfigEntrySecretValues = useCallback((entry: ModelLoadBalancingConfigEntry) => { + const result = { ...entry } + extendedSecretFormSchemas.forEach(({ variable }) => { + if (entry.id && result.credentials[variable] === originalConfigMap[entry.id]?.credentials?.[variable]) + result.credentials[variable] = '[__HIDDEN__]' + }) + return result + }, [extendedSecretFormSchemas, originalConfigMap]) + + const handleSave = async () => { + try { + setLoading(true) + const res = await saveCredentials( + false, + provider.provider, + { + __model_name: model.model, + __model_type: model.model_type, + }, + { + ...draftConfig, + enabled: Boolean(draftConfig?.enabled), + configs: draftConfig!.configs.map(encodeConfigEntrySecretValues), + }, + ) + if (res.result === 'success') { + notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + mutate() + onSave?.(provider.provider) + onClose?.() + } + } + finally { + setLoading(false) + } + } + return ( - +
) diff --git a/web/app/components/header/account-setting/model-provider-page/utils.ts b/web/app/components/header/account-setting/model-provider-page/utils.ts index bba5fccc6d163d..6425f836b2b191 100644 --- a/web/app/components/header/account-setting/model-provider-page/utils.ts +++ b/web/app/components/header/account-setting/model-provider-page/utils.ts @@ -3,6 +3,7 @@ import type { CredentialFormSchemaRadio, CredentialFormSchemaTextInput, FormValue, + ModelLoadBalancingConfig, } from './declarations' import { FormTypeEnum, @@ -12,6 +13,7 @@ import { import { deleteModelProvider, setModelProvider, + validateModelLoadBalancingCredentials, validateModelProvider, } from '@/service/common' @@ -54,12 +56,37 @@ export const validateCredentials = async (predefined: boolean, provider: string, } } -export const saveCredentials = async (predefined: boolean, provider: string, v: FormValue) => { +export const validateLoadBalancingCredentials = async (predefined: boolean, provider: string, v: FormValue): Promise<{ + status: ValidatedStatus + message?: string +}> => { + const { __model_name, __model_type, ...credentials } = v + try { + const res = await validateModelLoadBalancingCredentials({ + url: `/workspaces/current/model-providers/${provider}/models/load-balancing-configs/credentials-validate`, + body: { + model: __model_name, + model_type: __model_type, + credentials, + }, + }) + if (res.result === 'success') + return Promise.resolve({ status: ValidatedStatus.Success }) + else + return Promise.resolve({ status: ValidatedStatus.Error, message: res.error || 'error' }) + } + catch (e: any) { + return Promise.resolve({ status: ValidatedStatus.Error, message: e.message }) + } +} + +export const saveCredentials = async (predefined: boolean, provider: string, v: FormValue, loadBalancing?: ModelLoadBalancingConfig) => { let body, url if (predefined) { body = { credentials: v, + load_balancing: loadBalancing, } url = `/workspaces/current/model-providers/${provider}` } @@ -69,6 +96,7 @@ export const saveCredentials = async (predefined: boolean, provider: string, v: model: __model_name, model_type: __model_type, credentials, + load_balancing: loadBalancing, } url = `/workspaces/current/model-providers/${provider}/models` } diff --git a/web/context/modal-context.tsx b/web/context/modal-context.tsx index 4aba9608727d6a..6ceb985c84da74 100644 --- a/web/context/modal-context.tsx +++ b/web/context/modal-context.tsx @@ -29,6 +29,7 @@ export type ModalState = { payload: T onCancelCallback?: () => void onSaveCallback?: (newPayload: T) => void + onRemoveCallback?: (newPayload: T) => void onValidateBeforeSaveCallback?: (newPayload: T) => boolean } @@ -39,6 +40,7 @@ export type ModelModalType = { } export type LoadBalancingEntryModalType = ModelModalType & { entry?: ModelLoadBalancingConfigEntry + index?: number } export type ModalContextState = { setShowAccountSettingModal: Dispatch | null>> @@ -116,18 +118,20 @@ export const ModalContextProvider = ({ }, [showModelModal]) const handleCancelModelLoadBalancingEntryModal = useCallback(() => { + showModelLoadBalancingEntryModal?.onCancelCallback?.() setShowModelLoadBalancingEntryModal(null) - if (showModelLoadBalancingEntryModal?.onCancelCallback) - showModelLoadBalancingEntryModal.onCancelCallback() }, [showModelLoadBalancingEntryModal]) const handleSaveModelLoadBalancingEntryModal = useCallback((entry: ModelLoadBalancingConfigEntry) => { - if (showModelLoadBalancingEntryModal?.onSaveCallback) { - showModelLoadBalancingEntryModal.onSaveCallback({ - ...showModelLoadBalancingEntryModal.payload, - entry, - }) - } + showModelLoadBalancingEntryModal?.onSaveCallback?.({ + ...showModelLoadBalancingEntryModal.payload, + entry, + }) + setShowModelLoadBalancingEntryModal(null) + }, [showModelLoadBalancingEntryModal]) + + const handleRemoveModelLoadBalancingEntry = useCallback(() => { + showModelLoadBalancingEntryModal?.onRemoveCallback?.(showModelLoadBalancingEntryModal.payload) setShowModelLoadBalancingEntryModal(null) }, [showModelLoadBalancingEntryModal]) @@ -244,6 +248,7 @@ export const ModalContextProvider = ({ entry={showModelLoadBalancingEntryModal.payload.entry} onCancel={handleCancelModelLoadBalancingEntryModal} onSave={handleSaveModelLoadBalancingEntryModal} + onRemove={handleRemoveModelLoadBalancingEntry} /> ) } diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index efc82d11b81c36..939436c79eaf60 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -345,7 +345,7 @@ const translation = { addConfig: '增加配置', loadBalancingLeastKeyWarning: '至少启用 2 个 Key 以使用负载均衡', loadBalancingInfo: '默认情况下，负载平衡使用 Round-robin 策略。如果触发速率限制，将应用 1 分钟的冷却时间', - upgradeForLoadBalancing: '升级以解锁负载均衡', + upgradeForLoadBalancing: '升级以解锁负载均衡功能', }, dataSource: { add: '添加数据源', diff --git a/web/service/common.ts b/web/service/common.ts index 36419b68801841..d9dca649e64841 100644 --- a/web/service/common.ts +++ b/web/service/common.ts @@ -197,6 +197,10 @@ export const validateModelProvider: Fetcher(url, { body }) } +export const validateModelLoadBalancingCredentials: Fetcher = ({ url, body }) => { + return post(url, { body }) +} + export const setModelProvider: Fetcher = ({ url, body }) => { return post(url, { body }) } From 9d3ce3c051d905de06a589cdabcd4f771461dc31 Mon Sep 17 00:00:00 2001 From: takatost Date: Mon, 3 Jun 2024 10:36:56 +0800 Subject: [PATCH 074/273] add config_from --- api/controllers/console/workspace/models.py | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/api/controllers/console/workspace/models.py b/api/controllers/console/workspace/models.py index 55f2ca9292bad0..305b8d13742a25 100644 --- a/api/controllers/console/workspace/models.py +++ b/api/controllers/console/workspace/models.py @@ -107,6 +107,7 @@ def post(self, provider: str): choices=[mt.value for mt in ModelType], location='json') parser.add_argument('credentials', type=dict, required=False, nullable=True, location='json') parser.add_argument('load_balancing', type=dict, required=False, nullable=True, location='json') + parser.add_argument('config_from', type=str, required=False, nullable=True, delocation='json') args = parser.parse_args() model_load_balancing_service = ModelLoadBalancingService() @@ -141,18 +142,19 @@ def post(self, provider: str): model_type=args['model_type'] ) - model_provider_service = ModelProviderService() - - try: - model_provider_service.save_model_credentials( - tenant_id=tenant_id, - provider=provider, - model=args['model'], - model_type=args['model_type'], - credentials=args['credentials'] - ) - except CredentialsValidateFailedError as ex: - raise ValueError(str(ex)) + if args.get('config_from', '') != 'predefined-model': + model_provider_service = ModelProviderService() + + try: + model_provider_service.save_model_credentials( + tenant_id=tenant_id, + provider=provider, + model=args['model'], + model_type=args['model_type'], + credentials=args['credentials'] + ) + except CredentialsValidateFailedError as ex: + raise ValueError(str(ex)) return {'result': 'success'}, 200 From 2ca91ec4cd72b50934a05b97a8607d128fdbcd5d Mon Sep 17 00:00:00 2001 From: takatost Date: Mon, 3 Jun 2024 10:51:16 +0800 Subject: [PATCH 075/273] fix --- api/controllers/console/workspace/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/console/workspace/models.py b/api/controllers/console/workspace/models.py index 305b8d13742a25..76ae6a4ab9e0ee 100644 --- a/api/controllers/console/workspace/models.py +++ b/api/controllers/console/workspace/models.py @@ -107,7 +107,7 @@ def post(self, provider: str): choices=[mt.value for mt in ModelType], location='json') parser.add_argument('credentials', type=dict, required=False, nullable=True, location='json') parser.add_argument('load_balancing', type=dict, required=False, nullable=True, location='json') - parser.add_argument('config_from', type=str, required=False, nullable=True, delocation='json') + parser.add_argument('config_from', type=str, required=False, nullable=True, location='json') args = parser.parse_args() model_load_balancing_service = ModelLoadBalancingService() From 54b22f6f4e67ed03770f9abc7dac147a9eb3e0ae Mon Sep 17 00:00:00 2001 From: nite-knite Date: Mon, 3 Jun 2024 13:23:05 +0800 Subject: [PATCH 076/273] feat: update model load balancing api --- .../model-load-balancing-modal.tsx | 12 ++++++------ .../account-setting/model-provider-page/utils.ts | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx index 274fe3e7835192..bb0c5d302a2176 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx @@ -6,7 +6,7 @@ import type { ModelItem, ModelLoadBalancingConfig, ModelLoadBalancingConfigEntry import { FormTypeEnum } from '../declarations' import ModelIcon from '../model-icon' import ModelName from '../model-name' -import { saveCredentials } from '../utils' +import { savePredefinedLoadBalancingConfig } from '../utils' import ModelLoadBalancingConfigs from './model-load-balancing-configs' import Modal from '@/app/components/base/modal' import Button from '@/app/components/base/button' @@ -78,13 +78,13 @@ const ModelLoadBalancingModal = ({ provider, model, open = false, onClose, onSav const handleSave = async () => { try { setLoading(true) - const res = await saveCredentials( - false, + const res = await savePredefinedLoadBalancingConfig( provider.provider, - { - __model_name: model.model, + ({ + ...(data?.credentials ?? {}), __model_type: model.model_type, - }, + __model_name: model.model, + }), { ...draftConfig, enabled: Boolean(draftConfig?.enabled), diff --git a/web/app/components/header/account-setting/model-provider-page/utils.ts b/web/app/components/header/account-setting/model-provider-page/utils.ts index 0b6c41e329f45f..8cad3997638754 100644 --- a/web/app/components/header/account-setting/model-provider-page/utils.ts +++ b/web/app/components/header/account-setting/model-provider-page/utils.ts @@ -6,6 +6,7 @@ import type { ModelLoadBalancingConfig, } from './declarations' import { + ConfigurationMethodEnum, FormTypeEnum, MODEL_TYPE_TEXT, ModelTypeEnum, @@ -84,6 +85,7 @@ export const saveCredentials = async (predefined: boolean, provider: string, v: if (predefined) { body = { + config_from: ConfigurationMethodEnum.predefinedModel, credentials: v, load_balancing: loadBalancing, } @@ -103,6 +105,20 @@ export const saveCredentials = async (predefined: boolean, provider: string, v: return setModelProvider({ url, body }) } +export const savePredefinedLoadBalancingConfig = async (provider: string, v: FormValue, loadBalancing?: ModelLoadBalancingConfig) => { + const { __model_name, __model_type, ...credentials } = v + const body = { + config_from: ConfigurationMethodEnum.predefinedModel, + model: __model_name, + model_type: __model_type, + credentials, + load_balancing: loadBalancing, + } + const url = `/workspaces/current/model-providers/${provider}/models` + + return setModelProvider({ url, body }) +} + export const removeCredentials = async (predefined: boolean, provider: string, v: FormValue) => { let url = '' let body From d3cfd5d3df71d6048df7678c6467244dc4c37ca4 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Mon, 3 Jun 2024 17:04:32 +0800 Subject: [PATCH 077/273] add file detail --- api/fields/document_fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/fields/document_fields.py b/api/fields/document_fields.py index 94d905eafe00d3..e8215255b35d5b 100644 --- a/api/fields/document_fields.py +++ b/api/fields/document_fields.py @@ -8,6 +8,7 @@ 'position': fields.Integer, 'data_source_type': fields.String, 'data_source_info': fields.Raw(attribute='data_source_info_dict'), + 'data_source_detail_dict': fields.Raw(attribute='data_source_detail_dict'), 'dataset_process_rule_id': fields.String, 'name': fields.String, 'created_from': fields.String, @@ -31,6 +32,7 @@ 'position': fields.Integer, 'data_source_type': fields.String, 'data_source_info': fields.Raw(attribute='data_source_info_dict'), + 'data_source_detail_dict': fields.Raw(attribute='data_source_detail_dict'), 'dataset_process_rule_id': fields.String, 'name': fields.String, 'created_from': fields.String, From 7b23a9d83f79f70773b324c7db197b0b481339c8 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 4 Jun 2024 10:49:54 +0800 Subject: [PATCH 078/273] feat: show file extenstion --- web/app/components/datasets/documents/list.tsx | 5 +++-- web/models/datasets.ts | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 4254b98088da75..4566287fb81aca 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -406,7 +406,8 @@ const DocumentList: FC = ({ embeddingAvailable, documents = {localDocs.map((doc) => { - const documentType = 'txt'// TODO: wait for api. doc.name.split('.').pop() || 'txt' + const isFile = doc.data_source_type === DataSourceType.FILE + const fileType = isFile ? doc.data_source_detail_dict?.upload_file.extension : '' return = ({ embeddingAvailable, documents = { doc?.data_source_type === DataSourceType.NOTION ? - :
+ :
} { doc.name diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 585c0473d4f31a..1f02a43184d805 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -178,6 +178,12 @@ export type SimpleDocumentDetail = InitialDocumentDetail & { updated_at: number hit_count: number dataset_process_rule_id?: string + data_source_detail_dict?: { + upload_file: { + name: string + extension: string + } + } } export type DocumentListResponse = { From 10e0f43e416c0dd7b6d9daddf66a15d41e1f45a4 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 4 Jun 2024 11:46:03 +0800 Subject: [PATCH 079/273] chore: crawel text --- .../data-source-page/panel/config-item.tsx | 10 +++++----- .../account-setting/data-source-page/panel/index.tsx | 2 +- web/i18n/en-US/common.ts | 3 +++ web/i18n/zh-Hans/common.ts | 3 +++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx index 2e6e0c64c63958..376c4aea7b3318 100644 --- a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx +++ b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx @@ -48,11 +48,11 @@ const ConfigItem: FC = ({ ? : } -
+
{ payload.isActive - ? t('common.dataSource.notion.connected') - : t('common.dataSource.notion.disconnected') + ? t(isNotion ? 'common.dataSource.notion.connected' : 'common.dataSource.website.active') + : t(isNotion ? 'common.dataSource.notion.disconnected' : 'common.dataSource.website.inactive') }
@@ -66,8 +66,8 @@ const ConfigItem: FC = ({ { isWebsite && ( -
- +
+
) } diff --git a/web/app/components/header/account-setting/data-source-page/panel/index.tsx b/web/app/components/header/account-setting/data-source-page/panel/index.tsx index 75bef952f8ede0..b0f6f4ad13ac95 100644 --- a/web/app/components/header/account-setting/data-source-page/panel/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/panel/index.tsx @@ -110,7 +110,7 @@ const Panel: FC = ({ isConfigured && (
- {t('common.dataSource.notion.connectedWorkspace')} + {isNotion ? t('common.dataSource.notion.connectedWorkspace') : t('common.dataSource.website.configuredCrawlers')}
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index 6984b89072607c..c0a64cc0b6e56c 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -362,6 +362,9 @@ const translation = { title: 'Website', description: 'Import content from websites using web crawler.', with: 'With', + configuredCrawlers: 'Configured crawlers', + active: 'Active', + inactive: 'Inactive', }, }, plugin: { diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index 64d0a367bc41a6..c3e82cb7aaa488 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -362,6 +362,9 @@ const translation = { title: '网站', description: '使用网络爬虫从网站导入内容。', with: '使用', + configuredCrawlers: '已配置的爬虫', + active: '可用', + inactive: '不可用', }, }, plugin: { From 43b169baa1f0906a10fa40a92d72f3c9c5bdd379 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Tue, 4 Jun 2024 13:51:14 +0800 Subject: [PATCH 080/273] DocumentRenameApi http method --- api/controllers/console/datasets/datasets_document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 5597a88ef3938d..2c1b5c6a4c47d8 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -929,7 +929,7 @@ class DocumentRenameApi(DocumentResource): @login_required @account_initialization_required @marshal_with(document_fields) - def put(self, dataset_id, document_id): + def post(self, dataset_id, document_id): # The role of the current user in the ta table must be admin or owner if not current_user.is_admin_or_owner: raise Forbidden() From df97c6b291ddecebb3e42eb41fbf19f251ddce8c Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 4 Jun 2024 16:48:38 +0800 Subject: [PATCH 081/273] feat: firecrawl no data --- .../assets/vender/line/others/icon-3-dots.svg | 5 +++ .../src/vender/line/others/Icon3Dots.json | 39 +++++++++++++++++++ .../src/vender/line/others/Icon3Dots.tsx | 16 ++++++++ .../icons/src/vender/line/others/index.ts | 1 + web/app/components/datasets/create/index.tsx | 2 +- .../datasets/create/step-one/index.tsx | 9 +++-- .../create/website/import-firecrawl.tsx | 15 +++++++ .../datasets/create/website/index.tsx | 18 +++++++++ .../datasets/create/website/no-data.tsx | 36 +++++++++++++++++ web/i18n/en-US/dataset-creation.ts | 5 +++ web/i18n/zh-Hans/dataset-creation.ts | 5 +++ 11 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg create mode 100644 web/app/components/base/icons/src/vender/line/others/Icon3Dots.json create mode 100644 web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx create mode 100644 web/app/components/datasets/create/website/import-firecrawl.tsx create mode 100644 web/app/components/datasets/create/website/index.tsx create mode 100644 web/app/components/datasets/create/website/no-data.tsx diff --git a/web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg b/web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg new file mode 100644 index 00000000000000..bba42851f61829 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/src/vender/line/others/Icon3Dots.json b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.json new file mode 100644 index 00000000000000..0942222f39e0a4 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.json @@ -0,0 +1,39 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "16", + "height": "16", + "viewBox": "0 0 16 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Icon-3-dots" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon", + "d": "M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103", + "stroke": "currentColor", + "stroke-width": "2", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + } + ] + }, + "name": "Icon3Dots" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx new file mode 100644 index 00000000000000..1f9eb767a8f2fb --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Icon3Dots.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Icon3Dots' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/others/index.ts b/web/app/components/base/icons/src/vender/line/others/index.ts index 648792f22b40f0..554f14b55bd650 100644 --- a/web/app/components/base/icons/src/vender/line/others/index.ts +++ b/web/app/components/base/icons/src/vender/line/others/index.ts @@ -3,4 +3,5 @@ export { default as Colors } from './Colors' export { default as DragHandle } from './DragHandle' export { default as Exchange02 } from './Exchange02' export { default as FileCode } from './FileCode' +export { default as Icon3Dots } from './Icon3Dots' export { default as Tools } from './Tools' diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index ce32f29fc528fd..83d74b55baefd6 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -23,7 +23,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { const { t } = useTranslation() const { setShowAccountSettingModal } = useModalContext() const [hasConnection, setHasConnection] = useState(true) - const [dataSourceType, setDataSourceType] = useState(DataSourceType.FILE) + const [dataSourceType, setDataSourceType] = useState(DataSourceType.WEB) // TODO: for test. DataSourceType.FILE const [step, setStep] = useState(1) const [indexingTypeCache, setIndexTypeCache] = useState('') const [fileList, setFiles] = useState([]) diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index 66df1dd2083466..7d46d5eb5d0bc1 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -6,6 +6,7 @@ import FilePreview from '../file-preview' import FileUploader from '../file-uploader' import NotionPagePreview from '../notion-page-preview' import EmptyDatasetCreationModal from '../empty-dataset-creation-modal' +import Website from '../website' import s from './index.module.css' import type { FileItem } from '@/models/datasets' import type { NotionPage } from '@/models/common' @@ -150,10 +151,9 @@ const StepOne = ({ {t('datasetCreation.stepOne.dataSourceType.notion')}
changeType(DataSourceType.WEB)} + className={cn(s.dataSourceItem, dataSourceType === DataSourceType.WEB ? s.active : s.disabled)} + onClick={() => changeType(DataSourceType.WEB)} > - Coming soon {t('datasetCreation.stepOne.dataSourceType.web')}
@@ -201,6 +201,9 @@ const StepOne = ({ )} )} + {dataSourceType === DataSourceType.WEB && ( + + )} {!datasetId && ( <>
diff --git a/web/app/components/datasets/create/website/import-firecrawl.tsx b/web/app/components/datasets/create/website/import-firecrawl.tsx new file mode 100644 index 00000000000000..7fafaf32bc93da --- /dev/null +++ b/web/app/components/datasets/create/website/import-firecrawl.tsx @@ -0,0 +1,15 @@ +'use client' +import type { FC } from 'react' +import React from 'react' + +type Props = { + +} + +const ImportFireCrawl: FC = () => { + return ( +
+
+ ) +} +export default React.memo(ImportFireCrawl) diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx new file mode 100644 index 00000000000000..cd82d45f7c588c --- /dev/null +++ b/web/app/components/datasets/create/website/index.tsx @@ -0,0 +1,18 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import NoData from './no-data' + +type Props = { + +} + +const WebsitePreview: FC = () => { + const handleOnConfig = useCallback(() => { }, []) + return ( +
+ +
+ ) +} +export default React.memo(WebsitePreview) diff --git a/web/app/components/datasets/create/website/no-data.tsx b/web/app/components/datasets/create/website/no-data.tsx new file mode 100644 index 00000000000000..35eb35f7095dac --- /dev/null +++ b/web/app/components/datasets/create/website/no-data.tsx @@ -0,0 +1,36 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import { useTranslation } from 'react-i18next' +import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others' +import Button from '@/app/components/base/button' + +const I18N_PREFIX = 'datasetCreation.stepOne.website' + +type Props = { + onConfig: () => void +} + +const NoData: FC = ({ + onConfig, +}) => { + const { t } = useTranslation() + + return ( +
+
+ 🔥 +
+
+ {t(`${I18N_PREFIX}.fireCrawlNotConfigured`)} +
+ {t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`)} +
+
+ +
+ ) +} +export default React.memo(NoData) diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 8923170f7f4539..9c7f44f1d0472c 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -50,6 +50,11 @@ const translation = { confirmButton: 'Create', failed: 'Creation failed', }, + website: { + fireCrawlNotConfigured: 'Firecrawl is not configured', + fireCrawlNotConfiguredDescription: 'Configure Firecrawl with API key to use it.', + configure: 'Configure', + }, }, stepTwo: { segmentation: 'Chunk settings', diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index d36850dc3dc833..208312f01f971d 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -50,6 +50,11 @@ const translation = { confirmButton: '创建', failed: '创建失败', }, + website: { + fireCrawlNotConfigured: 'Firecrawl 未配置', + fireCrawlNotConfiguredDescription: '请配置 Firecrawl 的 API 密钥以使用它。', + configure: 'Configure', + }, }, stepTwo: { segmentation: '分段设置', From 5aef25b5fe01fab8fa7064fe51ab12773be93aea Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 4 Jun 2024 16:53:25 +0800 Subject: [PATCH 082/273] feat: handle show config firecrawl --- web/app/components/datasets/create/website/index.tsx | 11 ++++++++++- .../data-source-page/data-source-website/index.tsx | 2 -- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx index cd82d45f7c588c..96af30faf3ab3b 100644 --- a/web/app/components/datasets/create/website/index.tsx +++ b/web/app/components/datasets/create/website/index.tsx @@ -2,13 +2,22 @@ import type { FC } from 'react' import React, { useCallback } from 'react' import NoData from './no-data' +import { useModalContext } from '@/context/modal-context' type Props = { } const WebsitePreview: FC = () => { - const handleOnConfig = useCallback(() => { }, []) + const { setShowAccountSettingModal } = useModalContext() + + const handleOnConfig = useCallback(() => { + setShowAccountSettingModal({ + payload: 'data-source', + }) + }, [setShowAccountSettingModal]) + + // TODO: on Hide return (
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx index c572f5d3b0dd9e..489af9842cbc03 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx @@ -57,8 +57,6 @@ const DataSourceWebsite: FC = () => { }, []) - console.log(list) - return ( Date: Tue, 4 Jun 2024 17:35:58 +0800 Subject: [PATCH 083/273] feat: add cooldown timer --- .../provider-added-card/cooldown-timer.tsx | 64 +++++++++++++++++++ .../model-load-balancing-configs.tsx | 20 ++++-- .../model-load-balancing-modal.tsx | 4 +- web/i18n/en-US/common.ts | 2 + web/i18n/zh-Hans/common.ts | 2 + 5 files changed, 83 insertions(+), 9 deletions(-) create mode 100644 web/app/components/header/account-setting/model-provider-page/provider-added-card/cooldown-timer.tsx diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/cooldown-timer.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/cooldown-timer.tsx new file mode 100644 index 00000000000000..09cbe5223061c6 --- /dev/null +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/cooldown-timer.tsx @@ -0,0 +1,64 @@ +import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useLatest } from 'ahooks' +import SimplePieChart from '@/app/components/base/simple-pie-chart' +import TooltipPlus from '@/app/components/base/tooltip-plus' + +export type CooldownTimerProps = { + secondsRemaining?: number + onFinish?: () => void +} + +const CooldownTimer = ({ secondsRemaining, onFinish }: CooldownTimerProps) => { + const { t } = useTranslation() + + const targetTime = useRef(Date.now()) + const [currentTime, setCurrentTime] = useState(targetTime.current) + const displayTime = useMemo( + () => Math.ceil((targetTime.current - currentTime) / 1000), + [currentTime], + ) + + const countdownTimeout = useRef() + const clearCountdown = useCallback(() => { + if (countdownTimeout.current) { + clearTimeout(countdownTimeout.current) + countdownTimeout.current = undefined + } + }, []) + + const onFinishRef = useLatest(onFinish) + + const countdown = useCallback(() => { + clearCountdown() + countdownTimeout.current = setTimeout(() => { + const now = Date.now() + if (now <= targetTime.current) { + setCurrentTime(Date.now()) + countdown() + } + else { + onFinishRef.current?.() + clearCountdown() + } + }, 1000) + }, [clearCountdown, onFinishRef]) + + useEffect(() => { + const now = Date.now() + targetTime.current = now + (secondsRemaining ?? 0) * 1000 + setCurrentTime(now) + countdown() + return clearCountdown + }, [clearCountdown, countdown, secondsRemaining]) + + return displayTime + ? ( + + + + ) + : null +} + +export default memo(CooldownTimer) diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx index 01005890bd95c5..b4e9ea7f06003c 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx @@ -4,9 +4,9 @@ import { useCallback } from 'react' import { useTranslation } from 'react-i18next' import type { ConfigurationMethodEnum, CustomConfigurationModelFixedFields, ModelLoadBalancingConfig, ModelLoadBalancingConfigEntry, ModelProvider } from '../declarations' import Indicator from '../../../indicator' +import CooldownTimer from './cooldown-timer' import TooltipPlus from '@/app/components/base/tooltip-plus' import Switch from '@/app/components/base/switch' -import SimplePieChart from '@/app/components/base/simple-pie-chart' import { Balance } from '@/app/components/base/icons/src/vender/line/financeAndECommerce' import { Edit02, HelpCircle, Plus02, Trash03 } from '@/app/components/base/icons/src/vender/line/general' import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' @@ -91,7 +91,6 @@ const ModelLoadBalancingConfigs = ({ }, onSaveCallback: ({ entry: result }) => { if (entry) { - // edit entry setDraftConfig(prev => ({ ...prev, enabled: !!prev?.enabled, @@ -126,6 +125,15 @@ const ModelLoadBalancingConfigs = ({ setShowModelLoadBalancingEntryModal, ]) + const clearCountdown = useCallback((index: number) => { + updateConfigEntry(index, ({ ttl: _, ...entry }) => { + return { + ...entry, + in_cooldown: false, + } + }) + }, [updateConfigEntry]) + if (!draftConfig) return null @@ -141,7 +149,7 @@ const ModelLoadBalancingConfigs = ({ onClick={(!withSwitch && !draftConfig.enabled) ? () => toggleModalBalancing(true) : undefined} >
-
+
@@ -151,7 +159,7 @@ const ModelLoadBalancingConfigs = ({
-
Todo
+
{t('common.modelProvider.loadBalancingDescription')}
{ withSwitch && ( @@ -175,9 +183,7 @@ const ModelLoadBalancingConfigs = ({
{(config.in_cooldown && Boolean(config.ttl)) ? ( - - - + clearCountdown(index)} /> ) : ( diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx index bb0c5d302a2176..b3a68d1bc0fc28 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx @@ -144,14 +144,14 @@ const ModelLoadBalancingModal = ({ provider, model, open = false, onClose, onSav onClick={draftConfig.enabled ? () => toggleModalBalancing(false) : undefined} >
-
+
{Boolean(model) && ( )}
{t('common.modelProvider.providerManaged')}
-
Todo
+
{t('common.modelProvider.providerManagedDescription')}
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index 465b9b9d5b01a3..57eb8904a43d97 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -336,10 +336,12 @@ const translation = { loadPresets: 'Load Presents', parameters: 'PARAMETERS', loadBalancing: 'Load balancing', + loadBalancingDescription: 'Reduce pressure with multiple sets of credentials.', loadBalancingHeadline: 'Load Balancing', configLoadBalancing: 'Config Load Balancing', modelHasBeenDeprecated: 'This model has been deprecated', providerManaged: 'Provider managed', + providerManagedDescription: 'Use the single set of credentials provided by the model provider.', defaultConfig: 'Default Config', apiKeyStatusNormal: 'APIKey status is normal', apiKeyRateLimit: 'Rate limit was reached, available after {{seconds}}s', diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index 939436c79eaf60..394180d391ba2d 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -335,10 +335,12 @@ const translation = { loadPresets: '加载预设', parameters: '参数', loadBalancing: '负载均衡', + loadBalancingDescription: '为了减轻单组凭据的压力，您可以为模型调用配置多组凭据。', loadBalancingHeadline: '负载均衡', configLoadBalancing: '设置负载均衡', modelHasBeenDeprecated: '该模型已废弃', providerManaged: '由模型供应商管理', + providerManagedDescription: '使用模型供应商提供的单组凭据', defaultConfig: '默认配置', apiKeyStatusNormal: 'API Key 正常', apiKeyRateLimit: '已达频率上限，{{seconds}}秒后恢复', From fdae549b267f3ec53ddb9762963b0c6942712540 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 4 Jun 2024 18:30:27 +0800 Subject: [PATCH 084/273] feat: firecrawl header --- .../create/website/firecrawl/base/field.tsx | 0 .../create/website/firecrawl/base/input.tsx | 25 +++++++++++ .../website/firecrawl/base/url-input.tsx | 41 ++++++++++++++++++ .../create/website/firecrawl/header.tsx | 42 +++++++++++++++++++ .../create/website/firecrawl/index.tsx | 32 ++++++++++++++ .../website/firecrawl/options-error.tsx | 0 .../create/website/firecrawl/options.tsx | 0 .../create/website/import-firecrawl.tsx | 15 ------- .../datasets/create/website/index.tsx | 19 +++++++-- web/i18n/en-US/dataset-creation.ts | 4 ++ web/i18n/zh-Hans/dataset-creation.ts | 6 ++- 11 files changed, 165 insertions(+), 19 deletions(-) create mode 100644 web/app/components/datasets/create/website/firecrawl/base/field.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/base/input.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/base/url-input.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/header.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/index.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/options-error.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/options.tsx delete mode 100644 web/app/components/datasets/create/website/import-firecrawl.tsx diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/web/app/components/datasets/create/website/firecrawl/base/input.tsx b/web/app/components/datasets/create/website/firecrawl/base/input.tsx new file mode 100644 index 00000000000000..0d3337848d41b0 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/base/input.tsx @@ -0,0 +1,25 @@ +'use client' +import type { FC } from 'react' +import React from 'react' + +type Props = { + value: string + onChange: (value: string) => void + placeholder?: string +} + +const Input: FC = ({ + value, + onChange, + placeholder = '', +}) => { + return ( + onChange(e.target.value)} + className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400' + placeholder={placeholder} + /> + ) +} +export default React.memo(Input) diff --git a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx new file mode 100644 index 00000000000000..915db90d5c5a77 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx @@ -0,0 +1,41 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback, useState } from 'react' +import { useTranslation } from 'react-i18next' +import Input from './input' +import Button from '@/app/components/base/button' + +const I18N_PREFIX = 'datasetCreation.stepOne.website' + +type Props = { + onRun: (url: string) => void +} + +const UrlInput: FC = ({ + onRun, +}) => { + const { t } = useTranslation() + const [url, setUrl] = useState('') + + const handleOnRun = useCallback(() => { + onRun(url) + }, [onRun, url]) + + return ( +
+ + +
+ ) +} +export default React.memo(UrlInput) diff --git a/web/app/components/datasets/create/website/firecrawl/header.tsx b/web/app/components/datasets/create/website/firecrawl/header.tsx new file mode 100644 index 00000000000000..d89cfce46a55ce --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/header.tsx @@ -0,0 +1,42 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import { useTranslation } from 'react-i18next' +import { Settings01 } from '@/app/components/base/icons/src/vender/line/general' +import { BookOpen01 } from '@/app/components/base/icons/src/vender/line/education' + +const I18N_PREFIX = 'datasetCreation.stepOne.website' + +type Props = { + onSetting: () => void +} + +const Header: FC = ({ + onSetting, +}) => { + const { t } = useTranslation() + + return ( +
+
+
{t(`${I18N_PREFIX}.firecrawlTitle`)}
+
+
+ +
+
+ + + {t(`${I18N_PREFIX}.firecrawlDoc`)} + +
+ ) +} +export default React.memo(Header) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx new file mode 100644 index 00000000000000..85dd5fbc7d6b00 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -0,0 +1,32 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import Header from './header' +import UrlInput from './base/url-input' +import { useModalContext } from '@/context/modal-context' + +type Props = { + +} + +const FireCrawl: FC = () => { + const { setShowAccountSettingModal } = useModalContext() + const handleSetting = useCallback(() => { + setShowAccountSettingModal({ + payload: 'data-source', + }) + }, [setShowAccountSettingModal]) + + const handleRun = useCallback((url: string) => { + console.log(url) + }, []) + return ( +
+
+
+ +
+
+ ) +} +export default React.memo(FireCrawl) diff --git a/web/app/components/datasets/create/website/firecrawl/options-error.tsx b/web/app/components/datasets/create/website/firecrawl/options-error.tsx new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/web/app/components/datasets/create/website/firecrawl/options.tsx b/web/app/components/datasets/create/website/firecrawl/options.tsx new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/web/app/components/datasets/create/website/import-firecrawl.tsx b/web/app/components/datasets/create/website/import-firecrawl.tsx deleted file mode 100644 index 7fafaf32bc93da..00000000000000 --- a/web/app/components/datasets/create/website/import-firecrawl.tsx +++ /dev/null @@ -1,15 +0,0 @@ -'use client' -import type { FC } from 'react' -import React from 'react' - -type Props = { - -} - -const ImportFireCrawl: FC = () => { - return ( -
-
- ) -} -export default React.memo(ImportFireCrawl) diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx index 96af30faf3ab3b..b85d986f798418 100644 --- a/web/app/components/datasets/create/website/index.tsx +++ b/web/app/components/datasets/create/website/index.tsx @@ -1,7 +1,8 @@ 'use client' import type { FC } from 'react' -import React, { useCallback } from 'react' +import React, { useCallback, useState } from 'react' import NoData from './no-data' +import Firecrawl from './firecrawl' import { useModalContext } from '@/context/modal-context' type Props = { @@ -10,6 +11,8 @@ type Props = { const WebsitePreview: FC = () => { const { setShowAccountSettingModal } = useModalContext() + const [isLoaded, setIsLoaded] = useState(false) + const [isConfigured, setIsConfigured] = useState(true) const handleOnConfig = useCallback(() => { setShowAccountSettingModal({ @@ -17,10 +20,20 @@ const WebsitePreview: FC = () => { }) }, [setShowAccountSettingModal]) - // TODO: on Hide + // TODO: on Hide account setting modal + + if (isLoaded) + return null + return (
- + {isConfigured + ? ( + + ) + : ( + + )}
) } diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 9c7f44f1d0472c..c3506485d5d8b4 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -54,6 +54,10 @@ const translation = { fireCrawlNotConfigured: 'Firecrawl is not configured', fireCrawlNotConfiguredDescription: 'Configure Firecrawl with API key to use it.', configure: 'Configure', + run: 'Run', + firecrawlTitle: 'Extract web content with 🔥Firecrawl', + firecrawlDoc: 'Firecrawl docs', + firecrawlDocLink: '', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 208312f01f971d..7956c98931db90 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -53,7 +53,11 @@ const translation = { website: { fireCrawlNotConfigured: 'Firecrawl 未配置', fireCrawlNotConfiguredDescription: '请配置 Firecrawl 的 API 密钥以使用它。', - configure: 'Configure', + configure: '配置', + run: '运行', + firecrawlTitle: '使用 🔥Firecrawl 提取网页内容', + firecrawlDoc: 'Firecrawl 文档', + firecrawlDocLink: '', }, }, stepTwo: { From b37a636ceeb83ac44b64835c52103bdb63c946d0 Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 4 Jun 2024 20:17:08 +0800 Subject: [PATCH 085/273] fix bugs --- api/services/model_load_balancing_service.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index 79438ec9b0b024..0d6770535ee33d 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -133,6 +133,9 @@ def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, # Get credential form schemas from model credential schema or provider credential schema credential_schemas = self._get_credential_schema(provider_configuration) + # Get decoding rsa key and cipher for decrypting credentials + decoding_rsa_key, decoding_cipher_rsa = encrypter.get_decrypt_decoding(tenant_id) + # fetch status and ttl for each config datas = [] for load_balancing_config in load_balancing_configs: @@ -152,6 +155,23 @@ def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, except JSONDecodeError: credentials = {} + # Get provider credential secret variables + credential_secret_variables = provider_configuration.extract_secret_variables( + credential_schemas.credential_form_schemas + ) + + # decrypt credentials + for variable in credential_secret_variables: + if variable in credentials: + try: + credentials[variable] = encrypter.decrypt_token_with_decoding( + credentials.get(variable), + decoding_rsa_key, + decoding_cipher_rsa + ) + except ValueError: + pass + # Obfuscate credentials credentials = provider_configuration.obfuscated_credentials( credentials=credentials, From 3481c75f983a4d5ebd0b76c3ccc73003a9c3d946 Mon Sep 17 00:00:00 2001 From: takatost Date: Tue, 4 Jun 2024 20:22:23 +0800 Subject: [PATCH 086/273] fix provider managed position --- api/services/model_load_balancing_service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/services/model_load_balancing_service.py b/api/services/model_load_balancing_service.py index 0d6770535ee33d..c684c2862b9745 100644 --- a/api/services/model_load_balancing_service.py +++ b/api/services/model_load_balancing_service.py @@ -129,6 +129,12 @@ def get_load_balancing_configs(self, tenant_id: str, provider: str, model: str, # prepend the inherit configuration load_balancing_configs.insert(0, inherit_config) + else: + # move the inherit configuration to the first + for i, load_balancing_config in enumerate(load_balancing_configs): + if load_balancing_config.name == '__inherit__': + inherit_config = load_balancing_configs.pop(i) + load_balancing_configs.insert(0, inherit_config) # Get credential form schemas from model credential schema or provider credential schema credential_schemas = self._get_credential_schema(provider_configuration) From 633c0c8bc961404e3be54b14032fde52732be03e Mon Sep 17 00:00:00 2001 From: nite-knite Date: Tue, 4 Jun 2024 21:31:51 +0800 Subject: [PATCH 087/273] feat: render with model load balancing modal with context --- .../model-provider-page/index.tsx | 4 +-- .../provider-added-card/model-list.tsx | 25 ++++++++----------- .../model-load-balancing-modal.tsx | 4 +-- web/context/modal-context.tsx | 11 ++++++++ 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/web/app/components/header/account-setting/model-provider-page/index.tsx b/web/app/components/header/account-setting/model-provider-page/index.tsx index b8f0caf93e6ae7..656c7b023979dd 100644 --- a/web/app/components/header/account-setting/model-provider-page/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/index.tsx @@ -19,7 +19,7 @@ import { } from './hooks' import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' import { useProviderContext } from '@/context/provider-context' -import { useModalContext } from '@/context/modal-context' +import { useModalContextSelector } from '@/context/modal-context' import { useEventEmitterContextContext } from '@/context/event-emitter' const ModelProviderPage = () => { @@ -33,7 +33,7 @@ const ModelProviderPage = () => { const { data: speech2textDefaultModel } = useDefaultModel(ModelTypeEnum.speech2text) const { data: ttsDefaultModel } = useDefaultModel(ModelTypeEnum.tts) const { modelProviders: providers } = useProviderContext() - const { setShowModelModal } = useModalContext() + const setShowModelModal = useModalContextSelector(state => state.setShowModelModal) const defaultModelNotConfigured = !textGenerationDefaultModel && !embeddingsDefaultModel && !speech2textDefaultModel && !rerankDefaultModel && !ttsDefaultModel const [configedProviders, notConfigedProviders] = useMemo(() => { const configedProviders: ModelProvider[] = [] diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx index 3bc6c265d35466..87f7ef91a4f6a0 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list.tsx @@ -1,5 +1,5 @@ import type { FC } from 'react' -import { useCallback, useState } from 'react' +import { useCallback } from 'react' import { useTranslation } from 'react-i18next' import type { CustomConfigurationModelFixedFields, @@ -12,8 +12,8 @@ import { // import Tab from './tab' import AddModelButton from './add-model-button' import ModelListItem from './model-list-item' -import ModelLoadBalancingModal from './model-load-balancing-modal' import { ChevronDownDouble } from '@/app/components/base/icons/src/vender/line/arrows' +import { useModalContextSelector } from '@/context/modal-context' type ModelListProps = { provider: ModelProvider @@ -33,10 +33,16 @@ const ModelList: FC = ({ const configurativeMethods = provider.configurate_methods.filter(method => method !== ConfigurationMethodEnum.fetchFromRemote) const isConfigurable = configurativeMethods.includes(ConfigurationMethodEnum.customizableModel) - const [balancingModel, setBalancingModel] = useState() + const setShowModelLoadBalancingModal = useModalContextSelector(state => state.setShowModelLoadBalancingModal) const onModifyLoadBalancing = useCallback((model: ModelItem) => { - setBalancingModel(model) - }, []) + setShowModelLoadBalancingModal({ + provider, + model: model!, + open: !!model, + onClose: () => setShowModelLoadBalancingModal(null), + onSave: onChange, + }) + }, [onChange, provider, setShowModelLoadBalancingModal]) return (
@@ -84,15 +90,6 @@ const ModelList: FC = ({ )) }
- {Boolean(balancingModel) && ( - setBalancingModel(undefined), - onSave: onChange, - }} /> - )}
) } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx index b3a68d1bc0fc28..b18b5c1077a4d0 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal.tsx @@ -46,9 +46,9 @@ const ModelLoadBalancingModal = ({ provider, model, open = false, onClose, onSav }, {} as Record) }, [originalConfig]) useEffect(() => { - if (originalConfig && !draftConfig) + if (originalConfig) setDraftConfig(originalConfig) - }, [draftConfig, originalConfig]) + }, [originalConfig]) const toggleModalBalancing = useCallback((enabled: boolean) => { if (draftConfig) { diff --git a/web/context/modal-context.tsx b/web/context/modal-context.tsx index b75dd76d3e2d0a..3547c45aacff82 100644 --- a/web/context/modal-context.tsx +++ b/web/context/modal-context.tsx @@ -24,6 +24,8 @@ import type { ExternalDataTool, } from '@/models/common' import ModelLoadBalancingEntryModal from '@/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal' +import type { ModelLoadBalancingModalProps } from '@/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal' +import ModelLoadBalancingModal from '@/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-modal' export type ModalState = { payload: T @@ -50,6 +52,7 @@ export type ModalContextState = { setShowPricingModal: () => void setShowAnnotationFullModal: () => void setShowModelModal: Dispatch | null>> + setShowModelLoadBalancingModal: Dispatch> setShowModelLoadBalancingEntryModal: Dispatch | null>> } const ModalContext = createContext({ @@ -60,6 +63,7 @@ const ModalContext = createContext({ setShowPricingModal: () => { }, setShowAnnotationFullModal: () => { }, setShowModelModal: () => { }, + setShowModelLoadBalancingModal: () => { }, setShowModelLoadBalancingEntryModal: () => { }, }) @@ -82,6 +86,7 @@ export const ModalContextProvider = ({ const [showModerationSettingModal, setShowModerationSettingModal] = useState | null>(null) const [showExternalDataToolModal, setShowExternalDataToolModal] = useState | null>(null) const [showModelModal, setShowModelModal] = useState | null>(null) + const [showModelLoadBalancingModal, setShowModelLoadBalancingModal] = useState(null) const [showModelLoadBalancingEntryModal, setShowModelLoadBalancingEntryModal] = useState | null>(null) const searchParams = useSearchParams() const router = useRouter() @@ -168,6 +173,7 @@ export const ModalContextProvider = ({ setShowPricingModal: () => setShowPricingModal(true), setShowAnnotationFullModal: () => setShowAnnotationFullModal(true), setShowModelModal, + setShowModelLoadBalancingModal, setShowModelLoadBalancingEntryModal, }}> <> @@ -239,6 +245,11 @@ export const ModalContextProvider = ({ /> ) } + { + Boolean(showModelLoadBalancingModal) && ( + + ) + } { !!showModelLoadBalancingEntryModal && ( Date: Tue, 4 Jun 2024 21:49:56 +0800 Subject: [PATCH 088/273] feat: update load balancing entry modal title --- .../model-modal/model-load-balancing-entry-modal.tsx | 2 +- web/i18n/en-US/common.ts | 1 + web/i18n/zh-Hans/common.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx index 51019f7bb13135..186a69d41cfee4 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx @@ -241,7 +241,7 @@ const ModelLoadBalancingEntryModal: FC = ({
-
{t('common.modelProvider.addConfig')}
+
{t(isEditMode ? 'common.modelProvider.editConfig' : 'common.modelProvider.addConfig')}
Date: Tue, 4 Jun 2024 22:48:20 +0800 Subject: [PATCH 089/273] just add --- .../website/firecrawl/base/options-wrap.tsx | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx new file mode 100644 index 00000000000000..9f1fe32b7f2742 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -0,0 +1,37 @@ +'use client' +import { useBoolean } from 'ahooks' +import type { FC } from 'react' +import React from 'react' +import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' + +type Props = { + children: React.ReactNode +} + +const OptionsWrap: FC = ({ + children, +}) => { + const [fold, { + setTrue: foldTrue, + setFalse: foldFalse, + toggle: foldToggle, + }] = useBoolean(false) + return ( +
+
+
+ +
Options
+
+ +
+ {!fold && ( +
+ {children} +
+ )} + +
+ ) +} +export default React.memo(OptionsWrap) From 7cb374bb8fdde12af9632db8d01c89c538de5c40 Mon Sep 17 00:00:00 2001 From: nite-knite Date: Tue, 4 Jun 2024 23:04:42 +0800 Subject: [PATCH 090/273] feat: update model load balancing entry modal form --- .../model-load-balancing-entry-modal.tsx | 18 ++++++++++-------- .../model-load-balancing-configs.tsx | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx index 186a69d41cfee4..a2c271265367c3 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-modal/model-load-balancing-entry-modal.tsx @@ -153,20 +153,21 @@ const ModelLoadBalancingEntryModal: FC = ({ showOnVariableMap, ] }, [formSchemas]) - const [result, setResult] = useState() + const [initialValue, setInitialValue] = useState() useEffect(() => { - if (entry && !result) { - setResult({ + if (entry && !initialValue) { + setInitialValue({ ...defaultFormSchemaValue, ...entry.credentials, id: entry.id, + name: entry.name, } as Record) } - }, [entry, defaultFormSchemaValue, result]) + }, [entry, defaultFormSchemaValue, initialValue]) const formSchemasValue = useMemo(() => ({ ...currentCustomConfigurationModelFixedFields, - ...result, - }), [currentCustomConfigurationModelFixedFields, result]) + ...initialValue, + }), [currentCustomConfigurationModelFixedFields, initialValue]) const initialFormSchemasValue: Record = useMemo(() => { return { ...defaultFormSchemaValue, @@ -214,10 +215,11 @@ const ModelLoadBalancingEntryModal: FC = ({ ) if (res.status === ValidatedStatus.Success) { // notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) + const { __model_type, __model_name, name, ...credentials } = value onSave({ ...(entry || {}), - name: value.name as string, - credentials: value as Record, + name: name as string, + credentials: credentials as Record, }) // onCancel() } diff --git a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx index b4e9ea7f06003c..ab089360dff786 100644 --- a/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx +++ b/web/app/components/header/account-setting/model-provider-page/provider-added-card/model-load-balancing-configs.tsx @@ -80,7 +80,7 @@ const ModelLoadBalancingConfigs = ({ const setShowModelLoadBalancingEntryModal = useModalContextSelector(state => state.setShowModelLoadBalancingEntryModal) - const toggleEntryModel = useCallback((index?: number, entry?: ModelLoadBalancingConfigEntry) => { + const toggleEntryModal = useCallback((index?: number, entry?: ModelLoadBalancingConfigEntry) => { setShowModelLoadBalancingEntryModal({ payload: { currentProvider: provider, @@ -91,6 +91,7 @@ const ModelLoadBalancingConfigs = ({ }, onSaveCallback: ({ entry: result }) => { if (entry) { + // edit setDraftConfig(prev => ({ ...prev, enabled: !!prev?.enabled, @@ -98,13 +99,13 @@ const ModelLoadBalancingConfigs = ({ })) } else { + // add setDraftConfig(prev => ({ ...prev, enabled: !!prev?.enabled, configs: (prev?.configs || []).concat([{ ...result!, enabled: true }]), })) } - // onRefreshData() }, onRemoveCallback: ({ index }) => { if (index !== undefined && (draftConfig?.configs?.length ?? 0) > index) { @@ -204,7 +205,7 @@ const ModelLoadBalancingConfigs = ({
toggleEntryModel(index, config)} + onClick={() => toggleEntryModal(index, config)} > @@ -231,7 +232,7 @@ const ModelLoadBalancingConfigs = ({
toggleEntryModel()} + onClick={() => toggleEntryModal()} >
{t('common.modelProvider.addConfig')} From baad0a7a5f903c57af37e2fa124b616660dbc208 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 5 Jun 2024 11:11:07 +0800 Subject: [PATCH 091/273] options temp --- .../create/website/firecrawl/base/options-wrap.tsx | 7 ++++++- .../components/datasets/create/website/firecrawl/index.tsx | 6 +++++- web/i18n/en-US/dataset-creation.ts | 1 + web/i18n/zh-Hans/dataset-creation.ts | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx index 9f1fe32b7f2742..32b4d3c4b5e9b6 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -2,8 +2,11 @@ import { useBoolean } from 'ahooks' import type { FC } from 'react' import React from 'react' +import { useTranslation } from 'react-i18next' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' +const I18N_PREFIX = 'datasetCreation.stepOne.website' + type Props = { children: React.ReactNode } @@ -11,6 +14,8 @@ type Props = { const OptionsWrap: FC = ({ children, }) => { + const { t } = useTranslation() + const [fold, { setTrue: foldTrue, setFalse: foldFalse, @@ -21,7 +26,7 @@ const OptionsWrap: FC = ({
-
Options
+
{t(`${I18N_PREFIX}.options`)}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 85dd5fbc7d6b00..d813ae6e94f2dd 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -3,8 +3,8 @@ import type { FC } from 'react' import React, { useCallback } from 'react' import Header from './header' import UrlInput from './base/url-input' +import OptionsWrap from './base/options-wrap' import { useModalContext } from '@/context/modal-context' - type Props = { } @@ -25,6 +25,10 @@ const FireCrawl: FC = () => {
+ + +
contents
+
) diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index c3506485d5d8b4..9bb86765520f59 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -58,6 +58,7 @@ const translation = { firecrawlTitle: 'Extract web content with 🔥Firecrawl', firecrawlDoc: 'Firecrawl docs', firecrawlDocLink: '', + options: 'Options', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 7956c98931db90..9c4793c0a393b4 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -58,6 +58,7 @@ const translation = { firecrawlTitle: '使用 🔥Firecrawl 提取网页内容', firecrawlDoc: 'Firecrawl 文档', firecrawlDocLink: '', + options: '选项', }, }, stepTwo: { From f36763fd0824fb19e1dba55490c92986e8edc1e1 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Wed, 5 Jun 2024 11:41:45 +0800 Subject: [PATCH 092/273] note editor --- .../(appDetailLayout)/[appId]/note/page.tsx | 21 ++++++++++++ .../components/workflow/nodes/constants.ts | 2 ++ .../workflow/note-node/constants.ts | 1 + .../components/workflow/note-node/index.tsx | 7 ++++ .../note-node/note-editor/context.tsx | 34 +++++++++++++++++++ .../workflow/note-node/note-editor/editor.tsx | 30 ++++++++++++++++ .../workflow/note-node/note-editor/index.tsx | 3 ++ .../note-node/note-editor/toolbar.tsx | 9 +++++ 8 files changed, 107 insertions(+) create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx create mode 100644 web/app/components/workflow/note-node/constants.ts create mode 100644 web/app/components/workflow/note-node/index.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/context.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/editor.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/index.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar.tsx diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx new file mode 100644 index 00000000000000..441751c4bb02a2 --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx @@ -0,0 +1,21 @@ +'use client' + +import { + NoteEditor, + NoteEditorContextProvider, + NoteEditorToolbar, +} from '@/app/components/workflow/note-node/note-editor' + +const Page = () => { + return ( +
+
+ + + + +
+
+ ) +} +export default Page diff --git a/web/app/components/workflow/nodes/constants.ts b/web/app/components/workflow/nodes/constants.ts index c67ce757ab2ce2..a97aa086edac9f 100644 --- a/web/app/components/workflow/nodes/constants.ts +++ b/web/app/components/workflow/nodes/constants.ts @@ -64,3 +64,5 @@ export const PanelComponentMap: Record> = { [BlockEnum.ParameterExtractor]: ParameterExtractorPanel, [BlockEnum.Iteration]: IterationPanel, } + +export const CUSTOM_NODE_TYPE = 'custom' diff --git a/web/app/components/workflow/note-node/constants.ts b/web/app/components/workflow/note-node/constants.ts new file mode 100644 index 00000000000000..a0f8cb03e5e9df --- /dev/null +++ b/web/app/components/workflow/note-node/constants.ts @@ -0,0 +1 @@ +export const NOTE_NODE_TYPE = 'custom-note' diff --git a/web/app/components/workflow/note-node/index.tsx b/web/app/components/workflow/note-node/index.tsx new file mode 100644 index 00000000000000..d8cc2c09a43504 --- /dev/null +++ b/web/app/components/workflow/note-node/index.tsx @@ -0,0 +1,7 @@ +const NoteNode = () => { + return ( +
+ ) +} + +export default NoteNode diff --git a/web/app/components/workflow/note-node/note-editor/context.tsx b/web/app/components/workflow/note-node/note-editor/context.tsx new file mode 100644 index 00000000000000..cebbb769db6526 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/context.tsx @@ -0,0 +1,34 @@ +'use client' + +import { memo } from 'react' +import { createContext } from 'use-context-selector' +import { CodeNode } from '@lexical/code' +import { LexicalComposer } from '@lexical/react/LexicalComposer' + +const NoteEditorContext = createContext({}) + +type NoteEditorContextProviderProps = { + children: JSX.Element | string | (JSX.Element | string)[] +} +export const NoteEditorContextProvider = memo(({ + children, +}: NoteEditorContextProviderProps) => { + const initialConfig = { + namespace: 'note-editor', + nodes: [ + CodeNode, + ], + onError: (error: Error) => { + throw error + }, + } + + return ( + + {children} + + ) +}) +NoteEditorContextProvider.displayName = 'NoteEditorContextProvider' + +export default NoteEditorContext diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx new file mode 100644 index 00000000000000..32039d0d2f6e94 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -0,0 +1,30 @@ +'use client' + +import { memo } from 'react' +import { RichTextPlugin } from '@lexical/react/LexicalRichTextPlugin' +import { ContentEditable } from '@lexical/react/LexicalContentEditable' +import LexicalErrorBoundary from '@lexical/react/LexicalErrorBoundary' +import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin' +// import TreeView from './plugins/tree-view' +import Placeholder from '@/app/components/base/prompt-editor/plugins/placeholder' + +type EditorProps = { + placeholder?: string +} +const Editor = ({ + placeholder, +}: EditorProps) => { + return ( +
+ } + placeholder={} + ErrorBoundary={LexicalErrorBoundary} + /> + + {/* */} +
+ ) +} + +export default memo(Editor) diff --git a/web/app/components/workflow/note-node/note-editor/index.tsx b/web/app/components/workflow/note-node/note-editor/index.tsx new file mode 100644 index 00000000000000..f3c7364e8e0691 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/index.tsx @@ -0,0 +1,3 @@ +export { NoteEditorContextProvider } from './context' +export { default as NoteEditor } from './editor' +export { default as NoteEditorToolbar } from './toolbar' diff --git a/web/app/components/workflow/note-node/note-editor/toolbar.tsx b/web/app/components/workflow/note-node/note-editor/toolbar.tsx new file mode 100644 index 00000000000000..89e62429e5a2ed --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar.tsx @@ -0,0 +1,9 @@ +import { memo } from 'react' + +const Toolbar = () => { + return ( +
+ ) +} + +export default memo(Toolbar) From 5a0bfb69a37ea13b5d5fc35ad75f05d1863510af Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 5 Jun 2024 14:32:14 +0800 Subject: [PATCH 093/273] feat: options main --- .../firecrawl/base/checkbox-with-label.tsx | 27 +++++++ .../create/website/firecrawl/base/field.tsx | 41 +++++++++++ .../create/website/firecrawl/base/input.tsx | 19 +++-- .../website/firecrawl/base/options-wrap.tsx | 26 +++++-- .../create/website/firecrawl/index.tsx | 19 +++-- .../create/website/firecrawl/options.tsx | 71 +++++++++++++++++++ web/models/datasets.ts | 9 +++ 7 files changed, 197 insertions(+), 15 deletions(-) create mode 100644 web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx diff --git a/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx new file mode 100644 index 00000000000000..f25e45724512e7 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx @@ -0,0 +1,27 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import cn from 'classnames' +import Checkbox from '@/app/components/base/checkbox' + +type Props = { + className?: string + isChecked: boolean + onChange: (isChecked: boolean) => void + label: string +} + +const CheckboxWithLabel: FC = ({ + className = '', + isChecked, + onChange, + label, +}) => { + return ( + + ) +} +export default React.memo(CheckboxWithLabel) diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx index e69de29bb2d1d6..d9fb18aa48c420 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx @@ -0,0 +1,41 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import cn from 'classnames' +import Input from './input' + +type Props = { + className?: string + label: string + value: string | number + onChange: (value: string | number) => void + isRequired?: boolean + placeholder?: string + isNumber?: boolean +} + +const Field: FC = ({ + className, + label, + value, + onChange, + isRequired = false, + placeholder = '', + isNumber = false, +}) => { + return ( +
+
+ {label} + {isRequired && *} +
+ +
+ ) +} +export default React.memo(Field) diff --git a/web/app/components/datasets/create/website/firecrawl/base/input.tsx b/web/app/components/datasets/create/website/firecrawl/base/input.tsx index 0d3337848d41b0..a042b1460a78ba 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/input.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/input.tsx @@ -1,22 +1,33 @@ 'use client' import type { FC } from 'react' -import React from 'react' +import React, { useCallback } from 'react' type Props = { - value: string - onChange: (value: string) => void + value: string | number + onChange: (value: string | number) => void placeholder?: string + isNumber?: boolean } const Input: FC = ({ value, onChange, placeholder = '', + isNumber = false, }) => { + const handleChange = useCallback((e: React.ChangeEvent) => { + const value = e.target.value + if (isNumber) { + onChange(parseFloat(value)) + return + } + onChange(value) + }, [isNumber, onChange]) return ( onChange(e.target.value)} + onChange={handleChange} className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400' placeholder={placeholder} /> diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx index 32b4d3c4b5e9b6..ff66cb0e614ca0 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -3,36 +3,48 @@ import { useBoolean } from 'ahooks' import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' +import cn from 'classnames' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' - +import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows' const I18N_PREFIX = 'datasetCreation.stepOne.website' type Props = { + className?: string children: React.ReactNode + errorMsg?: string } const OptionsWrap: FC = ({ + className = '', children, + errorMsg, }) => { const { t } = useTranslation() const [fold, { - setTrue: foldTrue, - setFalse: foldFalse, toggle: foldToggle, }] = useBoolean(false) return ( -
-
+
+
{t(`${I18N_PREFIX}.options`)}
- +
{!fold && (
- {children} + {!errorMsg + ? children + : ( +
+ {errorMsg} +
+ )}
)} diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index d813ae6e94f2dd..1746a1a2450bec 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -1,14 +1,23 @@ 'use client' import type { FC } from 'react' -import React, { useCallback } from 'react' +import React, { useCallback, useState } from 'react' import Header from './header' import UrlInput from './base/url-input' import OptionsWrap from './base/options-wrap' +import Options from './options' import { useModalContext } from '@/context/modal-context' +import type { CrawlOptions } from '@/models/datasets' type Props = { } - +const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { + crawl_sub_pages: true, + only_main_content: true, + includes: '', + excludes: '', + limit: 100, + max_depth: 3, +} const FireCrawl: FC = () => { const { setShowAccountSettingModal } = useModalContext() const handleSetting = useCallback(() => { @@ -17,6 +26,8 @@ const FireCrawl: FC = () => { }) }, [setShowAccountSettingModal]) + const [crawlOptions, setCrawlOptions] = useState(DEFAULT_CRAWL_OPTIONS) + const handleRun = useCallback((url: string) => { console.log(url) }, []) @@ -26,8 +37,8 @@ const FireCrawl: FC = () => {
- -
contents
+ +
diff --git a/web/app/components/datasets/create/website/firecrawl/options.tsx b/web/app/components/datasets/create/website/firecrawl/options.tsx index e69de29bb2d1d6..3062ba2d70772e 100644 --- a/web/app/components/datasets/create/website/firecrawl/options.tsx +++ b/web/app/components/datasets/create/website/firecrawl/options.tsx @@ -0,0 +1,71 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import cn from 'classnames' +import CheckboxWithLabel from './base/checkbox-with-label' +import Field from './base/field' +import type { CrawlOptions } from '@/models/datasets' + +type Props = { + className?: string + payload: CrawlOptions + onChange: (payload: CrawlOptions) => void +} + +const Options: FC = ({ + className = '', + payload, + onChange, +}) => { + const handleChange = useCallback((key: keyof CrawlOptions) => { + return (value: any) => { + onChange({ + ...payload, + [key]: value, + }) + } + }, [payload, onChange]) + return ( +
+ +
+ + +
+ +
+ + +
+ +
+ ) +} +export default React.memo(Options) diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 585c0473d4f31a..8a16b207d13c55 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -39,6 +39,15 @@ export type CustomFile = File & { created_at?: number } +export type CrawlOptions = { + crawl_sub_pages: boolean + only_main_content: boolean + includes: string + excludes: string + limit: number + max_depth: number +} + export type FileItem = { fileID: string file: CustomFile From b4a108960f11f0fc264422db8b1e245b6e0f6aa2 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 5 Jun 2024 14:50:28 +0800 Subject: [PATCH 094/273] feat: option styles --- .../create/website/firecrawl/base/field.tsx | 6 ++--- .../website/firecrawl/base/options-wrap.tsx | 2 +- .../create/website/firecrawl/index.tsx | 3 +-- .../create/website/firecrawl/options.tsx | 26 ++++++++++++++----- web/i18n/en-US/dataset-creation.ts | 6 +++++ web/i18n/zh-Hans/dataset-creation.ts | 6 +++++ 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx index d9fb18aa48c420..12eaefa9b85f03 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx @@ -25,9 +25,9 @@ const Field: FC = ({ }) => { return (
-
- {label} - {isRequired && *} +
+
{label}
+ {isRequired && *}
= ({ >
-
{t(`${I18N_PREFIX}.options`)}
+
{t(`${I18N_PREFIX}.options`)}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 1746a1a2450bec..e394ac51f3aea9 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -36,8 +36,7 @@ const FireCrawl: FC = () => {
- - +
diff --git a/web/app/components/datasets/create/website/firecrawl/options.tsx b/web/app/components/datasets/create/website/firecrawl/options.tsx index 3062ba2d70772e..b882febd9d3c25 100644 --- a/web/app/components/datasets/create/website/firecrawl/options.tsx +++ b/web/app/components/datasets/create/website/firecrawl/options.tsx @@ -2,10 +2,13 @@ import type { FC } from 'react' import React, { useCallback } from 'react' import cn from 'classnames' +import { useTranslation } from 'react-i18next' import CheckboxWithLabel from './base/checkbox-with-label' import Field from './base/field' import type { CrawlOptions } from '@/models/datasets' +const I18N_PREFIX = 'datasetCreation.stepOne.website' + type Props = { className?: string payload: CrawlOptions @@ -17,6 +20,8 @@ const Options: FC = ({ payload, onChange, }) => { + const { t } = useTranslation() + const handleChange = useCallback((key: keyof CrawlOptions) => { return (value: any) => { onChange({ @@ -27,11 +32,15 @@ const Options: FC = ({ }, [payload, onChange]) return (
- +
= ({ /> = ({
- +
) } diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 9bb86765520f59..56df52e89df810 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -59,6 +59,12 @@ const translation = { firecrawlDoc: 'Firecrawl docs', firecrawlDocLink: '', options: 'Options', + crawlSubPage: 'Crawl sub-pages', + limit: 'Limit', + maxDepth: 'Max depth', + excludePaths: 'Exclude paths', + includeOnlyPaths: 'Include only paths', + extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 9c4793c0a393b4..c8ab88147b9906 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -59,6 +59,12 @@ const translation = { firecrawlDoc: 'Firecrawl 文档', firecrawlDocLink: '', options: '选项', + crawlSubPage: '爬取子页面', + limit: '限制数量', + maxDepth: '最大深度', + excludePaths: '排除路径', + includeOnlyPaths: '仅包含路径', + extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）', }, }, stepTwo: { From 446f24cb9fea7a198b0e9c6ca10f8f040a571e80 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 5 Jun 2024 15:43:50 +0800 Subject: [PATCH 095/273] feat: valid crawl options --- .../create/website/firecrawl/base/input.tsx | 24 ++++++- .../website/firecrawl/base/url-input.tsx | 8 ++- .../create/website/firecrawl/index.tsx | 67 +++++++++++++++++-- web/i18n/en-US/common.ts | 4 ++ web/i18n/zh-Hans/common.ts | 4 ++ web/models/datasets.ts | 4 +- 6 files changed, 100 insertions(+), 11 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/input.tsx b/web/app/components/datasets/create/website/firecrawl/base/input.tsx index a042b1460a78ba..06249f57e7469a 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/input.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/input.tsx @@ -9,6 +9,8 @@ type Props = { isNumber?: boolean } +const MIN_VALUE = 1 + const Input: FC = ({ value, onChange, @@ -18,14 +20,34 @@ const Input: FC = ({ const handleChange = useCallback((e: React.ChangeEvent) => { const value = e.target.value if (isNumber) { - onChange(parseFloat(value)) + let numberValue = parseInt(value, 10) // integer only + if (isNaN(numberValue)) { + onChange('') + return + } + if (numberValue < MIN_VALUE) + numberValue = MIN_VALUE + + onChange(numberValue) return } onChange(value) }, [isNumber, onChange]) + + const otherOption = (() => { + if (isNumber) { + return { + min: MIN_VALUE, + } + } + return { + + } + })() return ( = ({ onRun, }) => { const { t } = useTranslation() - const [url, setUrl] = useState('') - + const [url, setUrl] = useState('https://docs.dify.ai') // TODO: for test + const handleUrlChange = useCallback((url: string | number) => { + setUrl(url as string) + }, []) const handleOnRun = useCallback(() => { onRun(url) }, [onRun, url]) @@ -25,7 +27,7 @@ const UrlInput: FC = ({
)} diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index cca2cdf06d0c4d..271c2d1064dbfa 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -2,6 +2,7 @@ import type { FC } from 'react' import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' +import cn from 'classnames' import Header from './header' import UrlInput from './base/url-input' import OptionsWrap from './base/options-wrap' @@ -13,6 +14,8 @@ import Toast from '@/app/components/base/toast' const ERROR_I18N_PREFIX = 'common.errorMsg' const I18N_PREFIX = 'datasetCreation.stepOne.website' +// const testCrawlErrorMsg = 'Firecrawl currently does not support social media scraping due to policy restrictions. We are actively working on building support for it.' + type Props = { } @@ -65,10 +68,11 @@ const FireCrawl: FC = () => { } }, [crawlOptions, t]) - const [isCrawlFinished, setIsCrawlFinished] = useState(false) + const [isCrawlFinished, setIsCrawlFinished] = useState(true) const [crawlErrorMsg, setCrawlErrorMsg] = useState('') - + const showCrawlError = isCrawlFinished && !!crawlErrorMsg const handleRun = useCallback((url: string) => { + setIsCrawlFinished(false) const { isValid, errorMsg } = checkValid(url) if (!isValid) { Toast.notify({ @@ -85,9 +89,9 @@ const FireCrawl: FC = () => { return (
-
+
- + {!isCrawlFinished ? ( diff --git a/web/app/components/datasets/create/website/firecrawl/options-error.tsx b/web/app/components/datasets/create/website/firecrawl/options-error.tsx deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 56df52e89df810..b059b6f0d1e885 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -65,6 +65,7 @@ const translation = { excludePaths: 'Exclude paths', includeOnlyPaths: 'Include only paths', extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)', + exceptionErrorTitle: 'An exception occurred while running Firecrawl job:', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index c8ab88147b9906..23df0a9d537732 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -65,6 +65,8 @@ const translation = { excludePaths: '排除路径', includeOnlyPaths: '仅包含路径', extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）', + exceptionErrorTitle: '运行 Firecrawl 时发生异常:', + }, }, stepTwo: { From a278e922f9fb77f40805f57df9b2fec6fca15035 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 5 Jun 2024 16:55:31 +0800 Subject: [PATCH 097/273] feat: crawl options struct --- .../website/firecrawl/crawled-result-item.tsx | 32 ++++++++++ .../website/firecrawl/crawled-result.tsx | 58 +++++++++++++++++++ .../create/website/firecrawl/index.tsx | 13 ++++- .../website/firecrawl/mock-crawl-result.ts | 24 ++++++++ web/models/datasets.ts | 7 +++ 5 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/crawled-result.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx new file mode 100644 index 00000000000000..6e3735186f19a6 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx @@ -0,0 +1,32 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets' +import Checkbox from '@/app/components/base/checkbox' + +type Props = { + payload: CrawlResultItemType + isChecked: boolean + onCheckChange: (checked: boolean) => void +} + +const CrawledResultItem: FC = ({ + payload, + isChecked, + onCheckChange, +}) => { + const handleCheckChange = useCallback(() => { + onCheckChange(!isChecked) + }, [isChecked, onCheckChange]) + return ( +
+
+ +
{payload.title}
+
Preview
+
+
{payload.source_url}
+
+ ) +} +export default React.memo(CrawledResultItem) diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx new file mode 100644 index 00000000000000..9df4b2b521679c --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -0,0 +1,58 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback } from 'react' +import CheckboxWithLabel from './base/checkbox-with-label' +import CrawledResultItem from './crawled-result-item' +import type { CrawlResultItem } from '@/models/datasets' + +type Props = { + list: CrawlResultItem[] + checkedList: CrawlResultItem[] + onSelectedChange: (selected: CrawlResultItem[]) => void +} + +const CrawledResult: FC = ({ + list, + checkedList, + onSelectedChange, +}) => { + const isCheckAll = checkedList.length === list.length + + const handleCheckedAll = useCallback(() => { + if (!isCheckAll) + onSelectedChange(list) + + else + onSelectedChange([]) + }, [isCheckAll, list, onSelectedChange]) + + const handleItemCheckChange = useCallback((item: CrawlResultItem) => { + return (checked: boolean) => { + if (checked) + onSelectedChange([...checkedList, item]) + + else + onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url)) + } + }, [checkedList, onSelectedChange]) + + return ( +
+
+ +
Scraped 10 pages in total within 12.4 seconds
+
+
+ {list.map(item => ( + checkedItem.source_url === item.source_url)} + onCheckChange={handleItemCheckChange(item)} + /> + ))} +
+
+ ) +} +export default React.memo(CrawledResult) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 271c2d1064dbfa..fe640f1d25bea9 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -7,8 +7,10 @@ import Header from './header' import UrlInput from './base/url-input' import OptionsWrap from './base/options-wrap' import Options from './options' +import mockCrawlResult from './mock-crawl-result' +import CrawledResult from './crawled-result' import { useModalContext } from '@/context/modal-context' -import type { CrawlOptions } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import Toast from '@/app/components/base/toast' const ERROR_I18N_PREFIX = 'common.errorMsg' @@ -69,6 +71,9 @@ const FireCrawl: FC = () => { }, [crawlOptions, t]) const [isCrawlFinished, setIsCrawlFinished] = useState(true) + const [crawlResult, setCrawlResult] = useState(mockCrawlResult) + const [checkedCrawlResult, setCheckedCrawlResult] = useState([]) + const [crawlErrorMsg, setCrawlErrorMsg] = useState('') const showCrawlError = isCrawlFinished && !!crawlErrorMsg const handleRun = useCallback((url: string) => { @@ -97,7 +102,11 @@ const FireCrawl: FC = () => { ) : ( -
Result list
+ )}
diff --git a/web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts b/web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts new file mode 100644 index 00000000000000..8fd5e6636f1617 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts @@ -0,0 +1,24 @@ +import type { CrawlResultItem } from '@/models/datasets' + +const result: CrawlResultItem[] = [ + { + title: 'Start the frontend Docker container separately', + markdown: 'Markdown 1', + description: 'Description 1', + source_url: 'https://example.com/1', + }, + { + title: 'Advanced Tool Integration', + markdown: 'Markdown 2', + description: 'Description 2', + source_url: 'https://example.com/2', + }, + { + title: 'Local Source Code Start | English | Dify', + markdown: 'Markdown 3', + description: 'Description 3', + source_url: 'https://example.com/3', + }, +] + +export default result diff --git a/web/models/datasets.ts b/web/models/datasets.ts index d3b752aaf576af..1edb63748faadd 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -48,6 +48,13 @@ export type CrawlOptions = { max_depth: number | string } +export type CrawlResultItem = { + title: string + markdown: string + description: string + source_url: string +} + export type FileItem = { fileID: string file: CustomFile From 67322e7cd52c9af756a90545088a92d19c0e4c91 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Wed, 5 Jun 2024 18:17:59 +0800 Subject: [PATCH 098/273] note editor --- .../(appDetailLayout)/[appId]/note/page.tsx | 2 +- .../assets/vender/line/editor/bold-01.svg | 5 + .../vender/line/editor/dotpoints-01.svg | 5 + .../vender/line/editor/strikethrough-01.svg | 5 + .../assets/vender/line/editor/title-case.svg | 8 ++ .../assets/vender/line/general/Workflow.zip | Bin 478 -> 0 bytes .../assets/vender/line/general/link-01.svg | 5 + .../icons/src/vender/line/editor/Bold01.json | 39 ++++++++ .../icons/src/vender/line/editor/Bold01.tsx | 16 ++++ .../src/vender/line/editor/Dotpoints01.json | 39 ++++++++ .../src/vender/line/editor/Dotpoints01.tsx | 16 ++++ .../vender/line/editor/Strikethrough01.json | 39 ++++++++ .../vender/line/editor/Strikethrough01.tsx | 16 ++++ .../src/vender/line/editor/TitleCase.json | 53 +++++++++++ .../src/vender/line/editor/TitleCase.tsx | 16 ++++ .../icons/src/vender/line/editor/index.ts | 4 + .../icons/src/vender/line/general/Link01.json | 39 ++++++++ .../icons/src/vender/line/general/Link01.tsx | 16 ++++ .../icons/src/vender/line/general/index.ts | 1 + .../workflow/note-node/note-editor/editor.tsx | 6 +- .../note-node/note-editor/toolbar.tsx | 9 -- .../note-editor/toolbar/color-picker.tsx | 88 ++++++++++++++++++ .../note-node/note-editor/toolbar/command.tsx | 60 ++++++++++++ .../note-node/note-editor/toolbar/divider.tsx | 7 ++ .../toolbar/font-size-selector.tsx | 77 +++++++++++++++ .../note-node/note-editor/toolbar/index.tsx | 27 ++++++ .../note-editor/toolbar/operator.tsx | 66 +++++++++++++ 27 files changed, 651 insertions(+), 13 deletions(-) create mode 100644 web/app/components/base/icons/assets/vender/line/editor/bold-01.svg create mode 100644 web/app/components/base/icons/assets/vender/line/editor/dotpoints-01.svg create mode 100644 web/app/components/base/icons/assets/vender/line/editor/strikethrough-01.svg create mode 100644 web/app/components/base/icons/assets/vender/line/editor/title-case.svg delete mode 100644 web/app/components/base/icons/assets/vender/line/general/Workflow.zip create mode 100644 web/app/components/base/icons/assets/vender/line/general/link-01.svg create mode 100644 web/app/components/base/icons/src/vender/line/editor/Bold01.json create mode 100644 web/app/components/base/icons/src/vender/line/editor/Bold01.tsx create mode 100644 web/app/components/base/icons/src/vender/line/editor/Dotpoints01.json create mode 100644 web/app/components/base/icons/src/vender/line/editor/Dotpoints01.tsx create mode 100644 web/app/components/base/icons/src/vender/line/editor/Strikethrough01.json create mode 100644 web/app/components/base/icons/src/vender/line/editor/Strikethrough01.tsx create mode 100644 web/app/components/base/icons/src/vender/line/editor/TitleCase.json create mode 100644 web/app/components/base/icons/src/vender/line/editor/TitleCase.tsx create mode 100644 web/app/components/base/icons/src/vender/line/general/Link01.json create mode 100644 web/app/components/base/icons/src/vender/line/general/Link01.tsx delete mode 100644 web/app/components/workflow/note-node/note-editor/toolbar.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar/command.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar/divider.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar/font-size-selector.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar/index.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx index 441751c4bb02a2..388b8668dddbf6 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx @@ -8,7 +8,7 @@ import { const Page = () => { return ( -
+
diff --git a/web/app/components/base/icons/assets/vender/line/editor/bold-01.svg b/web/app/components/base/icons/assets/vender/line/editor/bold-01.svg new file mode 100644 index 00000000000000..5e9e20816bf029 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/editor/bold-01.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/assets/vender/line/editor/dotpoints-01.svg b/web/app/components/base/icons/assets/vender/line/editor/dotpoints-01.svg new file mode 100644 index 00000000000000..dddb80780affae --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/editor/dotpoints-01.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/assets/vender/line/editor/strikethrough-01.svg b/web/app/components/base/icons/assets/vender/line/editor/strikethrough-01.svg new file mode 100644 index 00000000000000..5935b4dd7dda82 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/editor/strikethrough-01.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/assets/vender/line/editor/title-case.svg b/web/app/components/base/icons/assets/vender/line/editor/title-case.svg new file mode 100644 index 00000000000000..c3a13dd7ef8fd5 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/editor/title-case.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/web/app/components/base/icons/assets/vender/line/general/Workflow.zip b/web/app/components/base/icons/assets/vender/line/general/Workflow.zip deleted file mode 100644 index 05631f93b04db33a2cb267793513d60df912c8ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 478 zcmWIWW@h1HU|`^2@SYYG!8vj9F)v02hC(I=21%f3cz#iKT26ksesV@?a<*=AW>IoZ zs$OwfdTG$XzGel1z2CJt_P&wpV&dAh(s!F}?k3-Fd!{dQT$K}R>g8=;?|5{X?>DB; z_deY7nGyN@I+MiI*GA12O$(;1()cP7Kh@;+vuOtfS{U396sm`YO0Jn5=iLt;MS=U#^qgW=N^SGFMjFxOL3MIlk^*9mr5Va1P$NID!)=VgxiIWD7Ob6VUtmQ^g&K|O2DbV zzvmveaA|o^v4>M4lgVOJZuHNVa*@J$24>F<5A`g48Eg0OX5FD?KJ|oa)^_`D^A|kt z?)SVu>$=_6f4q+CcR$lH-f<-5^v&JsVRdC1^HzFY4U)OdQ+9UgIr$g!-LB5#f1}Ud zYFkwK>h05IAD-6#%~Z)#nk({af2id3LsPs?U%WnhXT_&->5Y+p7z4Z+nM4?HM*%Pv ZkO0mI3GilR1F2&KLU$nD2#iGr1^}H_#GL>D diff --git a/web/app/components/base/icons/assets/vender/line/general/link-01.svg b/web/app/components/base/icons/assets/vender/line/general/link-01.svg new file mode 100644 index 00000000000000..60fccd84f2d400 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/general/link-01.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/src/vender/line/editor/Bold01.json b/web/app/components/base/icons/src/vender/line/editor/Bold01.json new file mode 100644 index 00000000000000..6a807dacdfd575 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/Bold01.json @@ -0,0 +1,39 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "17", + "height": "16", + "viewBox": "0 0 17 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Icon" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon_2", + "d": "M4.5 7.99996H9.83333C11.3061 7.99996 12.5 6.80605 12.5 5.33329C12.5 3.86053 11.3061 2.66663 9.83333 2.66663H4.5V7.99996ZM4.5 7.99996H10.5C11.9728 7.99996 13.1667 9.19387 13.1667 10.6666C13.1667 12.1394 11.9728 13.3333 10.5 13.3333H4.5V7.99996Z", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + } + ] + }, + "name": "Bold01" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/editor/Bold01.tsx b/web/app/components/base/icons/src/vender/line/editor/Bold01.tsx new file mode 100644 index 00000000000000..5fb2c4b6203b45 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/Bold01.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Bold01.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Bold01' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/editor/Dotpoints01.json b/web/app/components/base/icons/src/vender/line/editor/Dotpoints01.json new file mode 100644 index 00000000000000..926824bddac19b --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/Dotpoints01.json @@ -0,0 +1,39 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "17", + "height": "16", + "viewBox": "0 0 17 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Icon" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon_2", + "d": "M14.5 8.00004L6.5 8.00004M14.5 4.00004L6.5 4.00004M14.5 12L6.5 12M3.83333 8.00004C3.83333 8.36823 3.53486 8.66671 3.16667 8.66671C2.79848 8.66671 2.5 8.36823 2.5 8.00004C2.5 7.63185 2.79848 7.33337 3.16667 7.33337C3.53486 7.33337 3.83333 7.63185 3.83333 8.00004ZM3.83333 4.00004C3.83333 4.36823 3.53486 4.66671 3.16667 4.66671C2.79848 4.66671 2.5 4.36823 2.5 4.00004C2.5 3.63185 2.79848 3.33337 3.16667 3.33337C3.53486 3.33337 3.83333 3.63185 3.83333 4.00004ZM3.83333 12C3.83333 12.3682 3.53486 12.6667 3.16667 12.6667C2.79848 12.6667 2.5 12.3682 2.5 12C2.5 11.6319 2.79848 11.3334 3.16667 11.3334C3.53486 11.3334 3.83333 11.6319 3.83333 12Z", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + } + ] + }, + "name": "Dotpoints01" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/editor/Dotpoints01.tsx b/web/app/components/base/icons/src/vender/line/editor/Dotpoints01.tsx new file mode 100644 index 00000000000000..34cdbb71449a18 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/Dotpoints01.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Dotpoints01.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Dotpoints01' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/editor/Strikethrough01.json b/web/app/components/base/icons/src/vender/line/editor/Strikethrough01.json new file mode 100644 index 00000000000000..fda981321ad565 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/Strikethrough01.json @@ -0,0 +1,39 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "17", + "height": "16", + "viewBox": "0 0 17 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Icon" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon_2", + "d": "M4.5 10.6666C4.5 12.1394 5.69391 13.3333 7.16667 13.3333H9.83333C11.3061 13.3333 12.5 12.1394 12.5 10.6666C12.5 9.19387 11.3061 7.99996 9.83333 7.99996M12.5 5.33329C12.5 3.86053 11.3061 2.66663 9.83333 2.66663H7.16667C5.69391 2.66663 4.5 3.86053 4.5 5.33329M2.5 7.99996H14.5", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + } + ] + }, + "name": "Strikethrough01" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/editor/Strikethrough01.tsx b/web/app/components/base/icons/src/vender/line/editor/Strikethrough01.tsx new file mode 100644 index 00000000000000..7aee626ceecf04 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/Strikethrough01.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Strikethrough01.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Strikethrough01' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/editor/TitleCase.json b/web/app/components/base/icons/src/vender/line/editor/TitleCase.json new file mode 100644 index 00000000000000..6f7e587ee575b9 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/TitleCase.json @@ -0,0 +1,53 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "17", + "height": "16", + "viewBox": "0 0 17 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Icon" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Vector" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "d": "M2.0922 12.4865C2.57616 12.4865 2.84839 12.2445 3.01778 11.6638L3.47754 10.3026H6.62933L7.0891 11.6819C7.25243 12.2506 7.52463 12.4865 8.03887 12.4865C8.5712 12.4865 8.95232 12.1295 8.95232 11.6275C8.95232 11.4459 8.92208 11.2827 8.83743 11.0467L6.44179 4.54954C6.18167 3.83569 5.7582 3.52112 5.04436 3.52112C4.35471 3.52112 3.9252 3.84779 3.67112 4.55559L1.28762 11.0467C1.20897 11.2705 1.16663 11.4762 1.16663 11.6275C1.16663 12.1538 1.52355 12.4865 2.0922 12.4865ZM3.8768 8.88703L5.00806 5.31177H5.05041L6.20586 8.88703H3.8768Z", + "fill": "currentColor" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "M12.1068 12.4744C12.9174 12.4744 13.7281 12.0691 14.091 11.3795H14.1273V11.7122C14.1636 12.2324 14.4963 12.4986 14.9742 12.4986C15.4764 12.4986 15.8333 12.1961 15.8333 11.6093V7.91309C15.8333 6.60636 14.7504 5.74734 13.0868 5.74734C11.7438 5.74734 10.7033 6.22525 10.4008 6.99957C10.3403 7.13269 10.3101 7.25973 10.3101 7.39885C10.3101 7.79813 10.6186 8.07638 11.0481 8.07638C11.3324 8.07638 11.5563 7.9675 11.7499 7.74973C12.1431 7.24157 12.4697 7.06613 13.0081 7.06613C13.6736 7.06613 14.0971 7.41701 14.0971 8.02198V8.4515L12.4637 8.54823C10.8424 8.64503 9.93506 9.32864 9.93506 10.5083C9.93506 11.6759 10.8727 12.4744 12.1068 12.4744ZM12.6876 11.1979C12.0947 11.1979 11.6954 10.8955 11.6954 10.4115C11.6954 9.95176 12.0705 9.65528 12.7299 9.60695L14.0971 9.52224V9.99408C14.0971 10.6958 13.4619 11.1979 12.6876 11.1979Z", + "fill": "currentColor" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "TitleCase" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/editor/TitleCase.tsx b/web/app/components/base/icons/src/vender/line/editor/TitleCase.tsx new file mode 100644 index 00000000000000..a1e073e48e3852 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/editor/TitleCase.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './TitleCase.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'TitleCase' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/editor/index.ts b/web/app/components/base/icons/src/vender/line/editor/index.ts index aafadaf5d2846c..01b46196a7868a 100644 --- a/web/app/components/base/icons/src/vender/line/editor/index.ts +++ b/web/app/components/base/icons/src/vender/line/editor/index.ts @@ -1,11 +1,15 @@ export { default as AlignLeft } from './AlignLeft' export { default as BezierCurve03 } from './BezierCurve03' +export { default as Bold01 } from './Bold01' export { default as Colors } from './Colors' export { default as Cursor02C } from './Cursor02C' +export { default as Dotpoints01 } from './Dotpoints01' export { default as Hand02 } from './Hand02' export { default as ImageIndentLeft } from './ImageIndentLeft' export { default as LeftIndent02 } from './LeftIndent02' export { default as LetterSpacing01 } from './LetterSpacing01' +export { default as Strikethrough01 } from './Strikethrough01' +export { default as TitleCase } from './TitleCase' export { default as TypeSquare } from './TypeSquare' export { default as ZoomIn } from './ZoomIn' export { default as ZoomOut } from './ZoomOut' diff --git a/web/app/components/base/icons/src/vender/line/general/Link01.json b/web/app/components/base/icons/src/vender/line/general/Link01.json new file mode 100644 index 00000000000000..b49b5776a07209 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/general/Link01.json @@ -0,0 +1,39 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "17", + "height": "16", + "viewBox": "0 0 17 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Icon" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon_2", + "d": "M8.97167 12.2427L8.02886 13.1855C6.72711 14.4872 4.61656 14.4872 3.31481 13.1855C2.01306 11.8837 2.01306 9.77317 3.31481 8.47142L4.25762 7.52861M12.7429 8.47142L13.6857 7.52861C14.9875 6.22687 14.9875 4.11632 13.6857 2.81457C12.384 1.51282 10.2734 1.51282 8.97167 2.81457L8.02886 3.75738M6.16693 10.3333L10.8336 5.66667", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + } + ] + }, + "name": "Link01" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/general/Link01.tsx b/web/app/components/base/icons/src/vender/line/general/Link01.tsx new file mode 100644 index 00000000000000..ddddab1bd87e96 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/general/Link01.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './Link01.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'Link01' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/general/index.ts b/web/app/components/base/icons/src/vender/line/general/index.ts index 8369117eedf262..8288f78aadf6ab 100644 --- a/web/app/components/base/icons/src/vender/line/general/index.ts +++ b/web/app/components/base/icons/src/vender/line/general/index.ts @@ -14,6 +14,7 @@ export { default as Edit05 } from './Edit05' export { default as Hash02 } from './Hash02' export { default as HelpCircle } from './HelpCircle' export { default as InfoCircle } from './InfoCircle' +export { default as Link01 } from './Link01' export { default as Link03 } from './Link03' export { default as LinkExternal01 } from './LinkExternal01' export { default as LinkExternal02 } from './LinkExternal02' diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index 32039d0d2f6e94..2f341098ffd0fe 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -5,7 +5,7 @@ import { RichTextPlugin } from '@lexical/react/LexicalRichTextPlugin' import { ContentEditable } from '@lexical/react/LexicalContentEditable' import LexicalErrorBoundary from '@lexical/react/LexicalErrorBoundary' import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin' -// import TreeView from './plugins/tree-view' +import TreeView from '@/app/components/base/prompt-editor/plugins/tree-view' import Placeholder from '@/app/components/base/prompt-editor/plugins/placeholder' type EditorProps = { @@ -17,12 +17,12 @@ const Editor = ({ return (
} + contentEditable={} placeholder={} ErrorBoundary={LexicalErrorBoundary} /> - {/* */} +
) } diff --git a/web/app/components/workflow/note-node/note-editor/toolbar.tsx b/web/app/components/workflow/note-node/note-editor/toolbar.tsx deleted file mode 100644 index 89e62429e5a2ed..00000000000000 --- a/web/app/components/workflow/note-node/note-editor/toolbar.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import { memo } from 'react' - -const Toolbar = () => { - return ( -
- ) -} - -export default memo(Toolbar) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx new file mode 100644 index 00000000000000..ff6de436bd559c --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx @@ -0,0 +1,88 @@ +import { + memo, + useState, +} from 'react' +import cn from 'classnames' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' + +const COLOR_LIST = [ + { + key: 'blue', + inner: '#D1E9FF', + outer: '#2E90FA', + }, + { + key: 'cyan', + inner: '#CFF9FE', + outer: '#06AED4', + }, + { + key: 'green', + inner: '#D3F8DF', + outer: '#16B364', + }, + { + key: 'yellow', + inner: '#FEF7C3', + outer: '#EAAA08', + }, + { + key: 'pink', + inner: '#FCE7F6', + outer: '#EE46BC', + }, + { + key: 'violet', + inner: '#ECE9FE', + outer: '#875BF7', + }, +] + +const ColorPicker = () => { + const [open, setOpen] = useState(false) + + return ( + + setOpen(!open)}> +
+
+
+ + +
+ { + COLOR_LIST.map(color => ( +
+
+
+
+ )) + } +
+ + + ) +} + +export default memo(ColorPicker) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx new file mode 100644 index 00000000000000..54aef3628d965e --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx @@ -0,0 +1,60 @@ +import { memo, useMemo } from 'react' +import { + $getSelection, + $isRangeSelection, + $setSelection, +} from 'lexical' +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' +import { Link01 } from '@/app/components/base/icons/src/vender/line/general' +import { + Bold01, + Dotpoints01, + Strikethrough01, +} from '@/app/components/base/icons/src/vender/line/editor' + +type CommandProps = { + type: 'bold' | 'strikethrough' | 'link' | 'bullet' +} +const Command = ({ + type, +}: CommandProps) => { + const [editor] = useLexicalComposerContext() + + const icon = useMemo(() => { + switch (type) { + case 'bold': + return + case 'strikethrough': + return + case 'link': + return + case 'bullet': + return + } + }, [type]) + + const handleClick = () => { + if (type === 'bold') + return + + if (type === 'link') { + editor.update(() => { + const selection = $getSelection() + + if ($isRangeSelection(selection) && !selection.isCollapsed()) + $setSelection(selection) + }) + } + } + + return ( +
+ {icon} +
+ ) +} + +export default memo(Command) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/divider.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/divider.tsx new file mode 100644 index 00000000000000..aefdb46b0aae82 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar/divider.tsx @@ -0,0 +1,7 @@ +const Divider = () => { + return ( +
+ ) +} + +export default Divider diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/font-size-selector.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/font-size-selector.tsx new file mode 100644 index 00000000000000..d7aff03d2b62a9 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar/font-size-selector.tsx @@ -0,0 +1,77 @@ +import { + memo, + useState, +} from 'react' +import cn from 'classnames' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' +import { TitleCase } from '@/app/components/base/icons/src/vender/line/editor' +import { ChevronDown } from '@/app/components/base/icons/src/vender/line/arrows' +import { Check } from '@/app/components/base/icons/src/vender/line/general' + +const FONT_SIZE_LIST = [ + { + key: 'Small', + value: 12, + }, + { + key: 'Medium', + value: 14, + }, + { + key: 'Large', + value: 16, + }, +] +const FontSizeSelector = () => { + const [value] = useState(FONT_SIZE_LIST[0].key) + const [open, setOpen] = useState(false) + + return ( + + setOpen(!open)}> +
+ + {value} + +
+ + +
+ { + FONT_SIZE_LIST.map(font => ( +
+
+ {font.key} +
+ { + value === font.key && ( + + ) + } +
+ )) + } +
+ + + ) +} + +export default memo(FontSizeSelector) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx new file mode 100644 index 00000000000000..70b52afe191355 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx @@ -0,0 +1,27 @@ +import { memo } from 'react' +import Divider from './divider' +import ColorPicker from './color-picker' +import FontSizeSelector from './font-size-selector' +import Command from './command' +import Operator from './operator' + +const Toolbar = () => { + return ( +
+ + + + +
+ + + + +
+ + +
+ ) +} + +export default memo(Toolbar) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx new file mode 100644 index 00000000000000..593af4844b60d5 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx @@ -0,0 +1,66 @@ +import { + memo, + useState, +} from 'react' +import cn from 'classnames' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' +import { DotsHorizontal } from '@/app/components/base/icons/src/vender/line/general' +import Switch from '@/app/components/base/switch' + +const Operator = () => { + const [open, setOpen] = useState(false) + + return ( + + setOpen(!open)}> +
+ +
+ + +
+
+
+
Copy
+
C
+
+
+
Duplicate
+
D
+
+
+
+
+
+
Show Author
+ +
+
+
+
+
+
Delete
+
Backspace
+
+
+
+ + + ) +} + +export default memo(Operator) From eff80c8d27a3d6b12c141bb69bd477c7aa6e4dee Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 5 Jun 2024 18:20:29 +0800 Subject: [PATCH 099/273] feat: scrawl loading --- .../icons/assets/public/other/row-struct.svg | 5 ++ .../icons/src/public/other/RowStruct.json | 56 ++++++++++++++++++ .../base/icons/src/public/other/RowStruct.tsx | 16 +++++ .../base/icons/src/public/other/index.ts | 1 + .../website/firecrawl/base/options-wrap.tsx | 6 +- .../website/firecrawl/base/url-input.tsx | 9 ++- .../create/website/firecrawl/crawling.tsx | 37 ++++++++++++ .../create/website/firecrawl/index.tsx | 59 +++++++++++++------ web/i18n/en-US/dataset-creation.ts | 1 + web/i18n/zh-Hans/dataset-creation.ts | 2 +- 10 files changed, 168 insertions(+), 24 deletions(-) create mode 100644 web/app/components/base/icons/assets/public/other/row-struct.svg create mode 100644 web/app/components/base/icons/src/public/other/RowStruct.json create mode 100644 web/app/components/base/icons/src/public/other/RowStruct.tsx create mode 100644 web/app/components/datasets/create/website/firecrawl/crawling.tsx diff --git a/web/app/components/base/icons/assets/public/other/row-struct.svg b/web/app/components/base/icons/assets/public/other/row-struct.svg new file mode 100644 index 00000000000000..ba275ffeec9079 --- /dev/null +++ b/web/app/components/base/icons/assets/public/other/row-struct.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/src/public/other/RowStruct.json b/web/app/components/base/icons/src/public/other/RowStruct.json new file mode 100644 index 00000000000000..0d1ef43f4f5d5d --- /dev/null +++ b/web/app/components/base/icons/src/public/other/RowStruct.json @@ -0,0 +1,56 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "624", + "height": "48", + "viewBox": "0 0 624 48", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "x": "8", + "y": "7", + "width": "16", + "height": "16", + "rx": "5", + "fill": "#F2F4F7" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "x": "32", + "y": "10", + "width": "233", + "height": "10", + "rx": "3", + "fill": "#EAECF0" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "x": "32", + "y": "31", + "width": "345", + "height": "6", + "rx": "3", + "fill": "#F2F4F7" + }, + "children": [] + } + ] + }, + "name": "RowStruct" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/other/RowStruct.tsx b/web/app/components/base/icons/src/public/other/RowStruct.tsx new file mode 100644 index 00000000000000..ef5ab8c62df79a --- /dev/null +++ b/web/app/components/base/icons/src/public/other/RowStruct.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './RowStruct.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'RowStruct' + +export default Icon diff --git a/web/app/components/base/icons/src/public/other/index.ts b/web/app/components/base/icons/src/public/other/index.ts index adf723edb7db55..257ba59b0d0131 100644 --- a/web/app/components/base/icons/src/public/other/index.ts +++ b/web/app/components/base/icons/src/public/other/index.ts @@ -1,2 +1,3 @@ export { default as Icon3Dots } from './Icon3Dots' export { default as DefaultToolIcon } from './DefaultToolIcon' +export { default as RowStruct } from './RowStruct' diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx index 8a27012ebb5227..af427249aa8291 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -12,6 +12,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website' type Props = { className?: string children: React.ReactNode + isFilledFull?: boolean errorMsg?: string } @@ -19,6 +20,7 @@ const OptionsWrap: FC = ({ className = '', children, errorMsg, + isFilledFull = false, }) => { const { t } = useTranslation() @@ -38,11 +40,11 @@ const OptionsWrap: FC = ({
{!fold && ( -
+
{!errorMsg ? children : ( - + )}
)} diff --git a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx index 1b572eb68e6f41..90acc769be6ce8 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx @@ -8,10 +8,12 @@ import Button from '@/app/components/base/button' const I18N_PREFIX = 'datasetCreation.stepOne.website' type Props = { + isRunning: boolean onRun: (url: string) => void } const UrlInput: FC = ({ + isRunning, onRun, }) => { const { t } = useTranslation() @@ -20,8 +22,10 @@ const UrlInput: FC = ({ setUrl(url as string) }, []) const handleOnRun = useCallback(() => { + if (isRunning) + return onRun(url) - }, [onRun, url]) + }, [isRunning, onRun, url]) return (
@@ -34,8 +38,9 @@ const UrlInput: FC = ({ type='primary' onClick={handleOnRun} className='ml-2 !h-8 text-[13px] font-medium' + loading={isRunning} > - {t(`${I18N_PREFIX}.run`)} + {!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
) diff --git a/web/app/components/datasets/create/website/firecrawl/crawling.tsx b/web/app/components/datasets/create/website/firecrawl/crawling.tsx new file mode 100644 index 00000000000000..97b2b01d2eeb61 --- /dev/null +++ b/web/app/components/datasets/create/website/firecrawl/crawling.tsx @@ -0,0 +1,37 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import cn from 'classnames' +import { useTranslation } from 'react-i18next' +import { RowStruct } from '@/app/components/base/icons/src/public/other' + +type Props = { + className?: string + crawledNum: number + totalNum: number +} + +const Crawling: FC = ({ + className = '', + crawledNum, + totalNum, +}) => { + const { t } = useTranslation() + + return ( +
+
+ {t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum} +
+ +
+ {['', '', '', ''].map((item, index) => ( +
+ +
+ ))} +
+
+ ) +} +export default React.memo(Crawling) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index fe640f1d25bea9..234ecc4e4b629e 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -9,9 +9,11 @@ import OptionsWrap from './base/options-wrap' import Options from './options' import mockCrawlResult from './mock-crawl-result' import CrawledResult from './crawled-result' +import Crawling from './crawling' import { useModalContext } from '@/context/modal-context' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import Toast from '@/app/components/base/toast' +import { sleep } from '@/utils' const ERROR_I18N_PREFIX = 'common.errorMsg' const I18N_PREFIX = 'datasetCreation.stepOne.website' @@ -30,8 +32,15 @@ const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { max_depth: 2, } +enum Step { + init = 'init', + running = 'running', + finished = 'finished', +} + const FireCrawl: FC = () => { const { t } = useTranslation() + const [step, setStep] = useState(Step.running) const { setShowAccountSettingModal } = useModalContext() const handleSetting = useCallback(() => { @@ -70,14 +79,15 @@ const FireCrawl: FC = () => { } }, [crawlOptions, t]) - const [isCrawlFinished, setIsCrawlFinished] = useState(true) + const isInit = step === Step.init + const isCrawlFinished = step === Step.finished + const isRunning = step === Step.running const [crawlResult, setCrawlResult] = useState(mockCrawlResult) const [checkedCrawlResult, setCheckedCrawlResult] = useState([]) const [crawlErrorMsg, setCrawlErrorMsg] = useState('') - const showCrawlError = isCrawlFinished && !!crawlErrorMsg - const handleRun = useCallback((url: string) => { - setIsCrawlFinished(false) + const showCrawlError = step === Step.finished && !!crawlErrorMsg + const handleRun = useCallback(async (url: string) => { const { isValid, errorMsg } = checkValid(url) if (!isValid) { Toast.notify({ @@ -86,28 +96,39 @@ const FireCrawl: FC = () => { }) return } + setStep(Step.running) // TODO: crawl - setIsCrawlFinished(true) + await sleep(2000) + setCrawlResult(mockCrawlResult) // TODO: + + setStep(Step.finished) setCrawlErrorMsg('') }, [checkValid]) return (
-
- - - {!isCrawlFinished - ? ( - - ) - : ( - - )} +
+ + + {isInit && } + {isRunning + && } + {isCrawlFinished && ( + + )}
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index b059b6f0d1e885..87b1e57862f812 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -66,6 +66,7 @@ const translation = { includeOnlyPaths: 'Include only paths', extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)', exceptionErrorTitle: 'An exception occurred while running Firecrawl job:', + totalPageScraped: 'Total pages scraped:', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 23df0a9d537732..a0be0f0cc100a2 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -66,7 +66,7 @@ const translation = { includeOnlyPaths: '仅包含路径', extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）', exceptionErrorTitle: '运行 Firecrawl 时发生异常:', - + totalPageScraped: '抓取页面总数:', }, }, stepTwo: { From 0448c49c7ac8a8a3f89f0ea71242d05466904cd8 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 5 Jun 2024 18:33:40 +0800 Subject: [PATCH 100/273] feat: result --- .../firecrawl/base/checkbox-with-label.tsx | 4 +++- .../website/firecrawl/base/options-wrap.tsx | 2 +- .../website/firecrawl/crawled-result.tsx | 22 ++++++++++++++----- .../create/website/firecrawl/index.tsx | 3 +-- web/i18n/en-US/dataset-creation.ts | 3 +++ web/i18n/zh-Hans/dataset-creation.ts | 3 +++ 6 files changed, 28 insertions(+), 9 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx index f25e45724512e7..ed5d2efd511711 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx @@ -9,6 +9,7 @@ type Props = { isChecked: boolean onChange: (isChecked: boolean) => void label: string + labelClassName?: string } const CheckboxWithLabel: FC = ({ @@ -16,11 +17,12 @@ const CheckboxWithLabel: FC = ({ isChecked, onChange, label, + labelClassName, }) => { return ( ) } diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx index af427249aa8291..61c89fc10745df 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -44,7 +44,7 @@ const OptionsWrap: FC = ({ {!errorMsg ? children : ( - + )}
)} diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx index 9df4b2b521679c..48cca13b93ad3b 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -1,10 +1,13 @@ 'use client' import type { FC } from 'react' import React, { useCallback } from 'react' +import { useTranslation } from 'react-i18next' import CheckboxWithLabel from './base/checkbox-with-label' import CrawledResultItem from './crawled-result-item' import type { CrawlResultItem } from '@/models/datasets' +const I18N_PREFIX = 'datasetCreation.stepOne.website' + type Props = { list: CrawlResultItem[] checkedList: CrawlResultItem[] @@ -16,6 +19,8 @@ const CrawledResult: FC = ({ checkedList, onSelectedChange, }) => { + const { t } = useTranslation() + const isCheckAll = checkedList.length === list.length const handleCheckedAll = useCallback(() => { @@ -37,12 +42,19 @@ const CrawledResult: FC = ({ }, [checkedList, onSelectedChange]) return ( -
-
- -
Scraped 10 pages in total within 12.4 seconds
+
+
+ +
{t(`${I18N_PREFIX}.scrapTimeInfo`, { + total: list.length, + time: '12.4 seconds', + })}
-
+
{list.map(item => ( = () => { const { t } = useTranslation() - const [step, setStep] = useState(Step.running) + const [step, setStep] = useState(Step.finished) const { setShowAccountSettingModal } = useModalContext() const handleSetting = useCallback(() => { @@ -86,7 +86,6 @@ const FireCrawl: FC = () => { const [checkedCrawlResult, setCheckedCrawlResult] = useState([]) const [crawlErrorMsg, setCrawlErrorMsg] = useState('') - const showCrawlError = step === Step.finished && !!crawlErrorMsg const handleRun = useCallback(async (url: string) => { const { isValid, errorMsg } = checkValid(url) if (!isValid) { diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 87b1e57862f812..50b8669b4b25ce 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -67,6 +67,9 @@ const translation = { extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)', exceptionErrorTitle: 'An exception occurred while running Firecrawl job:', totalPageScraped: 'Total pages scraped:', + selectAll: 'Select All', + resetAll: 'Reset All', + scrapTimeInfo: 'Scraped {{total}} pages in total within {{time}}', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index a0be0f0cc100a2..e801c44261ce5d 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -67,6 +67,9 @@ const translation = { extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）', exceptionErrorTitle: '运行 Firecrawl 时发生异常:', totalPageScraped: '抓取页面总数:', + selectAll: '全选', + resetAll: '重置全部', + scrapTimeInfo: '总共在 {{time}} 内抓取了 {{total}} 个页面', }, }, stepTwo: { From c8294ac6e55a51a0a9972fbb3fba3a77022834f5 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 11:10:31 +0800 Subject: [PATCH 101/273] feat: crawl result --- .../website/firecrawl/crawled-result-item.tsx | 16 ++++++++++++---- .../create/website/firecrawl/crawled-result.tsx | 11 ++++++++++- web/i18n/en-US/dataset-creation.ts | 1 + web/i18n/zh-Hans/dataset-creation.ts | 1 + 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx index 6e3735186f19a6..1730314b477924 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx @@ -1,31 +1,39 @@ 'use client' import type { FC } from 'react' import React, { useCallback } from 'react' +import { useTranslation } from 'react-i18next' +import cn from 'classnames' import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets' import Checkbox from '@/app/components/base/checkbox' type Props = { payload: CrawlResultItemType isChecked: boolean + isPreview: boolean onCheckChange: (checked: boolean) => void + onPreview: () => void } const CrawledResultItem: FC = ({ + isPreview, payload, isChecked, onCheckChange, + onPreview, }) => { + const { t } = useTranslation() + const handleCheckChange = useCallback(() => { onCheckChange(!isChecked) }, [isChecked, onCheckChange]) return ( -
+
- +
{payload.title}
-
Preview
+
{t('datasetCreation.stepOne.website.preview')}
-
{payload.source_url}
+
{payload.source_url}
) } diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx index 48cca13b93ad3b..5e92ce45a1bc1a 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -41,6 +41,13 @@ const CrawledResult: FC = ({ } }, [checkedList, onSelectedChange]) + const [previewIndex, setPreviewIndex] = React.useState(-1) + const handlePreview = useCallback((index: number) => { + return () => { + setPreviewIndex(index) + } + }, []) + return (
@@ -55,9 +62,11 @@ const CrawledResult: FC = ({ })}
- {list.map(item => ( + {list.map((item, index) => ( checkedItem.source_url === item.source_url)} onCheckChange={handleItemCheckChange(item)} diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 50b8669b4b25ce..baa32587d92582 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -70,6 +70,7 @@ const translation = { selectAll: 'Select All', resetAll: 'Reset All', scrapTimeInfo: 'Scraped {{total}} pages in total within {{time}}', + preview: 'Preview', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index e801c44261ce5d..c8866fbd3b2d1b 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -70,6 +70,7 @@ const translation = { selectAll: '全选', resetAll: '重置全部', scrapTimeInfo: '总共在 {{time}} 内抓取了 {{total}} 个页面', + preview: '预览', }, }, stepTwo: { From 4aaa6fa2cde21e508dfebe06885150b865985c6a Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 11:30:12 +0800 Subject: [PATCH 102/273] feat: add login by mail evn --- web/Dockerfile | 1 + web/app/layout.tsx | 3 ++- web/app/signin/normalForm.tsx | 3 ++- web/config/index.ts | 2 ++ web/docker/entrypoint.sh | 1 + 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/web/Dockerfile b/web/Dockerfile index f2fc4af2f81e27..7570783f9fb465 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -38,6 +38,7 @@ ENV EDITION SELF_HOSTED ENV DEPLOY_ENV PRODUCTION ENV CONSOLE_API_URL http://127.0.0.1:5001 ENV APP_API_URL http://127.0.0.1:5001 +ENV SUPPORT_MAIL_LOGIN true ENV PORT 3000 # set timezone diff --git a/web/app/layout.tsx b/web/app/layout.tsx index ca665fb360a366..fedf66045adac1 100644 --- a/web/app/layout.tsx +++ b/web/app/layout.tsx @@ -39,11 +39,12 @@ const LocaleLayout = ({ data-api-prefix={process.env.NEXT_PUBLIC_API_PREFIX} data-pubic-api-prefix={process.env.NEXT_PUBLIC_PUBLIC_API_PREFIX} data-public-edition={process.env.NEXT_PUBLIC_EDITION} + data-public-support-mail-login={process.env.NEXT_PUBLIC_SUPPORT_MAIL_LOGIN} data-public-sentry-dsn={process.env.NEXT_PUBLIC_SENTRY_DSN} data-public-maintenance-notice={process.env.NEXT_PUBLIC_MAINTENANCE_NOTICE} data-public-site-about={process.env.NEXT_PUBLIC_SITE_ABOUT} > - + {children} diff --git a/web/app/signin/normalForm.tsx b/web/app/signin/normalForm.tsx index aba658b3ce2298..e9026e368791d8 100644 --- a/web/app/signin/normalForm.tsx +++ b/web/app/signin/normalForm.tsx @@ -7,7 +7,7 @@ import useSWR from 'swr' import Link from 'next/link' import Toast from '../components/base/toast' import style from './page.module.css' -import { IS_CE_EDITION, apiPrefix } from '@/config' +import { IS_CE_EDITION, SUPPORT_MAIL_LOGIN, apiPrefix } from '@/config' import Button from '@/app/components/base/button' import { login, oauth } from '@/service/common' import { getPurifyHref } from '@/utils' @@ -61,6 +61,7 @@ function reducer(state: IState, action: IAction) { } const NormalForm = () => { + console.log(SUPPORT_MAIL_LOGIN) const { t } = useTranslation() const router = useRouter() diff --git a/web/config/index.ts b/web/config/index.ts index 933c479b7c9e6e..16f458d58023fd 100644 --- a/web/config/index.ts +++ b/web/config/index.ts @@ -33,6 +33,8 @@ export const PUBLIC_API_PREFIX: string = publicApiPrefix const EDITION = process.env.NEXT_PUBLIC_EDITION || globalThis.document?.body?.getAttribute('data-public-edition') || 'SELF_HOSTED' export const IS_CE_EDITION = EDITION === 'SELF_HOSTED' +export const SUPPORT_MAIL_LOGIN = !!(process.env.NEXT_PUBLIC_SUPPORT_MAIL_LOGIN || globalThis.document?.body?.getAttribute('data-public-support-mail-login')) + export const TONE_LIST = [ { id: 1, diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 9c30bce74889a7..fd50bb61d1ba90 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -14,6 +14,7 @@ set -e export NEXT_PUBLIC_DEPLOY_ENV=${DEPLOY_ENV} export NEXT_PUBLIC_EDITION=${EDITION} +export NEXT_PUBLIC_SUPPORT_MAIL_LOGIN=${SUPPORT_MAIL_LOGIN} export NEXT_PUBLIC_API_PREFIX=${CONSOLE_API_URL}/console/api export NEXT_PUBLIC_PUBLIC_API_PREFIX=${APP_API_URL}/api From 0f5f1150b6d902f85b3d531d571331f6aff5ddea Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 13:43:37 +0800 Subject: [PATCH 103/273] feat: use email login --- web/app/signin/normalForm.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/app/signin/normalForm.tsx b/web/app/signin/normalForm.tsx index e9026e368791d8..f6abf0d6c0f383 100644 --- a/web/app/signin/normalForm.tsx +++ b/web/app/signin/normalForm.tsx @@ -61,8 +61,9 @@ function reducer(state: IState, action: IAction) { } const NormalForm = () => { - console.log(SUPPORT_MAIL_LOGIN) const { t } = useTranslation() + const useEmailLogin = IS_CE_EDITION || SUPPORT_MAIL_LOGIN + const router = useRouter() const [state, dispatch] = useReducer(reducer, { @@ -151,7 +152,7 @@ const NormalForm = () => {
- {!IS_CE_EDITION && ( + {!useEmailLogin && (
@@ -195,7 +196,7 @@ const NormalForm = () => { )} { - IS_CE_EDITION && <> + useEmailLogin && <> {/*
) } diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx index 5e92ce45a1bc1a..bf451e2eaa8a71 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -12,12 +12,14 @@ type Props = { list: CrawlResultItem[] checkedList: CrawlResultItem[] onSelectedChange: (selected: CrawlResultItem[]) => void + onPreview: (payload: CrawlResultItem) => void } const CrawledResult: FC = ({ list, checkedList, onSelectedChange, + onPreview, }) => { const { t } = useTranslation() @@ -45,8 +47,9 @@ const CrawledResult: FC = ({ const handlePreview = useCallback((index: number) => { return () => { setPreviewIndex(index) + onPreview(list[index]) } - }, []) + }, [list, onPreview]) return (
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 7c27d96803a84e..1007551978d09e 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -21,8 +21,9 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website' // const testCrawlErrorMsg = 'Firecrawl currently does not support social media scraping due to policy restrictions. We are actively working on building support for it.' type Props = { - + onPreview: (payload: CrawlResultItem) => void } + const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { crawl_sub_pages: true, only_main_content: true, @@ -38,7 +39,9 @@ enum Step { finished = 'finished', } -const FireCrawl: FC = () => { +const FireCrawl: FC = ({ + onPreview, +}) => { const { t } = useTranslation() const [step, setStep] = useState(Step.finished) @@ -126,6 +129,7 @@ const FireCrawl: FC = () => { list={crawlResult} checkedList={checkedCrawlResult} onSelectedChange={setCheckedCrawlResult} + onPreview={onPreview} /> )} diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx index b85d986f798418..d3f94e314995ca 100644 --- a/web/app/components/datasets/create/website/index.tsx +++ b/web/app/components/datasets/create/website/index.tsx @@ -4,12 +4,15 @@ import React, { useCallback, useState } from 'react' import NoData from './no-data' import Firecrawl from './firecrawl' import { useModalContext } from '@/context/modal-context' +import type { CrawlResultItem } from '@/models/datasets' type Props = { - + onPreview: (payload: CrawlResultItem) => void } -const WebsitePreview: FC = () => { +const WebsitePreview: FC = ({ + onPreview, +}) => { const { setShowAccountSettingModal } = useModalContext() const [isLoaded, setIsLoaded] = useState(false) const [isConfigured, setIsConfigured] = useState(true) @@ -29,7 +32,7 @@ const WebsitePreview: FC = () => {
{isConfigured ? ( - + ) : ( diff --git a/web/app/components/datasets/create/website/preview.tsx b/web/app/components/datasets/create/website/preview.tsx new file mode 100644 index 00000000000000..322ce43b174d5b --- /dev/null +++ b/web/app/components/datasets/create/website/preview.tsx @@ -0,0 +1,41 @@ +'use client' +import React from 'react' +import { useTranslation } from 'react-i18next' +import cn from 'classnames' +import { XMarkIcon } from '@heroicons/react/20/solid' +import s from '../file-preview/index.module.css' +import type { CrawlResultItem } from '@/models/datasets' + +type IProps = { + payload: CrawlResultItem + hidePreview: () => void +} + +const WebsitePreview = ({ + payload, + hidePreview, +}: IProps) => { + const { t } = useTranslation() + + return ( +
+
+
+ {t('datasetCreation.stepOne.pagePreview')} +
+ +
+
+
+ {payload.title} +
+
{payload.source_url}
+
+
+
{payload.markdown}
+
+
+ ) +} + +export default WebsitePreview From b9349b2a72c81675a35c4f2817cfaa86f66fcad6 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 6 Jun 2024 15:02:08 +0800 Subject: [PATCH 105/273] add total and current page number --- api/core/rag/extractor/firecrawl/firecrawl_app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index d30eb4b9a538e9..7bb18bfc845a99 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -63,6 +63,8 @@ def check_crawl_status(self, job_id) -> dict: if crawl_status_response.get('status') != 'completed': return { 'status': crawl_status_response.get('status'), + 'total': crawl_status_response.get('total'), + 'current': crawl_status_response.get('current'), 'data': [] } else: @@ -78,6 +80,8 @@ def check_crawl_status(self, job_id) -> dict: url_data_list.append(url_data) return { 'status': 'completed', + 'total': crawl_status_response.get('total'), + 'current': crawl_status_response.get('current'), 'data': url_data_list } else: From 1d3dc21c4ebb37348e4dcc759b9d6be2344bde5d Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 15:15:09 +0800 Subject: [PATCH 106/273] feat: finish step one --- web/app/components/datasets/create/index.tsx | 6 +++++- .../components/datasets/create/step-one/index.tsx | 12 ++++++++++-- .../datasets/create/website/firecrawl/index.tsx | 7 +++++-- web/app/components/datasets/create/website/index.tsx | 10 +++++++++- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 83d74b55baefd6..0a29252a871379 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -8,7 +8,7 @@ import StepOne from './step-one' import StepTwo from './step-two' import StepThree from './step-three' import { DataSourceType } from '@/models/datasets' -import type { DataSet, FileItem, createDocumentResponse } from '@/models/datasets' +import type { CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets' import { fetchDataSource } from '@/service/common' import { fetchDatasetDetail } from '@/service/datasets' import type { NotionPage } from '@/models/common' @@ -36,6 +36,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { setNotionPages(value) } + const [websitePages, setWebsitePages] = useState([]) + console.log(websitePages) const updateFileList = (preparedFiles: FileItem[]) => { setFiles(preparedFiles) } @@ -121,6 +123,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { notionPages={notionPages} updateNotionPages={updateNotionPages} onStepChange={nextStep} + websitePages={websitePages} + updateWebsitePages={setWebsitePages} />} {(step === 2 && (!datasetId || (datasetId && !!detail))) && void onStepChange: () => void changeType: (type: DataSourceType) => void + websitePages?: CrawlResultItem[] + updateWebsitePages: (value: CrawlResultItem[]) => void } type NotionConnectorProps = { @@ -62,6 +64,8 @@ const StepOne = ({ updateFile, notionPages = [], updateNotionPages, + websitePages = [], + updateWebsitePages, }: IStepOneProps) => { const { dataset } = useDatasetDetailContext() const [showModal, setShowModal] = useState(false) @@ -210,14 +214,18 @@ const StepOne = ({ {dataSourceType === DataSourceType.WEB && ( <>
- +
{isShowVectorSpaceFull && (
)} - + )} {!datasetId && ( diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 1007551978d09e..80e36392483a80 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -22,6 +22,8 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website' type Props = { onPreview: (payload: CrawlResultItem) => void + checkedCrawlResult: CrawlResultItem[] + onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void } const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { @@ -41,6 +43,8 @@ enum Step { const FireCrawl: FC = ({ onPreview, + checkedCrawlResult, + onCheckedCrawlResultChange, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.finished) @@ -86,7 +90,6 @@ const FireCrawl: FC = ({ const isCrawlFinished = step === Step.finished const isRunning = step === Step.running const [crawlResult, setCrawlResult] = useState(mockCrawlResult) - const [checkedCrawlResult, setCheckedCrawlResult] = useState([]) const [crawlErrorMsg, setCrawlErrorMsg] = useState('') const handleRun = useCallback(async (url: string) => { @@ -128,7 +131,7 @@ const FireCrawl: FC = ({ )} diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx index d3f94e314995ca..799bcb76ecf1ae 100644 --- a/web/app/components/datasets/create/website/index.tsx +++ b/web/app/components/datasets/create/website/index.tsx @@ -8,10 +8,14 @@ import type { CrawlResultItem } from '@/models/datasets' type Props = { onPreview: (payload: CrawlResultItem) => void + checkedCrawlResult: CrawlResultItem[] + onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void } const WebsitePreview: FC = ({ onPreview, + checkedCrawlResult, + onCheckedCrawlResultChange, }) => { const { setShowAccountSettingModal } = useModalContext() const [isLoaded, setIsLoaded] = useState(false) @@ -32,7 +36,11 @@ const WebsitePreview: FC = ({
{isConfigured ? ( - + ) : ( From 042bc4f7b34275f63b9e700ef27b602c31d1ff0a Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 15:53:06 +0800 Subject: [PATCH 107/273] feat: step two --- web/app/components/datasets/create/index.tsx | 1 + .../datasets/create/step-two/index.module.css | 1 + .../datasets/create/step-two/index.tsx | 45 ++++++++++++++++++- web/i18n/en-US/dataset-creation.ts | 2 + web/i18n/zh-Hans/dataset-creation.ts | 2 + web/models/datasets.ts | 7 ++- 6 files changed, 56 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 0a29252a871379..1d7d3c6b0e9a9e 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -134,6 +134,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { dataSourceType={dataSourceType} files={fileList.map(file => file.file)} notionPages={notionPages} + websitePages={websitePages} onStepChange={changeStep} updateIndexingTypeCache={updateIndexingTypeCache} updateResultCache={updateResultCache} diff --git a/web/app/components/datasets/create/step-two/index.module.css b/web/app/components/datasets/create/step-two/index.module.css index b5089b5fa327b4..24a62c8e3c9054 100644 --- a/web/app/components/datasets/create/step-two/index.module.css +++ b/web/app/components/datasets/create/step-two/index.module.css @@ -323,6 +323,7 @@ } .sourceContent { + width: 0; flex: 1 1 auto; } diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 90b798025ed2f3..35930b77f99666 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -12,7 +12,7 @@ import RetrievalMethodInfo from '../../common/retrieval-method-info' import PreviewItem, { PreviewType } from './preview-item' import LanguageSelect from './language-select' import s from './index.module.css' -import type { CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' +import type { CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' import { createDocument, createFirstDocument, @@ -44,6 +44,7 @@ import TooltipPlus from '@/app/components/base/tooltip-plus' import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { LanguagesSupported } from '@/i18n/language' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel' type ValueOf = T[keyof T] type StepTwoProps = { @@ -56,6 +57,7 @@ type StepTwoProps = { dataSourceType: DataSourceType files: CustomFile[] notionPages?: NotionPage[] + websitePages?: CrawlResultItem[] onStepChange?: (delta: number) => void updateIndexingTypeCache?: (type: string) => void updateResultCache?: (res: createDocumentResponse) => void @@ -82,6 +84,7 @@ const StepTwo = ({ dataSourceType, files, notionPages = [], + websitePages = [], onStepChange, updateIndexingTypeCache, updateResultCache, @@ -242,6 +245,14 @@ const StepTwo = ({ }) as NotionInfo[] } + const getWebsiteInfo = () => { + return { + provider: 'firecrawl', + job_id: 'xxx', + urls: websitePages.map(page => page.source_url), + } + } + const getFileIndexingEstimateParams = (docForm: DocForm): IndexingEstimateParams | undefined => { if (dataSourceType === DataSourceType.FILE) { return { @@ -271,6 +282,19 @@ const StepTwo = ({ dataset_id: datasetId as string, } } + if (dataSourceType === DataSourceType.WEB) { + return { + info_list: { + data_source_type: dataSourceType, + website_info_list: getWebsiteInfo(), + }, + indexing_technique: getIndexing_technique() as string, + process_rule: getProcessRule(), + doc_form: docForm, + doc_language: docLanguage, + dataset_id: datasetId as string, + } + } } const { modelList: rerankModelList, @@ -335,6 +359,9 @@ const StepTwo = ({ } if (dataSourceType === DataSourceType.NOTION) params.data_source.info_list.notion_info_list = getNotionInfo() + + if (dataSourceType === DataSourceType.WEB) + params.data_source.info_list.website_info_list = getWebsiteInfo() } return params } @@ -819,6 +846,22 @@ const StepTwo = ({
)} + {dataSourceType === DataSourceType.WEB && ( + <> +
{t('datasetCreation.stepTwo.websiteSource')}
+
+ + {websitePages[0].source_url} + {websitePages.length > 1 && ( + + {t('datasetCreation.stepTwo.other')} + {websitePages.length - 1} + {t('datasetCreation.stepTwo.webpageUnit')} + + )} +
+ + )}
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index baa32587d92582..c539319a0d01f7 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -108,9 +108,11 @@ const translation = { calculating: 'Calculating...', fileSource: 'Preprocess documents', notionSource: 'Preprocess pages', + websiteSource: 'Preprocess website', other: 'and other ', fileUnit: ' files', notionUnit: ' pages', + webpageUnit: ' pages', previousStep: 'Previous step', nextStep: 'Save & Process', save: 'Save & Process', diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index c8866fbd3b2d1b..15de59b3e5750c 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -108,9 +108,11 @@ const translation = { calculating: '计算中...', fileSource: '预处理文档', notionSource: '预处理页面', + websiteSource: '预处理页面', other: '和其他 ', fileUnit: ' 个文件', notionUnit: ' 个页面', + webpageUnit: ' 个页面', previousStep: '上一步', nextStep: '保存并处理', save: '保存并处理', diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 1edb63748faadd..47b786c7c9b8f8 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -5,7 +5,7 @@ import type { Tag } from '@/app/components/base/tag-management/constant' export enum DataSourceType { FILE = 'upload_file', NOTION = 'notion_import', - WEB = 'web_import', + WEB = 'website', } export type DataSet = { @@ -229,6 +229,11 @@ export type DataSource = { file_info_list?: { file_ids: string[] } + website_info_list?: { + provider: string + job_id: string + urls: string[] + } } } From 131f1522556fd8a5a987669786bb573d0347f8d2 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 6 Jun 2024 15:57:09 +0800 Subject: [PATCH 108/273] merge migration --- .../versions/7b45942e39bb_add_api_key_auth_binding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py b/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py index 1591e87f7ee592..30992e00c9c01b 100644 --- a/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py +++ b/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = '7b45942e39bb' -down_revision = '47cc7df8c4f3' +down_revision = '4e99a8df00ff' branch_labels = None depends_on = None From dcf988231499b3fcc13af1c97c09832a45b0148b Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 16:25:12 +0800 Subject: [PATCH 109/273] feat: list page --- web/app/components/datasets/create/index.tsx | 1 - web/app/components/datasets/documents/list.tsx | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index cc21618e13aa79..30ee370be54753 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -37,7 +37,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { } const [websitePages, setWebsitePages] = useState([]) - console.log(websitePages) const updateFileList = (preparedFiles: FileItem[]) => { setFiles(preparedFiles) } diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 4566287fb81aca..611fdd3241ad05 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -13,6 +13,7 @@ import cn from 'classnames' import dayjs from 'dayjs' import { Edit03 } from '../../base/icons/src/vender/solid/general' import TooltipPlus from '../../base/tooltip-plus' +import { Globe01 } from '../../base/icons/src/vender/line/mapsAndTravel' import s from './style.module.css' import RenameModal from './rename-modal' import Switch from '@/app/components/base/switch' @@ -418,10 +419,10 @@ const DocumentList: FC = ({ embeddingAvailable, documents =
- { - doc?.data_source_type === DataSourceType.NOTION - ? - :
+ {doc?.data_source_type === DataSourceType.NOTION && + } + {doc?.data_source_type === DataSourceType.FILE &&
} + {doc?.data_source_type === DataSourceType.WEB && } { doc.name @@ -450,6 +451,7 @@ const DocumentList: FC = ({ embeddingAvailable, documents = { + // TODO: Maybe add website (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) && doc?.data_source_type === DataSourceType.NOTION) ? : From c9dda6ead92e3b2876b5ff0dddc4936e29e92de5 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 17:21:29 +0800 Subject: [PATCH 110/273] feat: config firecrawl main --- .../create/website/firecrawl/base/field.tsx | 4 +- .../config-firecrawl-modal.tsx | 139 ++++++++++++++++++ .../data-source-website/index.tsx | 40 ++--- web/models/common.ts | 6 + web/service/datasets.ts | 4 + 5 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx index 12eaefa9b85f03..9cf978d0af191d 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx @@ -7,6 +7,7 @@ import Input from './input' type Props = { className?: string label: string + labelClassName?: string value: string | number onChange: (value: string | number) => void isRequired?: boolean @@ -17,6 +18,7 @@ type Props = { const Field: FC = ({ className, label, + labelClassName, value, onChange, isRequired = false, @@ -26,7 +28,7 @@ const Field: FC = ({ return (
-
{label}
+
{label}
{isRequired && *}
void + onSaved: () => void +} + +const DEFAULT_BASE_URL = 'https://api.firecrawl.dev' + +const ConfigFirecrawlModal: FC = ({ + onCancel, + onSaved, +}) => { + const { t } = useTranslation() + const [config, setConfig] = useState({ + api_key: '', + base_url: '', + }) + + const handleConfigChange = useCallback((key: string) => { + return (value: string | number) => { + setConfig(prev => ({ ...prev, [key]: value as string })) + } + }, []) + + const handleSave = useCallback(async () => { + if (!config.api_key) { + const errorMsg = t('common.errorMsg.fieldRequired', { + field: 'API Key', + }) + Toast.notify({ + type: 'error', + message: errorMsg, + }) + return + } + const postData = { + category: 'website', + provider: 'firecrawl', + credentials: { + auth_type: 'bearer', + config: { + api_key: config.api_key, + base_url: config.base_url || DEFAULT_BASE_URL, + }, + }, + } + await createFirecrawlApiKey(postData) + Toast.notify({ + type: 'success', + message: t('common.operation.saveSuccess'), + }) + onSaved() + }, [config.api_key, config.base_url, onSaved, t]) + + return ( + + + + + + ) +} +export default React.memo(ConfigFirecrawlModal) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx index 489af9842cbc03..d3da70ce96cde6 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx @@ -5,6 +5,7 @@ import { useBoolean } from 'ahooks' import cn from 'classnames' import Panel from '../panel' import { DataSourceType } from '../panel/types' +import ConfigFirecrawlModal from './config-firecrawl-modal' import { fetchWebsiteDataSource } from '@/service/common' import type { DataSourceWebsiteItem, @@ -18,7 +19,7 @@ import { type Props = {} -const isUseMock = true +const isUseMock = false const mockList: DataSourceWebsiteItem[] = [ { id: '1', @@ -51,29 +52,34 @@ const DataSourceWebsite: FC = () => { const [isShowConfig, { setTrue: showConfig, setFalse: hideConfig, - }] = useBoolean(false) + }] = useBoolean(true) const handleRemove = useCallback(() => { }, []) return ( - 0} - onConfigure={showConfig} - readonly={!isCurrentWorkspaceManager} - configuredList={list.map(item => ({ - id: item.id, - logo: ({ className }: { className: string }) => ( -
🔥
- ), - name: 'FireCrawl', - isActive: true, - }))} - onRemove={handleRemove} + <> + 0} + onConfigure={showConfig} + readonly={!isCurrentWorkspaceManager} + configuredList={list.map(item => ({ + id: item.id, + logo: ({ className }: { className: string }) => ( +
🔥
+ ), + name: 'FireCrawl', + isActive: true, + }))} + onRemove={handleRemove} + /> + {isShowConfig && ( + + )} + - /> ) } export default React.memo(DataSourceWebsite) diff --git a/web/models/common.ts b/web/models/common.ts index a8322e2478206f..ec10ff2b88c4ae 100644 --- a/web/models/common.ts +++ b/web/models/common.ts @@ -185,6 +185,12 @@ export type WebsiteCredentials = { api_key: string } } + +export type FirecrawlConfig = { + api_key: string + base_url: string +} + export type DataSourceWebsiteItem = { id: string category: DataSourceCategory.website diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 302b16f6f55567..200785b02db01c 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -227,6 +227,10 @@ export const fetchDatasetApiBaseUrl: Fetcher<{ api_base_url: string }, string> = return get<{ api_base_url: string }>(url) } +export const createFirecrawlApiKey: Fetcher> = (body) => { + return post('api-key-auth/data-source/binding', { body }) +} + type FileTypesRes = { allowed_extensions: string[] } From a1e73bb2b183a123fbda5d2b4603bffc2b1cf4bf Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 17:49:14 +0800 Subject: [PATCH 111/273] chore: config i18n --- .../create/website/firecrawl/index.tsx | 2 -- .../config-firecrawl-modal.tsx | 27 ++++++++++++++----- .../data-source-website/index.tsx | 2 +- web/i18n/en-US/dataset-creation.ts | 6 +++++ web/i18n/zh-Hans/dataset-creation.ts | 6 +++++ 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 80e36392483a80..8bee207976dc0c 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -18,8 +18,6 @@ import { sleep } from '@/utils' const ERROR_I18N_PREFIX = 'common.errorMsg' const I18N_PREFIX = 'datasetCreation.stepOne.website' -// const testCrawlErrorMsg = 'Firecrawl currently does not support social media scraping due to policy restrictions. We are actively working on building support for it.' - type Props = { onPreview: (payload: CrawlResultItem) => void checkedCrawlResult: CrawlResultItem[] diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx index e8d23fb3eae798..f2f71d0b1d2975 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx @@ -18,6 +18,8 @@ type Props = { onSaved: () => void } +const I18N_PREFIX = 'datasetCreation.firecrawl' + const DEFAULT_BASE_URL = 'https://api.firecrawl.dev' const ConfigFirecrawlModal: FC = ({ @@ -37,10 +39,21 @@ const ConfigFirecrawlModal: FC = ({ }, []) const handleSave = useCallback(async () => { - if (!config.api_key) { - const errorMsg = t('common.errorMsg.fieldRequired', { - field: 'API Key', - }) + let errorMsg = '' + if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://')))) + errorMsg = t('common.errorMsg.urlError') + if (!errorMsg) { + if (!config.api_key) { + errorMsg = t('common.errorMsg.fieldRequired', { + field: 'API Key', + }) + } + else if (!config.api_key.startsWith('fc-')) { + errorMsg = t(`${I18N_PREFIX}.apiKeyFormatError`) + } + } + + if (errorMsg) { Toast.notify({ type: 'error', message: errorMsg, @@ -73,7 +86,7 @@ const ConfigFirecrawlModal: FC = ({
-
{t('common.modelProvider.editConfig')}
+
{t(`${I18N_PREFIX}.configFirecrawl`)}
@@ -90,12 +103,12 @@ const ConfigFirecrawlModal: FC = ({ isRequired value={config.api_key} onChange={handleConfigChange('api_key')} - placeholder={t('common.modelProvider.apiKey')!} + placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!} />
- Get your API key from firecrawl.dev + {t(`${I18N_PREFIX}.getApiKeyLinkText`)}
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx index d3da70ce96cde6..e56d19eece3870 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx @@ -52,7 +52,7 @@ const DataSourceWebsite: FC = () => { const [isShowConfig, { setTrue: showConfig, setFalse: hideConfig, - }] = useBoolean(true) + }] = useBoolean(false) const handleRemove = useCallback(() => { diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 8e63a466a78ba3..466b0afc3ad16c 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -11,6 +11,12 @@ const translation = { error: { unavailable: 'This Knowledge is not available', }, + firecrawl: { + configFirecrawl: 'Configure 🔥Firecrawl', + apiKeyPlaceholder: 'API key from firecrawl.dev, starting with "fc-"', + apiKeyFormatError: 'API key should start with "fc-"', + getApiKeyLinkText: 'Get your API key from firecrawl.dev', + }, stepOne: { filePreview: 'File Preview', pagePreview: 'Page Preview', diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 15de59b3e5750c..015d3b61c8fd29 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -11,6 +11,12 @@ const translation = { error: { unavailable: '该知识库不可用', }, + firecrawl: { + configFirecrawl: '配置 🔥Firecrawl', + apiKeyPlaceholder: '从 firecrawl.dev 获取 API Key，以 "fc-" 开头', + apiKeyFormatError: 'API Key 应以 "fc-" 开头', + getApiKeyLinkText: '从 firecrawl.dev 获取您的 API Key', + }, stepOne: { filePreview: '文件预览', pagePreview: '页面预览', From 05674c5e3dec9339d20ae2c6c99c830ed987476a Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 6 Jun 2024 18:06:15 +0800 Subject: [PATCH 112/273] optimize firecrawl error msg --- api/controllers/console/auth/data_source_bearer_auth.py | 6 +++++- api/services/auth/firecrawl.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index ab0cdbdc23126c..f4712036732302 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -5,6 +5,7 @@ from controllers.console import api from libs.login import login_required from services.auth.api_key_auth_service import ApiKeyAuthService +from .error import ApiKeyAuthFailedError from ..setup import setup_required from ..wraps import account_initialization_required @@ -40,7 +41,10 @@ def post(self): parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') args = parser.parse_args() ApiKeyAuthService.validate_api_key_auth_args(args) - ApiKeyAuthService.create_provider_auth(current_user.current_tenant_id, args) + try: + ApiKeyAuthService.create_provider_auth(current_user.current_tenant_id, args) + except Exception as e: + raise ApiKeyAuthFailedError(str(e)) return {'result': 'success'}, 200 diff --git a/api/services/auth/firecrawl.py b/api/services/auth/firecrawl.py index ee219a1bcb445b..69e3fb43c79dab 100644 --- a/api/services/auth/firecrawl.py +++ b/api/services/auth/firecrawl.py @@ -1,3 +1,5 @@ +import json + import requests from services.auth.api_key_auth_base import ApiKeyAuthBase @@ -48,4 +50,7 @@ def _handle_error(self, response): error_message = response.json().get('error', 'Unknown error occurred') raise Exception(f'Failed to authorize. Status code: {response.status_code}. Error: {error_message}') else: + if response.text: + error_message = json.loads(response.text).get('error', 'Unknown error occurred') + raise Exception(f'Failed to authorize. Status code: {response.status_code}. Error: {error_message}') raise Exception(f'Unexpected error occurred while trying to authorize. Status code: {response.status_code}') From 980ce12976cb88820b6ec0b3cc3ecca8a990d633 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 6 Jun 2024 18:19:06 +0800 Subject: [PATCH 113/273] feat: fetch and remove --- .../data-source-website/index.tsx | 43 +++++++------------ web/models/common.ts | 1 + web/service/datasets.ts | 8 ++++ 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx index e56d19eece3870..4992598296c321 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx @@ -1,50 +1,34 @@ 'use client' import type { FC } from 'react' import React, { useCallback, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import cn from 'classnames' import Panel from '../panel' import { DataSourceType } from '../panel/types' import ConfigFirecrawlModal from './config-firecrawl-modal' -import { fetchWebsiteDataSource } from '@/service/common' +import { fetchFirecrawlApiKey, removeFirecrawlApiKey } from '@/service/datasets' + import type { DataSourceWebsiteItem, } from '@/models/common' import { useAppContext } from '@/context/app-context' import { - DataSourceCategory, WebsiteProvider, } from '@/models/common' +import Toast from '@/app/components/base/toast' type Props = {} -const isUseMock = false -const mockList: DataSourceWebsiteItem[] = [ - { - id: '1', - category: DataSourceCategory.website, - provider: WebsiteProvider.fireCrawl, - credentials: { - auth_type: 'bearer', - config: { - base_url: 'https://xxx', - api_key: '123456', - }, - }, - created_at: 1627584000, - updated_at: 1627584000, - }, -] - const DataSourceWebsite: FC = () => { + const { t } = useTranslation() const { isCurrentWorkspaceManager } = useAppContext() - const [list, setList] = useState(isUseMock ? mockList : []) + const [list, setList] = useState([]) useEffect(() => { (async () => { - const { data } = await fetchWebsiteDataSource() - const list = data.settings.filter(item => item.provider === WebsiteProvider.fireCrawl) - + const res = await fetchFirecrawlApiKey() as any + const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled) setList(list) })() }, []) @@ -54,9 +38,14 @@ const DataSourceWebsite: FC = () => { setFalse: hideConfig, }] = useBoolean(false) - const handleRemove = useCallback(() => { - - }, []) + const handleRemove = useCallback(async () => { + await removeFirecrawlApiKey(list[0].id) + setList([]) + Toast.notify({ + type: 'success', + message: t('common.operation.saveSuccess'), + }) + }, [list, t]) return ( <> diff --git a/web/models/common.ts b/web/models/common.ts index ec10ff2b88c4ae..da52b0c7f75f45 100644 --- a/web/models/common.ts +++ b/web/models/common.ts @@ -196,6 +196,7 @@ export type DataSourceWebsiteItem = { category: DataSourceCategory.website provider: WebsiteProvider credentials: WebsiteCredentials + disabled: boolean created_at: number updated_at: number } diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 200785b02db01c..b620f2133ecfa2 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -227,10 +227,18 @@ export const fetchDatasetApiBaseUrl: Fetcher<{ api_base_url: string }, string> = return get<{ api_base_url: string }>(url) } +export const fetchFirecrawlApiKey = () => { + return get('api-key-auth/data-source') +} + export const createFirecrawlApiKey: Fetcher> = (body) => { return post('api-key-auth/data-source/binding', { body }) } +export const removeFirecrawlApiKey: Fetcher = (id: string) => { + return del(`api-key-auth/data-source/${id}`) +} + type FileTypesRes = { allowed_extensions: string[] } From 1ce9844e5a43e678fbbbb9cd407669742f325f0b Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 6 Jun 2024 18:23:46 +0800 Subject: [PATCH 114/273] optimize firecrawl error msg --- api/controllers/console/auth/data_source_bearer_auth.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index f4712036732302..14fc1197c6ddb5 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -5,8 +5,6 @@ from controllers.console import api from libs.login import login_required from services.auth.api_key_auth_service import ApiKeyAuthService -from .error import ApiKeyAuthFailedError - from ..setup import setup_required from ..wraps import account_initialization_required @@ -44,7 +42,7 @@ def post(self): try: ApiKeyAuthService.create_provider_auth(current_user.current_tenant_id, args) except Exception as e: - raise ApiKeyAuthFailedError(str(e)) + return {'error': str(e)}, 500 return {'result': 'success'}, 200 From 43f4e3a5a13c68293b7ab3c6004d5ec569bca685 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 6 Jun 2024 18:26:34 +0800 Subject: [PATCH 115/273] optimize firecrawl error msg --- api/controllers/console/auth/data_source_bearer_auth.py | 4 +++- api/controllers/console/auth/error.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 api/controllers/console/auth/error.py diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index 14fc1197c6ddb5..dd6f631a11e9b0 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -5,6 +5,8 @@ from controllers.console import api from libs.login import login_required from services.auth.api_key_auth_service import ApiKeyAuthService +from controllers.console.auth.error import ApiKeyAuthFailedError + from ..setup import setup_required from ..wraps import account_initialization_required @@ -42,7 +44,7 @@ def post(self): try: ApiKeyAuthService.create_provider_auth(current_user.current_tenant_id, args) except Exception as e: - return {'error': str(e)}, 500 + raise ApiKeyAuthFailedError(str(e)) return {'result': 'success'}, 200 diff --git a/api/controllers/console/auth/error.py b/api/controllers/console/auth/error.py new file mode 100644 index 00000000000000..c55ff8707d224f --- /dev/null +++ b/api/controllers/console/auth/error.py @@ -0,0 +1,7 @@ +from libs.exception import BaseHTTPException + + +class ApiKeyAuthFailedError(BaseHTTPException): + error_code = 'auth_failed' + description = "{message}" + code = 500 From 2715a5a339a7cbb61286e8090097409db1fefa3a Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 6 Jun 2024 18:31:14 +0800 Subject: [PATCH 116/273] optimize firecrawl error msg --- api/controllers/console/auth/data_source_bearer_auth.py | 3 ++- api/controllers/console/auth/error.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 api/controllers/console/auth/error.py diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index 14fc1197c6ddb5..eddb67f4bc30b9 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -1,6 +1,7 @@ from flask_login import current_user from flask_restful import Resource, reqparse from werkzeug.exceptions import Forbidden +from controllers.console.auth.error import ApiKeyAuthFailedError from controllers.console import api from libs.login import login_required @@ -42,7 +43,7 @@ def post(self): try: ApiKeyAuthService.create_provider_auth(current_user.current_tenant_id, args) except Exception as e: - return {'error': str(e)}, 500 + raise ApiKeyAuthFailedError(str(e)) return {'result': 'success'}, 200 diff --git a/api/controllers/console/auth/error.py b/api/controllers/console/auth/error.py new file mode 100644 index 00000000000000..c55ff8707d224f --- /dev/null +++ b/api/controllers/console/auth/error.py @@ -0,0 +1,7 @@ +from libs.exception import BaseHTTPException + + +class ApiKeyAuthFailedError(BaseHTTPException): + error_code = 'auth_failed' + description = "{message}" + code = 500 From 637546aac70515184a41fea6d139b1dbf9551348 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 11:30:32 +0800 Subject: [PATCH 117/273] feat: crawl api --- .../create/website/firecrawl/index.tsx | 39 ++++++++++++++++--- web/service/datasets.ts | 17 ++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 8bee207976dc0c..db5c4592bda613 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -13,6 +13,7 @@ import Crawling from './crawling' import { useModalContext } from '@/context/modal-context' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import Toast from '@/app/components/base/toast' +import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets' import { sleep } from '@/utils' const ERROR_I18N_PREFIX = 'common.errorMsg' @@ -45,8 +46,7 @@ const FireCrawl: FC = ({ onCheckedCrawlResultChange, }) => { const { t } = useTranslation() - const [step, setStep] = useState(Step.finished) - + const [step, setStep] = useState(Step.init) const { setShowAccountSettingModal } = useModalContext() const handleSetting = useCallback(() => { setShowAccountSettingModal({ @@ -90,6 +90,25 @@ const FireCrawl: FC = ({ const [crawlResult, setCrawlResult] = useState(mockCrawlResult) const [crawlErrorMsg, setCrawlErrorMsg] = useState('') + + const waitForCrawlFinished = useCallback(async (jobId: string) => { + const res = await checkFirecrawlTaskStatus(jobId) as any + if (res.status === 'completed') { + return { + isError: false, + data: res.data, + } + } + if (res.status === 'error') { + return { + isError: true, + errorMessage: res.errorMessage, // TODO: wait for the api structure + } + } + await sleep(2500) + return await waitForCrawlFinished(jobId) + }, []) + const handleRun = useCallback(async (url: string) => { const { isValid, errorMsg } = checkValid(url) if (!isValid) { @@ -100,13 +119,21 @@ const FireCrawl: FC = ({ return } setStep(Step.running) - // TODO: crawl - await sleep(2000) - setCrawlResult(mockCrawlResult) // TODO: + const res = await createFirecrawlTask({ + url, + options: crawlOptions, + }) as any + const jobId = res.job_id + const { isError, data, errorMessage } = await waitForCrawlFinished(jobId) + if (isError) { + setCrawlErrorMsg(errorMessage) + setCrawlResult(data) + } setStep(Step.finished) + setCrawlResult(data) setCrawlErrorMsg('') - }, [checkValid]) + }, [checkValid, crawlOptions, waitForCrawlFinished]) return (
diff --git a/web/service/datasets.ts b/web/service/datasets.ts index b620f2133ecfa2..91397756048a19 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -239,6 +239,23 @@ export const removeFirecrawlApiKey: Fetcher = (id: strin return del(`api-key-auth/data-source/${id}`) } +export const createFirecrawlTask: Fetcher> = (body) => { + return post('website/crawl', { + body: { + ...body, + provider: 'firecrawl', + }, + }) +} + +export const checkFirecrawlTaskStatus: Fetcher = (jobId: string) => { + return get(`website/crawl/status/${jobId}`, { + params: { + provider: 'firecrawl', + }, + }) +} + type FileTypesRes = { allowed_extensions: string[] } From 06764b2129641536a1655e1bbac6b110fcbfb3a8 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 7 Jun 2024 14:23:45 +0800 Subject: [PATCH 118/273] add delete binding --- .../console/auth/data_source_bearer_auth.py | 15 +++++++++++++++ api/services/auth/api_key_auth_service.py | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index eddb67f4bc30b9..f93e9180ad5ba6 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -47,5 +47,20 @@ def post(self): return {'result': 'success'}, 200 +class ApiKeyAuthDataSourceBindingDelete(Resource): + @setup_required + @login_required + @account_initialization_required + def delete(self, binding_id): + # The role of the current user in the table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() + + ApiKeyAuthService.delete_provider_auth(current_user.current_tenant_id, binding_id) + + return {'result': 'success'}, 200 + + api.add_resource(ApiKeyAuthDataSource, '/api-key-auth/data-source') api.add_resource(ApiKeyAuthDataSourceBinding, '/api-key-auth/data-source/binding') +api.add_resource(ApiKeyAuthDataSourceBindingDelete, '/api-key-auth/data-source/') diff --git a/api/services/auth/api_key_auth_service.py b/api/services/auth/api_key_auth_service.py index 6cab79f6924202..43d0fbf98f2df7 100644 --- a/api/services/auth/api_key_auth_service.py +++ b/api/services/auth/api_key_auth_service.py @@ -45,6 +45,16 @@ def get_auth_credentials(tenant_id: str, category: str, provider: str): credentials = json.loads(data_source_api_key_bindings.credentials) return credentials + @staticmethod + def delete_provider_auth(tenant_id: str, binding_id: str): + data_source_api_key_binding = db.session.query(DataSourceApiKeyAuthBinding).filter( + DataSourceApiKeyAuthBinding.tenant_id == tenant_id, + DataSourceApiKeyAuthBinding.id == binding_id + ).first() + if data_source_api_key_binding: + db.session.delete(data_source_api_key_binding) + db.session.commit() + @classmethod def validate_api_key_auth_args(cls, args): if 'category' not in args or not args['category']: From a5f2b4a6e7d7a90c42715d664b0eaef6656afec8 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 14:40:20 +0800 Subject: [PATCH 119/273] feat: firecrawel init datasets --- web/app/components/datasets/create/index.tsx | 3 ++ .../datasets/create/step-one/index.tsx | 3 ++ .../datasets/create/step-two/index.tsx | 4 ++- .../create/website/firecrawl/index.tsx | 5 ++- .../datasets/create/website/index.tsx | 33 ++++++++++++++----- web/service/common.ts | 5 --- 6 files changed, 37 insertions(+), 16 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 30ee370be54753..21acb4a70bddc7 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -40,6 +40,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { const updateFileList = (preparedFiles: FileItem[]) => { setFiles(preparedFiles) } + const [fireCrawlJobId, setFireCrawlJobId] = useState('') const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => { const targetIndex = list.findIndex(file => file.fileID === fileItem.fileID) @@ -124,6 +125,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { onStepChange={nextStep} websitePages={websitePages} updateWebsitePages={setWebsitePages} + onFireCrawlJobIdChange={setFireCrawlJobId} />} {(step === 2 && (!datasetId || (datasetId && !!detail))) && { files={fileList.map(file => file.file)} notionPages={notionPages} websitePages={websitePages} + fireCrawlJobId={fireCrawlJobId} onStepChange={changeStep} updateIndexingTypeCache={updateIndexingTypeCache} updateResultCache={updateResultCache} diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index 857cf88f589935..c013bc94a50d8c 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -33,6 +33,7 @@ type IStepOneProps = { changeType: (type: DataSourceType) => void websitePages?: CrawlResultItem[] updateWebsitePages: (value: CrawlResultItem[]) => void + onFireCrawlJobIdChange: (jobId: string) => void } type NotionConnectorProps = { @@ -66,6 +67,7 @@ const StepOne = ({ updateNotionPages, websitePages = [], updateWebsitePages, + onFireCrawlJobIdChange, }: IStepOneProps) => { const { dataset } = useDatasetDetailContext() const [showModal, setShowModal] = useState(false) @@ -218,6 +220,7 @@ const StepOne = ({ onPreview={setCurrentWebsite} checkedCrawlResult={websitePages} onCheckedCrawlResultChange={updateWebsitePages} + onJobIdChange={onFireCrawlJobIdChange} />
{isShowVectorSpaceFull && ( diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index a0d085fd104edf..0ffc9da2a6be09 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -58,6 +58,7 @@ type StepTwoProps = { files: CustomFile[] notionPages?: NotionPage[] websitePages?: CrawlResultItem[] + fireCrawlJobId?: string onStepChange?: (delta: number) => void updateIndexingTypeCache?: (type: string) => void updateResultCache?: (res: createDocumentResponse) => void @@ -85,6 +86,7 @@ const StepTwo = ({ files, notionPages = [], websitePages = [], + fireCrawlJobId = '', onStepChange, updateIndexingTypeCache, updateResultCache, @@ -248,7 +250,7 @@ const StepTwo = ({ const getWebsiteInfo = () => { return { provider: 'firecrawl', - job_id: 'xxx', + job_id: fireCrawlJobId, urls: websitePages.map(page => page.source_url), } } diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index db5c4592bda613..808c444d701018 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -23,6 +23,7 @@ type Props = { onPreview: (payload: CrawlResultItem) => void checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void + onJobIdChange: (jobId: string) => void } const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { @@ -44,6 +45,7 @@ const FireCrawl: FC = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, + onJobIdChange, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) @@ -124,6 +126,7 @@ const FireCrawl: FC = ({ options: crawlOptions, }) as any const jobId = res.job_id + onJobIdChange(jobId) const { isError, data, errorMessage } = await waitForCrawlFinished(jobId) if (isError) { setCrawlErrorMsg(errorMessage) @@ -133,7 +136,7 @@ const FireCrawl: FC = ({ setStep(Step.finished) setCrawlResult(data) setCrawlErrorMsg('') - }, [checkValid, crawlOptions, waitForCrawlFinished]) + }, [checkValid, crawlOptions, onJobIdChange, waitForCrawlFinished]) return (
diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx index 799bcb76ecf1ae..9230a999fe5573 100644 --- a/web/app/components/datasets/create/website/index.tsx +++ b/web/app/components/datasets/create/website/index.tsx @@ -1,45 +1,60 @@ 'use client' import type { FC } from 'react' -import React, { useCallback, useState } from 'react' +import React, { useCallback, useEffect, useState } from 'react' import NoData from './no-data' import Firecrawl from './firecrawl' import { useModalContext } from '@/context/modal-context' import type { CrawlResultItem } from '@/models/datasets' +import { fetchFirecrawlApiKey } from '@/service/datasets' +import { type DataSourceWebsiteItem, WebsiteProvider } from '@/models/common' type Props = { onPreview: (payload: CrawlResultItem) => void checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void + onJobIdChange: (jobId: string) => void } -const WebsitePreview: FC = ({ +const Website: FC = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, + onJobIdChange, }) => { const { setShowAccountSettingModal } = useModalContext() const [isLoaded, setIsLoaded] = useState(false) - const [isConfigured, setIsConfigured] = useState(true) + const [isSetFirecrawlApiKey, setIsSetFirecrawlApiKey] = useState(false) + const checkSetApiKey = useCallback(async () => { + const res = await fetchFirecrawlApiKey() as any + const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled) + setIsSetFirecrawlApiKey(list.length > 0) + }, []) + useEffect(() => { + checkSetApiKey().then(() => { + setIsLoaded(true) + }) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) const handleOnConfig = useCallback(() => { setShowAccountSettingModal({ payload: 'data-source', + onCancelCallback: checkSetApiKey, }) - }, [setShowAccountSettingModal]) - - // TODO: on Hide account setting modal + }, [checkSetApiKey, setShowAccountSettingModal]) - if (isLoaded) + if (!isLoaded) return null return (
- {isConfigured + {isSetFirecrawlApiKey ? ( ) : ( @@ -48,4 +63,4 @@ const WebsitePreview: FC = ({
) } -export default React.memo(WebsitePreview) +export default React.memo(Website) diff --git a/web/service/common.ts b/web/service/common.ts index fa0b2ffc5ffa16..a68aeb2256391b 100644 --- a/web/service/common.ts +++ b/web/service/common.ts @@ -6,7 +6,6 @@ import type { CodeBasedExtension, CommonResponse, DataSourceNotion, - DataSourceWebsite, FileUploadConfigResponse, ICurrentWorkspace, IWorkspace, @@ -146,10 +145,6 @@ export const updateDataSourceNotionAction: Fetcher(url) } -export const fetchWebsiteDataSource = () => { - return get<{ data: DataSourceWebsite }>('/api-key-auth/data-source') -} - export const fetchPluginProviders: Fetcher = (url) => { return get(url) } From e10c625ced863c99b9935baf16ade41b2c32f89f Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 7 Jun 2024 14:46:30 +0800 Subject: [PATCH 120/273] add delete binding --- api/models/dataset.py | 2 +- api/services/website_service.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/models/dataset.py b/api/models/dataset.py index 7f98bbde153494..5fe653fdcb4487 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -270,7 +270,7 @@ class Document(db.Model): 255), nullable=False, server_default=db.text("'text_model'::character varying")) doc_language = db.Column(db.String(255), nullable=True) - DATA_SOURCES = ['upload_file', 'notion_import'] + DATA_SOURCES = ['upload_file', 'notion_import', 'website'] @property def display_status(self): diff --git a/api/services/website_service.py b/api/services/website_service.py index 6d46a1626e4c7a..6158de749d089f 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -97,6 +97,8 @@ def get_crawl_status(cls, job_id: str, provider: str) -> dict: crawl_status_data = { 'status': result.get('status', 'active'), 'job_id': job_id, + 'total': result.get('total', 0), + 'current': result.get('current', 0), 'data': result.get('data', []) } else: From 2e3b9001fc956e44535c363a82d90007ed949674 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 14:56:55 +0800 Subject: [PATCH 121/273] feat: handle error show --- .../create/website/firecrawl/base/error-message.tsx | 6 ++++-- .../create/website/firecrawl/base/options-wrap.tsx | 8 ++++---- .../datasets/create/website/firecrawl/index.tsx | 13 +++++++------ web/i18n/en-US/dataset-creation.ts | 2 +- web/i18n/zh-Hans/dataset-creation.ts | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx b/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx index 6e78286fe69a7d..3af234e09f5180 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx @@ -7,7 +7,7 @@ import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/aler type Props = { className?: string title: string - errorMsg: string + errorMsg?: string } const ErrorMessage: FC = ({ @@ -21,7 +21,9 @@ const ErrorMessage: FC = ({
{title}
-
{errorMsg}
+ {errorMsg && ( +
{errorMsg}
+ )}
) } diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx index 61c89fc10745df..aef5097c75d46b 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -13,13 +13,13 @@ type Props = { className?: string children: React.ReactNode isFilledFull?: boolean - errorMsg?: string + hasError?: boolean } const OptionsWrap: FC = ({ className = '', children, - errorMsg, + hasError, isFilledFull = false, }) => { const { t } = useTranslation() @@ -41,10 +41,10 @@ const OptionsWrap: FC = ({
{!fold && (
- {!errorMsg + {!hasError ? children : ( - + )}
)} diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 808c444d701018..106ef12cd6b4f8 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -91,7 +91,7 @@ const FireCrawl: FC = ({ const isRunning = step === Step.running const [crawlResult, setCrawlResult] = useState(mockCrawlResult) - const [crawlErrorMsg, setCrawlErrorMsg] = useState('') + const [crawlHasError, setCrawlHasError] = useState(false) const waitForCrawlFinished = useCallback(async (jobId: string) => { const res = await checkFirecrawlTaskStatus(jobId) as any @@ -102,9 +102,9 @@ const FireCrawl: FC = ({ } } if (res.status === 'error') { + // can't get the error message from the firecrawl api return { isError: true, - errorMessage: res.errorMessage, // TODO: wait for the api structure } } await sleep(2500) @@ -120,6 +120,7 @@ const FireCrawl: FC = ({ }) return } + setCrawlHasError(false) setStep(Step.running) const res = await createFirecrawlTask({ url, @@ -127,15 +128,15 @@ const FireCrawl: FC = ({ }) as any const jobId = res.job_id onJobIdChange(jobId) - const { isError, data, errorMessage } = await waitForCrawlFinished(jobId) + const { isError, data } = await waitForCrawlFinished(jobId) if (isError) { - setCrawlErrorMsg(errorMessage) + setCrawlHasError(true) setCrawlResult(data) } setStep(Step.finished) setCrawlResult(data) - setCrawlErrorMsg('') + setCrawlHasError(false) }, [checkValid, crawlOptions, onJobIdChange, waitForCrawlFinished]) return ( @@ -146,7 +147,7 @@ const FireCrawl: FC = ({ {isInit && } {isRunning diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 466b0afc3ad16c..caff36ccb7c59e 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -71,7 +71,7 @@ const translation = { excludePaths: 'Exclude paths', includeOnlyPaths: 'Include only paths', extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)', - exceptionErrorTitle: 'An exception occurred while running Firecrawl job:', + exceptionErrorTitle: 'An exception occurred while running Firecrawl job', totalPageScraped: 'Total pages scraped:', selectAll: 'Select All', resetAll: 'Reset All', diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 015d3b61c8fd29..46143ecc4cc5a6 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -71,7 +71,7 @@ const translation = { excludePaths: '排除路径', includeOnlyPaths: '仅包含路径', extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）', - exceptionErrorTitle: '运行 Firecrawl 时发生异常:', + exceptionErrorTitle: '运行 Firecrawl 时发生异常', totalPageScraped: '抓取页面总数:', selectAll: '全选', resetAll: '重置全部', From ead40f678758a0dd80691278bfc68a2275df5c54 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 7 Jun 2024 15:04:00 +0800 Subject: [PATCH 122/273] add delete binding --- api/services/dataset_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 1af49bea4b4f98..0873c11259760a 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -1031,8 +1031,8 @@ def data_source_args_validate(cls, args: dict): 'notion_info_list']: raise ValueError("Notion source info is required") if args['data_source']['type'] == 'website': - if 'website_info' not in args['data_source']['info_list'] or not args['data_source']['info_list'][ - 'website_info']: + if 'website_info_list' not in args['data_source']['info_list'] or not args['data_source']['info_list'][ + 'website_info_list']: raise ValueError("Website source info is required") @classmethod From cd01ff7c03272bee5fe6f39e8ee0a0a4283655a0 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 15:24:14 +0800 Subject: [PATCH 123/273] feat: handle prossed info --- .../create/website/firecrawl/index.tsx | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 106ef12cd6b4f8..5150d4bd5e9502 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -7,7 +7,6 @@ import Header from './header' import UrlInput from './base/url-input' import OptionsWrap from './base/options-wrap' import Options from './options' -import mockCrawlResult from './mock-crawl-result' import CrawledResult from './crawled-result' import Crawling from './crawling' import { useModalContext } from '@/context/modal-context' @@ -89,8 +88,11 @@ const FireCrawl: FC = ({ const isInit = step === Step.init const isCrawlFinished = step === Step.finished const isRunning = step === Step.running - const [crawlResult, setCrawlResult] = useState(mockCrawlResult) - + const [crawlResult, setCrawlResult] = useState<{ + current: number + total: number + data: CrawlResultItem[] + } | undefined>(undefined) const [crawlHasError, setCrawlHasError] = useState(false) const waitForCrawlFinished = useCallback(async (jobId: string) => { @@ -98,18 +100,24 @@ const FireCrawl: FC = ({ if (res.status === 'completed') { return { isError: false, - data: res.data, + data: { + ...res, + total: Math.min(res.total, parseFloat(crawlOptions.limit as string)), + }, } } if (res.status === 'error') { // can't get the error message from the firecrawl api return { isError: true, + data: { + data: [], + }, } } await sleep(2500) return await waitForCrawlFinished(jobId) - }, []) + }, [crawlOptions.limit]) const handleRun = useCallback(async (url: string) => { const { isValid, errorMsg } = checkValid(url) @@ -153,12 +161,12 @@ const FireCrawl: FC = ({ {isRunning && } {isCrawlFinished && ( Date: Fri, 7 Jun 2024 15:34:31 +0800 Subject: [PATCH 124/273] chore: remove popup --- web/app/(commonLayout)/apps/Apps.tsx | 8 ----- web/app/components/datasets/create/index.tsx | 36 +++++++++++--------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx index f4bac7e44bbf7f..4efd4ec4e38fa0 100644 --- a/web/app/(commonLayout)/apps/Apps.tsx +++ b/web/app/(commonLayout)/apps/Apps.tsx @@ -24,7 +24,6 @@ import SearchInput from '@/app/components/base/search-input' import { useStore as useTagStore } from '@/app/components/base/tag-management/store' import TagManagementModal from '@/app/components/base/tag-management' import TagFilter from '@/app/components/base/tag-management/filter' -import { useModalContext } from '@/context/modal-context' const getKey = ( pageIndex: number, @@ -50,13 +49,6 @@ const getKey = ( } const Apps = () => { - const { setShowAccountSettingModal } = useModalContext() - useEffect(() => { - setShowAccountSettingModal({ - payload: 'data-source', - }) - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) const { t } = useTranslation() const { isCurrentWorkspaceManager } = useAppContext() const showTagManagementModal = useTagStore(s => s.showTagManagementModal) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 21acb4a70bddc7..51408cce8e59e5 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -110,23 +110,25 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
- {step === 1 && setShowAccountSettingModal({ payload: 'data-source' })} - datasetId={datasetId} - dataSourceType={dataSourceType} - dataSourceTypeDisable={!!detail?.data_source_type} - changeType={setDataSourceType} - files={fileList} - updateFile={updateFile} - updateFileList={updateFileList} - notionPages={notionPages} - updateNotionPages={updateNotionPages} - onStepChange={nextStep} - websitePages={websitePages} - updateWebsitePages={setWebsitePages} - onFireCrawlJobIdChange={setFireCrawlJobId} - />} +
+ setShowAccountSettingModal({ payload: 'data-source' })} + datasetId={datasetId} + dataSourceType={dataSourceType} + dataSourceTypeDisable={!!detail?.data_source_type} + changeType={setDataSourceType} + files={fileList} + updateFile={updateFile} + updateFileList={updateFileList} + notionPages={notionPages} + updateNotionPages={updateNotionPages} + onStepChange={nextStep} + websitePages={websitePages} + updateWebsitePages={setWebsitePages} + onFireCrawlJobIdChange={setFireCrawlJobId} + /> +
{(step === 2 && (!datasetId || (datasetId && !!detail))) && setShowAccountSettingModal({ payload: 'provider' })} From 6e86c73e6c7a34781979ba0643dc286226feab04 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 15:45:47 +0800 Subject: [PATCH 125/273] fix: process show error --- web/app/components/datasets/create/index.tsx | 2 +- .../create/website/firecrawl/index.tsx | 39 +++++++++++++------ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 51408cce8e59e5..4a25b6699e5e1c 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -110,7 +110,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
-
+
setShowAccountSettingModal({ payload: 'data-source' })} diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 5150d4bd5e9502..da54e45695d580 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -96,18 +96,35 @@ const FireCrawl: FC = ({ const [crawlHasError, setCrawlHasError] = useState(false) const waitForCrawlFinished = useCallback(async (jobId: string) => { - const res = await checkFirecrawlTaskStatus(jobId) as any - if (res.status === 'completed') { - return { - isError: false, - data: { - ...res, - total: Math.min(res.total, parseFloat(crawlOptions.limit as string)), - }, + try { + const res = await checkFirecrawlTaskStatus(jobId) as any + if (res.status === 'completed') { + return { + isError: false, + data: { + ...res, + total: Math.min(res.total, parseFloat(crawlOptions.limit as string)), + }, + } } + if (res.status === 'error') { + // can't get the error message from the firecrawl api + return { + isError: true, + data: { + data: [], + }, + } + } + // update the progress + setCrawlResult({ + ...res, + total: Math.min(res.total, parseFloat(crawlOptions.limit as string)), + }) + await sleep(2500) + return await waitForCrawlFinished(jobId) } - if (res.status === 'error') { - // can't get the error message from the firecrawl api + catch (e) { return { isError: true, data: { @@ -115,8 +132,6 @@ const FireCrawl: FC = ({ }, } } - await sleep(2500) - return await waitForCrawlFinished(jobId) }, [crawlOptions.limit]) const handleRun = useCallback(async (url: string) => { From 8e8418cad5e04db76ae2ecc029568e2ad4cd1d47 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 15:49:26 +0800 Subject: [PATCH 126/273] fix: other error status --- web/app/components/datasets/create/website/firecrawl/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index da54e45695d580..e87137a2916d38 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -107,7 +107,7 @@ const FireCrawl: FC = ({ }, } } - if (res.status === 'error') { + if (res.status === 'error' || !res.status) { // can't get the error message from the firecrawl api return { isError: true, From be91a909479f52320ce95451039cd3539469cda2 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 7 Jun 2024 18:08:14 +0800 Subject: [PATCH 127/273] fix firecrawl issue --- api/controllers/console/datasets/datasets.py | 15 +++++++++++++++ .../console/datasets/datasets_document.py | 13 +++++++++++++ api/models/dataset.py | 4 ++-- api/services/dataset_service.py | 10 +++++++--- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 3f666949c8cd51..52577334acd8b5 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -311,6 +311,21 @@ def post(self): document_model=args['doc_form'] ) extract_settings.append(extract_setting) + elif args['info_list']['data_source_type'] == 'website_crawl': + website_info_list = args['info_list']['website_info_list'] + for url in website_info_list['urls']: + extract_setting = ExtractSetting( + datasource_type="website", + website_info={ + "provider": website_info_list['provider'], + "job_id": website_info_list['job_id'], + "url": url, + "mode": 'crawl', + "only_main_content": website_info_list['only_main_content'] + }, + document_model=args['doc_form'] + ) + extract_settings.append(extract_setting) else: raise ValueError('Data source type not support') indexing_runner = IndexingRunner() diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index a4b2423d62db0a..d88123adf640b5 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -465,6 +465,19 @@ def get(self, dataset_id, batch): document_model=document.doc_form ) extract_settings.append(extract_setting) + elif document.data_source_type == 'website_crawl': + extract_setting = ExtractSetting( + datasource_type="website", + website_info={ + "provider": data_source_info['provider'], + "job_id": data_source_info['job_id'], + "url": data_source_info['url'], + "mode": data_source_info['mode'], + "only_main_content": data_source_info['only_main_content'] + }, + document_model=document.doc_form + ) + extract_settings.append(extract_setting) else: raise ValueError('Data source type not support') diff --git a/api/models/dataset.py b/api/models/dataset.py index 5fe653fdcb4487..9f8b15be1a45ec 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -270,7 +270,7 @@ class Document(db.Model): 255), nullable=False, server_default=db.text("'text_model'::character varying")) doc_language = db.Column(db.String(255), nullable=True) - DATA_SOURCES = ['upload_file', 'notion_import', 'website'] + DATA_SOURCES = ['upload_file', 'notion_import', 'website_crawl'] @property def display_status(self): @@ -322,7 +322,7 @@ def data_source_detail_dict(self): 'created_at': file_detail.created_at.timestamp() } } - elif self.data_source_type == 'notion_import': + elif self.data_source_type == 'notion_import' or self.data_source_type == 'website_crawl': return json.loads(self.data_source_info) return {} diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 0873c11259760a..33b96749f39047 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -749,7 +749,9 @@ def save_document_with_dataset_id(dataset: Dataset, document_data: dict, data_source_info = { 'url': url, 'provider': website_info['provider'], - 'job_id': website_info['job_id'] + 'job_id': website_info['job_id'], + 'only_main_content': website_info.get('only_main_content', False), + 'mode': 'crawl', } document = DocumentService.build_document(dataset, dataset_process_rule.id, document_data["data_source"]["type"], @@ -887,7 +889,9 @@ def update_document_with_dataset_id(dataset: Dataset, document_data: dict, data_source_info = { 'url': url, 'provider': website_info['provider'], - 'job_id': website_info['job_id'] + 'job_id': website_info['job_id'], + 'only_main_content': website_info.get('only_main_content', False), + 'mode': 'crawl', } document.data_source_type = document_data["data_source"]["type"] document.data_source_info = json.dumps(data_source_info) @@ -1030,7 +1034,7 @@ def data_source_args_validate(cls, args: dict): if 'notion_info_list' not in args['data_source']['info_list'] or not args['data_source']['info_list'][ 'notion_info_list']: raise ValueError("Notion source info is required") - if args['data_source']['type'] == 'website': + if args['data_source']['type'] == 'website_crawl': if 'website_info_list' not in args['data_source']['info_list'] or not args['data_source']['info_list'][ 'website_info_list']: raise ValueError("Website source info is required") From 592f795aecdfbc55b186c930a42f000145941a10 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 18:10:32 +0800 Subject: [PATCH 128/273] chore: add only_main_content to api --- web/app/components/datasets/create/index.tsx | 16 +++++++++++++++- .../datasets/create/step-one/index.tsx | 8 +++++++- .../datasets/create/step-two/index.tsx | 5 ++++- .../create/website/firecrawl/crawled-result.tsx | 2 +- .../datasets/create/website/firecrawl/index.tsx | 16 +++++----------- .../components/datasets/create/website/index.tsx | 8 +++++++- web/models/datasets.ts | 2 +- 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 4a25b6699e5e1c..7bfc3895210d93 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -8,7 +8,7 @@ import StepOne from './step-one' import StepTwo from './step-two' import StepThree from './step-three' import { DataSourceType } from '@/models/datasets' -import type { CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets' import { fetchDataSource } from '@/service/common' import { fetchDatasetDetail } from '@/service/datasets' import type { NotionPage } from '@/models/common' @@ -19,6 +19,15 @@ type DatasetUpdateFormProps = { datasetId?: string } +const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { + crawl_sub_pages: true, + only_main_content: true, + includes: '', + excludes: '', + limit: 10, + max_depth: 2, +} + const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { const { t } = useTranslation() const { setShowAccountSettingModal } = useModalContext() @@ -37,6 +46,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { } const [websitePages, setWebsitePages] = useState([]) + const [crawlOptions, setCrawlOptions] = useState(DEFAULT_CRAWL_OPTIONS) + const updateFileList = (preparedFiles: FileItem[]) => { setFiles(preparedFiles) } @@ -127,6 +138,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { websitePages={websitePages} updateWebsitePages={setWebsitePages} onFireCrawlJobIdChange={setFireCrawlJobId} + crawlOptions={crawlOptions} + onCrawlOptionsChange={setCrawlOptions} />
{(step === 2 && (!datasetId || (datasetId && !!detail))) && { onStepChange={changeStep} updateIndexingTypeCache={updateIndexingTypeCache} updateResultCache={updateResultCache} + crawlOptions={crawlOptions} />} {step === 3 && void onFireCrawlJobIdChange: (jobId: string) => void + crawlOptions: CrawlOptions + onCrawlOptionsChange: (payload: CrawlOptions) => void } type NotionConnectorProps = { @@ -68,6 +70,8 @@ const StepOne = ({ websitePages = [], updateWebsitePages, onFireCrawlJobIdChange, + crawlOptions, + onCrawlOptionsChange, }: IStepOneProps) => { const { dataset } = useDatasetDetailContext() const [showModal, setShowModal] = useState(false) @@ -221,6 +225,8 @@ const StepOne = ({ checkedCrawlResult={websitePages} onCheckedCrawlResultChange={updateWebsitePages} onJobIdChange={onFireCrawlJobIdChange} + crawlOptions={crawlOptions} + onCrawlOptionsChange={onCrawlOptionsChange} />
{isShowVectorSpaceFull && ( diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 0ffc9da2a6be09..b62ed879ad522c 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -12,7 +12,7 @@ import RetrievalMethodInfo from '../../common/retrieval-method-info' import PreviewItem, { PreviewType } from './preview-item' import LanguageSelect from './language-select' import s from './index.module.css' -import type { CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' import { createDocument, createFirstDocument, @@ -58,6 +58,7 @@ type StepTwoProps = { files: CustomFile[] notionPages?: NotionPage[] websitePages?: CrawlResultItem[] + crawlOptions?: CrawlOptions fireCrawlJobId?: string onStepChange?: (delta: number) => void updateIndexingTypeCache?: (type: string) => void @@ -86,6 +87,7 @@ const StepTwo = ({ files, notionPages = [], websitePages = [], + crawlOptions, fireCrawlJobId = '', onStepChange, updateIndexingTypeCache, @@ -252,6 +254,7 @@ const StepTwo = ({ provider: 'firecrawl', job_id: fireCrawlJobId, urls: websitePages.map(page => page.source_url), + only_main_content: crawlOptions?.only_main_content, } } diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx index bf451e2eaa8a71..0a10ac52bac215 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -61,7 +61,7 @@ const CrawledResult: FC = ({ />
{t(`${I18N_PREFIX}.scrapTimeInfo`, { total: list.length, - time: '12.4 seconds', + time: '12.4 seconds', // TODO toFixed(1) })}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index e87137a2916d38..eb985fe9957b82 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -23,15 +23,8 @@ type Props = { checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void -} - -const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { - crawl_sub_pages: true, - only_main_content: true, - includes: '', - excludes: '', - limit: 10, - max_depth: 2, + crawlOptions: CrawlOptions + onCrawlOptionsChange: (payload: CrawlOptions) => void } enum Step { @@ -45,6 +38,8 @@ const FireCrawl: FC = ({ checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, + crawlOptions, + onCrawlOptionsChange, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) @@ -55,7 +50,6 @@ const FireCrawl: FC = ({ }) }, [setShowAccountSettingModal]) - const [crawlOptions, setCrawlOptions] = useState(DEFAULT_CRAWL_OPTIONS) const checkValid = useCallback((url: string) => { let errorMsg = '' if (!url) { @@ -172,7 +166,7 @@ const FireCrawl: FC = ({ isFilledFull={!isInit} hasError={isCrawlFinished && crawlHasError} > - {isInit && } + {isInit && } {isRunning && void onJobIdChange: (jobId: string) => void + crawlOptions: CrawlOptions + onCrawlOptionsChange: (payload: CrawlOptions) => void } const Website: FC = ({ @@ -20,6 +22,8 @@ const Website: FC = ({ checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, + crawlOptions, + onCrawlOptionsChange, }) => { const { setShowAccountSettingModal } = useModalContext() const [isLoaded, setIsLoaded] = useState(false) @@ -55,6 +59,8 @@ const Website: FC = ({ checkedCrawlResult={checkedCrawlResult} onCheckedCrawlResultChange={onCheckedCrawlResultChange} onJobIdChange={onJobIdChange} + crawlOptions={crawlOptions} + onCrawlOptionsChange={onCrawlOptionsChange} /> ) : ( diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 101ce2d7a3799f..d417ec4943b807 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -5,7 +5,7 @@ import type { Tag } from '@/app/components/base/tag-management/constant' export enum DataSourceType { FILE = 'upload_file', NOTION = 'notion_import', - WEB = 'website', + WEB = 'website_crawl', } export type DataSet = { From 537f7ec3e265f29ea71e9fa4a796adffc92a0737 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 7 Jun 2024 18:10:32 +0800 Subject: [PATCH 129/273] chore: add only_main_content to api --- web/app/components/datasets/create/index.tsx | 16 +++++++++++++++- .../datasets/create/step-one/index.tsx | 8 +++++++- .../datasets/create/step-two/index.tsx | 5 ++++- .../create/website/firecrawl/crawled-result.tsx | 2 +- .../datasets/create/website/firecrawl/index.tsx | 16 +++++----------- .../components/datasets/create/website/index.tsx | 8 +++++++- web/models/datasets.ts | 2 +- 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 4a25b6699e5e1c..7bfc3895210d93 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -8,7 +8,7 @@ import StepOne from './step-one' import StepTwo from './step-two' import StepThree from './step-three' import { DataSourceType } from '@/models/datasets' -import type { CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets' import { fetchDataSource } from '@/service/common' import { fetchDatasetDetail } from '@/service/datasets' import type { NotionPage } from '@/models/common' @@ -19,6 +19,15 @@ type DatasetUpdateFormProps = { datasetId?: string } +const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { + crawl_sub_pages: true, + only_main_content: true, + includes: '', + excludes: '', + limit: 10, + max_depth: 2, +} + const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { const { t } = useTranslation() const { setShowAccountSettingModal } = useModalContext() @@ -37,6 +46,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { } const [websitePages, setWebsitePages] = useState([]) + const [crawlOptions, setCrawlOptions] = useState(DEFAULT_CRAWL_OPTIONS) + const updateFileList = (preparedFiles: FileItem[]) => { setFiles(preparedFiles) } @@ -127,6 +138,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { websitePages={websitePages} updateWebsitePages={setWebsitePages} onFireCrawlJobIdChange={setFireCrawlJobId} + crawlOptions={crawlOptions} + onCrawlOptionsChange={setCrawlOptions} />
{(step === 2 && (!datasetId || (datasetId && !!detail))) && { onStepChange={changeStep} updateIndexingTypeCache={updateIndexingTypeCache} updateResultCache={updateResultCache} + crawlOptions={crawlOptions} />} {step === 3 && void onFireCrawlJobIdChange: (jobId: string) => void + crawlOptions: CrawlOptions + onCrawlOptionsChange: (payload: CrawlOptions) => void } type NotionConnectorProps = { @@ -68,6 +70,8 @@ const StepOne = ({ websitePages = [], updateWebsitePages, onFireCrawlJobIdChange, + crawlOptions, + onCrawlOptionsChange, }: IStepOneProps) => { const { dataset } = useDatasetDetailContext() const [showModal, setShowModal] = useState(false) @@ -221,6 +225,8 @@ const StepOne = ({ checkedCrawlResult={websitePages} onCheckedCrawlResultChange={updateWebsitePages} onJobIdChange={onFireCrawlJobIdChange} + crawlOptions={crawlOptions} + onCrawlOptionsChange={onCrawlOptionsChange} />
{isShowVectorSpaceFull && ( diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 0ffc9da2a6be09..b62ed879ad522c 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -12,7 +12,7 @@ import RetrievalMethodInfo from '../../common/retrieval-method-info' import PreviewItem, { PreviewType } from './preview-item' import LanguageSelect from './language-select' import s from './index.module.css' -import type { CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' import { createDocument, createFirstDocument, @@ -58,6 +58,7 @@ type StepTwoProps = { files: CustomFile[] notionPages?: NotionPage[] websitePages?: CrawlResultItem[] + crawlOptions?: CrawlOptions fireCrawlJobId?: string onStepChange?: (delta: number) => void updateIndexingTypeCache?: (type: string) => void @@ -86,6 +87,7 @@ const StepTwo = ({ files, notionPages = [], websitePages = [], + crawlOptions, fireCrawlJobId = '', onStepChange, updateIndexingTypeCache, @@ -252,6 +254,7 @@ const StepTwo = ({ provider: 'firecrawl', job_id: fireCrawlJobId, urls: websitePages.map(page => page.source_url), + only_main_content: crawlOptions?.only_main_content, } } diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx index bf451e2eaa8a71..0a10ac52bac215 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -61,7 +61,7 @@ const CrawledResult: FC = ({ />
{t(`${I18N_PREFIX}.scrapTimeInfo`, { total: list.length, - time: '12.4 seconds', + time: '12.4 seconds', // TODO toFixed(1) })}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index e87137a2916d38..eb985fe9957b82 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -23,15 +23,8 @@ type Props = { checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void -} - -const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { - crawl_sub_pages: true, - only_main_content: true, - includes: '', - excludes: '', - limit: 10, - max_depth: 2, + crawlOptions: CrawlOptions + onCrawlOptionsChange: (payload: CrawlOptions) => void } enum Step { @@ -45,6 +38,8 @@ const FireCrawl: FC = ({ checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, + crawlOptions, + onCrawlOptionsChange, }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) @@ -55,7 +50,6 @@ const FireCrawl: FC = ({ }) }, [setShowAccountSettingModal]) - const [crawlOptions, setCrawlOptions] = useState(DEFAULT_CRAWL_OPTIONS) const checkValid = useCallback((url: string) => { let errorMsg = '' if (!url) { @@ -172,7 +166,7 @@ const FireCrawl: FC = ({ isFilledFull={!isInit} hasError={isCrawlFinished && crawlHasError} > - {isInit && } + {isInit && } {isRunning && void onJobIdChange: (jobId: string) => void + crawlOptions: CrawlOptions + onCrawlOptionsChange: (payload: CrawlOptions) => void } const Website: FC = ({ @@ -20,6 +22,8 @@ const Website: FC = ({ checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, + crawlOptions, + onCrawlOptionsChange, }) => { const { setShowAccountSettingModal } = useModalContext() const [isLoaded, setIsLoaded] = useState(false) @@ -55,6 +59,8 @@ const Website: FC = ({ checkedCrawlResult={checkedCrawlResult} onCheckedCrawlResultChange={onCheckedCrawlResultChange} onJobIdChange={onJobIdChange} + crawlOptions={crawlOptions} + onCrawlOptionsChange={onCrawlOptionsChange} /> ) : ( diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 101ce2d7a3799f..d417ec4943b807 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -5,7 +5,7 @@ import type { Tag } from '@/app/components/base/tag-management/constant' export enum DataSourceType { FILE = 'upload_file', NOTION = 'notion_import', - WEB = 'website', + WEB = 'website_crawl', } export type DataSet = { From affbcf463bed165fe2577efe1ccbbb38dbba4150 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Fri, 7 Jun 2024 21:32:55 +0800 Subject: [PATCH 130/273] note editor --- .../(appDetailLayout)/[appId]/note/page.tsx | 5 +- .../note-node/note-editor/context.tsx | 26 ++++-- .../workflow/note-node/note-editor/editor.tsx | 30 ++++++- .../plugins/format-detector-plugin/hooks.ts | 63 ++++++++++++++ .../plugins/format-detector-plugin/index.tsx | 9 ++ .../plugins/link-editor-plugin/component.tsx | 82 +++++++++++++++++++ .../plugins/link-editor-plugin/index.tsx | 15 ++++ .../workflow/note-node/note-editor/store.ts | 42 ++++++++++ .../note-node/note-editor/toolbar/command.tsx | 49 +++++------ .../note-node/note-editor/toolbar/hooks.ts | 61 ++++++++++++++ .../workflow/note-node/note-editor/utils.ts | 19 +++++ 11 files changed, 364 insertions(+), 37 deletions(-) create mode 100644 web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts create mode 100644 web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/index.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/store.ts create mode 100644 web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts create mode 100644 web/app/components/workflow/note-node/note-editor/utils.ts diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx index 388b8668dddbf6..54cdd2ac94c562 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx @@ -12,7 +12,10 @@ const Page = () => {
- +
+ +
+
diff --git a/web/app/components/workflow/note-node/note-editor/context.tsx b/web/app/components/workflow/note-node/note-editor/context.tsx index cebbb769db6526..453b76d8a1de7f 100644 --- a/web/app/components/workflow/note-node/note-editor/context.tsx +++ b/web/app/components/workflow/note-node/note-editor/context.tsx @@ -1,11 +1,17 @@ 'use client' -import { memo } from 'react' -import { createContext } from 'use-context-selector' +import { + createContext, + memo, + useRef, +} from 'react' import { CodeNode } from '@lexical/code' import { LexicalComposer } from '@lexical/react/LexicalComposer' +import { LinkNode } from '@lexical/link' +import { createNoteEditorStore } from './store' -const NoteEditorContext = createContext({}) +type NoteEditorStore = ReturnType +const NoteEditorContext = createContext(null) type NoteEditorContextProviderProps = { children: JSX.Element | string | (JSX.Element | string)[] @@ -13,10 +19,16 @@ type NoteEditorContextProviderProps = { export const NoteEditorContextProvider = memo(({ children, }: NoteEditorContextProviderProps) => { + const storeRef = useRef() + + if (!storeRef.current) + storeRef.current = createNoteEditorStore() + const initialConfig = { namespace: 'note-editor', nodes: [ CodeNode, + LinkNode, ], onError: (error: Error) => { throw error @@ -24,9 +36,11 @@ export const NoteEditorContextProvider = memo(({ } return ( - - {children} - + + + {children} + + ) }) NoteEditorContextProvider.displayName = 'NoteEditorContextProvider' diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index 2f341098ffd0fe..40ef58f8d9d630 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -1,10 +1,16 @@ 'use client' -import { memo } from 'react' +import { + memo, + useState, +} from 'react' import { RichTextPlugin } from '@lexical/react/LexicalRichTextPlugin' import { ContentEditable } from '@lexical/react/LexicalContentEditable' +import { LinkPlugin } from '@lexical/react/LexicalLinkPlugin' import LexicalErrorBoundary from '@lexical/react/LexicalErrorBoundary' import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin' +import LinkEditorPlugin from './plugins/link-editor-plugin' +import FormatDetectorPlugin from './plugins/format-detector-plugin' import TreeView from '@/app/components/base/prompt-editor/plugins/tree-view' import Placeholder from '@/app/components/base/prompt-editor/plugins/placeholder' @@ -12,15 +18,33 @@ type EditorProps = { placeholder?: string } const Editor = ({ - placeholder, + placeholder = 'write you note...', }: EditorProps) => { + const [containerElement, setContainerElement] = useState(null) + const onRef = (_containerElement: HTMLDivElement) => { + if (_containerElement !== null) + setContainerElement(_containerElement) + } + return (
} + contentEditable={ +
+ +
+ } placeholder={} ErrorBoundary={LexicalErrorBoundary} /> + + +
diff --git a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts new file mode 100644 index 00000000000000..3b85a723e30883 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts @@ -0,0 +1,63 @@ +import { + useCallback, + useEffect, +} from 'react' +import { + $getSelection, + $isRangeSelection, +} from 'lexical' +import { mergeRegister } from '@lexical/utils' +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' +import { $isLinkNode } from '@lexical/link' +import { getSelectedNode } from '../../utils' +import { useNoteEditorStore } from '../../store' + +export const useFormatDetector = () => { + const [editor] = useLexicalComposerContext() + const noteEditorStore = useNoteEditorStore() + + const handleFormat = useCallback(() => { + editor.getEditorState().read(() => { + // Should not to pop up the floating toolbar when using IME input + if (editor.isComposing()) + return + + const selection = $getSelection() + if (!$isRangeSelection(selection)) + return + + const node = getSelectedNode(selection) + const { + setIsBold, + setIsStrikeThrough, + setIsLink, + } = noteEditorStore.getState() + setIsBold(selection.hasFormat('bold')) + setIsStrikeThrough(selection.hasFormat('strikethrough')) + const parent = node.getParent() + if ($isLinkNode(parent) || $isLinkNode(node)) + setIsLink(true) + else + setIsLink(false) + }) + }, [editor, noteEditorStore]) + + useEffect(() => { + document.addEventListener('selectionchange', handleFormat) + return () => { + document.removeEventListener('selectionchange', handleFormat) + } + }, [handleFormat]) + + useEffect(() => { + return mergeRegister( + editor.registerUpdateListener(() => { + handleFormat() + }), + ) + }, [editor, handleFormat]) + + return { + handleFormat, + } +} diff --git a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/index.tsx b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/index.tsx new file mode 100644 index 00000000000000..3a2585c4b5ab6f --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/index.tsx @@ -0,0 +1,9 @@ +import { useFormatDetector } from './hooks' + +const FormatDetectorPlugin = () => { + useFormatDetector() + + return null +} + +export default FormatDetectorPlugin diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx new file mode 100644 index 00000000000000..a6b0fe08d6c603 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx @@ -0,0 +1,82 @@ +import { + memo, + useCallback, + useState, +} from 'react' +import { + FloatingPortal, + flip, + offset, + shift, + useFloating, +} from '@floating-ui/react' +import { useTranslation } from 'react-i18next' +import cn from 'classnames' +import { useStore } from '../../store' +import Button from '@/app/components/base/button' + +type LinkEditorComponentProps = { + containerElement: HTMLDivElement | null + linkUrl?: string + showInput?: boolean +} +const LinkEditorComponent = ({ + containerElement, + linkUrl, + showInput = true, +}: LinkEditorComponentProps) => { + const { t } = useTranslation() + const [url, setUrl] = useState(linkUrl) + const anchorElement = useStore(s => s.anchorElement) + const { refs, floatingStyles, elements } = useFloating({ + placement: 'top', + middleware: [ + offset(4), + shift(), + flip(), + ], + }) + + const handleConfirm = useCallback(() => {}, []) + + if (!anchorElement) + return null + + if (!elements.reference && anchorElement) + refs.setReference(anchorElement) + + return ( + <> + { + elements.reference && ( + +
+ { + showInput && ( + <> + setUrl(e.target.value)} + /> + + + ) + } +
+ + ) + } + + ) +} + +export default memo(LinkEditorComponent) diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx new file mode 100644 index 00000000000000..3c5207526c4365 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx @@ -0,0 +1,15 @@ +import { memo } from 'react' +import LinkEditorComponent from './component' + +type LinkEditorPluginProps = { + containerElement: HTMLDivElement | null +} +const LinkEditorPlugin = ({ + containerElement, +}: LinkEditorPluginProps) => { + return ( + + ) +} + +export default memo(LinkEditorPlugin) diff --git a/web/app/components/workflow/note-node/note-editor/store.ts b/web/app/components/workflow/note-node/note-editor/store.ts new file mode 100644 index 00000000000000..08f5ed9ddcd456 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/store.ts @@ -0,0 +1,42 @@ +import { useContext } from 'react' +import { + useStore as useZustandStore, +} from 'zustand' +import { createStore } from 'zustand/vanilla' +import NoteEditorContext from './context' + +type Shape = { + anchorElement: HTMLElement | null + setAnchorElement: (anchorElement: HTMLElement | null) => void + isBold: boolean + setIsBold: (isBold: boolean) => void + isStrikeThrough: boolean + setIsStrikeThrough: (isStrikeThrough: boolean) => void + isLink: boolean + setIsLink: (isLink: boolean) => void +} + +export const createNoteEditorStore = () => { + return createStore(set => ({ + anchorElement: null, + setAnchorElement: anchorElement => set(() => ({ anchorElement })), + isBold: false, + setIsBold: isBold => set(() => ({ isBold })), + isStrikeThrough: false, + setIsStrikeThrough: isStrikeThrough => set(() => ({ isStrikeThrough })), + isLink: false, + setIsLink: isLink => set(() => ({ isLink })), + })) +} + +export function useStore(selector: (state: Shape) => T): T { + const store = useContext(NoteEditorContext) + if (!store) + throw new Error('Missing NoteEditorContext.Provider in the tree') + + return useZustandStore(store, selector) +} + +export const useNoteEditorStore = () => { + return useContext(NoteEditorContext)! +} diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx index 54aef3628d965e..07ec23ab31e3d7 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx @@ -1,10 +1,10 @@ -import { memo, useMemo } from 'react' import { - $getSelection, - $isRangeSelection, - $setSelection, -} from 'lexical' -import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' + memo, + useMemo, +} from 'react' +import cn from 'classnames' +import { useStore } from '../store' +import { useCommand } from './hooks' import { Link01 } from '@/app/components/base/icons/src/vender/line/general' import { Bold01, @@ -18,39 +18,34 @@ type CommandProps = { const Command = ({ type, }: CommandProps) => { - const [editor] = useLexicalComposerContext() + const isBold = useStore(s => s.isBold) + const isStrikeThrough = useStore(s => s.isStrikeThrough) + const isLink = useStore(s => s.isLink) + console.log(isBold) + const { handleCommand } = useCommand() const icon = useMemo(() => { switch (type) { case 'bold': - return + return case 'strikethrough': - return + return case 'link': - return + return case 'bullet': return } - }, [type]) - - const handleClick = () => { - if (type === 'bold') - return - - if (type === 'link') { - editor.update(() => { - const selection = $getSelection() - - if ($isRangeSelection(selection) && !selection.isCollapsed()) - $setSelection(selection) - }) - } - } + }, [type, isBold, isStrikeThrough, isLink]) return (
handleCommand(type)} > {icon}
diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts new file mode 100644 index 00000000000000..e2200669fa8736 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts @@ -0,0 +1,61 @@ +import { + useCallback, +} from 'react' +import { + $getSelection, + $isRangeSelection, + FORMAT_TEXT_COMMAND, +} from 'lexical' +import { + $isLinkNode, + TOGGLE_LINK_COMMAND, +} from '@lexical/link' +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' +import { useNoteEditorStore } from '../store' +import { getSelectedNode } from '../utils' + +export const useCommand = () => { + const [editor] = useLexicalComposerContext() + const noteEditorStore = useNoteEditorStore() + + const handleCommand = useCallback((type: string) => { + if (type === 'bold') + editor.dispatchCommand(FORMAT_TEXT_COMMAND, 'bold') + + if (type === 'strikethrough') + editor.dispatchCommand(FORMAT_TEXT_COMMAND, 'strikethrough') + if (type === 'link') { + editor.update(() => { + const selection = $getSelection() + + if ($isRangeSelection(selection) && !selection.isCollapsed()) { + const node = getSelectedNode(selection) + const parent = node.getParent() + + if ($isLinkNode(parent) || $isLinkNode(node)) + editor.dispatchCommand(TOGGLE_LINK_COMMAND, null) + else + editor.dispatchCommand(TOGGLE_LINK_COMMAND, 'https://') + + // setTimeout(() => { + // const nativeSelection = window.getSelection() + // editor.getEditorState().read(() => { + // const node = getSelectedNode(selection) + // console.log(node, 'node') + // }) + + // if (nativeSelection?.focusNode) { + // const { setAnchorElement } = noteEditorStore.getState() + // const parent = nativeSelection.focusNode.parentElement + // setAnchorElement(parent) + // } + // }) + } + }) + } + }, [editor, noteEditorStore]) + + return { + handleCommand, + } +} diff --git a/web/app/components/workflow/note-node/note-editor/utils.ts b/web/app/components/workflow/note-node/note-editor/utils.ts new file mode 100644 index 00000000000000..16802dc1158d65 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/utils.ts @@ -0,0 +1,19 @@ +import { $isAtNodeEnd } from '@lexical/selection' +import type { ElementNode, RangeSelection, TextNode } from 'lexical' + +export function getSelectedNode( + selection: RangeSelection, +): TextNode | ElementNode { + const anchor = selection.anchor + const focus = selection.focus + const anchorNode = selection.anchor.getNode() + const focusNode = selection.focus.getNode() + if (anchorNode === focusNode) + return anchorNode + + const isBackward = selection.isBackward() + if (isBackward) + return $isAtNodeEnd(focus) ? anchorNode : focusNode + else + return $isAtNodeEnd(anchor) ? anchorNode : focusNode +} From bf6b6ebea74e47512930872d74569a5cb3603aab Mon Sep 17 00:00:00 2001 From: Pascal M <11357019+perzeuss@users.noreply.github.com> Date: Sat, 8 Jun 2024 01:30:13 +0200 Subject: [PATCH 131/273] feat: add dataset delete endpoint (#5035) --- .../service_api/dataset/dataset.py | 36 ++++++++++++++++-- .../datasets/template/template.en.mdx | 37 +++++++++++++++++++ .../datasets/template/template.zh.mdx | 37 +++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index bf08291d7b9f87..dcbfa88d99487d 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -1,5 +1,6 @@ from flask import request from flask_restful import marshal, reqparse +from werkzeug.exceptions import NotFound import services.dataset_service from controllers.service_api import api @@ -19,10 +20,12 @@ def _validate_name(name): return name -class DatasetApi(DatasetApiResource): - """Resource for get datasets.""" +class DatasetListApi(DatasetApiResource): + """Resource for datasets.""" def get(self, tenant_id): + """Resource for getting datasets.""" + page = request.args.get('page', default=1, type=int) limit = request.args.get('limit', default=20, type=int) provider = request.args.get('provider', default="vendor") @@ -65,9 +68,9 @@ def get(self, tenant_id): } return response, 200 - """Resource for datasets.""" def post(self, tenant_id): + """Resource for creating datasets.""" parser = reqparse.RequestParser() parser.add_argument('name', nullable=False, required=True, help='type is required. Name must be between 1 to 40 characters.', @@ -89,6 +92,31 @@ def post(self, tenant_id): return marshal(dataset, dataset_detail_fields), 200 +class DatasetApi(DatasetApiResource): + """Resource for dataset.""" + + def delete(self, _, dataset_id): + """ + Deletes a dataset given its ID. + + Args: + dataset_id (UUID): The ID of the dataset to be deleted. + + Returns: + dict: A dictionary with a key 'result' and a value 'success' + if the dataset was successfully deleted. Omitted in HTTP response. + int: HTTP status code 204 indicating that the operation was successful. + + Raises: + NotFound: If the dataset with the given ID does not exist. + """ + + dataset_id_str = str(dataset_id) -api.add_resource(DatasetApi, '/datasets') + if DatasetService.delete_dataset(dataset_id_str, current_user): + return {'result': 'success'}, 204 + else: + raise NotFound("Dataset not found.") +api.add_resource(DatasetListApi, '/datasets') +api.add_resource(DatasetApi, '/datasets/') diff --git a/web/app/(commonLayout)/datasets/template/template.en.mdx b/web/app/(commonLayout)/datasets/template/template.en.mdx index 3775d30d246e1c..36395d391de1b3 100644 --- a/web/app/(commonLayout)/datasets/template/template.en.mdx +++ b/web/app/(commonLayout)/datasets/template/template.en.mdx @@ -345,6 +345,43 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from --- + + + + ### Params + + + Knowledge ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}' \ + --header 'Authorization: Bearer {api_key}' + ``` + + + ```text {{ title: 'Response' }} + 204 No Content + ``` + + + + +--- + + + + ### Path + + + 知识库 ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}' \ + --header 'Authorization: Bearer {api_key}' + ``` + + + ```text {{ title: 'Response' }} + 204 No Content + ``` + + + + +--- + Date: Tue, 11 Jun 2024 11:49:38 +0800 Subject: [PATCH 132/273] fix: api success message --- .../data-source-website/config-firecrawl-modal.tsx | 2 +- .../data-source-page/data-source-website/index.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx index f2f71d0b1d2975..b01a684d08deff 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx @@ -74,7 +74,7 @@ const ConfigFirecrawlModal: FC = ({ await createFirecrawlApiKey(postData) Toast.notify({ type: 'success', - message: t('common.operation.saveSuccess'), + message: t('common.api.create'), }) onSaved() }, [config.api_key, config.base_url, onSaved, t]) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx index 4992598296c321..9a0a364da11684 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx @@ -43,7 +43,7 @@ const DataSourceWebsite: FC = () => { setList([]) Toast.notify({ type: 'success', - message: t('common.operation.saveSuccess'), + message: t('common.api.remove'), }) }, [list, t]) From 5870e5ff036e782006a0a1d728e5d5c0360b0b7a Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Tue, 11 Jun 2024 14:32:46 +0800 Subject: [PATCH 133/273] fix firecrawl issue --- api/controllers/console/datasets/datasets.py | 2 +- api/controllers/console/datasets/datasets_document.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 52577334acd8b5..a40038881f68a1 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -315,7 +315,7 @@ def post(self): website_info_list = args['info_list']['website_info_list'] for url in website_info_list['urls']: extract_setting = ExtractSetting( - datasource_type="website", + datasource_type="website_crawl", website_info={ "provider": website_info_list['provider'], "job_id": website_info_list['job_id'], diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index d88123adf640b5..910f1bc589567b 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -467,7 +467,7 @@ def get(self, dataset_id, batch): extract_settings.append(extract_setting) elif document.data_source_type == 'website_crawl': extract_setting = ExtractSetting( - datasource_type="website", + datasource_type="website_crawl", website_info={ "provider": data_source_info['provider'], "job_id": data_source_info['job_id'], From a0583a78d5294ccf0b445a59b5d93b8d074cecd3 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 14:53:57 +0800 Subject: [PATCH 134/273] fix: added key not refresh data --- .../data-source-website/index.tsx | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx index 9a0a364da11684..b6ac22436cd91f 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx @@ -25,12 +25,15 @@ const DataSourceWebsite: FC = () => { const { t } = useTranslation() const { isCurrentWorkspaceManager } = useAppContext() const [list, setList] = useState([]) + const checkSetApiKey = useCallback(async () => { + const res = await fetchFirecrawlApiKey() as any + const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled) + setList(list) + }, []) + useEffect(() => { - (async () => { - const res = await fetchFirecrawlApiKey() as any - const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled) - setList(list) - })() + checkSetApiKey() + // eslint-disable-next-line react-hooks/exhaustive-deps }, []) const [isShowConfig, { @@ -38,6 +41,11 @@ const DataSourceWebsite: FC = () => { setFalse: hideConfig, }] = useBoolean(false) + const handleAdded = useCallback(() => { + checkSetApiKey() + hideConfig() + }, [checkSetApiKey, hideConfig]) + const handleRemove = useCallback(async () => { await removeFirecrawlApiKey(list[0].id) setList([]) @@ -65,7 +73,7 @@ const DataSourceWebsite: FC = () => { onRemove={handleRemove} /> {isShowConfig && ( - + )} From 419b715fdcc47a661a4e5fc478b4845a5a70941f Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Tue, 11 Jun 2024 14:56:31 +0800 Subject: [PATCH 135/273] fix firecrawl issue --- api/controllers/console/datasets/datasets.py | 1 + api/controllers/console/datasets/datasets_document.py | 1 + api/core/indexing_runner.py | 1 + api/core/rag/extractor/entity/extract_setting.py | 1 + api/core/rag/extractor/extract_processor.py | 1 + .../rag/extractor/firecrawl/firecrawl_web_extractor.py | 4 +++- api/services/dataset_service.py | 2 +- api/services/website_service.py | 8 ++++---- 8 files changed, 13 insertions(+), 6 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index a40038881f68a1..1f49a7fc104dfb 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -320,6 +320,7 @@ def post(self): "provider": website_info_list['provider'], "job_id": website_info_list['job_id'], "url": url, + "tenant_id": current_user.current_tenant_id, "mode": 'crawl', "only_main_content": website_info_list['only_main_content'] }, diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 910f1bc589567b..210faa0aa6a467 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -472,6 +472,7 @@ def get(self, dataset_id, batch): "provider": data_source_info['provider'], "job_id": data_source_info['job_id'], "url": data_source_info['url'], + "tenant_id": current_user.current_tenant_id, "mode": data_source_info['mode'], "only_main_content": data_source_info['only_main_content'] }, diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 0ba0e8a933b083..c5f1d72a334334 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -384,6 +384,7 @@ def _extract(self, index_processor: BaseIndexProcessor, dataset_document: Datase website_info={ "provider": data_source_info['provider'], "job_id": data_source_info['job_id'], + "tenant_id": dataset_document.tenant_id, "url": data_source_info['url'], "mode": data_source_info['mode'], "only_main_content": data_source_info['only_main_content'] diff --git a/api/core/rag/extractor/entity/extract_setting.py b/api/core/rag/extractor/entity/extract_setting.py index 9bf99d06af75ef..4a3ecb7754ab2a 100644 --- a/api/core/rag/extractor/entity/extract_setting.py +++ b/api/core/rag/extractor/entity/extract_setting.py @@ -29,6 +29,7 @@ class WebsiteInfo(BaseModel): job_id: str url: str mode: str + tenant_id: str only_main_content: bool = False class Config: diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index 83a15a7ab0d09e..909bfdc137ff71 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -160,6 +160,7 @@ def extract(cls, extract_setting: ExtractSetting, is_automatic: bool = False, extractor = FirecrawlWebExtractor( url=extract_setting.website_info.url, job_id=extract_setting.website_info.job_id, + tenant_id=extract_setting.website_info.tenant_id, mode=extract_setting.website_info.mode, only_main_content=extract_setting.website_info.only_main_content ) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index 870ae8d272458e..8030d437130fed 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -19,12 +19,14 @@ def __init__( self, url: str, job_id: str, + tenant_id: str, mode: str = 'crawl', only_main_content: bool = False ): """Initialize with url, api_key, base_url and mode.""" self._url = url self.job_id = job_id + self.tenant_id = tenant_id self.mode = mode self.only_main_content = only_main_content @@ -32,7 +34,7 @@ def extract(self) -> list[Document]: """Extract content from the URL.""" documents = [] if self.mode == 'crawl': - crawl_data = WebsiteService.get_crawl_url_data(self.job_id, 'firecrawl', self._url) + crawl_data = WebsiteService.get_crawl_url_data(self.job_id, 'firecrawl', self._url, self.tenant_id) if crawl_data is None: return [] document = Document(page_content=crawl_data.get('markdown', ''), diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 33b96749f39047..fb3c791f680c8b 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -758,7 +758,7 @@ def save_document_with_dataset_id(dataset: Dataset, document_data: dict, document_data["doc_form"], document_data["doc_language"], data_source_info, created_from, position, - account, website_info['url'], batch) + account, url, batch) db.session.add(document) db.session.flush() document_ids.append(document.id) diff --git a/api/services/website_service.py b/api/services/website_service.py index 6158de749d089f..085e7492a5919f 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -106,14 +106,14 @@ def get_crawl_status(cls, job_id: str, provider: str) -> dict: return crawl_status_data @classmethod - def get_crawl_url_data(cls, job_id: str, provider: str, url: str) -> dict | None: - credentials = ApiKeyAuthService.get_auth_credentials(current_user.current_tenant_id, + def get_crawl_url_data(cls, job_id: str, provider: str, url: str, tenant_id: str) -> dict | None: + credentials = ApiKeyAuthService.get_auth_credentials(tenant_id, 'website', provider) if provider == 'firecrawl': # decrypt api_key api_key = encrypter.decrypt_token( - tenant_id=current_user.current_tenant_id, + tenant_id=tenant_id, token=credentials.get('config').get('api_key') ) firecrawl_app = FirecrawlApp(api_key=api_key, @@ -124,7 +124,7 @@ def get_crawl_url_data(cls, job_id: str, provider: str, url: str) -> dict | None data = result.get('data') if data: for item in data: - if item.get('data').get('source_url') == url: + if item.get('source_url') == url: return item return None else: From 4a56405b053ff0d775d21f59f3d112fd122b54c0 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 15:03:12 +0800 Subject: [PATCH 136/273] chore: add loading text --- .../config-firecrawl-modal.tsx | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx index b01a684d08deff..7fb60c389e4874 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx @@ -27,6 +27,7 @@ const ConfigFirecrawlModal: FC = ({ onSaved, }) => { const { t } = useTranslation() + const [isSaving, setIsSaving] = useState(false) const [config, setConfig] = useState({ api_key: '', base_url: '', @@ -39,6 +40,8 @@ const ConfigFirecrawlModal: FC = ({ }, []) const handleSave = useCallback(async () => { + if (isSaving) + return let errorMsg = '' if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://')))) errorMsg = t('common.errorMsg.urlError') @@ -71,13 +74,20 @@ const ConfigFirecrawlModal: FC = ({ }, }, } - await createFirecrawlApiKey(postData) - Toast.notify({ - type: 'success', - message: t('common.api.create'), - }) + try { + setIsSaving(true) + await createFirecrawlApiKey(postData) + Toast.notify({ + type: 'success', + message: t('common.api.success'), + }) + } + finally { + setIsSaving(false) + } + onSaved() - }, [config.api_key, config.base_url, onSaved, t]) + }, [config.api_key, config.base_url, onSaved, t, isSaving]) return ( @@ -122,6 +132,7 @@ const ConfigFirecrawlModal: FC = ({ className='h-9 text-sm font-medium' type='primary' onClick={handleSave} + loading={isSaving} > {t('common.operation.save')} From 36ef4d9c73f23dd07d38f6948c3d46943b8698c0 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 15:31:26 +0800 Subject: [PATCH 137/273] feat: web scrapy sync --- web/app/components/datasets/documents/list.tsx | 11 ++++++++--- web/service/datasets.ts | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 611fdd3241ad05..65de85bb798d54 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -27,7 +27,7 @@ import type { IndicatorProps } from '@/app/components/header/indicator' import Indicator from '@/app/components/header/indicator' import { asyncRunSafe } from '@/utils' import { formatNumber } from '@/utils/format' -import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument, unArchiveDocument } from '@/service/datasets' +import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument, syncWebsite, unArchiveDocument } from '@/service/datasets' import NotionIcon from '@/app/components/base/notion-icon' import ProgressBar from '@/app/components/base/progress-bar' import { DataSourceType, type DocumentDisplayStatus, type SimpleDocumentDetail } from '@/models/datasets' @@ -147,7 +147,12 @@ export const OperationAction: FC<{ opApi = disableDocument break case 'sync': - opApi = syncDocument + if (data_source_type === 'notion_import') + opApi = syncDocument + + else + opApi = syncWebsite + break default: opApi = deleteDocument @@ -250,7 +255,7 @@ export const OperationAction: FC<{ {t('datasetDocuments.list.action.settings')}
- {data_source_type === 'notion_import' && ( + {['notion_import', DataSourceType.WEB].includes(data_source_type) && (
onOperate('sync')}> {t('datasetDocuments.list.action.sync')} diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 91397756048a19..dc08b054d02c6d 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -152,6 +152,10 @@ export const syncDocument: Fetcher = ({ datasetId, return get(`/datasets/${datasetId}/documents/${documentId}/notion/sync`) } +export const syncWebsite: Fetcher = ({ datasetId, documentId }) => { + return get(`/datasets/${datasetId}/documents/${documentId}/website-sync`) +} + export const preImportNotionPages: Fetcher<{ notion_info: DataSourceNotionWorkspace[] }, { url: string; datasetId?: string }> = ({ url, datasetId }) => { return get<{ notion_info: DataSourceNotionWorkspace[] }>(url, { params: { dataset_id: datasetId } }) } From 658c9e1e24f297b009c107b451a471ee7885ff0f Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 15:48:39 +0800 Subject: [PATCH 138/273] feat: add crawl time --- .../datasets/create/website/firecrawl/crawled-result.tsx | 4 +++- .../components/datasets/create/website/firecrawl/index.tsx | 2 ++ web/i18n/en-US/dataset-creation.ts | 2 +- web/i18n/zh-Hans/dataset-creation.ts | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx index 0a10ac52bac215..2e35c6afe75fec 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -13,6 +13,7 @@ type Props = { checkedList: CrawlResultItem[] onSelectedChange: (selected: CrawlResultItem[]) => void onPreview: (payload: CrawlResultItem) => void + usedTime: number } const CrawledResult: FC = ({ @@ -20,6 +21,7 @@ const CrawledResult: FC = ({ checkedList, onSelectedChange, onPreview, + usedTime, }) => { const { t } = useTranslation() @@ -61,7 +63,7 @@ const CrawledResult: FC = ({ />
{t(`${I18N_PREFIX}.scrapTimeInfo`, { total: list.length, - time: '12.4 seconds', // TODO toFixed(1) + time: usedTime.toFixed(1), })}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index eb985fe9957b82..d641be6919d62e 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -86,6 +86,7 @@ const FireCrawl: FC = ({ current: number total: number data: CrawlResultItem[] + time_consuming: number } | undefined>(undefined) const [crawlHasError, setCrawlHasError] = useState(false) @@ -179,6 +180,7 @@ const FireCrawl: FC = ({ checkedList={checkedCrawlResult} onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} + usedTime={crawlResult?.time_consuming || 0} /> )} diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index caff36ccb7c59e..ef1639607af62d 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -75,7 +75,7 @@ const translation = { totalPageScraped: 'Total pages scraped:', selectAll: 'Select All', resetAll: 'Reset All', - scrapTimeInfo: 'Scraped {{total}} pages in total within {{time}}', + scrapTimeInfo: 'Scraped {{total}} pages in total within {{time}}s', preview: 'Preview', }, }, diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 46143ecc4cc5a6..4655b44aa795b0 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -75,7 +75,7 @@ const translation = { totalPageScraped: '抓取页面总数:', selectAll: '全选', resetAll: '重置全部', - scrapTimeInfo: '总共在 {{time}} 内抓取了 {{total}} 个页面', + scrapTimeInfo: '总共在 {{time}}秒 内抓取了 {{total}} 个页面', preview: '预览', }, }, From fc4212462bf69ba5875c36635745735c1fc1a963 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Tue, 11 Jun 2024 16:01:00 +0800 Subject: [PATCH 139/273] fix firecrawl issue --- .../console/datasets/datasets_document.py | 9 +++++---- .../firecrawl/firecrawl_web_extractor.py | 3 ++- api/services/website_service.py | 19 ++++++++++++++++--- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 210faa0aa6a467..0ddd749639d19f 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -256,7 +256,7 @@ def post(self, dataset_id): DocumentService.document_create_args_validate(args) try: - documents, batch = DocumentService. save_document_with_dataset_id(dataset, args, current_user) + documents, batch = DocumentService.save_document_with_dataset_id(dataset, args, current_user) except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) except QuotaExceededError: @@ -970,7 +970,7 @@ class WebsiteDocumentSyncApi(DocumentResource): @setup_required @login_required @account_initialization_required - def post(self, dataset_id, document_id): + def get(self, dataset_id, document_id): """sync website document.""" dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -990,7 +990,8 @@ def post(self, dataset_id, document_id): # sync document DocumentService.sync_website_document(dataset_id, document) - return {'result': 'success'}, 204 + return {'result': 'success'}, 200 + api.add_resource(GetProcessRuleApi, '/datasets/process-rule') api.add_resource(DatasetDocumentListApi, @@ -1021,4 +1022,4 @@ def post(self, dataset_id, document_id): api.add_resource(DocumentRenameApi, '/datasets//documents//rename') -api.add_resource(WebsiteDocumentSyncApi, '/datasets///website-sync') \ No newline at end of file +api.add_resource(WebsiteDocumentSyncApi, '/datasets//documents//website-sync') diff --git a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py index 8030d437130fed..8e2f107e5eb795 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_web_extractor.py @@ -46,7 +46,8 @@ def extract(self) -> list[Document]: ) documents.append(document) elif self.mode == 'scrape': - scrape_data = WebsiteService.get_scrape_url_data('firecrawl', self._url, self.only_main_content) + scrape_data = WebsiteService.get_scrape_url_data('firecrawl', self._url, self.tenant_id, + self.only_main_content) document = Document(page_content=scrape_data.get('markdown', ''), metadata={ diff --git a/api/services/website_service.py b/api/services/website_service.py index 085e7492a5919f..a1758d5bbd574c 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -1,3 +1,4 @@ +import datetime import json from typing import Any @@ -7,6 +8,7 @@ from core.helper import encrypter from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp from extensions.ext_database import db +from extensions.ext_redis import redis_client from models.source import DataSourceApiKeyAuthBinding from services.auth.api_key_auth_service import ApiKeyAuthService @@ -73,6 +75,9 @@ def crawl_url(cls, args: dict) -> dict: } } job_id = firecrawl_app.crawl_url(url, params) + website_crawl_time_cache_key = f'website_crawl_{job_id}' + time = str(datetime.datetime.now().timestamp()) + redis_client.setex(website_crawl_time_cache_key, 3600, time) return { 'status': 'active', 'job_id': job_id @@ -101,6 +106,14 @@ def get_crawl_status(cls, job_id: str, provider: str) -> dict: 'current': result.get('current', 0), 'data': result.get('data', []) } + if crawl_status_data['status'] == 'completed': + website_crawl_time_cache_key = f'website_crawl_{job_id}' + start_time = redis_client.get(website_crawl_time_cache_key) + if start_time: + end_time = datetime.datetime.now().timestamp() + time_consuming = abs(end_time - float(start_time)) + crawl_status_data['time_consuming'] = f"{time_consuming:.2f}" + redis_client.delete(website_crawl_time_cache_key) else: raise ValueError('Invalid provider') return crawl_status_data @@ -131,14 +144,14 @@ def get_crawl_url_data(cls, job_id: str, provider: str, url: str, tenant_id: str raise ValueError('Invalid provider') @classmethod - def get_scrape_url_data(cls, provider: str, url: str, only_main_content: bool) -> dict | None: - credentials = ApiKeyAuthService.get_auth_credentials(current_user.current_tenant_id, + def get_scrape_url_data(cls, provider: str, url: str, tenant_id: str, only_main_content: bool) -> dict | None: + credentials = ApiKeyAuthService.get_auth_credentials(tenant_id, 'website', provider) if provider == 'firecrawl': # decrypt api_key api_key = encrypter.decrypt_token( - tenant_id=current_user.current_tenant_id, + tenant_id=tenant_id, token=credentials.get('config').get('api_key') ) firecrawl_app = FirecrawlApp(api_key=api_key, From 251a19912166933a9cf8816b1bcb47ef7f22b2c9 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 16:19:30 +0800 Subject: [PATCH 140/273] feat: help link url --- web/app/components/datasets/create/website/firecrawl/header.tsx | 2 +- web/i18n/en-US/dataset-creation.ts | 2 +- web/i18n/zh-Hans/dataset-creation.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/header.tsx b/web/app/components/datasets/create/website/firecrawl/header.tsx index d89cfce46a55ce..ae46f3f210cedf 100644 --- a/web/app/components/datasets/create/website/firecrawl/header.tsx +++ b/web/app/components/datasets/create/website/firecrawl/header.tsx @@ -29,7 +29,7 @@ const Header: FC = ({
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index ef1639607af62d..a44e08deb47bd4 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -63,7 +63,7 @@ const translation = { run: 'Run', firecrawlTitle: 'Extract web content with 🔥Firecrawl', firecrawlDoc: 'Firecrawl docs', - firecrawlDocLink: '', + firecrawlDocLink: 'https://docs.dify.ai/guides/knowledge-base/sync_from_website', options: 'Options', crawlSubPage: 'Crawl sub-pages', limit: 'Limit', diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 4655b44aa795b0..b8c0b8b69724b4 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -63,7 +63,7 @@ const translation = { run: '运行', firecrawlTitle: '使用 🔥Firecrawl 提取网页内容', firecrawlDoc: 'Firecrawl 文档', - firecrawlDocLink: '', + firecrawlDocLink: 'https://docs.dify.ai/v/zh-hans/guides/knowledge-base/sync_from_website', options: '选项', crawlSubPage: '爬取子页面', limit: '限制数量', From d1c86bc195c31321bae9da15a1b842aa223c896f Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 16:30:36 +0800 Subject: [PATCH 141/273] fix: handle crawl error --- .../create/website/firecrawl/index.tsx | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index d641be6919d62e..94816ef797d5ec 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -140,21 +140,25 @@ const FireCrawl: FC = ({ } setCrawlHasError(false) setStep(Step.running) - const res = await createFirecrawlTask({ - url, - options: crawlOptions, - }) as any - const jobId = res.job_id - onJobIdChange(jobId) - const { isError, data } = await waitForCrawlFinished(jobId) - if (isError) { - setCrawlHasError(true) + try { + const res = await createFirecrawlTask({ + url, + options: crawlOptions, + }) as any + const jobId = res.job_id + onJobIdChange(jobId) + const { isError, data } = await waitForCrawlFinished(jobId) + if (isError) { + setCrawlHasError(true) + setCrawlResult(data) + } + setCrawlResult(data) + setCrawlHasError(false) + } + finally { + setStep(Step.finished) } - - setStep(Step.finished) - setCrawlResult(data) - setCrawlHasError(false) }, [checkValid, crawlOptions, onJobIdChange, waitForCrawlFinished]) return ( From a43c301397efe0327cab8fc7cf48ba18269062d8 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 17:20:42 +0800 Subject: [PATCH 142/273] feat: always can change options --- .../website/firecrawl/base/options-wrap.tsx | 24 ++++----- .../create/website/firecrawl/index.tsx | 51 ++++++++++++------- .../components/datasets/documents/list.tsx | 1 - 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx index aef5097c75d46b..09ea8f60f16696 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -1,10 +1,9 @@ 'use client' import { useBoolean } from 'ahooks' import type { FC } from 'react' -import React from 'react' +import React, { useEffect } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' -import ErrorMessage from './error-message' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows' const I18N_PREFIX = 'datasetCreation.stepOne.website' @@ -12,21 +11,26 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website' type Props = { className?: string children: React.ReactNode - isFilledFull?: boolean - hasError?: boolean + controlFoldOptions?: number } const OptionsWrap: FC = ({ className = '', children, - hasError, - isFilledFull = false, + controlFoldOptions, }) => { const { t } = useTranslation() const [fold, { toggle: foldToggle, + setTrue: foldHide, }] = useBoolean(false) + + useEffect(() => { + if (controlFoldOptions) + foldHide() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [controlFoldOptions]) return (
= ({
{!fold && ( -
- {!hasError - ? children - : ( - - )} +
+ {children}
)} diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 94816ef797d5ec..725e87f2c7396c 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React, { useCallback, useState } from 'react' +import React, { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' import Header from './header' @@ -9,6 +9,7 @@ import OptionsWrap from './base/options-wrap' import Options from './options' import CrawledResult from './crawled-result' import Crawling from './crawling' +import ErrorMessage from './base/error-message' import { useModalContext } from '@/context/modal-context' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import Toast from '@/app/components/base/toast' @@ -43,6 +44,11 @@ const FireCrawl: FC = ({ }) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) + const [controlFoldOptions, setControlFoldOptions] = useState(0) + useEffect(() => { + if (step !== Step.init) + setControlFoldOptions(Date.now()) + }, [step]) const { setShowAccountSettingModal } = useModalContext() const handleSetting = useCallback(() => { setShowAccountSettingModal({ @@ -89,6 +95,7 @@ const FireCrawl: FC = ({ time_consuming: number } | undefined>(undefined) const [crawlHasError, setCrawlHasError] = useState(false) + const showError = isCrawlFinished && crawlHasError const waitForCrawlFinished = useCallback(async (jobId: string) => { try { @@ -168,26 +175,32 @@ const FireCrawl: FC = ({ - {isInit && } - {isRunning - && } - {isCrawlFinished && ( - - )} + + {!isInit && ( +
+ {isRunning + && } + {showError && ( + + )} + {isCrawlFinished && !showError + && + } +
+ )}
) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 65de85bb798d54..aa4d035de983f7 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -456,7 +456,6 @@ const DocumentList: FC = ({ embeddingAvailable, documents = { - // TODO: Maybe add website (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) && doc?.data_source_type === DataSourceType.NOTION) ? : From a35335df10978d010ba9a82b9b9cb7b049104349 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 17:42:13 +0800 Subject: [PATCH 143/273] chore: split options from other painel --- .../create/website/firecrawl/base/options-wrap.tsx | 4 ++-- .../datasets/create/website/firecrawl/crawled-result.tsx | 5 ++++- .../components/datasets/create/website/firecrawl/index.tsx | 7 ++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx index 09ea8f60f16696..ca58fe6cacf2d0 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx @@ -32,7 +32,7 @@ const OptionsWrap: FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [controlFoldOptions]) return ( -
+
= ({
{!fold && ( -
+
{children}
)} diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx index 2e35c6afe75fec..ebda7952d9fcef 100644 --- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx +++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx @@ -2,6 +2,7 @@ import type { FC } from 'react' import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' +import cn from 'classnames' import CheckboxWithLabel from './base/checkbox-with-label' import CrawledResultItem from './crawled-result-item' import type { CrawlResultItem } from '@/models/datasets' @@ -9,6 +10,7 @@ import type { CrawlResultItem } from '@/models/datasets' const I18N_PREFIX = 'datasetCreation.stepOne.website' type Props = { + className?: string list: CrawlResultItem[] checkedList: CrawlResultItem[] onSelectedChange: (selected: CrawlResultItem[]) => void @@ -17,6 +19,7 @@ type Props = { } const CrawledResult: FC = ({ + className = '', list, checkedList, onSelectedChange, @@ -54,7 +57,7 @@ const CrawledResult: FC = ({ }, [list, onPreview]) return ( -
+
= ({ return (
-
+
{!isInit && ( -
+
{isRunning && = ({ )} {isCrawlFinished && !showError && Date: Tue, 11 Jun 2024 17:51:23 +0800 Subject: [PATCH 144/273] fix: popover pos loc --- web/app/components/datasets/documents/list.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index aa4d035de983f7..732f2a53d9a066 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -288,7 +288,7 @@ export const OperationAction: FC<{
} btnClassName={open => cn(isListScene ? s.actionIconWrapperList : s.actionIconWrapperDetail, open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')} - className={`!w-[200px] h-fit !z-20 ${className}`} + className={`flex justify-end !w-[200px] h-fit !z-20 ${className}`} /> )} {showModal && setShowModal(false)} className={s.delModal} closable> From 2aa95129f58af6720bbb06b6dc9d1673bf5e7cfb Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 18:06:01 +0800 Subject: [PATCH 145/273] fix: apps not found to app list page --- .../(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx index 014b1094776bcb..7824e910c23491 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx @@ -106,6 +106,9 @@ const AppDetailLayout: FC = (props) => { setAppDetail(res) setNavigation(getNavigations(appId, isCurrentWorkspaceManager, res.mode)) } + }).catch((e: any) => { + if (e.status === 404) + router.replace('/apps') }) }, [appId, isCurrentWorkspaceManager]) From 6a6a950d5a4a68db2f656b7f7364e3b6600606c9 Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 11 Jun 2024 18:31:23 +0800 Subject: [PATCH 146/273] chore: remove for test data --- web/app/components/datasets/create/index.tsx | 2 +- .../datasets/create/website/firecrawl/base/url-input.tsx | 2 +- web/app/components/datasets/create/website/firecrawl/index.tsx | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 7bfc3895210d93..ccb8e0668d9c4c 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -32,7 +32,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { const { t } = useTranslation() const { setShowAccountSettingModal } = useModalContext() const [hasConnection, setHasConnection] = useState(true) - const [dataSourceType, setDataSourceType] = useState(DataSourceType.WEB) // TODO: for test. DataSourceType.FILE + const [dataSourceType, setDataSourceType] = useState(DataSourceType.FILE) const [step, setStep] = useState(1) const [indexingTypeCache, setIndexTypeCache] = useState('') const [fileList, setFiles] = useState([]) diff --git a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx index 90acc769be6ce8..2240a378dcdcc3 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx @@ -17,7 +17,7 @@ const UrlInput: FC = ({ onRun, }) => { const { t } = useTranslation() - const [url, setUrl] = useState('https://docs.dify.ai') // TODO: for test + const [url, setUrl] = useState('') const handleUrlChange = useCallback((url: string | number) => { setUrl(url as string) }, []) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 1a726ee0996891..8e729b64dd1ec3 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -163,6 +163,9 @@ const FireCrawl: FC = ({ setCrawlResult(data) setCrawlHasError(false) } + catch (e) { + setCrawlHasError(true) + } finally { setStep(Step.finished) } From 66808f829ad6a85938ec3c14df7e3216c7f470f7 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Tue, 11 Jun 2024 18:41:21 +0800 Subject: [PATCH 147/273] note editor --- .../vender/line/general/link-broken-01.svg | 10 + .../src/vender/line/general/LinkBroken01.json | 66 ++++ .../src/vender/line/general/LinkBroken01.tsx | 16 + .../icons/src/vender/line/general/index.ts | 1 + .../note-node/note-editor/context.tsx | 2 + .../workflow/note-node/note-editor/editor.tsx | 4 +- .../plugins/format-detector-plugin/hooks.ts | 18 +- .../plugins/link-editor-plugin/component.tsx | 87 +++++- .../plugins/link-editor-plugin/hooks.ts | 110 +++++++ .../plugins/link-editor-plugin/index.tsx | 7 +- .../workflow/note-node/note-editor/store.ts | 50 ++- .../note-node/note-editor/theme/index.ts | 13 + .../note-node/note-editor/theme/theme.css | 12 + .../note-node/note-editor/toolbar/command.tsx | 21 +- .../note-node/note-editor/toolbar/hooks.ts | 24 +- .../workflow/note-node/note-editor/utils.ts | 3 + web/package.json | 4 +- web/yarn.lock | 295 ++++++++++-------- 18 files changed, 544 insertions(+), 199 deletions(-) create mode 100644 web/app/components/base/icons/assets/vender/line/general/link-broken-01.svg create mode 100644 web/app/components/base/icons/src/vender/line/general/LinkBroken01.json create mode 100644 web/app/components/base/icons/src/vender/line/general/LinkBroken01.tsx create mode 100644 web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts create mode 100644 web/app/components/workflow/note-node/note-editor/theme/index.ts create mode 100644 web/app/components/workflow/note-node/note-editor/theme/theme.css diff --git a/web/app/components/base/icons/assets/vender/line/general/link-broken-01.svg b/web/app/components/base/icons/assets/vender/line/general/link-broken-01.svg new file mode 100644 index 00000000000000..47a0560db12ec0 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/general/link-broken-01.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/web/app/components/base/icons/src/vender/line/general/LinkBroken01.json b/web/app/components/base/icons/src/vender/line/general/LinkBroken01.json new file mode 100644 index 00000000000000..92a1234744718a --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/general/LinkBroken01.json @@ -0,0 +1,66 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "12", + "height": "12", + "viewBox": "0 0 12 12", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Left Icon", + "clip-path": "url(#clip0_6246_47371)" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon", + "d": "M4.5 2V1M7.5 10V11M2 4.5H1M10 7.5H11M2.45711 2.45711L1.75 1.75M9.54289 9.54289L10.25 10.25M6 8.82843L4.93934 9.88909C4.15829 10.6701 2.89196 10.6701 2.11091 9.88909C1.32986 9.10804 1.32986 7.84171 2.11091 7.06066L3.17157 6M8.82843 6L9.88909 4.93934C10.6701 4.15829 10.6701 2.89196 9.88909 2.11091C9.10804 1.32986 7.84171 1.32986 7.06066 2.11091L6 3.17157", + "stroke": "currentColor", + "stroke-width": "1.25", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + }, + { + "type": "element", + "name": "defs", + "attributes": {}, + "children": [ + { + "type": "element", + "name": "clipPath", + "attributes": { + "id": "clip0_6246_47371" + }, + "children": [ + { + "type": "element", + "name": "rect", + "attributes": { + "width": "12", + "height": "12", + "fill": "white" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "LinkBroken01" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/general/LinkBroken01.tsx b/web/app/components/base/icons/src/vender/line/general/LinkBroken01.tsx new file mode 100644 index 00000000000000..10e1cee8eeb8dd --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/general/LinkBroken01.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './LinkBroken01.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'LinkBroken01' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/general/index.ts b/web/app/components/base/icons/src/vender/line/general/index.ts index 8288f78aadf6ab..350e54dbe15fa5 100644 --- a/web/app/components/base/icons/src/vender/line/general/index.ts +++ b/web/app/components/base/icons/src/vender/line/general/index.ts @@ -16,6 +16,7 @@ export { default as HelpCircle } from './HelpCircle' export { default as InfoCircle } from './InfoCircle' export { default as Link01 } from './Link01' export { default as Link03 } from './Link03' +export { default as LinkBroken01 } from './LinkBroken01' export { default as LinkExternal01 } from './LinkExternal01' export { default as LinkExternal02 } from './LinkExternal02' export { default as Loading02 } from './Loading02' diff --git a/web/app/components/workflow/note-node/note-editor/context.tsx b/web/app/components/workflow/note-node/note-editor/context.tsx index 453b76d8a1de7f..489e809a144dbe 100644 --- a/web/app/components/workflow/note-node/note-editor/context.tsx +++ b/web/app/components/workflow/note-node/note-editor/context.tsx @@ -9,6 +9,7 @@ import { CodeNode } from '@lexical/code' import { LexicalComposer } from '@lexical/react/LexicalComposer' import { LinkNode } from '@lexical/link' import { createNoteEditorStore } from './store' +import theme from './theme' type NoteEditorStore = ReturnType const NoteEditorContext = createContext(null) @@ -33,6 +34,7 @@ export const NoteEditorContextProvider = memo(({ onError: (error: Error) => { throw error }, + theme, } return ( diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index 40ef58f8d9d630..c24c4f4b1f26cb 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -6,8 +6,9 @@ import { } from 'react' import { RichTextPlugin } from '@lexical/react/LexicalRichTextPlugin' import { ContentEditable } from '@lexical/react/LexicalContentEditable' +import { ClickableLinkPlugin } from '@lexical/react/LexicalClickableLinkPlugin' import { LinkPlugin } from '@lexical/react/LexicalLinkPlugin' -import LexicalErrorBoundary from '@lexical/react/LexicalErrorBoundary' +import { LexicalErrorBoundary } from '@lexical/react/LexicalErrorBoundary' import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin' import LinkEditorPlugin from './plugins/link-editor-plugin' import FormatDetectorPlugin from './plugins/format-detector-plugin' @@ -40,6 +41,7 @@ const Editor = ({ placeholder={} ErrorBoundary={LexicalErrorBoundary} /> + { const handleFormat = useCallback(() => { editor.getEditorState().read(() => { - // Should not to pop up the floating toolbar when using IME input if (editor.isComposing()) return const selection = $getSelection() - if (!$isRangeSelection(selection)) + if (!($isRangeSelection(selection) && !selection?.isCollapsed())) return const node = getSelectedNode(selection) const { - setIsBold, - setIsStrikeThrough, - setIsLink, + setSelectedIsBold, + setSelectedIsStrikeThrough, + setSelectedLinkUrl, } = noteEditorStore.getState() - setIsBold(selection.hasFormat('bold')) - setIsStrikeThrough(selection.hasFormat('strikethrough')) + setSelectedIsBold(selection.hasFormat('bold')) + setSelectedIsStrikeThrough(selection.hasFormat('strikethrough')) const parent = node.getParent() if ($isLinkNode(parent) || $isLinkNode(node)) - setIsLink(true) + setSelectedLinkUrl($isLinkNode(parent) ? parent.getURL() : (node as LinkNode).getURL()) else - setIsLink(false) + setSelectedLinkUrl('') }) }, [editor, noteEditorStore]) diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx index a6b0fe08d6c603..1ca7160eb29764 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx @@ -1,6 +1,6 @@ import { memo, - useCallback, + useEffect, useState, } from 'react' import { @@ -13,21 +13,30 @@ import { import { useTranslation } from 'react-i18next' import cn from 'classnames' import { useStore } from '../../store' +import { useLink } from './hooks' import Button from '@/app/components/base/button' +import { + Edit03, + LinkBroken01, + LinkExternal01, +} from '@/app/components/base/icons/src/vender/line/general' type LinkEditorComponentProps = { containerElement: HTMLDivElement | null - linkUrl?: string - showInput?: boolean } const LinkEditorComponent = ({ containerElement, - linkUrl, - showInput = true, }: LinkEditorComponentProps) => { const { t } = useTranslation() - const [url, setUrl] = useState(linkUrl) - const anchorElement = useStore(s => s.anchorElement) + const { + handleSaveLink, + handleUnlink, + } = useLink() + const selectedLinkUrl = useStore(s => s.selectedLinkUrl) + const linkAnchorElement = useStore(s => s.linkAnchorElement) + const linkOperatorShow = useStore(s => s.linkOperatorShow) + const setLinkOperatorShow = useStore(s => s.setLinkOperatorShow) + const [url, setUrl] = useState(selectedLinkUrl) const { refs, floatingStyles, elements } = useFloating({ placement: 'top', middleware: [ @@ -37,14 +46,14 @@ const LinkEditorComponent = ({ ], }) - const handleConfirm = useCallback(() => {}, []) + useEffect(() => { + if (linkAnchorElement) + refs.setReference(linkAnchorElement) + }, [linkAnchorElement, refs]) - if (!anchorElement) + if (!linkAnchorElement) return null - if (!elements.reference && anchorElement) - refs.setReference(anchorElement) - return ( <> { @@ -53,21 +62,65 @@ const LinkEditorComponent = ({
{ - showInput && ( + !linkOperatorShow && ( <> setUrl(e.target.value)} + placeholder='Enter URL...' /> - + + + ) + } + { + linkOperatorShow && ( + <> +
+ +
Open
+ + {url} + +
+
+
setLinkOperatorShow(false)} + > + + Edit +
+
+ + Unlink +
) } diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts new file mode 100644 index 00000000000000..5e08196eaa18ff --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts @@ -0,0 +1,110 @@ +import { + useCallback, + useEffect, + useState, +} from 'react' +import type { RangeSelection } from 'lexical' +import { + $getSelection, + $isRangeSelection, + CLICK_COMMAND, + COMMAND_PRIORITY_CRITICAL, + COMMAND_PRIORITY_LOW, + SELECTION_CHANGE_COMMAND, +} from 'lexical' +import { + mergeRegister, +} from '@lexical/utils' +import { + $isLinkNode, + TOGGLE_LINK_COMMAND, +} from '@lexical/link' +import type { LinkNode } from '@lexical/link' +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' +import { useNoteEditorStore } from '../../store' +import { getSelectedNode } from '../../utils' + +export const useOpenLink = () => { + const [editor] = useLexicalComposerContext() + const [lastSelection, setLastSelection] = useState(null) + const noteEditorStore = useNoteEditorStore() + + useEffect(() => { + return mergeRegister( + editor.registerCommand( + SELECTION_CHANGE_COMMAND, + () => { + const selection = $getSelection() + + if ($isRangeSelection(selection)) + setLastSelection(selection) + return false + }, + COMMAND_PRIORITY_CRITICAL, + ), + editor.registerCommand( + CLICK_COMMAND, + (payload) => { + const selection = $getSelection() + + if ($isRangeSelection(selection) && selection.isCollapsed()) { + const node = getSelectedNode(selection) + const parent = node.getParent() + + if ($isLinkNode(parent) || $isLinkNode(node)) { + const linkUrl = ((parent || node) as LinkNode).getURL() + if (payload.metaKey || payload.ctrlKey) { + window.open(linkUrl, '_blank') + return true + } + else { + const { + setLinkAnchorElement, + setLinkOperatorShow, + setSelectedLinkUrl, + } = noteEditorStore.getState() + setLinkAnchorElement(true) + setSelectedLinkUrl(linkUrl) + setLinkOperatorShow(true) + } + } + else { + const { + setLinkAnchorElement, + setLinkOperatorShow, + } = noteEditorStore.getState() + setLinkAnchorElement() + setLinkOperatorShow(false) + } + } + return false + }, + COMMAND_PRIORITY_LOW, + ), + ) + }, [editor, noteEditorStore, lastSelection]) +} + +export const useLink = () => { + const [editor] = useLexicalComposerContext() + const noteEditorStore = useNoteEditorStore() + + const handleSaveLink = useCallback((url: string) => { + editor.dispatchCommand(TOGGLE_LINK_COMMAND, url) + + const { setLinkAnchorElement } = noteEditorStore.getState() + setLinkAnchorElement() + }, [editor, noteEditorStore]) + + const handleUnlink = useCallback(() => { + editor.dispatchCommand(TOGGLE_LINK_COMMAND, null) + + const { setLinkAnchorElement } = noteEditorStore.getState() + setLinkAnchorElement() + }, [editor, noteEditorStore]) + + return { + handleSaveLink, + handleUnlink, + } +} diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx index 3c5207526c4365..789745ecf8bdd1 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/index.tsx @@ -1,4 +1,7 @@ -import { memo } from 'react' +import { + memo, +} from 'react' +import { useOpenLink } from './hooks' import LinkEditorComponent from './component' type LinkEditorPluginProps = { @@ -7,6 +10,8 @@ type LinkEditorPluginProps = { const LinkEditorPlugin = ({ containerElement, }: LinkEditorPluginProps) => { + useOpenLink() + return ( ) diff --git a/web/app/components/workflow/note-node/note-editor/store.ts b/web/app/components/workflow/note-node/note-editor/store.ts index 08f5ed9ddcd456..89d7ae66a492f9 100644 --- a/web/app/components/workflow/note-node/note-editor/store.ts +++ b/web/app/components/workflow/note-node/note-editor/store.ts @@ -6,26 +6,44 @@ import { createStore } from 'zustand/vanilla' import NoteEditorContext from './context' type Shape = { - anchorElement: HTMLElement | null - setAnchorElement: (anchorElement: HTMLElement | null) => void - isBold: boolean - setIsBold: (isBold: boolean) => void - isStrikeThrough: boolean - setIsStrikeThrough: (isStrikeThrough: boolean) => void - isLink: boolean - setIsLink: (isLink: boolean) => void + linkAnchorElement: HTMLElement | null + setLinkAnchorElement: (open?: boolean) => void + linkOperatorShow: boolean + setLinkOperatorShow: (linkOperatorShow: boolean) => void + selectedIsBold: boolean + setSelectedIsBold: (selectedIsBold: boolean) => void + selectedIsStrikeThrough: boolean + setSelectedIsStrikeThrough: (selectedIsStrikeThrough: boolean) => void + selectedLinkUrl: string + setSelectedLinkUrl: (selectedLinkUrl: string) => void } export const createNoteEditorStore = () => { return createStore(set => ({ - anchorElement: null, - setAnchorElement: anchorElement => set(() => ({ anchorElement })), - isBold: false, - setIsBold: isBold => set(() => ({ isBold })), - isStrikeThrough: false, - setIsStrikeThrough: isStrikeThrough => set(() => ({ isStrikeThrough })), - isLink: false, - setIsLink: isLink => set(() => ({ isLink })), + linkAnchorElement: null, + setLinkAnchorElement: (open) => { + if (open) { + setTimeout(() => { + const nativeSelection = window.getSelection() + + if (nativeSelection?.focusNode) { + const parent = nativeSelection.focusNode.parentElement + set(() => ({ linkAnchorElement: parent })) + } + }) + } + else { + set(() => ({ linkAnchorElement: null })) + } + }, + linkOperatorShow: false, + setLinkOperatorShow: linkOperatorShow => set(() => ({ linkOperatorShow })), + selectedIsBold: false, + setSelectedIsBold: selectedIsBold => set(() => ({ selectedIsBold })), + selectedIsStrikeThrough: false, + setSelectedIsStrikeThrough: selectedIsStrikeThrough => set(() => ({ selectedIsStrikeThrough })), + selectedLinkUrl: '', + setSelectedLinkUrl: selectedLinkUrl => set(() => ({ selectedLinkUrl })), })) } diff --git a/web/app/components/workflow/note-node/note-editor/theme/index.ts b/web/app/components/workflow/note-node/note-editor/theme/index.ts new file mode 100644 index 00000000000000..c7431fe1b9c3b5 --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/theme/index.ts @@ -0,0 +1,13 @@ +import type { EditorThemeClasses } from 'lexical' + +import './theme.css' + +const theme: EditorThemeClasses = { + paragraph: 'note-editor-theme_paragraph', + link: 'note-editor-theme_link', + text: { + strikethrough: 'note-editor-theme_text-strikethrough', + }, +} + +export default theme diff --git a/web/app/components/workflow/note-node/note-editor/theme/theme.css b/web/app/components/workflow/note-node/note-editor/theme/theme.css new file mode 100644 index 00000000000000..96abe1c43cdd1c --- /dev/null +++ b/web/app/components/workflow/note-node/note-editor/theme/theme.css @@ -0,0 +1,12 @@ +.note-editor-theme_paragraph { + font-size: 12px; +} + +.note-editor-theme_link { + text-decoration: underline; + cursor: pointer; +} + +.note-editor-theme_text-strikethrough { + text-decoration: line-through; +} \ No newline at end of file diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx index 07ec23ab31e3d7..fc0cb4d2a70cfc 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx @@ -18,32 +18,31 @@ type CommandProps = { const Command = ({ type, }: CommandProps) => { - const isBold = useStore(s => s.isBold) - const isStrikeThrough = useStore(s => s.isStrikeThrough) - const isLink = useStore(s => s.isLink) - console.log(isBold) + const selectedIsBold = useStore(s => s.selectedIsBold) + const selectedIsStrikeThrough = useStore(s => s.selectedIsStrikeThrough) + const selectedLinkUrl = useStore(s => s.selectedLinkUrl) const { handleCommand } = useCommand() const icon = useMemo(() => { switch (type) { case 'bold': - return + return case 'strikethrough': - return + return case 'link': - return + return case 'bullet': return } - }, [type, isBold, isStrikeThrough, isLink]) + }, [type, selectedIsBold, selectedIsStrikeThrough, selectedLinkUrl]) return (
handleCommand(type)} > diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts index e2200669fa8736..9181ce908bd9dc 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts @@ -24,6 +24,7 @@ export const useCommand = () => { if (type === 'strikethrough') editor.dispatchCommand(FORMAT_TEXT_COMMAND, 'strikethrough') + if (type === 'link') { editor.update(() => { const selection = $getSelection() @@ -32,24 +33,11 @@ export const useCommand = () => { const node = getSelectedNode(selection) const parent = node.getParent() - if ($isLinkNode(parent) || $isLinkNode(node)) - editor.dispatchCommand(TOGGLE_LINK_COMMAND, null) - else - editor.dispatchCommand(TOGGLE_LINK_COMMAND, 'https://') - - // setTimeout(() => { - // const nativeSelection = window.getSelection() - // editor.getEditorState().read(() => { - // const node = getSelectedNode(selection) - // console.log(node, 'node') - // }) - - // if (nativeSelection?.focusNode) { - // const { setAnchorElement } = noteEditorStore.getState() - // const parent = nativeSelection.focusNode.parentElement - // setAnchorElement(parent) - // } - // }) + if (!($isLinkNode(parent) && $isLinkNode(node))) + editor.dispatchCommand(TOGGLE_LINK_COMMAND, '') + + const { setLinkAnchorElement } = noteEditorStore.getState() + setLinkAnchorElement(true) } }) } diff --git a/web/app/components/workflow/note-node/note-editor/utils.ts b/web/app/components/workflow/note-node/note-editor/utils.ts index 16802dc1158d65..9871263bf1e096 100644 --- a/web/app/components/workflow/note-node/note-editor/utils.ts +++ b/web/app/components/workflow/note-node/note-editor/utils.ts @@ -6,8 +6,11 @@ export function getSelectedNode( ): TextNode | ElementNode { const anchor = selection.anchor const focus = selection.focus + // console.log(selection, 'selection') const anchorNode = selection.anchor.getNode() + // console.log(anchorNode, 'anchorNode') const focusNode = selection.focus.getNode() + // console.log(focusNode, 'focusNode') if (anchorNode === focusNode) return anchorNode diff --git a/web/package.json b/web/package.json index 81f9f83a3d4130..7ff952ae05e93c 100644 --- a/web/package.json +++ b/web/package.json @@ -23,7 +23,7 @@ "@headlessui/react": "^1.7.13", "@heroicons/react": "^2.0.16", "@hookform/resolvers": "^3.3.4", - "@lexical/react": "^0.12.2", + "@lexical/react": "^0.16.0", "@mdx-js/loader": "^2.3.0", "@mdx-js/react": "^2.3.0", "@monaco-editor/react": "^4.6.0", @@ -47,7 +47,7 @@ "js-cookie": "^3.0.1", "katex": "^0.16.10", "lamejs": "^1.2.1", - "lexical": "^0.12.2", + "lexical": "^0.16.0", "lodash-es": "^4.17.21", "mermaid": "10.4.0", "negotiator": "^0.6.3", diff --git a/web/yarn.lock b/web/yarn.lock index e75fa8d0686142..d8aa078e6a3b14 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -414,159 +414,206 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" -"@lexical/clipboard@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/clipboard/-/clipboard-0.12.2.tgz" - integrity sha512-RldmfZquuJJJCJ5WquCyoJ1/eZ+AnNgdksqvd+G+Yn/GyJl/+O3dnHM0QVaDSPvh/PynLFcCtz/57ySLo2kQxQ== +"@lexical/clipboard@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/clipboard/-/clipboard-0.16.0.tgz#3ae0d87a56bd3518de077e45b0c1bbba2f356193" + integrity sha512-eYMJ6jCXpWBVC05Mu9HLMysrBbfi++xFfsm+Yo7A6kYGrqYUhpXqjJkYnw1xdZYL3bV73Oe4ByVJuq42GU+Mqw== dependencies: - "@lexical/html" "0.12.2" - "@lexical/list" "0.12.2" - "@lexical/selection" "0.12.2" - "@lexical/utils" "0.12.2" + "@lexical/html" "0.16.0" + "@lexical/list" "0.16.0" + "@lexical/selection" "0.16.0" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/code@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/code/-/code-0.12.2.tgz" - integrity sha512-w2JeJdnMUtYnC/Fx78sL3iJBt9Ug8pFSDOcI9ay/BkMQFQV8oqq1iyuLLBBJSG4FAM8b2DXrVdGklRQ+jTfTVw== +"@lexical/code@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/code/-/code-0.16.0.tgz#225030342e3c361e5541c750033323007a947880" + integrity sha512-1EKCBSFV745UI2zn5v75sKcvVdmd+y2JtZhw8CItiQkRnBLv4l4d/RZYy+cKOuXJGsoBrKtxXn5sl7HebwQbPw== dependencies: - "@lexical/utils" "0.12.2" + "@lexical/utils" "0.16.0" + lexical "0.16.0" prismjs "^1.27.0" -"@lexical/dragon@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/dragon/-/dragon-0.12.2.tgz" - integrity sha512-Mt8NLzTOt+VgQtc2DKDbHBwKeRlvKqbLqRIMYUVk60gol+YV7NpVBsP1PAMuYYjrTQLhlckBSC32H1SUHZRavA== +"@lexical/devtools-core@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/devtools-core/-/devtools-core-0.16.0.tgz#326c8e2995ce6e6e9e1fc4654ee2affbecdbd46d" + integrity sha512-Jt8p0J0UoMHf3UMh3VdyrXbLLwpEZuMqihTmbPRpwo+YQ6NGQU35QgwY2K0DpPAThpxL/Cm7uaFqGOy8Kjrhqw== + dependencies: + "@lexical/html" "0.16.0" + "@lexical/link" "0.16.0" + "@lexical/mark" "0.16.0" + "@lexical/table" "0.16.0" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/hashtag@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/hashtag/-/hashtag-0.12.2.tgz" - integrity sha512-2vYzIu5Ldf+eYdUrNA2m80c3N3MF3vJ0fIJzpl5QyX8OdViggEWl1bh+lKtw1Ju0H0CUyDIXdDLZ2apW3WDkTA== +"@lexical/dragon@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/dragon/-/dragon-0.16.0.tgz#de083903701af2bb5264309b565d613c3eec06a0" + integrity sha512-Yr29SFZzOPs+S6UrEZaXnnso1fJGVfZOXVJQZbyzlspqJpSHXVH7InOXYHWN6JSWQ8Hs/vU3ksJXwqz+0TCp2g== dependencies: - "@lexical/utils" "0.12.2" + lexical "0.16.0" -"@lexical/history@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/history/-/history-0.12.2.tgz" - integrity sha512-PM/EDjnUyBPMWh1UiYb7T+FLbvTk14HwUWLXvZxn72S6Kj8ExH/PfLbWZWLCFL8RfzvbP407VwfSN8S0bF5H6g== +"@lexical/hashtag@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/hashtag/-/hashtag-0.16.0.tgz#ea0187060a114678753adaf0a15aad59d4f49a71" + integrity sha512-2EdAvxYVYqb0nv6vgxCRgE8ip7yez5p0y0oeUyxmdbcfZdA+Jl90gYH3VdevmZ5Bk3wE0/fIqiLD+Bb5smqjCQ== dependencies: - "@lexical/utils" "0.12.2" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/html@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/html/-/html-0.12.2.tgz" - integrity sha512-LWUO6OKhDtDZa9X1spHAqzsp+4EF01exis4cz5H9y2sHi7EofogXnRCadZ+fa07NVwPVTZWsStkk5qdSe/NEzg== +"@lexical/history@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/history/-/history-0.16.0.tgz#f83f2e331957208c5c8186d98f2f84681d936cec" + integrity sha512-xwFxgDZGviyGEqHmgt6A6gPhsyU/yzlKRk9TBUVByba3khuTknlJ1a80H5jb+OYcrpiElml7iVuGYt+oC7atCA== dependencies: - "@lexical/selection" "0.12.2" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/link@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/link/-/link-0.12.2.tgz" - integrity sha512-etOIONa7uyRDmwg8GN52kDlf8thD2Zk1LOFLeocHWz1V8fe3i2unGUek5s/rNPkc6ynpPpNsHdN1VEghOLCCmw== +"@lexical/html@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/html/-/html-0.16.0.tgz#98477ed0dee4c7d910608f4e4de3fbd5eeecdffe" + integrity sha512-okxn3q/1qkUpCZNEFRI39XeJj4YRjb6prm3WqZgP4d39DI1W24feeTZJjYRCW+dc3NInwFaolU3pNA2MGkjRtg== dependencies: - "@lexical/utils" "0.12.2" + "@lexical/selection" "0.16.0" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/list@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/list/-/list-0.12.2.tgz" - integrity sha512-3CyWtYQC+IlK4cK/oiD8Uz1gSXD8UcKGOF2vVsDXkMU06O6zvHNmHZOnVJqA0JVNgZAoR9dMR1fi2xd4iuCAiw== +"@lexical/link@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/link/-/link-0.16.0.tgz#f137ab3071206ed3c3a8b8a302ed66b084399ed1" + integrity sha512-ppvJSh/XGqlzbeymOiwcXJcUcrqgQqTK2QXTBAZq7JThtb0WsJxYd2CSLSN+Ycu23prnwqOqILcU0+34+gAVFw== dependencies: - "@lexical/utils" "0.12.2" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/mark@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/mark/-/mark-0.12.2.tgz" - integrity sha512-ub+37PDfmThsqAWipRTrwqpgE+83ckqJ5C3mKQUBZvhZfVZW1rEUXZnKjFh2Q3eZK6iT7zVgoVJWJS9ZgEEyag== +"@lexical/list@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/list/-/list-0.16.0.tgz#ed97733633492e89c68ad51a1d455b63ce5aa1c0" + integrity sha512-nBx/DMM7nCgnOzo1JyNnVaIrk/Xi5wIPNi8jixrEV6w9Om2K6dHutn/79Xzp2dQlNGSLHEDjky6N2RyFgmXh0g== dependencies: - "@lexical/utils" "0.12.2" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/markdown@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/markdown/-/markdown-0.12.2.tgz" - integrity sha512-F2jTFtBp7Q+yoA11BeUOEcxhROzW+HUhUGdsn20pSLhuxsWRj3oUuryWFeNKFofpzTCVoqU6dwpaMNMI2mL/sQ== +"@lexical/mark@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/mark/-/mark-0.16.0.tgz#e87d92845c8bd231ef47106c5d44e7e10d2a3934" + integrity sha512-WMR4nqygSgIQ6Vdr5WAzohxBGjH+m44dBNTbWTGZGVlRvPzvBT6tieCoxFqpceIq/ko67HGTCNoFj2cMKVwgIA== dependencies: - "@lexical/code" "0.12.2" - "@lexical/link" "0.12.2" - "@lexical/list" "0.12.2" - "@lexical/rich-text" "0.12.2" - "@lexical/text" "0.12.2" - "@lexical/utils" "0.12.2" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/offset@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/offset/-/offset-0.12.2.tgz" - integrity sha512-rZLZXfOBmpmM8A2UZsX3cr/CQYw5F/ou67AbaKI0WImb5sjnIgICZqzu9VFUnkKlVNUurEpplV3UG3D1YYh1OQ== +"@lexical/markdown@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/markdown/-/markdown-0.16.0.tgz#fd2d2759d9d5554d9899c3e1fb30a868bfa162a2" + integrity sha512-7HQLFrBbpY68mcq4A6C1qIGmjgA+fAByditi2WRe7tD2eoIKb/B5baQAnDKis0J+m5kTaCBmdlT6csSzyOPzeQ== + dependencies: + "@lexical/code" "0.16.0" + "@lexical/link" "0.16.0" + "@lexical/list" "0.16.0" + "@lexical/rich-text" "0.16.0" + "@lexical/text" "0.16.0" + "@lexical/utils" "0.16.0" + lexical "0.16.0" + +"@lexical/offset@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/offset/-/offset-0.16.0.tgz#bb3bc695ed403db0795f095330c68cdc5cbbec4b" + integrity sha512-4TqPEC2qA7sgO8Tm65nOWnhJ8dkl22oeuGv9sUB+nhaiRZnw3R45mDelg23r56CWE8itZnvueE7TKvV+F3OXtQ== + dependencies: + lexical "0.16.0" -"@lexical/overflow@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/overflow/-/overflow-0.12.2.tgz" - integrity sha512-UgE5j3ukO6qRFRpH4T7m/DvnodE9nCtImD7QinyGdsTa0hi5xlRnl0FUo605vH+vz7xEsUNAGwQXYPX9Sc/vig== +"@lexical/overflow@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/overflow/-/overflow-0.16.0.tgz#31b791f7f7005ea4b160f3ae8083a2b3de05cfdc" + integrity sha512-a7gtIRxleEuMN9dj2yO4CdezBBfIr9Mq+m7G5z62+xy7VL7cfMfF+xWjy3EmDYDXS4vOQgAXAUgO4oKz2AKGhQ== + dependencies: + lexical "0.16.0" -"@lexical/plain-text@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/plain-text/-/plain-text-0.12.2.tgz" - integrity sha512-Lcg6+ngRnX70//kz34azYhID3bvW66HSHCfu5UPhCXT+vQ/Jkd/InhRKajBwWXpaJxMM1huoi3sjzVDb3luNtw== +"@lexical/plain-text@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/plain-text/-/plain-text-0.16.0.tgz#b903bfb59fb6629ded24194e1bef451df3383393" + integrity sha512-BK7/GSOZUHRJTbNPkpb9a/xN9z+FBCdunTsZhnOY8pQ7IKws3kuMO2Tk1zXfTd882ZNAxFdDKNdLYDSeufrKpw== + dependencies: + "@lexical/clipboard" "0.16.0" + "@lexical/selection" "0.16.0" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/react@^0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/react/-/react-0.12.2.tgz" - integrity sha512-ZBUvf5xmhiYWBw8pPrhYmLAEwFWrbF/cd15y76TUKD9l/2zDwwPs6nJQxBzfz3ei65r2/nnavLDV8W3QfvxfUA== - dependencies: - "@lexical/clipboard" "0.12.2" - "@lexical/code" "0.12.2" - "@lexical/dragon" "0.12.2" - "@lexical/hashtag" "0.12.2" - "@lexical/history" "0.12.2" - "@lexical/link" "0.12.2" - "@lexical/list" "0.12.2" - "@lexical/mark" "0.12.2" - "@lexical/markdown" "0.12.2" - "@lexical/overflow" "0.12.2" - "@lexical/plain-text" "0.12.2" - "@lexical/rich-text" "0.12.2" - "@lexical/selection" "0.12.2" - "@lexical/table" "0.12.2" - "@lexical/text" "0.12.2" - "@lexical/utils" "0.12.2" - "@lexical/yjs" "0.12.2" +"@lexical/react@^0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/react/-/react-0.16.0.tgz#0bd3ae63ceb5ad8b77e8c0e8ba7df1a0369462f0" + integrity sha512-WKFQbI0/m1YkLjL5t90YLJwjGcl5QRe6mkfm3ljQuL7Ioj3F92ZN/J2gHFVJ9iC8/lJs6Zzw6oFjiP8hQxJf9Q== + dependencies: + "@lexical/clipboard" "0.16.0" + "@lexical/code" "0.16.0" + "@lexical/devtools-core" "0.16.0" + "@lexical/dragon" "0.16.0" + "@lexical/hashtag" "0.16.0" + "@lexical/history" "0.16.0" + "@lexical/link" "0.16.0" + "@lexical/list" "0.16.0" + "@lexical/mark" "0.16.0" + "@lexical/markdown" "0.16.0" + "@lexical/overflow" "0.16.0" + "@lexical/plain-text" "0.16.0" + "@lexical/rich-text" "0.16.0" + "@lexical/selection" "0.16.0" + "@lexical/table" "0.16.0" + "@lexical/text" "0.16.0" + "@lexical/utils" "0.16.0" + "@lexical/yjs" "0.16.0" + lexical "0.16.0" react-error-boundary "^3.1.4" -"@lexical/rich-text@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/rich-text/-/rich-text-0.12.2.tgz" - integrity sha512-igsEuv7CwBOAj5c8jeE41cnx6zkhI/Bkbu4W7shT6S6lNA/3cnyZpAMlgixwyK5RoqjGRCT+IJK5l6yBxQfNkw== +"@lexical/rich-text@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/rich-text/-/rich-text-0.16.0.tgz#5b9ea6ceb1ea034fa7adf1770bd7fa6af1571d1d" + integrity sha512-AGTD6yJZ+kj2TNah1r7/6vyufs6fZANeSvv9x5eG+WjV4uyUJYkd1qR8C5gFZHdkyr+bhAcsAXvS039VzAxRrQ== + dependencies: + "@lexical/clipboard" "0.16.0" + "@lexical/selection" "0.16.0" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/selection@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/selection/-/selection-0.12.2.tgz" - integrity sha512-h+g3oOnihHKIyLTyG6uLCEVR/DmUEVdCcZO1iAoGsuW7nwWiWNPWj6oZ3Cw5J1Mk5u62DHnkkVDQsVSZbAwmtg== +"@lexical/selection@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/selection/-/selection-0.16.0.tgz#8e09edb1e555e79c646a0105beab58ac21fc7158" + integrity sha512-trT9gQVJ2j6AwAe7tHJ30SRuxCpV6yR9LFtggxphHsXSvJYnoHC0CXh1TF2jHl8Gd5OsdWseexGLBE4Y0V3gwQ== + dependencies: + lexical "0.16.0" -"@lexical/table@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/table/-/table-0.12.2.tgz" - integrity sha512-tiAmTq6RKHDVER9v589Ajm9/RL+WTF1WschrH6HHVCtil6cfJfTJeJ+MF45+XEzB9fkqy2LfrScAfWxqLjVePA== +"@lexical/table@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/table/-/table-0.16.0.tgz#68592afbb0f9c0d9bf42bebaae626b8129fc470d" + integrity sha512-A66K779kxdr0yH2RwT2itsMnkzyFLFNPXyiWGLobCH8ON4QPuBouZvjbRHBe8Pe64yJ0c1bRDxSbTqUi9Wt3Gg== dependencies: - "@lexical/utils" "0.12.2" + "@lexical/utils" "0.16.0" + lexical "0.16.0" -"@lexical/text@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/text/-/text-0.12.2.tgz" - integrity sha512-HyuIGuQvVi5djJKKBf+jYEBjK+0Eo9cKHf6WS7dlFozuCZvcCQEJkFy2yceWOwIVk+f2kptVQ5uO7aiZHExH2A== +"@lexical/text@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/text/-/text-0.16.0.tgz#fc4789591f8aaa4a33bc1814280bc8725fd036a9" + integrity sha512-9ilaOhuNIIGHKC8g8j3K/mEvJ09af9B6RKbm3GNoRcf/WNHD4dEFWNTEvgo/3zCzAS8EUBI6UINmfQQWlMjdIQ== + dependencies: + lexical "0.16.0" -"@lexical/utils@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/utils/-/utils-0.12.2.tgz" - integrity sha512-xW4y4l2Yd37+qLwkBvBGyzsKCA9wnh1ljphBJeR2vreT193i2gaIwuku2ZKlER14VHw4192qNJF7vUoAEmwurQ== +"@lexical/utils@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/utils/-/utils-0.16.0.tgz#6ad5785c53347aed5b39c980240c09b21c4a7469" + integrity sha512-GWmFEmd7o3GHqJBaEwzuZQbfTNI3Gg8ReGuHMHABgrkhZ8j2NggoRBlxsQLG0f7BewfTMVwbye22yBPq78775w== dependencies: - "@lexical/list" "0.12.2" - "@lexical/selection" "0.12.2" - "@lexical/table" "0.12.2" + "@lexical/list" "0.16.0" + "@lexical/selection" "0.16.0" + "@lexical/table" "0.16.0" + lexical "0.16.0" -"@lexical/yjs@0.12.2": - version "0.12.2" - resolved "https://registry.npmjs.org/@lexical/yjs/-/yjs-0.12.2.tgz" - integrity sha512-OPJhkJD1Mp9W80mfLzASTB3OFWFMzJteUYA+eSyDgiX9zNi1VGxAqmIITTkDvnCMa+qvw4EfhGeGezpjx6Og4A== +"@lexical/yjs@0.16.0": + version "0.16.0" + resolved "https://registry.yarnpkg.com/@lexical/yjs/-/yjs-0.16.0.tgz#e27bec25c12e90f7768b980da08f2d2d9919d25b" + integrity sha512-YIJr87DfAXTwoVHDjR7cci//hr4r/a61Nn95eo2JNwbTqQo65Gp8rwJivqVxNfvKZmRdwHTKgvdEDoBmI/tGog== dependencies: - "@lexical/offset" "0.12.2" + "@lexical/offset" "0.16.0" + lexical "0.16.0" "@mdx-js/loader@^2.3.0": version "2.3.0" @@ -4287,10 +4334,10 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" -lexical@^0.12.2: - version "0.12.2" - resolved "https://registry.npmjs.org/lexical/-/lexical-0.12.2.tgz" - integrity sha512-Kxavd+ETjxtVwG/hvPd6WZfXD44sLOKe9Vlkwxy7lBQ1qZArS+rZfs+u5iXwXe6tX9f2PIM0u3RHsrCEDDE0fw== +lexical@0.16.0, lexical@^0.16.0: + version "0.16.0" + resolved "https://registry.yarnpkg.com/lexical/-/lexical-0.16.0.tgz#0515d4003cbfba5a5e0e3e50f32f65076a6b89e2" + integrity sha512-Skn45Qhriazq4fpAtwnAB11U//GKc4vjzx54xsV3TkDLDvWpbL4Z9TNRwRoN3g7w8AkWnqjeOSODKkrjgfRSrg== lilconfig@2.1.0, lilconfig@^2.0.5, lilconfig@^2.1.0: version "2.1.0" From 35d391233bd987f7d3fe7230b52de201d83e497b Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 10:16:14 +0800 Subject: [PATCH 148/273] fix: websit hover chore --- web/app/components/datasets/create/step-one/index.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index fa945d043cc118..6999b8d151965d 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -167,7 +167,10 @@ const StepOne = ({ {t('datasetCreation.stepOne.dataSourceType.notion')}
changeType(DataSourceType.WEB)} > From b3fd6644e43461fe3314d9eee15b5667345e6e8a Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 10:17:00 +0800 Subject: [PATCH 149/273] chore: code beaty --- web/app/components/datasets/create/step-one/index.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index 6999b8d151965d..56a668b7e4b00e 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -170,7 +170,8 @@ const StepOne = ({ className={cn( s.dataSourceItem, dataSourceType === DataSourceType.WEB && s.active, - dataSourceTypeDisable && dataSourceType !== DataSourceType.WEB && s.disabled)} + dataSourceTypeDisable && dataSourceType !== DataSourceType.WEB && s.disabled, + )} onClick={() => changeType(DataSourceType.WEB)} > From 777674ca19744ce4cafa8ee55957ea2c558cf7cf Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 10:23:02 +0800 Subject: [PATCH 150/273] fix: time cosuming not right --- .../components/datasets/create/website/firecrawl/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 8e729b64dd1ec3..9948d6d49982f3 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -92,7 +92,7 @@ const FireCrawl: FC = ({ current: number total: number data: CrawlResultItem[] - time_consuming: number + time_consuming: number | string } | undefined>(undefined) const [crawlHasError, setCrawlHasError] = useState(false) const showError = isCrawlFinished && crawlHasError @@ -200,7 +200,7 @@ const FireCrawl: FC = ({ checkedList={checkedCrawlResult} onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} - usedTime={crawlResult?.time_consuming || 0} + usedTime={parseFloat(crawlResult?.time_consuming as string) || 0} /> }
From 8543457624b693e24a8105c3a51379fbb047a56f Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 10:47:43 +0800 Subject: [PATCH 151/273] feat: edit page add web crawl --- web/app/components/datasets/create/step-one/index.tsx | 7 ++++--- web/app/components/datasets/documents/index.tsx | 5 ++++- web/i18n/en-US/dataset-documents.ts | 3 ++- web/i18n/zh-Hans/dataset-documents.ts | 1 + 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index 56a668b7e4b00e..3e4f72f16ab032 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -56,7 +56,7 @@ export const NotionConnector = ({ onSetting }: NotionConnectorProps) => { const StepOne = ({ datasetId, - dataSourceType, + dataSourceType: inCreatePageDataSourceType, dataSourceTypeDisable, changeType, hasConnection, @@ -103,7 +103,8 @@ const StepOne = ({ } const shouldShowDataSourceTypeList = !datasetId || (datasetId && !dataset?.data_source_type) - + const isInCreatePage = shouldShowDataSourceTypeList + const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : dataset?.data_source_type const { plan, enableBilling } = useProviderContext() const allFileLoaded = (files.length > 0 && files.every(file => file.file.id)) const hasNotin = notionPages.length > 0 @@ -223,7 +224,7 @@ const StepOne = ({ )} {dataSourceType === DataSourceType.WEB && ( <> -
+
= ({ datasetId }) => { const [notionPageSelectorModalVisible, setNotionPageSelectorModalVisible] = useState(false) const [timerCanRun, setTimerCanRun] = useState(true) const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION + const isDataSourceWeb = dataset?.data_source_type === DataSourceType.WEB + const isDataSourceFile = dataset?.data_source_type === DataSourceType.FILE const embeddingAvailable = !!dataset?.embedding_available const query = useMemo(() => { @@ -211,7 +213,8 @@ const Documents: FC = ({ datasetId }) => { )}
diff --git a/web/i18n/en-US/dataset-documents.ts b/web/i18n/en-US/dataset-documents.ts index b431965323cd27..8988b9dc184c65 100644 --- a/web/i18n/en-US/dataset-documents.ts +++ b/web/i18n/en-US/dataset-documents.ts @@ -2,8 +2,9 @@ const translation = { list: { title: 'Documents', desc: 'All files of the Knowledge are shown here, and the entire Knowledge can be linked to Dify citations or indexed via the Chat plugin.', - addFile: 'add file', + addFile: 'Add file', addPages: 'Add Pages', + addUrl: 'Add URL', table: { header: { fileName: 'FILE NAME', diff --git a/web/i18n/zh-Hans/dataset-documents.ts b/web/i18n/zh-Hans/dataset-documents.ts index 9ea5e7aa0f58d6..2f68f04d1df485 100644 --- a/web/i18n/zh-Hans/dataset-documents.ts +++ b/web/i18n/zh-Hans/dataset-documents.ts @@ -4,6 +4,7 @@ const translation = { desc: '知识库的所有文件都在这里显示，整个知识库都可以链接到 Dify 引用或通过 Chat 插件进行索引。', addFile: '添加文件', addPages: '添加页面', + addUrl: '添加 URL', table: { header: { fileName: '文件名', From 77cff58c4766b02583b68470c58db0039457406f Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Wed, 12 Jun 2024 12:43:32 +0800 Subject: [PATCH 152/273] note editor --- .../plugins/format-detector-plugin/hooks.ts | 30 ++++---- .../plugins/link-editor-plugin/hooks.ts | 18 +---- .../note-editor/toolbar/color-picker.tsx | 40 +++++++--- .../toolbar/font-size-selector.tsx | 36 +++++---- .../note-node/note-editor/toolbar/hooks.ts | 73 +++++++++++++++++++ .../note-node/note-editor/toolbar/index.tsx | 8 +- 6 files changed, 147 insertions(+), 58 deletions(-) diff --git a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts index e14688abd5859d..486497d30202d9 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts @@ -23,22 +23,22 @@ export const useFormatDetector = () => { return const selection = $getSelection() - if (!($isRangeSelection(selection) && !selection?.isCollapsed())) - return - const node = getSelectedNode(selection) - const { - setSelectedIsBold, - setSelectedIsStrikeThrough, - setSelectedLinkUrl, - } = noteEditorStore.getState() - setSelectedIsBold(selection.hasFormat('bold')) - setSelectedIsStrikeThrough(selection.hasFormat('strikethrough')) - const parent = node.getParent() - if ($isLinkNode(parent) || $isLinkNode(node)) - setSelectedLinkUrl($isLinkNode(parent) ? parent.getURL() : (node as LinkNode).getURL()) - else - setSelectedLinkUrl('') + if ($isRangeSelection(selection)) { + const node = getSelectedNode(selection) + const { + setSelectedIsBold, + setSelectedIsStrikeThrough, + setSelectedLinkUrl, + } = noteEditorStore.getState() + setSelectedIsBold(selection.hasFormat('bold')) + setSelectedIsStrikeThrough(selection.hasFormat('strikethrough')) + const parent = node.getParent() + if ($isLinkNode(parent) || $isLinkNode(node)) + setSelectedLinkUrl($isLinkNode(parent) ? parent.getURL() : (node as LinkNode).getURL()) + else + setSelectedLinkUrl('') + } }) }, [editor, noteEditorStore]) diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts index 5e08196eaa18ff..e93e8a3b3a4168 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts @@ -1,16 +1,12 @@ import { useCallback, useEffect, - useState, } from 'react' -import type { RangeSelection } from 'lexical' import { $getSelection, $isRangeSelection, CLICK_COMMAND, - COMMAND_PRIORITY_CRITICAL, COMMAND_PRIORITY_LOW, - SELECTION_CHANGE_COMMAND, } from 'lexical' import { mergeRegister, @@ -26,22 +22,10 @@ import { getSelectedNode } from '../../utils' export const useOpenLink = () => { const [editor] = useLexicalComposerContext() - const [lastSelection, setLastSelection] = useState(null) const noteEditorStore = useNoteEditorStore() useEffect(() => { return mergeRegister( - editor.registerCommand( - SELECTION_CHANGE_COMMAND, - () => { - const selection = $getSelection() - - if ($isRangeSelection(selection)) - setLastSelection(selection) - return false - }, - COMMAND_PRIORITY_CRITICAL, - ), editor.registerCommand( CLICK_COMMAND, (payload) => { @@ -82,7 +66,7 @@ export const useOpenLink = () => { COMMAND_PRIORITY_LOW, ), ) - }, [editor, noteEditorStore, lastSelection]) + }, [editor, noteEditorStore]) } export const useLink = () => { diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx index ff6de436bd559c..570b209dc35d7f 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx @@ -9,40 +9,54 @@ import { PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' -const COLOR_LIST = [ +export const COLOR_MAP = { + blue: '#D1E9FF', + cyan: '#CFF9FE', + green: '#D3F8DF', + yellow: '#FEF7C3', + pink: '#FCE7F6', + violet: '#ECE9FE', +} as Record +export const COLOR_LIST = [ { key: 'blue', - inner: '#D1E9FF', + inner: COLOR_MAP.blue, outer: '#2E90FA', }, { key: 'cyan', - inner: '#CFF9FE', + inner: COLOR_MAP.cyan, outer: '#06AED4', }, { key: 'green', - inner: '#D3F8DF', + inner: COLOR_MAP.green, outer: '#16B364', }, { key: 'yellow', - inner: '#FEF7C3', + inner: COLOR_MAP.yellow, outer: '#EAAA08', }, { key: 'pink', - inner: '#FCE7F6', + inner: COLOR_MAP.pink, outer: '#EE46BC', }, { key: 'violet', - inner: '#ECE9FE', + inner: COLOR_MAP.violet, outer: '#875BF7', }, ] -const ColorPicker = () => { +export type ColorPickerProps = { + onColorChange?: (color: string) => void +} +const ColorPicker = ({ + onColorChange, +}: ColorPickerProps) => { + const [color, setColor] = useState('blue') const [open, setOpen] = useState(false) return ( @@ -57,7 +71,10 @@ const ColorPicker = () => { 'flex items-center justify-center w-8 h-8 rounded-md cursor-pointer hover:bg-black/5', open && 'bg-black/5', )}> -
+
@@ -67,6 +84,11 @@ const ColorPicker = () => {
{ + setColor(color.key) + onColorChange?.(color.key) + setOpen(false) + }} >
{ - const [value] = useState(FONT_SIZE_LIST[0].key) - const [open, setOpen] = useState(false) + const { + fontSizeSelectorShow, + handleOpenFontSizeSelector, + fontSize, + handleFontSize, + } = useFontSize() return ( - setOpen(!open)}> + handleOpenFontSizeSelector(!fontSizeSelectorShow)}>
- {value} + {FONT_SIZE_LIST.find(font => font.value === fontSize)?.key || 'Small'}
@@ -54,6 +56,10 @@ const FontSizeSelector = () => {
{ + handleFontSize(font.value) + handleOpenFontSizeSelector(false) + }} >
{ {font.key}
{ - value === font.key && ( + fontSize === font.value && ( ) } diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts index 9181ce908bd9dc..94062aedf4b14e 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts @@ -1,11 +1,21 @@ import { useCallback, + useEffect, + useState, } from 'react' import { $getSelection, $isRangeSelection, + $setSelection, + COMMAND_PRIORITY_CRITICAL, FORMAT_TEXT_COMMAND, + SELECTION_CHANGE_COMMAND, } from 'lexical' +import { + $getSelectionStyleValueForProperty, + $patchStyleText, +} from '@lexical/selection' +import { mergeRegister } from '@lexical/utils' import { $isLinkNode, TOGGLE_LINK_COMMAND, @@ -47,3 +57,66 @@ export const useCommand = () => { handleCommand, } } + +export const useFontSize = () => { + const [editor] = useLexicalComposerContext() + const [fontSize, setFontSize] = useState('12px') + const [fontSizeSelectorShow, setFontSizeSelectorShow] = useState(false) + + const handleFontSize = useCallback((fontSize: string) => { + editor.update(() => { + const selection = $getSelection() + + if ($isRangeSelection(selection)) + $patchStyleText(selection, { 'font-size': fontSize }) + }) + }, [editor]) + + const handleOpenFontSizeSelector = useCallback((newFontSizeSelectorShow: boolean) => { + if (newFontSizeSelectorShow) { + editor.update(() => { + const selection = $getSelection() + + if ($isRangeSelection(selection)) + $setSelection(selection.clone()) + }) + } + setFontSizeSelectorShow(newFontSizeSelectorShow) + }, [editor]) + + useEffect(() => { + return mergeRegister( + editor.registerUpdateListener(() => { + editor.getEditorState().read(() => { + const selection = $getSelection() + + if ($isRangeSelection(selection)) { + const fontSize = $getSelectionStyleValueForProperty(selection, 'font-size', '12px') + setFontSize(fontSize) + } + }) + }), + editor.registerCommand( + SELECTION_CHANGE_COMMAND, + () => { + const selection = $getSelection() + + if ($isRangeSelection(selection)) { + const fontSize = $getSelectionStyleValueForProperty(selection, 'font-size', '12px') + setFontSize(fontSize) + } + + return false + }, + COMMAND_PRIORITY_CRITICAL, + ), + ) + }, [editor]) + + return { + fontSize, + handleFontSize, + fontSizeSelectorShow, + handleOpenFontSizeSelector, + } +} diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx index 70b52afe191355..cd8046fe1db85c 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx @@ -1,14 +1,18 @@ import { memo } from 'react' import Divider from './divider' +import type { ColorPickerProps } from './color-picker' import ColorPicker from './color-picker' import FontSizeSelector from './font-size-selector' import Command from './command' import Operator from './operator' -const Toolbar = () => { +type ToolbarProps = ColorPickerProps +const Toolbar = ({ + onColorChange, +}: ToolbarProps) => { return (
- + From 59a50bcce20d8eb6f01c132fb5f604438987229c Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 12:54:08 +0800 Subject: [PATCH 153/273] chore: remove login by password --- web/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/Dockerfile b/web/Dockerfile index 7570783f9fb465..c15f15cb6c873f 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -38,7 +38,7 @@ ENV EDITION SELF_HOSTED ENV DEPLOY_ENV PRODUCTION ENV CONSOLE_API_URL http://127.0.0.1:5001 ENV APP_API_URL http://127.0.0.1:5001 -ENV SUPPORT_MAIL_LOGIN true +ENV SUPPORT_MAIL_LOGIN false ENV PORT 3000 # set timezone From b2a7680673c799fa1aab9a9acf23ee66821c922d Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 13:00:42 +0800 Subject: [PATCH 154/273] chore: remove login by password --- web/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/web/Dockerfile b/web/Dockerfile index c15f15cb6c873f..f2fc4af2f81e27 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -38,7 +38,6 @@ ENV EDITION SELF_HOSTED ENV DEPLOY_ENV PRODUCTION ENV CONSOLE_API_URL http://127.0.0.1:5001 ENV APP_API_URL http://127.0.0.1:5001 -ENV SUPPORT_MAIL_LOGIN false ENV PORT 3000 # set timezone From cbf5032bec7fca0fe39bb8c0d52cb16ad61cbf7c Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 14:34:44 +0800 Subject: [PATCH 155/273] fix: first error not show error status --- .../components/datasets/create/website/firecrawl/index.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 9948d6d49982f3..601bf986fff8c0 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -152,16 +152,17 @@ const FireCrawl: FC = ({ url, options: crawlOptions, }) as any + console.log(res) const jobId = res.job_id onJobIdChange(jobId) const { isError, data } = await waitForCrawlFinished(jobId) if (isError) { setCrawlHasError(true) + } + else { setCrawlResult(data) + setCrawlHasError(false) } - - setCrawlResult(data) - setCrawlHasError(false) } catch (e) { setCrawlHasError(true) From 5a80b54cb786a29cc5bb17ab9e5097230c56b83d Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Wed, 12 Jun 2024 14:41:20 +0800 Subject: [PATCH 156/273] note editor --- .../vender/line/files/sticker-square.svg | 5 +++ .../src/vender/line/files/StickerSquare.json | 39 +++++++++++++++++++ .../src/vender/line/files/StickerSquare.tsx | 16 ++++++++ .../base/icons/src/vender/line/files/index.ts | 1 + .../components/workflow/note-node/hooks.ts | 17 ++++++++ .../components/workflow/note-node/index.tsx | 28 +++++++++++-- .../note-node/note-editor/context.tsx | 8 +++- .../workflow/note-node/note-editor/editor.tsx | 16 ++++++-- .../plugins/format-detector-plugin/hooks.ts | 7 ++++ .../workflow/note-node/note-editor/store.ts | 4 ++ .../note-node/note-editor/theme/index.ts | 4 ++ .../note-node/note-editor/theme/theme.css | 12 ++++++ .../note-node/note-editor/toolbar/command.tsx | 6 ++- .../note-node/note-editor/toolbar/hooks.ts | 4 ++ .../workflow/note-node/note-editor/utils.ts | 3 -- .../components/workflow/note-node/types.ts | 0 .../components/workflow/operator/control.tsx | 19 +++++++++ 17 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 web/app/components/base/icons/assets/vender/line/files/sticker-square.svg create mode 100644 web/app/components/base/icons/src/vender/line/files/StickerSquare.json create mode 100644 web/app/components/base/icons/src/vender/line/files/StickerSquare.tsx create mode 100644 web/app/components/workflow/note-node/hooks.ts create mode 100644 web/app/components/workflow/note-node/types.ts diff --git a/web/app/components/base/icons/assets/vender/line/files/sticker-square.svg b/web/app/components/base/icons/assets/vender/line/files/sticker-square.svg new file mode 100644 index 00000000000000..0104e7ce8a82e9 --- /dev/null +++ b/web/app/components/base/icons/assets/vender/line/files/sticker-square.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/web/app/components/base/icons/src/vender/line/files/StickerSquare.json b/web/app/components/base/icons/src/vender/line/files/StickerSquare.json new file mode 100644 index 00000000000000..92d721912ccfdc --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/files/StickerSquare.json @@ -0,0 +1,39 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "16", + "height": "16", + "viewBox": "0 0 16 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "sticker-square" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Icon", + "d": "M8.66667 2.33333V4.13333C8.66667 5.25344 8.66667 5.81349 8.88465 6.24131C9.0764 6.61764 9.38236 6.9236 9.75869 7.11535C10.1865 7.33333 10.7466 7.33333 11.8667 7.33333H13.6667M14 8.65882V10.8C14 11.9201 14 12.4802 13.782 12.908C13.5903 13.2843 13.2843 13.5903 12.908 13.782C12.4802 14 11.9201 14 10.8 14H5.2C4.0799 14 3.51984 14 3.09202 13.782C2.71569 13.5903 2.40973 13.2843 2.21799 12.908C2 12.4802 2 11.9201 2 10.8V5.2C2 4.0799 2 3.51984 2.21799 3.09202C2.40973 2.71569 2.71569 2.40973 3.09202 2.21799C3.51984 2 4.0799 2 5.2 2H7.34118C7.83036 2 8.07496 2 8.30513 2.05526C8.5092 2.10425 8.70429 2.18506 8.88324 2.29472C9.08507 2.4184 9.25802 2.59135 9.60393 2.93726L13.0627 6.39608C13.4086 6.74198 13.5816 6.91493 13.7053 7.11676C13.8149 7.29571 13.8957 7.4908 13.9447 7.69487C14 7.92505 14 8.16964 14 8.65882Z", + "stroke": "currentColor", + "stroke-width": "1.5", + "stroke-linecap": "round", + "stroke-linejoin": "round" + }, + "children": [] + } + ] + } + ] + }, + "name": "StickerSquare" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/vender/line/files/StickerSquare.tsx b/web/app/components/base/icons/src/vender/line/files/StickerSquare.tsx new file mode 100644 index 00000000000000..925edcdc10ee22 --- /dev/null +++ b/web/app/components/base/icons/src/vender/line/files/StickerSquare.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './StickerSquare.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'StickerSquare' + +export default Icon diff --git a/web/app/components/base/icons/src/vender/line/files/index.ts b/web/app/components/base/icons/src/vender/line/files/index.ts index 4c0ddc22895903..71b4890ce8a73b 100644 --- a/web/app/components/base/icons/src/vender/line/files/index.ts +++ b/web/app/components/base/icons/src/vender/line/files/index.ts @@ -9,3 +9,4 @@ export { default as FilePlus02 } from './FilePlus02' export { default as FileText } from './FileText' export { default as FileUpload } from './FileUpload' export { default as Folder } from './Folder' +export { default as StickerSquare } from './StickerSquare' diff --git a/web/app/components/workflow/note-node/hooks.ts b/web/app/components/workflow/note-node/hooks.ts new file mode 100644 index 00000000000000..55f6a95b2b8073 --- /dev/null +++ b/web/app/components/workflow/note-node/hooks.ts @@ -0,0 +1,17 @@ +import { useCallback } from 'react' +import type { EditorState } from 'lexical' + +export const useNote = () => { + const handleColorChange = useCallback(() => { + + }, []) + + const handleEditorChange = useCallback((editorState: EditorState) => { + + }, []) + + return { + handleColorChange, + handleEditorChange, + } +} diff --git a/web/app/components/workflow/note-node/index.tsx b/web/app/components/workflow/note-node/index.tsx index d8cc2c09a43504..7be4b584180976 100644 --- a/web/app/components/workflow/note-node/index.tsx +++ b/web/app/components/workflow/note-node/index.tsx @@ -1,7 +1,29 @@ -const NoteNode = () => { +import { memo } from 'react' +import type { NodeProps } from 'reactflow' +import { + NoteEditor, + NoteEditorContextProvider, + NoteEditorToolbar, +} from './note-editor' + +const NoteNode = ({ + id, + type, + data, +}: NodeProps) => { return ( -
+ +
+
+
+ +
+
+ +
+
+ ) } -export default NoteNode +export default memo(NoteNode) diff --git a/web/app/components/workflow/note-node/note-editor/context.tsx b/web/app/components/workflow/note-node/note-editor/context.tsx index 489e809a144dbe..f41ae4e06dfbf2 100644 --- a/web/app/components/workflow/note-node/note-editor/context.tsx +++ b/web/app/components/workflow/note-node/note-editor/context.tsx @@ -5,9 +5,12 @@ import { memo, useRef, } from 'react' -import { CodeNode } from '@lexical/code' import { LexicalComposer } from '@lexical/react/LexicalComposer' import { LinkNode } from '@lexical/link' +import { + ListItemNode, + ListNode, +} from '@lexical/list' import { createNoteEditorStore } from './store' import theme from './theme' @@ -28,8 +31,9 @@ export const NoteEditorContextProvider = memo(({ const initialConfig = { namespace: 'note-editor', nodes: [ - CodeNode, LinkNode, + ListNode, + ListItemNode, ], onError: (error: Error) => { throw error diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index c24c4f4b1f26cb..f8ad9239894b58 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -2,14 +2,18 @@ import { memo, + useCallback, useState, } from 'react' +import type { EditorState } from 'lexical' import { RichTextPlugin } from '@lexical/react/LexicalRichTextPlugin' import { ContentEditable } from '@lexical/react/LexicalContentEditable' import { ClickableLinkPlugin } from '@lexical/react/LexicalClickableLinkPlugin' import { LinkPlugin } from '@lexical/react/LexicalLinkPlugin' +import { ListPlugin } from '@lexical/react/LexicalListPlugin' import { LexicalErrorBoundary } from '@lexical/react/LexicalErrorBoundary' import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin' +import { OnChangePlugin } from '@lexical/react/LexicalOnChangePlugin' import LinkEditorPlugin from './plugins/link-editor-plugin' import FormatDetectorPlugin from './plugins/format-detector-plugin' import TreeView from '@/app/components/base/prompt-editor/plugins/tree-view' @@ -17,9 +21,11 @@ import Placeholder from '@/app/components/base/prompt-editor/plugins/placeholder type EditorProps = { placeholder?: string + onChange?: (editorState: EditorState) => void } const Editor = ({ placeholder = 'write you note...', + onChange, }: EditorProps) => { const [containerElement, setContainerElement] = useState(null) const onRef = (_containerElement: HTMLDivElement) => { @@ -27,6 +33,10 @@ const Editor = ({ setContainerElement(_containerElement) } + const handleEditorChange = useCallback((editorState: EditorState) => { + onChange?.(editorState) + }, [onChange]) + return (
- + + +
) diff --git a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts index 486497d30202d9..da202f9841c931 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts @@ -10,6 +10,7 @@ import { mergeRegister } from '@lexical/utils' import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' import type { LinkNode } from '@lexical/link' import { $isLinkNode } from '@lexical/link' +import { $isListItemNode } from '@lexical/list' import { getSelectedNode } from '../../utils' import { useNoteEditorStore } from '../../store' @@ -30,6 +31,7 @@ export const useFormatDetector = () => { setSelectedIsBold, setSelectedIsStrikeThrough, setSelectedLinkUrl, + setSelectedIsBullet, } = noteEditorStore.getState() setSelectedIsBold(selection.hasFormat('bold')) setSelectedIsStrikeThrough(selection.hasFormat('strikethrough')) @@ -38,6 +40,11 @@ export const useFormatDetector = () => { setSelectedLinkUrl($isLinkNode(parent) ? parent.getURL() : (node as LinkNode).getURL()) else setSelectedLinkUrl('') + + if ($isListItemNode(parent) || $isListItemNode(node)) + setSelectedIsBullet(true) + else + setSelectedIsBullet(false) } }) }, [editor, noteEditorStore]) diff --git a/web/app/components/workflow/note-node/note-editor/store.ts b/web/app/components/workflow/note-node/note-editor/store.ts index 89d7ae66a492f9..ffafb9f697b877 100644 --- a/web/app/components/workflow/note-node/note-editor/store.ts +++ b/web/app/components/workflow/note-node/note-editor/store.ts @@ -16,6 +16,8 @@ type Shape = { setSelectedIsStrikeThrough: (selectedIsStrikeThrough: boolean) => void selectedLinkUrl: string setSelectedLinkUrl: (selectedLinkUrl: string) => void + selectedIsBullet: boolean + setSelectedIsBullet: (selectedIsBullet: boolean) => void } export const createNoteEditorStore = () => { @@ -44,6 +46,8 @@ export const createNoteEditorStore = () => { setSelectedIsStrikeThrough: selectedIsStrikeThrough => set(() => ({ selectedIsStrikeThrough })), selectedLinkUrl: '', setSelectedLinkUrl: selectedLinkUrl => set(() => ({ selectedLinkUrl })), + selectedIsBullet: false, + setSelectedIsBullet: selectedIsBullet => set(() => ({ selectedIsBullet })), })) } diff --git a/web/app/components/workflow/note-node/note-editor/theme/index.ts b/web/app/components/workflow/note-node/note-editor/theme/index.ts index c7431fe1b9c3b5..42069d2359bbdd 100644 --- a/web/app/components/workflow/note-node/note-editor/theme/index.ts +++ b/web/app/components/workflow/note-node/note-editor/theme/index.ts @@ -4,6 +4,10 @@ import './theme.css' const theme: EditorThemeClasses = { paragraph: 'note-editor-theme_paragraph', + list: { + ul: 'note-editor-theme_list-ul', + listitem: 'note-editor-theme_list-li', + }, link: 'note-editor-theme_link', text: { strikethrough: 'note-editor-theme_text-strikethrough', diff --git a/web/app/components/workflow/note-node/note-editor/theme/theme.css b/web/app/components/workflow/note-node/note-editor/theme/theme.css index 96abe1c43cdd1c..8b04d85bb5a944 100644 --- a/web/app/components/workflow/note-node/note-editor/theme/theme.css +++ b/web/app/components/workflow/note-node/note-editor/theme/theme.css @@ -2,6 +2,18 @@ font-size: 12px; } +.note-editor-theme_list-ul { + font-size: 12px; + margin: 0; + padding: 0; + list-style: disc; +} + +.note-editor-theme_list-li { + margin-left: 18px; + margin-right: 8px; +} + .note-editor-theme_link { text-decoration: underline; cursor: pointer; diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx index fc0cb4d2a70cfc..2f060cefb6ef8c 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx @@ -21,6 +21,7 @@ const Command = ({ const selectedIsBold = useStore(s => s.selectedIsBold) const selectedIsStrikeThrough = useStore(s => s.selectedIsStrikeThrough) const selectedLinkUrl = useStore(s => s.selectedLinkUrl) + const selectedIsBullet = useStore(s => s.selectedIsBullet) const { handleCommand } = useCommand() const icon = useMemo(() => { @@ -32,9 +33,9 @@ const Command = ({ case 'link': return case 'bullet': - return + return } - }, [type, selectedIsBold, selectedIsStrikeThrough, selectedLinkUrl]) + }, [type, selectedIsBold, selectedIsStrikeThrough, selectedLinkUrl, selectedIsBullet]) return (
handleCommand(type)} > diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts index 94062aedf4b14e..414bd49a900b04 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts @@ -15,6 +15,7 @@ import { $getSelectionStyleValueForProperty, $patchStyleText, } from '@lexical/selection' +import { INSERT_UNORDERED_LIST_COMMAND } from '@lexical/list' import { mergeRegister } from '@lexical/utils' import { $isLinkNode, @@ -51,6 +52,9 @@ export const useCommand = () => { } }) } + + if (type === 'bullet') + editor.dispatchCommand(INSERT_UNORDERED_LIST_COMMAND, undefined) }, [editor, noteEditorStore]) return { diff --git a/web/app/components/workflow/note-node/note-editor/utils.ts b/web/app/components/workflow/note-node/note-editor/utils.ts index 9871263bf1e096..16802dc1158d65 100644 --- a/web/app/components/workflow/note-node/note-editor/utils.ts +++ b/web/app/components/workflow/note-node/note-editor/utils.ts @@ -6,11 +6,8 @@ export function getSelectedNode( ): TextNode | ElementNode { const anchor = selection.anchor const focus = selection.focus - // console.log(selection, 'selection') const anchorNode = selection.anchor.getNode() - // console.log(anchorNode, 'anchorNode') const focusNode = selection.focus.getNode() - // console.log(focusNode, 'focusNode') if (anchorNode === focusNode) return anchorNode diff --git a/web/app/components/workflow/note-node/types.ts b/web/app/components/workflow/note-node/types.ts new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/web/app/components/workflow/operator/control.tsx b/web/app/components/workflow/operator/control.tsx index 8903239e90bf7f..e5dc9b81707d6c 100644 --- a/web/app/components/workflow/operator/control.tsx +++ b/web/app/components/workflow/operator/control.tsx @@ -20,6 +20,7 @@ import { Hand02 as Hand02Solid, } from '@/app/components/base/icons/src/vender/solid/editor' import { OrganizeGrid } from '@/app/components/base/icons/src/vender/line/layout' +import { StickerSquare } from '@/app/components/base/icons/src/vender/line/files' const Control = () => { const { t } = useTranslation() @@ -75,9 +76,27 @@ const Control = () => { handleLayout() } + const addNote = () => { + if (getNodesReadOnly()) + return + + console.log('add note') + } + return (
+ +
+ +
+
Date: Wed, 12 Jun 2024 17:52:56 +0800 Subject: [PATCH 157/273] feat: tracing panel --- .../[appId]/overview/page.tsx | 2 + .../[appId]/overview/tracing/detail-popup.tsx | 0 .../[appId]/overview/tracing/panel.tsx | 63 +++++ .../[appId]/overview/tracing/tracing-icon.tsx | 28 +++ .../[appId]/overview/tracing/type.ts | 4 + .../public/tracing/langfuse-icon-big.svg | 32 +++ .../assets/public/tracing/langfuse-icon.svg | 32 +++ .../public/tracing/langsmith-icon-big.svg | 24 ++ .../assets/public/tracing/langsmith-icon.svg | 24 ++ .../assets/public/tracing/tracing-icon.svg | 6 + .../src/public/tracing/LangfuseIcon.json | 236 ++++++++++++++++++ .../icons/src/public/tracing/LangfuseIcon.tsx | 16 ++ .../src/public/tracing/LangfuseIconBig.json | 236 ++++++++++++++++++ .../src/public/tracing/LangfuseIconBig.tsx | 16 ++ .../src/public/tracing/LangsmithIcon.json | 188 ++++++++++++++ .../src/public/tracing/LangsmithIcon.tsx | 16 ++ .../src/public/tracing/LangsmithIconBig.json | 188 ++++++++++++++ .../src/public/tracing/LangsmithIconBig.tsx | 16 ++ .../icons/src/public/tracing/TracingIcon.json | 47 ++++ .../icons/src/public/tracing/TracingIcon.tsx | 16 ++ .../base/icons/src/public/tracing/index.ts | 5 + 21 files changed, 1195 insertions(+) create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/detail-popup.tsx create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts create mode 100644 web/app/components/base/icons/assets/public/tracing/langfuse-icon-big.svg create mode 100644 web/app/components/base/icons/assets/public/tracing/langfuse-icon.svg create mode 100644 web/app/components/base/icons/assets/public/tracing/langsmith-icon-big.svg create mode 100644 web/app/components/base/icons/assets/public/tracing/langsmith-icon.svg create mode 100644 web/app/components/base/icons/assets/public/tracing/tracing-icon.svg create mode 100644 web/app/components/base/icons/src/public/tracing/LangfuseIcon.json create mode 100644 web/app/components/base/icons/src/public/tracing/LangfuseIcon.tsx create mode 100644 web/app/components/base/icons/src/public/tracing/LangfuseIconBig.json create mode 100644 web/app/components/base/icons/src/public/tracing/LangfuseIconBig.tsx create mode 100644 web/app/components/base/icons/src/public/tracing/LangsmithIcon.json create mode 100644 web/app/components/base/icons/src/public/tracing/LangsmithIcon.tsx create mode 100644 web/app/components/base/icons/src/public/tracing/LangsmithIconBig.json create mode 100644 web/app/components/base/icons/src/public/tracing/LangsmithIconBig.tsx create mode 100644 web/app/components/base/icons/src/public/tracing/TracingIcon.json create mode 100644 web/app/components/base/icons/src/public/tracing/TracingIcon.tsx create mode 100644 web/app/components/base/icons/src/public/tracing/index.ts diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx index 8f924268f20032..0cc9175b54e8ff 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx @@ -1,6 +1,7 @@ import React from 'react' import ChartView from './chartView' import CardView from './cardView' +import TracingPanel from './tracing/panel' import { getLocaleOnServer, useTranslation as translate } from '@/i18n/server' import ApikeyInfoPanel from '@/app/components/app/overview/apikey-info-panel' @@ -19,6 +20,7 @@ const Overview = async ({ const { t } = await translate(locale, 'app-overview') return (
+
{t('overview.title')} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/detail-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/detail-popup.tsx new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx new file mode 100644 index 00000000000000..cf6b6435f3e161 --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -0,0 +1,63 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import { useTranslation } from 'react-i18next' +import { useBoolean } from 'ahooks' +import cn from 'classnames' +import type { TracingTool } from './type' +import TracingIcon from './tracing-icon' +import Button from '@/app/components/base/button' +import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' + +const ConfigBtn = ({ + className, +}: { + className?: string +}) => { + const { t } = useTranslation() + + return ( + + ) +} + +const Panel: FC = () => { + const { t } = useTranslation() + + const inUseTracingTool: TracingTool | undefined = undefined + const hasConfiguredTracing = !!inUseTracingTool + const [isFold, { + toggle: toggleFold, + }] = useBoolean(false) + + if (!isFold) { + return ( +
+
+ +
+
Tracing app performance
+
+ Configuring a Third-Party LLMOps provider and tracing app performance. +
+ + +
+
+
+
+ +
+ + +
+
+ ) + } + + return ( +
+
+ ) +} +export default React.memo(Panel) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx new file mode 100644 index 00000000000000..21553b4e88bdc4 --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx @@ -0,0 +1,28 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import cn from 'classnames' +import { TracingIcon as Icon } from '@/app/components/base/icons/src/public/tracing' + +type Props = { + className?: string + size: 'lg' | 'md' +} + +const sizeClassMap = { + lg: 'w-9 h-9 p-2 rounded-[10px]', + md: 'w-4 h-4 p-1 rounded-lg', +} + +const TracingIcon: FC = ({ + className, + size, +}) => { + const sizeClass = sizeClassMap[size] + return ( +
+ +
+ ) +} +export default React.memo(TracingIcon) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts new file mode 100644 index 00000000000000..1bb571a9997b51 --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts @@ -0,0 +1,4 @@ +export enum TracingTool { + langSmith = 'langSmith', + langfuse = 'langfuse', +} diff --git a/web/app/components/base/icons/assets/public/tracing/langfuse-icon-big.svg b/web/app/components/base/icons/assets/public/tracing/langfuse-icon-big.svg new file mode 100644 index 00000000000000..6ce0c27a722202 --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/langfuse-icon-big.svg @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/tracing/langfuse-icon.svg b/web/app/components/base/icons/assets/public/tracing/langfuse-icon.svg new file mode 100644 index 00000000000000..fe10082fc3d07f --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/langfuse-icon.svg @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/tracing/langsmith-icon-big.svg b/web/app/components/base/icons/assets/public/tracing/langsmith-icon-big.svg new file mode 100644 index 00000000000000..95e1ff423c6639 --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/langsmith-icon-big.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/tracing/langsmith-icon.svg b/web/app/components/base/icons/assets/public/tracing/langsmith-icon.svg new file mode 100644 index 00000000000000..8efa791d54348a --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/langsmith-icon.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/tracing/tracing-icon.svg b/web/app/components/base/icons/assets/public/tracing/tracing-icon.svg new file mode 100644 index 00000000000000..b58357f3e989a3 --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/tracing-icon.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/web/app/components/base/icons/src/public/tracing/LangfuseIcon.json b/web/app/components/base/icons/src/public/tracing/LangfuseIcon.json new file mode 100644 index 00000000000000..ab0b8fbc1c5dbc --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangfuseIcon.json @@ -0,0 +1,236 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "74", + "height": "16", + "viewBox": "0 0 74 16", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Clip path group" + }, + "children": [ + { + "type": "element", + "name": "mask", + "attributes": { + "id": "mask0_20135_12984", + "style": "mask-type:luminance", + "maskUnits": "userSpaceOnUse", + "x": "0", + "y": "0", + "width": "96", + "height": "16" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "clip0_823_291" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector", + "d": "M95.5733 0H0V16H95.5733V0Z", + "fill": "white" + }, + "children": [] + } + ] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "mask": "url(#mask0_20135_12984)" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Group" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_2", + "d": "M21.2832 11.5431V3.72656H22.3735V10.4972H26.3932V11.5431H21.2832ZM27.6995 7.44766C27.9198 6.31372 28.8889 5.5761 30.1224 5.5761C31.543 5.5761 32.4791 6.40179 32.4791 8.02014V10.233C32.4791 10.4862 32.5893 10.5963 32.8316 10.5963H33.0849V11.5431H32.7765C32.0717 11.5431 31.6532 11.1688 31.543 10.6513C31.3228 11.1908 30.64 11.6752 29.7259 11.6752C28.5475 11.6752 27.6004 11.0587 27.6004 10.0128C27.6004 8.80179 28.4924 8.46051 29.836 8.2073L31.4109 7.89904C31.3999 7.0073 30.8933 6.56693 30.1114 6.56693C29.4506 6.56693 28.966 6.96326 28.8338 7.52473L27.6995 7.44766ZM28.7237 9.99078C28.7347 10.3981 29.0871 10.7394 29.8581 10.7394C30.7391 10.7394 31.4329 10.1229 31.4329 9.07702V8.82381L30.1774 9.04399C29.3625 9.18711 28.7237 9.25317 28.7237 9.99078ZM34.5453 5.70821H35.5255L35.5585 6.68803C35.8669 5.93941 36.5166 5.5761 37.2986 5.5761C38.5981 5.5761 39.2369 6.5339 39.2369 7.78895V11.5431H38.1686V8.06418C38.1686 7.02931 37.8272 6.48987 37.0232 6.48987C36.1752 6.48987 35.6136 7.02931 35.6136 8.06418V11.5431H34.5453V5.70821ZM43.2303 11.2348C41.7876 11.2348 40.7634 10.0789 40.7634 8.43849C40.7634 6.74308 41.7876 5.5761 43.2303 5.5761C44.0122 5.5761 44.6951 5.99445 44.9594 6.59996L44.9704 5.70821H45.9946V10.9045C45.9836 12.5009 44.9704 13.3266 43.4065 13.3266C42.129 13.3266 41.2039 12.655 40.9286 11.6422L42.0519 11.5651C42.2832 12.0715 42.7347 12.3688 43.4065 12.3688C44.3536 12.3688 44.9153 11.9394 44.9263 11.1357V10.266C44.629 10.8275 43.9241 11.2348 43.2303 11.2348ZM41.8867 8.42748C41.8867 9.5284 42.4704 10.299 43.4286 10.299C44.3647 10.299 44.9373 9.5284 44.9483 8.42748C44.9704 7.33757 44.3867 6.56693 43.4286 6.56693C42.4704 6.56693 41.8867 7.33757 41.8867 8.42748ZM48.9967 5.455C48.9967 4.3761 49.5364 3.72656 50.7258 3.72656H52.3337V4.67335H50.7038C50.3293 4.67335 50.065 4.95959 50.065 5.43298V6.08253H52.2566V7.02931H50.065V11.5431H48.9967V7.02931H47.4659V6.08253H48.9967V5.455ZM58.9041 11.5431H57.8909L57.8798 10.5963C57.5715 11.3229 56.9327 11.6752 56.1838 11.6752C54.9063 11.6752 54.2786 10.7174 54.2786 9.46234V5.70821H55.3468V9.18711C55.3468 10.222 55.6883 10.7614 56.4592 10.7614C57.2851 10.7614 57.8358 10.222 57.8358 9.18711V5.70821H58.9041V11.5431ZM64.5277 7.53574C64.4065 6.91922 63.8338 6.56693 63.151 6.56693C62.5894 6.56693 62.0718 6.84216 62.0828 7.38161C62.0828 7.9651 62.7876 8.09721 63.4374 8.26234C64.5497 8.53757 65.662 8.94491 65.662 10.0348C65.662 11.1798 64.5607 11.6752 63.3493 11.6752C61.9837 11.6752 60.8713 10.9045 60.7832 9.69354L61.9066 9.62748C62.0167 10.277 62.6004 10.6844 63.3493 10.6844C63.933 10.6844 64.5387 10.5302 64.5387 9.97977C64.5387 9.4073 63.8008 9.30821 63.151 9.15409C62.0497 8.88987 60.9594 8.48253 60.9594 7.42565C60.9594 6.24766 62.0167 5.5761 63.2502 5.5761C64.4836 5.5761 65.4417 6.31372 65.629 7.46968L64.5277 7.53574ZM67.2104 8.62565C67.2104 6.76509 68.2787 5.5761 69.9196 5.5761C71.2302 5.5761 72.4196 6.42381 72.5077 8.52656V8.9339H68.3448C68.4329 10.0348 68.9945 10.6844 69.9196 10.6844C70.5033 10.6844 71.032 10.3431 71.2853 9.75959L72.4196 9.85867C72.0892 10.9706 71.087 11.6752 69.9196 11.6752C68.2787 11.6752 67.2104 10.4862 67.2104 8.62565ZM68.3778 8.07519H71.3403C71.1861 6.96326 70.5804 6.56693 69.9196 6.56693C69.0716 6.56693 68.532 7.1284 68.3778 8.07519Z", + "fill": "black" + }, + "children": [] + }, + { + "type": "element", + "name": "g", + "attributes": { + "id": "Clip path group_2" + }, + "children": [ + { + "type": "element", + "name": "mask", + "attributes": { + "id": "mask1_20135_12984", + "style": "mask-type:luminance", + "maskUnits": "userSpaceOnUse", + "x": "0", + "y": "-1", + "width": "17", + "height": "18" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "clip1_823_291" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_3", + "d": "M16.3621 -0.0512695H0.203125V16.1021H16.3621V-0.0512695Z", + "fill": "white" + }, + "children": [] + } + ] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "mask": "url(#mask1_20135_12984)" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Group_2" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_4", + "d": "M14.6259 11.2357C13.9141 12.1984 12.8241 12.8406 11.5941 12.9344C11.5558 12.937 11.5175 12.9397 11.4787 12.9419C10.0365 13.0136 8.94706 12.3558 8.22466 11.7452C6.94631 11.0687 5.94609 10.8983 5.36089 10.751C4.93532 10.6438 4.56293 10.4296 4.40334 10.3225C4.26183 10.2384 3.97722 10.0434 3.76496 9.67965C3.52716 9.27204 3.51333 8.88257 3.51706 8.71705C3.641 8.70048 3.80113 8.68224 3.98839 8.67048C4.1416 8.66082 4.29002 8.65709 4.45654 8.65652C5.74819 8.65494 6.7499 8.71812 7.47874 9.0417C7.87295 9.21632 8.23842 9.4488 8.56395 9.73215C8.98265 10.0975 9.83862 10.6749 10.8935 10.4778C11.0276 10.4526 11.1563 10.4194 11.2803 10.3787C11.6601 10.3241 12.3097 10.2801 13.0747 10.4831C13.8008 10.676 14.3232 11.0092 14.6259 11.2357Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "1.70667", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_5", + "d": "M14.53 4.60662C14.2091 4.19101 13.819 3.79812 13.3584 3.53003C12.8675 3.2445 12.2411 2.99862 11.4835 2.93199C9.63248 2.76913 8.36691 3.79548 8.13634 3.98954C7.84947 4.25868 6.70187 5.21101 5.32048 5.73977C5.07981 5.82545 4.61653 6.02793 4.20477 6.48007C3.87909 6.83749 3.7197 7.20339 3.6416 7.43076C3.80631 7.45351 3.97632 7.46992 4.15164 7.47994C5.49102 7.55452 6.64184 7.56193 7.39466 7.19337C7.89196 6.95015 8.32815 6.60377 8.70431 6.1982C9.38222 5.4669 10.3709 5.14067 11.271 5.36436C11.6843 5.42197 12.4042 5.46588 13.2368 5.21101C13.8116 5.03492 14.2399 4.81337 14.53 4.60662Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "1.70667", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_6", + "d": "M1.96963 4.91518C1.45614 4.65135 1.05528 4.347 0.781876 4.10874C0.629046 3.97549 0.391602 4.08476 0.391602 4.28837V5.95295C0.391602 6.02543 0.424389 6.09419 0.480445 6.13896L1.16264 6.79512C1.19859 6.53125 1.2758 6.17255 1.44926 5.77597C1.61267 5.40184 1.80886 5.11558 1.96963 4.91518Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "1.70667", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_7", + "d": "M12.9521 8.63005C13.0302 8.38964 13.0735 8.13742 13.0799 7.8804C13.0853 7.67736 13.0617 7.6015 13.0264 7.4049C13.3895 7.34397 13.8428 7.24459 14.2561 7.1377C14.6929 7.02499 15.0158 6.89407 15.3789 6.76367C15.4318 7.01747 15.4874 7.14092 15.5067 7.32899C15.5248 7.50642 15.5361 7.69019 15.5392 7.8804C15.5489 8.47138 15.4767 9.0073 15.3655 9.47698C15.0233 9.29954 14.617 9.11577 14.1492 8.95439C13.714 8.8037 13.3093 8.70115 12.9521 8.63005Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "1.70667", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_8", + "d": "M0.766014 12.1447C0.609481 12.2583 0.391602 12.1463 0.391602 11.9516V9.90721C0.391602 9.84531 0.415399 9.78667 0.456648 9.74292C0.477272 9.72104 0.49631 9.70877 0.504771 9.70397L1.18061 9.41382C1.23032 9.6848 1.3123 10.0091 1.44926 10.3622C1.6095 10.775 1.79987 11.1094 1.96963 11.3638C1.56825 11.6241 1.16686 11.8844 0.766014 12.1447Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "1.70667", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_9", + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M5.11863 3.21273C6.47036 3.1457 7.48116 3.71166 8.00219 4.08992C7.60778 4.43817 6.54047 5.27853 5.27812 5.76389C5.19298 5.79436 5.08001 5.83945 4.95131 5.90513C4.03786 6.35487 3.49469 7.25118 3.47044 8.20872C3.46637 8.3699 3.4746 8.53046 3.49592 8.68826C3.49361 8.68857 3.49131 8.68888 3.48903 8.68918C3.48531 8.85338 3.49914 9.23978 3.73679 9.64428C3.94894 10.0051 4.23338 10.1986 4.37481 10.282C4.44499 10.3288 4.55634 10.3962 4.69529 10.466C4.8585 10.5529 5.03948 10.6258 5.2391 10.6822C5.26968 10.6911 5.30062 10.6995 5.33181 10.7072C5.40448 10.7254 5.48364 10.7442 5.56903 10.7644C6.17131 10.9074 7.08394 11.1238 8.20285 11.7118C8.31591 11.8066 8.43766 11.9022 8.56827 11.9956C8.52858 12.0311 8.49519 12.0621 8.46819 12.0875C8.23747 12.2826 6.97098 13.3142 5.11863 13.1505C4.36047 13.0836 3.73309 12.8364 3.24236 12.5494C2.4156 12.0663 1.79088 11.302 1.45008 10.4075C1.2305 9.83086 1.03909 9.08515 1.02527 8.20765C1.01304 7.45826 1.1332 6.79817 1.29696 6.25074C1.79833 4.57812 3.26043 3.35145 5.00327 3.22017L5.00335 3.22016C5.0416 3.21751 5.07986 3.21485 5.11863 3.21273ZM14.5861 11.1844C14.2827 10.9597 13.7622 10.6316 13.0411 10.4415C12.2766 10.2401 11.6274 10.2837 11.2478 10.3378C11.1239 10.3782 10.9952 10.4112 10.8613 10.4362C9.80694 10.6318 8.95148 10.0588 8.53303 9.69637C8.45168 9.62603 8.36781 9.55891 8.28165 9.49501C8.56326 9.2476 8.87288 9.03413 9.21043 8.8683C9.96382 8.49841 11.1154 8.50582 12.4558 8.58025C14.3028 8.68336 15.5788 9.56295 16.0882 9.96688C16.145 10.0121 16.1775 10.0801 16.1775 10.1524V11.8123C16.1775 12.0158 15.9388 12.1247 15.7851 11.9914C15.5098 11.7531 15.1049 11.4483 14.5861 11.1844ZM8.66435 6.22472C8.54326 6.35584 8.41593 6.48083 8.28237 6.59819C8.54101 6.79004 8.82057 6.95244 9.11629 7.08249C9.84473 7.40351 10.8459 7.46623 12.1367 7.46465C14.0301 7.46199 15.4241 6.74925 16.0637 6.36126C16.1344 6.31822 16.1775 6.2417 16.1775 6.15878V4.12158C16.1775 3.92758 15.9585 3.81597 15.8011 3.92917C15.4285 4.19722 14.8745 4.53933 14.1601 4.80844C13.9028 4.96005 13.5822 5.11485 13.2001 5.23242C12.367 5.48857 11.6466 5.44446 11.2329 5.38654C10.3323 5.16172 9.34277 5.48964 8.66435 6.22472Z", + "fill": "#E11312" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_10", + "d": "M8.00166 4.09005L8.13707 4.2433L8.32826 4.07447L8.12183 3.92461L8.00166 4.09005ZM5.11809 3.21286L5.10798 3.00864L5.10745 3.00866L5.10692 3.0087L5.11809 3.21286ZM5.27759 5.76403L5.34647 5.95659L5.34877 5.95577L5.35102 5.9549L5.27759 5.76403ZM4.95078 5.90527L5.04115 6.08868L5.04247 6.08807L5.04379 6.0874L4.95078 5.90527ZM3.49538 8.6884L3.52217 8.89108L3.72555 8.86425L3.69809 8.661L3.49538 8.6884ZM3.4885 8.68932L3.46154 8.48664L3.28798 8.50969L3.28401 8.68467L3.4885 8.68932ZM4.37427 10.2822L4.48774 10.112L4.48307 10.1089L4.47824 10.1061L4.37427 10.2822ZM4.69475 10.4661L4.79093 10.2857L4.78879 10.2845L4.78663 10.2834L4.69475 10.4661ZM5.23857 10.6823L5.29549 10.486L5.29487 10.4858L5.29421 10.4856L5.23857 10.6823ZM8.20232 11.7119L8.33384 11.5553L8.31701 11.5412L8.29748 11.5309L8.20232 11.7119ZM8.56773 11.9957L8.70411 12.1481L8.89429 11.978L8.68678 11.8295L8.56773 11.9957ZM8.46766 12.0877L8.59975 12.2438L8.60404 12.2402L8.60808 12.2364L8.46766 12.0877ZM3.24183 12.5496L3.34511 12.3731L3.34505 12.373L3.24183 12.5496ZM1.02474 8.20779L1.22926 8.20456L1.22925 8.20446L1.02474 8.20779ZM1.29642 6.25088L1.10049 6.19214L1.10045 6.1923L1.29642 6.25088ZM5.00274 3.2203L4.98903 3.01629L4.9882 3.01635L4.98737 3.01641L5.00274 3.2203ZM5.00281 3.2203L5.01652 3.42431L5.01698 3.42428L5.00281 3.2203ZM13.0406 10.4417L13.0928 10.2439H13.0927L13.0406 10.4417ZM14.5855 11.1845L14.4638 11.3488L14.4775 11.359L14.4928 11.3667L14.5855 11.1845ZM11.2473 10.338L11.2183 10.1356L11.2007 10.1381L11.1838 10.1436L11.2473 10.338ZM10.8607 10.4363L10.8981 10.6373H10.8982L10.8607 10.4363ZM8.5325 9.6965L8.66648 9.54197L8.66627 9.54177L8.5325 9.6965ZM8.28112 9.49515L8.14612 9.34159L7.95594 9.50864L8.15931 9.65939L8.28112 9.49515ZM12.4553 8.58039L12.4667 8.37622H12.4666L12.4553 8.58039ZM16.0877 9.96702L16.2149 9.80692L16.2148 9.80687L16.0877 9.96702ZM15.7846 11.9915L15.9187 11.8371L15.9185 11.8369L15.7846 11.9915ZM8.28183 6.59833L8.14678 6.44477L7.95666 6.61177L8.15998 6.76257L8.28183 6.59833ZM9.11576 7.08262L9.19829 6.89553L9.19814 6.89548L9.11576 7.08262ZM12.1362 7.46478L12.1365 7.66925H12.1365L12.1362 7.46478ZM16.0632 6.3614L16.1693 6.53622L16.1696 6.53607L16.0632 6.3614ZM14.1596 4.80857L14.0874 4.61723L14.0709 4.62346L14.0557 4.63242L14.1596 4.80857ZM11.2324 5.38667L11.1828 5.58506L11.1933 5.58767L11.204 5.58915L11.2324 5.38667ZM8.12183 3.92461C7.57989 3.53114 6.52347 2.93845 5.10798 3.00864L5.12822 3.41708C6.41618 3.35322 7.38138 3.89245 7.88144 4.25549L8.12183 3.92461ZM5.35102 5.9549C6.64538 5.45722 7.73371 4.59944 8.13707 4.2433L7.86625 3.9368C7.48074 4.27718 6.43449 5.10015 5.20416 5.5732L5.35102 5.9549ZM5.04379 6.0874C5.16309 6.02647 5.26772 5.98471 5.34647 5.95659L5.20871 5.57152C5.11717 5.60423 4.99585 5.65269 4.85776 5.72318L5.04379 6.0874ZM3.67439 8.21402C3.69676 7.3308 4.19746 6.50412 5.04115 6.08868L4.8604 5.72186C3.87719 6.20595 3.29156 7.17188 3.26543 8.2037L3.67439 8.21402ZM3.69809 8.661C3.6783 8.51455 3.67058 8.36487 3.67439 8.21402L3.26543 8.2037C3.2611 8.3752 3.26984 8.5467 3.29268 8.71575L3.69809 8.661ZM3.51546 8.892C3.5177 8.8917 3.51993 8.89139 3.52217 8.89108L3.4686 8.48566C3.46623 8.48597 3.46387 8.48633 3.46154 8.48664L3.51546 8.892ZM3.91263 9.54085C3.70211 9.18256 3.68969 8.83956 3.69299 8.69392L3.28401 8.68467C3.27987 8.86752 3.29509 9.29732 3.55989 9.74798L3.91263 9.54085ZM4.47824 10.1061C4.35261 10.032 4.10041 9.86028 3.91261 9.54079L3.55989 9.74798C3.79637 10.1503 4.11309 10.3655 4.2703 10.4583L4.47824 10.1061ZM4.78663 10.2834C4.6552 10.2174 4.55104 10.1543 4.48774 10.112L4.26081 10.4523C4.33787 10.5037 4.45643 10.5752 4.60289 10.6488L4.78663 10.2834ZM5.29421 10.4856C5.10788 10.4329 4.94058 10.3654 4.79093 10.2857L4.59858 10.6466C4.77536 10.7407 4.97 10.819 5.18294 10.8791L5.29421 10.4856ZM5.38088 10.509C5.35225 10.5019 5.32376 10.4941 5.29549 10.486L5.18161 10.8787C5.21454 10.8883 5.24788 10.8973 5.28168 10.9058L5.38088 10.509ZM5.61575 10.5656C5.53005 10.5453 5.45212 10.5268 5.38088 10.509L5.28168 10.9058C5.35572 10.9243 5.43616 10.9433 5.52125 10.9635L5.61575 10.5656ZM8.29748 11.5309C7.155 10.9306 6.22187 10.7094 5.61575 10.5656L5.52125 10.9635C6.11975 11.1055 7.01177 11.3174 8.10715 11.8929L8.29748 11.5309ZM8.68678 11.8295C8.56093 11.7394 8.44327 11.6471 8.33384 11.5553L8.07085 11.8685C8.18744 11.9664 8.31338 12.0652 8.44864 12.162L8.68678 11.8295ZM8.60808 12.2364C8.63406 12.2119 8.66607 12.1821 8.70411 12.1481L8.4313 11.8434C8.39004 11.8803 8.35526 11.9126 8.32724 11.939L8.60808 12.2364ZM5.10009 13.3543C7.03682 13.5255 8.35798 12.4482 8.59975 12.2438L8.33558 11.9315C8.11585 12.1173 6.90412 13.1032 5.13615 12.947L5.10009 13.3543ZM3.13854 12.726C3.65082 13.0256 4.30703 13.2843 5.10011 13.3544L5.13615 12.947C4.4129 12.8831 3.8143 12.6475 3.34511 12.3731L3.13854 12.726ZM1.25838 10.4804C1.61483 11.416 2.26927 12.2181 3.1386 12.7261L3.34505 12.373C2.56087 11.9148 1.96586 11.1883 1.64069 10.3349L1.25838 10.4804ZM0.820219 8.21101C0.834481 9.11662 1.03203 9.88594 1.25838 10.4804L1.64071 10.3349C1.4279 9.77599 1.24263 9.05395 1.22926 8.20456L0.820219 8.21101ZM1.10045 6.1923C0.93163 6.75664 0.807599 7.43774 0.820219 8.21116L1.22925 8.20446C1.21742 7.47904 1.3337 6.83991 1.49239 6.30946L1.10045 6.1923ZM4.98737 3.01641C3.15623 3.15434 1.62504 4.44222 1.10049 6.19214L1.49236 6.30956C1.97055 4.7143 3.36357 3.54883 5.0181 3.4242L4.98737 3.01641ZM4.9891 3.01629L4.98903 3.01629L5.01644 3.42432L5.01652 3.42431L4.9891 3.01629ZM5.10692 3.0087C5.0664 3.01091 5.02666 3.01368 4.98864 3.01632L5.01698 3.42428C5.05547 3.42161 5.09225 3.41906 5.12929 3.41703L5.10692 3.0087ZM12.9885 10.6393C13.6767 10.8208 14.1738 11.134 14.4638 11.3488L14.7073 11.0202C14.3904 10.7855 13.8465 10.4426 13.0928 10.2439L12.9885 10.6393ZM11.2762 10.5404C11.6387 10.4886 12.2586 10.4471 12.9885 10.6393L13.0927 10.2439C12.2935 10.0333 11.6151 10.0789 11.2183 10.1356L11.2762 10.5404ZM10.8982 10.6373C11.0409 10.6107 11.1782 10.5756 11.3107 10.5324L11.1838 10.1436C11.0685 10.1812 10.9485 10.2119 10.8232 10.2353L10.8982 10.6373ZM8.39858 9.85098C8.83155 10.2261 9.75005 10.8503 10.8981 10.6373L10.8234 10.2353C9.86276 10.4135 9.07035 9.89182 8.66648 9.54197L8.39858 9.85098ZM8.15931 9.65939C8.24138 9.72027 8.32126 9.78422 8.39873 9.85118L8.66627 9.54177C8.58108 9.46816 8.49323 9.39782 8.40297 9.3309L8.15931 9.65939ZM9.1197 8.68492C8.76425 8.85959 8.43969 9.08364 8.14612 9.34159L8.41617 9.64876C8.68576 9.41187 8.98056 9.20894 9.30011 9.05195L9.1197 8.68492ZM12.4666 8.37622C11.7952 8.33895 11.162 8.31784 10.5994 8.35373C10.0385 8.38951 9.53134 8.4828 9.1197 8.68492L9.30011 9.05195C9.64185 8.88418 10.0872 8.79621 10.6255 8.76186C11.1622 8.72761 11.7749 8.74739 12.4439 8.78455L12.4666 8.37622ZM16.2148 9.80687C15.6896 9.39035 14.3735 8.4827 12.4667 8.37622L12.4438 8.78455C14.231 8.88428 15.467 9.73586 15.9605 10.1272L16.2148 9.80687ZM16.3815 10.1525C16.3815 10.0185 16.3211 9.89131 16.2149 9.80692L15.9604 10.1271C15.9679 10.1331 15.9724 10.1419 15.9724 10.1525H16.3815ZM16.3815 11.8124V10.1525H15.9724V11.8124H16.3815ZM15.6504 12.1459C15.9368 12.3945 16.3815 12.1909 16.3815 11.8124H15.9724C15.9724 11.822 15.9699 11.8273 15.9676 11.8307C15.9648 11.8349 15.9601 11.8393 15.9534 11.8423C15.9468 11.8453 15.9404 11.846 15.9355 11.8455C15.9315 11.8449 15.926 11.8434 15.9187 11.8371L15.6504 12.1459ZM14.4928 11.3667C14.9936 11.6215 15.3848 11.916 15.6507 12.1461L15.9185 11.8369C15.6338 11.5905 15.2152 11.2754 14.6783 11.0023L14.4928 11.3667ZM8.41683 6.75194C8.55613 6.62956 8.68852 6.49957 8.81416 6.36354L8.51353 6.08612C8.39694 6.21239 8.27472 6.33236 8.14678 6.44477L8.41683 6.75194ZM9.19814 6.89548C8.91638 6.77157 8.65006 6.61683 8.40369 6.43414L8.15998 6.76257C8.43089 6.96352 8.7237 7.13359 9.03343 7.26982L9.19814 6.89548ZM12.136 7.26031C10.8405 7.26189 9.88163 7.19672 9.19829 6.89553L9.03328 7.26972C9.80676 7.61062 10.8502 7.67084 12.1365 7.66925L12.136 7.26031ZM15.9571 6.18662C15.3346 6.56423 13.9777 7.2577 12.136 7.26031L12.1365 7.66925C14.0813 7.66655 15.5126 6.93458 16.1693 6.53622L15.9571 6.18662ZM15.9724 6.15892C15.9724 6.17047 15.9666 6.18085 15.9568 6.18678L16.1696 6.53607C16.3012 6.45591 16.3815 6.31319 16.3815 6.15892H15.9724ZM15.9724 4.12171V6.15892H16.3815V4.12171H15.9724ZM15.92 4.09528C15.9427 4.07894 15.9724 4.09516 15.9724 4.12171H16.3815C16.3815 3.76028 15.9731 3.55327 15.6811 3.76334L15.92 4.09528ZM14.2317 4.99991C14.9668 4.72302 15.5366 4.37113 15.92 4.09528L15.6811 3.76334C15.3193 4.02358 14.7812 4.35591 14.0874 4.61723L14.2317 4.99991ZM13.2597 5.42798C13.6594 5.30504 13.9946 5.14315 14.2634 4.98473L14.0557 4.63242C13.8099 4.77723 13.5039 4.92497 13.1394 5.03712L13.2597 5.42798ZM11.204 5.58915C11.6356 5.64963 12.3885 5.69589 13.2597 5.42798L13.1395 5.03711C12.3443 5.28157 11.6564 5.23961 11.2608 5.18419L11.204 5.58915ZM8.81416 6.36354C9.44768 5.67713 10.3626 5.38033 11.1828 5.58506L11.2819 5.18828C10.3008 4.94339 9.23685 5.30248 8.51348 6.08617L8.81416 6.36354Z", + "fill": "black" + }, + "children": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "name": "LangfuseIcon" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/LangfuseIcon.tsx b/web/app/components/base/icons/src/public/tracing/LangfuseIcon.tsx new file mode 100644 index 00000000000000..38e763eb61d006 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangfuseIcon.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './LangfuseIcon.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'LangfuseIcon' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/LangfuseIconBig.json b/web/app/components/base/icons/src/public/tracing/LangfuseIconBig.json new file mode 100644 index 00000000000000..0fee622bd89cea --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangfuseIconBig.json @@ -0,0 +1,236 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "111", + "height": "24", + "viewBox": "0 0 111 24", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Clip path group" + }, + "children": [ + { + "type": "element", + "name": "mask", + "attributes": { + "id": "mask0_20135_18315", + "style": "mask-type:luminance", + "maskUnits": "userSpaceOnUse", + "x": "0", + "y": "0", + "width": "144", + "height": "24" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "clip0_823_291" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector", + "d": "M143.36 0H0V24H143.36V0Z", + "fill": "white" + }, + "children": [] + } + ] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "mask": "url(#mask0_20135_18315)" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Group" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_2", + "d": "M31.9258 17.3144V5.5896H33.5612V15.7456H39.5908V17.3144H31.9258ZM41.5502 11.1713C41.8806 9.47033 43.3343 8.36391 45.1845 8.36391C47.3155 8.36391 48.7197 9.60244 48.7197 12.03V15.3492C48.7197 15.729 48.8849 15.8942 49.2483 15.8942H49.6283V17.3144H49.1657C48.1085 17.3144 47.4807 16.7529 47.3155 15.9768C46.9852 16.7859 45.9609 17.5125 44.5898 17.5125C42.8222 17.5125 41.4016 16.5878 41.4016 15.019C41.4016 13.2024 42.7396 12.6905 44.755 12.3107L47.1173 11.8483C47.1008 10.5107 46.3409 9.85015 45.168 9.85015C44.1768 9.85015 43.45 10.4446 43.2517 11.2868L41.5502 11.1713ZM43.0865 14.9859C43.1031 15.5969 43.6317 16.1089 44.7881 16.1089C46.1096 16.1089 47.1503 15.1841 47.1503 13.6153V13.2355L45.2671 13.5657C44.0447 13.7804 43.0865 13.8795 43.0865 14.9859ZM51.8189 8.56208H53.2892L53.3387 10.0318C53.8013 8.90887 54.7759 8.36391 55.9488 8.36391C57.8981 8.36391 58.8563 9.80061 58.8563 11.6832V17.3144H57.2539V12.096C57.2539 10.5437 56.7418 9.73455 55.5358 9.73455C54.2638 9.73455 53.4213 10.5437 53.4213 12.096V17.3144H51.8189V8.56208ZM64.8465 16.852C62.6824 16.852 61.1461 15.118 61.1461 12.6575C61.1461 10.1144 62.6824 8.36391 64.8465 8.36391C66.0193 8.36391 67.0436 8.99143 67.44 9.89969L67.4565 8.56208H68.9929V16.3566C68.9763 18.7511 67.4565 19.9896 65.1108 19.9896C63.1945 19.9896 61.8069 18.9823 61.3939 17.463L63.0789 17.3474C63.4258 18.107 64.1031 18.5529 65.1108 18.5529C66.5315 18.5529 67.3739 17.9089 67.3905 16.7034V15.3988C66.9444 16.241 65.8872 16.852 64.8465 16.852ZM62.8311 12.641C62.8311 14.2924 63.7066 15.4483 65.1438 15.4483C66.548 15.4483 67.407 14.2924 67.4235 12.641C67.4565 11.0061 66.581 9.85015 65.1438 9.85015C63.7066 9.85015 62.8311 11.0061 62.8311 12.641ZM73.4961 8.18226C73.4961 6.56391 74.3055 5.5896 76.0897 5.5896H78.5015V7.00978H76.0566C75.495 7.00978 75.0985 7.43914 75.0985 8.14923V9.12354H78.3859V10.5437H75.0985V17.3144H73.4961V10.5437H71.1999V9.12354H73.4961V8.18226ZM88.3571 17.3144H86.8373L86.8207 15.8942C86.3582 16.9841 85.4001 17.5125 84.2767 17.5125C82.3605 17.5125 81.4189 16.0758 81.4189 14.1933V8.56208H83.0212V13.7804C83.0212 15.3327 83.5334 16.1419 84.6897 16.1419C85.9287 16.1419 86.7547 15.3327 86.7547 13.7804V8.56208H88.3571V17.3144ZM96.7925 11.3034C96.6108 10.3786 95.7518 9.85015 94.7275 9.85015C93.885 9.85015 93.1086 10.263 93.1251 11.0722C93.1251 11.9474 94.1824 12.1456 95.1571 12.3933C96.8255 12.8061 98.494 13.4171 98.494 15.052C98.494 16.7694 96.842 17.5125 95.0249 17.5125C92.9765 17.5125 91.308 16.3566 91.1758 14.5401L92.8608 14.441C93.026 15.4153 93.9016 16.0263 95.0249 16.0263C95.9004 16.0263 96.809 15.7951 96.809 14.9694C96.809 14.1107 95.7022 13.9621 94.7275 13.7309C93.0756 13.3346 91.4402 12.7235 91.4402 11.1382C91.4402 9.37125 93.026 8.36391 94.8762 8.36391C96.7264 8.36391 98.1636 9.47033 98.4444 11.2043L96.7925 11.3034ZM100.817 12.9382C100.817 10.1474 102.419 8.36391 104.88 8.36391C106.846 8.36391 108.63 9.63547 108.763 12.7896V13.4006H102.518C102.65 15.052 103.493 16.0263 104.88 16.0263C105.756 16.0263 106.549 15.5144 106.929 14.6391L108.63 14.7878C108.135 16.4557 106.632 17.5125 104.88 17.5125C102.419 17.5125 100.817 15.729 100.817 12.9382ZM102.568 12.1125H107.011C106.78 10.4446 105.872 9.85015 104.88 9.85015C103.608 9.85015 102.799 10.6924 102.568 12.1125Z", + "fill": "black" + }, + "children": [] + }, + { + "type": "element", + "name": "g", + "attributes": { + "id": "Clip path group_2" + }, + "children": [ + { + "type": "element", + "name": "mask", + "attributes": { + "id": "mask1_20135_18315", + "style": "mask-type:luminance", + "maskUnits": "userSpaceOnUse", + "x": "0", + "y": "-1", + "width": "25", + "height": "26" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "clip1_823_291" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_3", + "d": "M24.5471 -0.0771484H0.308594V24.1529H24.5471V-0.0771484Z", + "fill": "white" + }, + "children": [] + } + ] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "mask": "url(#mask1_20135_18315)" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Group_2" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_4", + "d": "M21.9423 16.8532C20.8746 18.2972 19.2396 19.2605 17.3946 19.4012C17.3372 19.4051 17.2797 19.4092 17.2215 19.4124C15.0582 19.5201 13.424 18.5334 12.3404 17.6174C10.4229 16.6027 8.92256 16.3471 8.04475 16.1261C7.4064 15.9654 6.84782 15.644 6.60842 15.4833C6.39617 15.3572 5.96925 15.0647 5.65086 14.5191C5.29416 13.9077 5.27342 13.3235 5.279 13.0752C5.46493 13.0504 5.70512 13.023 5.98601 13.0054C6.21582 12.9909 6.43845 12.9853 6.68823 12.9844C8.62571 12.982 10.1283 13.0768 11.2215 13.5622C11.8128 13.8241 12.361 14.1728 12.8493 14.5979C13.4774 15.1459 14.7613 16.012 16.3437 15.7164C16.5448 15.6786 16.7379 15.6287 16.9239 15.5677C17.4935 15.4857 18.4679 15.4198 19.6154 15.7243C20.7046 16.0136 21.4882 16.5134 21.9423 16.8532Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "2.56", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_5", + "d": "M21.8003 6.90944C21.319 6.28602 20.7339 5.69669 20.043 5.29456C19.3066 4.86626 18.367 4.49744 17.2306 4.3975C14.4541 4.15321 12.5557 5.69273 12.2099 5.98382C11.7796 6.38753 10.0582 7.81602 7.98609 8.60917C7.62509 8.73768 6.93016 9.0414 6.31253 9.71961C5.82401 10.2557 5.58492 10.8046 5.46777 11.1457C5.71483 11.1798 5.96985 11.2044 6.23284 11.2194C8.2419 11.3313 9.96813 11.3424 11.0974 10.7896C11.8433 10.4247 12.4976 9.90517 13.0618 9.29681C14.0787 8.19987 15.5618 7.71051 16.9118 8.04605C17.5318 8.13247 18.6117 8.19833 19.8605 7.81602C20.7228 7.55189 21.3652 7.21957 21.8003 6.90944Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "2.56", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_6", + "d": "M2.95884 7.37229C2.1886 6.97653 1.58732 6.52001 1.17721 6.16263C0.947963 5.96275 0.591797 6.12665 0.591797 6.43206V8.92893C0.591797 9.03766 0.640978 9.14079 0.725063 9.20796L1.74835 10.1922C1.80229 9.79638 1.9181 9.25834 2.17829 8.66347C2.4234 8.10227 2.71769 7.67288 2.95884 7.37229Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "2.56", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_7", + "d": "M19.4326 12.9446C19.5497 12.584 19.6146 12.2056 19.6243 11.8201C19.6323 11.5156 19.597 11.4018 19.5441 11.1069C20.0886 11.0155 20.7686 10.8664 21.3886 10.7061C22.0438 10.537 22.5282 10.3406 23.0727 10.145C23.1521 10.5257 23.2355 10.7109 23.2644 10.993C23.2916 11.2591 23.3085 11.5348 23.3132 11.8201C23.3277 12.7066 23.2194 13.5105 23.0526 14.215C22.5394 13.9488 21.9299 13.6732 21.2282 13.4311C20.5754 13.2051 19.9683 13.0512 19.4326 12.9446Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "2.56", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_8", + "d": "M1.15342 18.2166C0.918616 18.3871 0.591797 18.2191 0.591797 17.927V14.8605C0.591797 14.7676 0.627493 14.6796 0.689366 14.614C0.720303 14.5812 0.748859 14.5628 0.761552 14.5556L1.77532 14.1204C1.84988 14.5268 1.97284 15.0133 2.17829 15.5429C2.41864 16.1621 2.7042 16.6637 2.95884 17.0454C2.35676 17.4358 1.75469 17.8263 1.15342 18.2166Z", + "fill": "#0A60B5", + "stroke": "black", + "stroke-width": "2.56", + "stroke-miterlimit": "10" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_9", + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M7.68233 4.81872C9.70993 4.71818 11.2261 5.56713 12.0077 6.13451C11.4161 6.65689 9.81509 7.91742 7.92157 8.64547C7.79386 8.69117 7.62441 8.7588 7.43136 8.85733C6.06118 9.53194 5.24643 10.8764 5.21006 12.3127C5.20395 12.5545 5.21629 12.7953 5.24827 13.032C5.24481 13.0325 5.24136 13.033 5.23794 13.0334C5.23236 13.2797 5.2531 13.8593 5.60958 14.4661C5.9278 15.0073 6.35446 15.2975 6.5666 15.4227C6.67187 15.4928 6.83891 15.5939 7.04732 15.6986C7.29214 15.829 7.56361 15.9383 7.86305 16.0229C7.90892 16.0362 7.95532 16.0488 8.00211 16.0605C8.11111 16.0877 8.22985 16.1159 8.35794 16.1463C9.26137 16.3607 10.6303 16.6854 12.3087 17.5673C12.4783 17.7095 12.6609 17.8529 12.8568 17.993C12.7973 18.0463 12.7472 18.0927 12.7067 18.1309C12.3606 18.4235 10.4609 19.9709 7.68233 19.7254C6.5451 19.625 5.60404 19.2542 4.86793 18.8238C3.62779 18.0991 2.69071 16.9526 2.17951 15.6109C1.85014 14.7459 1.56303 13.6274 1.5423 12.3111C1.52395 11.187 1.70419 10.1969 1.94983 9.37575C2.70188 6.86682 4.89504 5.0268 7.5093 4.82989L7.50941 4.82988C7.5668 4.82589 7.62418 4.82191 7.68233 4.81872ZM21.8835 16.7762C21.4284 16.4391 20.6476 15.947 19.5661 15.6619C18.4192 15.3597 17.4455 15.4251 16.8761 15.5064C16.6902 15.567 16.4972 15.6164 16.2963 15.6539C14.7148 15.9473 13.4316 15.0879 12.8039 14.5442C12.6819 14.4387 12.5561 14.338 12.4269 14.2422C12.8493 13.871 13.3137 13.5508 13.82 13.3021C14.9501 12.7473 16.6775 12.7584 18.6881 12.87C21.4586 13.0247 23.3726 14.3441 24.1367 14.95C24.222 15.0177 24.2707 15.1198 24.2707 15.2282V17.718C24.2707 18.0233 23.9125 18.1867 23.682 17.9867C23.2692 17.6292 22.6618 17.1721 21.8835 16.7762ZM13.0009 9.33672C12.8193 9.5334 12.6283 9.72087 12.4279 9.89693C12.8159 10.1847 13.2353 10.4283 13.6788 10.6234C14.7715 11.1049 16.2732 11.199 18.2095 11.1966C21.0495 11.1926 23.1406 10.1235 24.1 9.54153C24.206 9.47696 24.2707 9.36218 24.2707 9.23781V6.182C24.2707 5.89101 23.9421 5.72359 23.706 5.8934C23.1472 6.29546 22.3162 6.80863 21.2445 7.21229C20.8586 7.43971 20.3776 7.6719 19.8046 7.84826C18.5548 8.23249 17.4742 8.16632 16.8538 8.07944C15.5028 7.74222 14.0186 8.2341 13.0009 9.33672Z", + "fill": "#E11312" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_10", + "d": "M12.0069 6.13459L12.21 6.36447L12.4968 6.11122L12.1871 5.88642L12.0069 6.13459ZM7.68154 4.8188L7.66636 4.51247L7.66557 4.51251L7.66477 4.51255L7.68154 4.8188ZM7.92078 8.64555L8.0241 8.9344L8.02755 8.93317L8.03092 8.93187L7.92078 8.64555ZM7.43056 8.85741L7.56612 9.13253L7.56811 9.13161L7.57008 9.13062L7.43056 8.85741ZM5.24747 13.0321L5.28765 13.3361L5.59271 13.2959L5.55152 12.991L5.24747 13.0321ZM5.23715 13.0335L5.1967 12.7295L4.93636 12.764L4.93041 13.0265L5.23715 13.0335ZM6.56581 15.4228L6.736 15.1676L6.729 15.1629L6.72175 15.1587L6.56581 15.4228ZM7.04653 15.6987L7.19079 15.428L7.18759 15.4263L7.18433 15.4247L7.04653 15.6987ZM7.86225 16.023L7.94762 15.7285L7.9467 15.7282L7.94571 15.7279L7.86225 16.023ZM12.3079 17.5673L12.5052 17.3324L12.4799 17.3112L12.4506 17.2958L12.3079 17.5673ZM12.856 17.9931L13.0606 18.2216L13.3458 17.9664L13.0346 17.7437L12.856 17.9931ZM12.7059 18.131L12.904 18.3652L12.9105 18.3598L12.9165 18.3541L12.7059 18.131ZM4.86713 18.8239L5.02207 18.5591L5.02197 18.559L4.86713 18.8239ZM1.5415 12.3112L1.84828 12.3064L1.84827 12.3062L1.5415 12.3112ZM1.94903 9.37583L1.65512 9.28773L1.65507 9.28796L1.94903 9.37583ZM7.5085 4.82997L7.48794 4.52395L7.48669 4.52403L7.48544 4.52413L7.5085 4.82997ZM7.50862 4.82996L7.52918 5.13598L7.52987 5.13593L7.50862 4.82996ZM19.5653 15.662L19.6435 15.3654H19.6435L19.5653 15.662ZM21.8827 16.7763L21.7001 17.0227L21.7207 17.038L21.7436 17.0496L21.8827 16.7763ZM16.8753 15.5065L16.8319 15.2028L16.8055 15.2066L16.7801 15.2149L16.8753 15.5065ZM16.2955 15.654L16.3515 15.9555H16.3517L16.2955 15.654ZM12.8031 14.5443L13.0041 14.3125L13.0038 14.3122L12.8031 14.5443ZM12.4261 14.2422L12.2236 14.0119L11.9383 14.2625L12.2434 14.4886L12.4261 14.2422ZM18.6873 12.8701L18.7044 12.5638H18.7042L18.6873 12.8701ZM24.1359 14.95L24.3267 14.7099L24.3266 14.7098L24.1359 14.95ZM23.6813 17.9868L23.8824 17.7552L23.8821 17.7549L23.6813 17.9868ZM12.4271 9.89701L12.2246 9.66667L11.9394 9.91717L12.2444 10.1434L12.4271 9.89701ZM13.678 10.6234L13.8018 10.3428L13.8016 10.3427L13.678 10.6234ZM18.2087 11.1967L18.2091 11.5034H18.2092L18.2087 11.1967ZM24.0992 9.54161L24.2584 9.80384L24.2588 9.80361L24.0992 9.54161ZM21.2437 7.21237L21.1355 6.92536L21.1107 6.9347L21.088 6.94815L21.2437 7.21237ZM16.853 8.07952L16.7786 8.37711L16.7943 8.38102L16.8104 8.38324L16.853 8.07952ZM12.1871 5.88642C11.3742 5.29623 9.7896 4.40718 7.66636 4.51247L7.69672 5.12514C9.62867 5.02934 11.0765 5.83819 11.8266 6.38275L12.1871 5.88642ZM8.03092 8.93187C9.97246 8.18534 11.605 6.89867 12.21 6.36447L11.8038 5.90471C11.2255 6.41528 9.65613 7.64973 7.81063 8.35932L8.03092 8.93187ZM7.57008 9.13062C7.74904 9.03922 7.90597 8.97657 8.0241 8.9344L7.81746 8.35679C7.68016 8.40586 7.49818 8.47855 7.29104 8.58429L7.57008 9.13062ZM5.51598 12.3205C5.54953 10.9957 6.30059 9.75569 7.56612 9.13253L7.29499 8.5823C5.82017 9.30843 4.94173 10.7573 4.90255 12.3051L5.51598 12.3205ZM5.55152 12.991C5.52184 12.7713 5.51027 12.5468 5.51598 12.3205L4.90255 12.3051C4.89604 12.5623 4.90915 12.8196 4.94341 13.0731L5.55152 12.991ZM5.27759 13.3375C5.28094 13.3371 5.28429 13.3366 5.28765 13.3361L5.20729 12.728C5.20373 12.7285 5.2002 12.729 5.1967 12.7295L5.27759 13.3375ZM5.87334 14.3108C5.55756 13.7734 5.53893 13.2588 5.54388 13.0404L4.93041 13.0265C4.92419 13.3008 4.94703 13.9455 5.34423 14.6215L5.87334 14.3108ZM6.72175 15.1587C6.53331 15.0475 6.15501 14.7899 5.87331 14.3107L5.34423 14.6215C5.69895 15.2249 6.17402 15.5477 6.40985 15.6869L6.72175 15.1587ZM7.18433 15.4247C6.98719 15.3256 6.83096 15.2309 6.736 15.1676L6.39562 15.678C6.51119 15.755 6.68904 15.8623 6.90873 15.9728L7.18433 15.4247ZM7.94571 15.7279C7.66622 15.6489 7.41526 15.5476 7.19079 15.428L6.90227 15.9694C7.16743 16.1106 7.4594 16.2279 7.7788 16.3182L7.94571 15.7279ZM8.07572 15.763C8.03277 15.7523 7.99004 15.7407 7.94762 15.7285L7.7768 16.3176C7.8262 16.3319 7.87621 16.3455 7.92691 16.3581L8.07572 15.763ZM8.42802 15.8479C8.29947 15.8174 8.18257 15.7897 8.07572 15.763L7.92691 16.3581C8.03798 16.3859 8.15864 16.4145 8.28627 16.4448L8.42802 15.8479ZM12.4506 17.2958C10.7369 16.3954 9.3372 16.0636 8.42802 15.8479L8.28627 16.4448C9.18402 16.6578 10.522 16.9755 12.1651 17.8389L12.4506 17.2958ZM13.0346 17.7437C12.8458 17.6086 12.6693 17.4702 12.5052 17.3324L12.1107 17.8023C12.2855 17.949 12.4745 18.0973 12.6774 18.2425L13.0346 17.7437ZM12.9165 18.3541C12.9555 18.3173 13.0035 18.2727 13.0606 18.2216L12.6514 17.7646C12.5895 17.8199 12.5373 17.8684 12.4953 17.908L12.9165 18.3541ZM7.65454 20.031C10.5596 20.2878 12.5414 18.6718 12.904 18.3652L12.5078 17.8968C12.1782 18.1755 10.3606 19.6543 7.70861 19.42L7.65454 20.031ZM4.7122 19.0885C5.48062 19.538 6.46494 19.9259 7.65455 20.0311L7.70861 19.42C6.62375 19.3242 5.72585 18.9707 5.02207 18.5591L4.7122 19.0885ZM1.89197 15.7201C2.42664 17.1235 3.4083 18.3266 4.71229 19.0886L5.02197 18.559C3.84569 17.8717 2.95318 16.7819 2.46543 15.5018L1.89197 15.7201ZM1.23472 12.316C1.25612 13.6744 1.55244 14.8284 1.89197 15.7201L2.46546 15.5019C2.14624 14.6635 1.86835 13.5804 1.84828 12.3064L1.23472 12.316ZM1.65507 9.28796C1.40184 10.1345 1.21579 11.1561 1.23472 12.3163L1.84827 12.3062C1.83052 11.2181 2.00495 10.2594 2.24298 9.4637L1.65507 9.28796ZM7.48544 4.52413C4.73874 4.73102 2.44195 6.66285 1.65512 9.28773L2.24293 9.46385C2.96021 7.07095 5.04976 5.32275 7.53155 5.13581L7.48544 4.52413ZM7.48805 4.52394L7.48794 4.52395L7.52906 5.13599L7.52918 5.13598L7.48805 4.52394ZM7.66477 4.51255C7.60399 4.51588 7.54439 4.52003 7.48736 4.52399L7.52987 5.13593C7.58761 5.13192 7.64276 5.1281 7.69834 5.12505L7.66477 4.51255ZM19.4871 15.9585C20.5195 16.2307 21.2651 16.7006 21.7001 17.0227L22.0654 16.5298C21.5901 16.1778 20.7741 15.6634 19.6435 15.3654L19.4871 15.9585ZM16.9186 15.8101C17.4624 15.7325 18.3923 15.6701 19.4871 15.9585L19.6435 15.3654C18.4447 15.0495 17.427 15.1179 16.8319 15.2028L16.9186 15.8101ZM16.3517 15.9555C16.5658 15.9156 16.7717 15.8629 16.9704 15.7981L16.7801 15.2149C16.6071 15.2713 16.4271 15.3174 16.2392 15.3524L16.3517 15.9555ZM12.6023 14.776C13.2517 15.3386 14.6295 16.275 16.3515 15.9555L16.2395 15.3524C14.7985 15.6197 13.6099 14.8372 13.0041 14.3125L12.6023 14.776ZM12.2434 14.4886C12.3665 14.5799 12.4863 14.6758 12.6025 14.7763L13.0038 14.3122C12.876 14.2017 12.7442 14.0962 12.6089 13.9959L12.2434 14.4886ZM13.6839 13.0269C13.1508 13.2889 12.6639 13.625 12.2236 14.0119L12.6286 14.4726C13.033 14.1173 13.4752 13.8129 13.9546 13.5774L13.6839 13.0269ZM18.7042 12.5638C17.6973 12.5079 16.7474 12.4763 15.9035 12.5301C15.0621 12.5838 14.3014 12.7237 13.6839 13.0269L13.9546 13.5774C14.4672 13.3258 15.1352 13.1938 15.9426 13.1423C16.7477 13.0909 17.6667 13.1206 18.6702 13.1763L18.7042 12.5638ZM24.3266 14.7098C23.5387 14.085 21.5647 12.7236 18.7044 12.5638L18.6702 13.1763C21.3509 13.3259 23.2049 14.6033 23.9452 15.1903L24.3266 14.7098ZM24.5767 15.2283C24.5767 15.0273 24.4861 14.8365 24.3267 14.7099L23.945 15.1902C23.9563 15.1992 23.9631 15.2124 23.9631 15.2283H24.5767ZM24.5767 17.7181V15.2283H23.9631V17.7181H24.5767ZM23.4801 18.2183C23.9096 18.5912 24.5767 18.2859 24.5767 17.7181H23.9631C23.9631 17.7326 23.9593 17.7405 23.9559 17.7456C23.9516 17.7519 23.9445 17.7584 23.9345 17.7629C23.9246 17.7675 23.915 17.7685 23.9076 17.7677C23.9016 17.7669 23.8933 17.7646 23.8824 17.7552L23.4801 18.2183ZM21.7436 17.0496C22.4948 17.4318 23.0817 17.8734 23.4804 18.2186L23.8821 17.7549C23.4551 17.3852 22.8272 16.9126 22.0218 16.5029L21.7436 17.0496ZM12.6296 10.1274C12.8386 9.94385 13.0372 9.74886 13.2256 9.54483L12.7747 9.1287C12.5998 9.31809 12.4165 9.49805 12.2246 9.66667L12.6296 10.1274ZM13.8016 10.3427C13.379 10.1569 12.9795 9.92476 12.6099 9.65072L12.2444 10.1434C12.6507 10.4448 13.0899 10.6999 13.5545 10.9042L13.8016 10.3427ZM18.2083 10.89C16.2651 10.8924 14.8268 10.7946 13.8018 10.3428L13.5543 10.9041C14.7145 11.4154 16.2797 11.5058 18.2091 11.5034L18.2083 10.89ZM23.94 9.27945C23.0063 9.84586 20.971 10.8861 18.2083 10.89L18.2092 11.5034C21.1263 11.4993 23.2733 10.4014 24.2584 9.80384L23.94 9.27945ZM23.9631 9.23789C23.9631 9.25522 23.9542 9.27078 23.9396 9.27968L24.2588 9.80361C24.4563 9.68338 24.5767 9.4693 24.5767 9.23789H23.9631ZM23.9631 6.18208V9.23789H24.5767V6.18208H23.9631ZM23.8844 6.14243C23.9185 6.11792 23.9631 6.14225 23.9631 6.18208H24.5767C24.5767 5.63993 23.9641 5.32941 23.526 5.64453L23.8844 6.14243ZM21.3519 7.49938C22.4546 7.08404 23.3093 6.55621 23.8844 6.14243L23.526 5.64453C22.9834 6.03488 22.1762 6.53338 21.1355 6.92536L21.3519 7.49938ZM19.894 8.14148C20.4934 7.95707 20.9962 7.71423 21.3995 7.4766L21.088 6.94815C20.7192 7.16536 20.2602 7.38697 19.7135 7.55519L19.894 8.14148ZM16.8104 8.38324C17.4579 8.47395 18.5872 8.54334 19.894 8.14148L19.7136 7.55517C18.5209 7.92187 17.4889 7.85892 16.8955 7.7758L16.8104 8.38324ZM13.2256 9.54483C14.1759 8.5152 15.5483 8.07001 16.7786 8.37711L16.9273 7.78194C15.4556 7.41459 13.8597 7.95323 12.7746 9.12877L13.2256 9.54483Z", + "fill": "black" + }, + "children": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "name": "LangfuseIconBig" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/LangfuseIconBig.tsx b/web/app/components/base/icons/src/public/tracing/LangfuseIconBig.tsx new file mode 100644 index 00000000000000..d1d3d001b9960a --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangfuseIconBig.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './LangfuseIconBig.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'LangfuseIconBig' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/LangsmithIcon.json b/web/app/components/base/icons/src/public/tracing/LangsmithIcon.json new file mode 100644 index 00000000000000..04d480bd2019e7 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangsmithIcon.json @@ -0,0 +1,188 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "84", + "height": "14", + "viewBox": "0 0 84 14", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Clip path group" + }, + "children": [ + { + "type": "element", + "name": "mask", + "attributes": { + "id": "mask0_20135_16592", + "style": "mask-type:luminance", + "maskUnits": "userSpaceOnUse", + "x": "0", + "y": "0", + "width": "84", + "height": "14" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "a" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector", + "d": "M83.2164 0.600098H0.799805V13.4001H83.2164V0.600098Z", + "fill": "white" + }, + "children": [] + } + ] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "mask": "url(#mask0_20135_16592)" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Group" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_2", + "d": "M31.0264 3.12256V10.8845H36.3737V9.71251H32.2403V3.12256H31.0264Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_3", + "d": "M39.2238 4.96436C38.0585 4.96436 37.1871 5.51066 36.8333 6.46298C36.8108 6.52391 36.7427 6.70772 36.7427 6.70772L37.7416 7.35386L37.8773 7.00007C38.1087 6.39693 38.5367 6.11584 39.2238 6.11584C39.911 6.11584 40.3042 6.44916 40.297 7.10554C40.297 7.13216 40.295 7.21255 40.295 7.21255C40.295 7.21255 39.3856 7.36 39.0109 7.43936C37.4119 7.77728 36.7422 8.38759 36.7422 9.38599C36.7422 9.91796 37.0376 10.494 37.5767 10.817C37.9003 11.0106 38.3227 11.0838 38.7892 11.0838C39.0959 11.0838 39.3938 11.0382 39.6698 10.9542C40.297 10.7459 40.4721 10.3363 40.4721 10.3363V10.8718H41.511V7.04308C41.511 5.74157 40.6559 4.96436 39.2238 4.96436ZM40.3011 9.05012C40.3011 9.45255 39.8628 10.0193 38.8419 10.0193C38.5536 10.0193 38.3494 9.94304 38.2132 9.82938C38.0309 9.67732 37.971 9.45869 37.9961 9.26567C38.0068 9.1817 38.0575 9.00096 38.2454 8.84429C38.4374 8.68404 38.7769 8.56935 39.3012 8.45517C39.7323 8.36148 40.3016 8.25805 40.3016 8.25805V9.05063L40.3011 9.05012Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_4", + "d": "M45.3523 4.96438C45.2079 4.96438 45.0671 4.97462 44.9304 4.99356C44.0001 5.13334 43.7277 5.60591 43.7277 5.60591L43.7287 5.13334H42.5645V10.8729H43.7784V7.68924C43.7784 6.60739 44.5674 6.11484 45.3006 6.11484C46.0932 6.11484 46.4782 6.54083 46.4782 7.41788V10.8729H47.6921V7.25097C47.6921 5.8399 46.7956 4.96387 45.3528 4.96387L45.3523 4.96438Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_5", + "d": "M52.7575 5.12922V5.72058C52.7575 5.72058 52.4601 4.96436 51.1067 4.96436C49.4253 4.96436 48.3809 6.12455 48.3809 7.99284C48.3809 9.04704 48.7178 9.877 49.3122 10.4013C49.7745 10.8088 50.392 11.0177 51.1272 11.0321C51.6387 11.0418 51.97 10.9025 52.1769 10.7709C52.5742 10.518 52.7217 10.2779 52.7217 10.2779C52.7217 10.2779 52.7048 10.4658 52.6741 10.7203C52.6521 10.9046 52.6106 11.0341 52.6106 11.0341C52.4258 11.692 51.885 12.0725 51.0965 12.0725C50.308 12.0725 49.8303 11.8129 49.7356 11.3014L48.5555 11.6536C48.7592 12.6367 49.6819 13.2234 51.0233 13.2234C51.9352 13.2234 52.65 12.9756 53.1482 12.4861C53.6505 11.9926 53.9054 11.2814 53.9054 10.3721V5.12871H52.7575V5.12922ZM52.6813 8.04455C52.6813 9.19348 52.1201 9.87956 51.18 9.87956C50.1729 9.87956 49.5953 9.19143 49.5953 7.99232C49.5953 6.79322 50.1729 6.11533 51.18 6.11533C52.0976 6.11533 52.6725 6.79834 52.6813 7.89812V8.04455Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_6", + "d": "M61.022 9.69984C61.1858 9.40237 61.2688 9.05165 61.2688 8.65689C61.2688 8.26214 61.1986 7.95904 61.0599 7.71379C60.9211 7.46752 60.7419 7.2663 60.5279 7.11526C60.3123 6.9632 60.0845 6.84339 59.852 6.7584C59.6186 6.67341 59.404 6.6048 59.2151 6.55513L57.8291 6.16857C57.654 6.12198 57.4789 6.0631 57.3079 5.99296C57.1354 5.9223 56.9884 5.82451 56.8722 5.70215C56.7534 5.57773 56.693 5.41594 56.693 5.21984C56.693 5.01402 56.7632 4.83124 56.9009 4.67661C57.0376 4.52352 57.2199 4.40474 57.4431 4.32333C57.6648 4.24192 57.909 4.2025 58.1691 4.20608C58.4364 4.21325 58.6919 4.26804 58.9279 4.36941C59.1649 4.47079 59.3687 4.62029 59.5336 4.81383C59.6938 5.00224 59.8049 5.23162 59.8643 5.49632L61.2042 5.26336C61.0901 4.80461 60.8955 4.40679 60.6252 4.08116C60.3497 3.74938 60.0026 3.49236 59.593 3.31776C59.1829 3.14266 58.7093 3.05204 58.185 3.04845C57.6689 3.04487 57.1912 3.1273 56.7688 3.2937C56.3474 3.45959 56.008 3.71252 55.7596 4.04532C55.5118 4.3776 55.3859 4.79437 55.3859 5.28384C55.3859 5.61869 55.4417 5.90285 55.5523 6.12916C55.6629 6.35597 55.8072 6.54439 55.9808 6.69031C56.1554 6.83674 56.3428 6.95245 56.5384 7.0354C56.7355 7.11885 56.9214 7.18644 57.0913 7.23559L59.0892 7.82644C59.2335 7.86996 59.3626 7.92218 59.4727 7.98157C59.5838 8.04148 59.6759 8.10906 59.7471 8.18228C59.8188 8.256 59.8741 8.341 59.9109 8.4352C59.9478 8.52941 59.9662 8.63284 59.9662 8.7424C59.9662 8.98765 59.8874 9.19808 59.7312 9.36653C59.5771 9.53344 59.3738 9.66247 59.1276 9.749C58.8823 9.83552 58.6176 9.87956 58.3401 9.87956C57.8716 9.87956 57.4518 9.75156 57.0929 9.49914C56.7391 9.25031 56.5 8.89498 56.3822 8.4434L55.0879 8.64C55.1678 9.12743 55.3516 9.55495 55.6342 9.91028C55.9219 10.2723 56.2947 10.5544 56.7411 10.7484C57.1886 10.943 57.6996 11.0418 58.2587 11.0418C58.6519 11.0418 59.0334 10.9916 59.3933 10.8923C59.7522 10.7935 60.0758 10.6429 60.3548 10.4448C60.6334 10.2477 60.8576 9.99629 61.0209 9.69882L61.022 9.69984Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_7", + "d": "M67.38 6.22747C67.5479 6.13173 67.7405 6.08309 67.9514 6.08309C68.2939 6.08309 68.5699 6.19777 68.7706 6.42459C68.9708 6.65038 69.0727 6.96629 69.0727 7.36513V10.8309H70.3158V7.04053C70.3158 6.4292 70.1453 5.92897 69.8094 5.55419C69.4741 5.18043 68.9846 4.99048 68.3543 4.99048C67.9734 4.99048 67.6237 5.07547 67.314 5.24289C67.0237 5.40008 66.7856 5.61921 66.6074 5.89365L66.5838 5.93L66.5634 5.89211C66.4226 5.63201 66.2229 5.41953 65.9694 5.26081C65.6832 5.08161 65.3218 4.99048 64.8958 4.99048C64.5082 4.99048 64.1534 5.07649 63.8421 5.24545C63.603 5.37499 63.3987 5.54446 63.2349 5.74824L63.1893 5.80507V5.13435H62.0967V10.8309H63.3506V7.31752C63.3506 6.95451 63.453 6.65499 63.6552 6.42766C63.858 6.19931 64.1309 6.0836 64.4667 6.0836C64.8026 6.0836 65.0903 6.19931 65.2916 6.42766C65.4918 6.65499 65.5931 6.97601 65.5931 7.38101V10.8309H66.8306V7.31752C66.8306 7.06254 66.8803 6.83931 66.9786 6.65345C67.0774 6.46709 67.2126 6.32424 67.3805 6.22798L67.38 6.22747Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_8", + "d": "M74.2724 9.3726C74.2796 9.6286 74.3487 9.88358 74.4787 10.1293C74.6257 10.3981 74.8438 10.5978 75.1269 10.7222C75.4126 10.8472 75.7408 10.9153 76.1028 10.924C76.4597 10.9327 76.8293 10.9025 77.2016 10.8344V9.80064C76.8514 9.85081 76.5339 9.86412 76.2585 9.83955C75.9682 9.81395 75.7541 9.68851 75.621 9.46732C75.5509 9.35366 75.513 9.20467 75.5074 9.02547C75.5022 8.84985 75.4992 8.64403 75.4992 8.4126V6.04563H77.2016V5.08204H75.4992V3.13184H74.2617V5.08204H73.209V6.04563H74.2617V8.48787C74.2617 8.81657 74.2652 9.11456 74.2724 9.37312V9.3726Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_9", + "d": "M80.8767 4.95543C80.7436 4.95543 80.6141 4.96414 80.4881 4.98052C79.5726 5.12337 79.3033 5.5898 79.3033 5.5898V5.4531H79.3028V3.11377H78.0889V10.8649H79.3028V7.68132C79.3028 6.5923 80.0918 6.09668 80.825 6.09668C81.6176 6.09668 82.0026 6.52267 82.0026 7.39972V10.8649H83.2165V7.23281C83.2165 5.8499 82.298 4.95595 80.8772 4.95595L80.8767 4.95543Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_10", + "d": "M72.3934 5.13281H71.1855V10.8775H72.3934V5.13281Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_11", + "d": "M71.7889 4.70524C72.2236 4.70524 72.5758 4.35291 72.5758 3.91829C72.5758 3.48368 72.2236 3.13135 71.7889 3.13135C71.3542 3.13135 71.002 3.48368 71.002 3.91829C71.002 4.35291 71.3542 4.70524 71.7889 4.70524Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_12", + "d": "M15.5941 10.4208C15.524 10.5375 15.313 10.5452 15.1333 10.4372C15.0412 10.3819 14.97 10.3056 14.9331 10.2226C14.8993 10.1474 14.8988 10.0762 14.9311 10.0224C14.968 9.96099 15.0442 9.92976 15.1344 9.92976C15.2152 9.92976 15.3074 9.95485 15.3924 10.0061C15.5721 10.1141 15.6648 10.304 15.5946 10.4208H15.5941ZM25.9939 7.0001C25.9939 10.5288 23.1226 13.4001 19.5939 13.4001H7.20859C3.67989 13.4001 0.808594 10.5293 0.808594 7.0001C0.808594 3.47088 3.67989 0.600098 7.20859 0.600098H19.5939C23.1231 0.600098 25.9939 3.47139 25.9939 7.0001ZM13.1427 10.2098C13.2435 10.0875 12.7776 9.74288 12.6824 9.61642C12.4888 9.4065 12.4878 9.10442 12.3573 8.85917C12.0378 8.11882 11.6707 7.3841 11.1571 6.75741C10.6144 6.07184 9.94472 5.50455 9.35643 4.86096C8.9197 4.41194 8.80296 3.77245 8.41743 3.28963C7.88597 2.50474 6.20559 2.29072 5.95931 3.3992C5.96034 3.43402 5.94959 3.45603 5.91937 3.47805C5.78318 3.57687 5.66184 3.69002 5.55995 3.82672C5.3106 4.17386 5.2722 4.76266 5.5835 5.07447C5.59374 4.91011 5.59937 4.75498 5.72942 4.63722C5.97007 4.84407 6.33358 4.91677 6.61262 4.76266C7.22907 5.64279 7.07547 6.86032 7.56494 7.80855C7.70011 8.0328 7.8363 8.26167 8.00987 8.45827C8.15067 8.67741 8.63707 8.93597 8.66574 9.13872C8.67086 9.48688 8.6299 9.8673 8.85825 10.1586C8.96577 10.3767 8.70158 10.5959 8.48859 10.5687C8.21211 10.6066 7.8747 10.3829 7.63252 10.5206C7.54702 10.6133 7.3796 10.5109 7.30587 10.6394C7.28027 10.706 7.14203 10.7997 7.22446 10.8637C7.31611 10.794 7.4011 10.7213 7.52449 10.7628C7.50606 10.8631 7.58542 10.8775 7.6484 10.9067C7.64635 10.9748 7.60641 11.0444 7.65864 11.1022C7.71956 11.0408 7.75592 10.9538 7.85268 10.9282C8.17422 11.3567 8.50139 10.4945 9.1972 10.8826C9.05588 10.8759 8.93044 10.8933 8.83521 11.0096C8.81166 11.0357 8.79169 11.0664 8.83316 11.1002C9.20846 10.858 9.20641 11.1831 9.45012 11.0833C9.63752 10.9855 9.82388 10.8631 10.0466 10.898C9.83003 10.9604 9.82132 11.1345 9.69435 11.2814C9.67284 11.304 9.6626 11.3296 9.68769 11.3669C10.1372 11.3291 10.1741 11.1796 10.5371 10.9963C10.8079 10.8309 11.0778 11.2318 11.3123 11.0034C11.364 10.9538 11.4346 10.9707 11.4986 10.964C11.4167 10.5273 10.5161 11.0439 10.5304 10.4581C10.8202 10.261 10.7537 9.88368 10.7731 9.57904C11.1064 9.76387 11.4771 9.87139 11.8038 10.048C11.9687 10.3143 12.2272 10.666 12.5718 10.643C12.581 10.6164 12.5892 10.5928 12.5989 10.5657C12.7034 10.5836 12.8375 10.6527 12.8949 10.5206C13.051 10.6839 13.2804 10.6757 13.4847 10.6338C13.6357 10.5109 13.2005 10.3358 13.1422 10.2093L13.1427 10.2098ZM17.8147 8.14595L17.1296 7.22128C16.5316 7.90531 16.1322 8.23863 16.1251 8.24477C16.1215 8.24835 15.74 8.61955 15.3924 8.93802C15.0514 9.25034 14.7821 9.49763 14.6449 9.76797C14.607 9.84272 14.5246 10.1182 14.6403 10.3936C14.7294 10.6066 14.9111 10.7582 15.1809 10.8442C15.2618 10.8698 15.3392 10.8811 15.4129 10.8811C15.8988 10.8811 16.2177 10.3936 16.2198 10.3895C16.2239 10.3839 16.6371 9.79357 17.1404 9.0481C17.3078 8.80029 17.4993 8.53815 17.8147 8.14544V8.14595ZM21.0357 10.2754C21.0357 10.1392 20.986 10.0076 20.8959 9.9057L20.8109 9.80944C20.2974 9.2273 18.979 7.73277 18.4659 7.15216C17.8218 6.42307 17.1015 5.49379 17.0416 5.41597L16.955 5.23677V4.92138C16.955 4.80567 16.932 4.69251 16.8874 4.58602L16.7041 4.15082C16.7016 4.14467 16.7006 4.13751 16.7016 4.13085L16.7088 4.07043C16.7098 4.06071 16.7144 4.052 16.7221 4.04535C16.8649 3.91939 17.3964 3.51082 18.2192 3.54512C18.3267 3.54973 18.3456 3.49085 18.3487 3.46576C18.3635 3.34493 18.0876 3.20259 17.8305 3.14986C17.4773 3.07767 16.5383 2.88618 15.7872 3.37923L15.7815 3.38333C15.2961 3.78883 14.906 4.09859 14.9019 4.10167L14.8932 4.11037C14.8876 4.11703 14.7504 4.28138 14.7836 4.49079C14.8051 4.62698 14.7345 4.67562 14.7304 4.67818C14.7263 4.68074 14.63 4.74115 14.5307 4.67306C14.4104 4.58295 14.2015 4.73757 14.159 4.77136L13.8436 5.04272L13.8375 5.04887C13.8318 5.05552 13.6967 5.21373 13.8774 5.46768C14.0336 5.68733 14.0888 5.76055 14.2245 5.92951C14.3623 6.10051 14.6106 6.31709 14.6239 6.32835C14.63 6.33347 14.7816 6.44816 14.9905 6.28842C15.162 6.15683 15.2997 6.03805 15.2997 6.03805C15.311 6.02883 15.4103 5.94691 15.4149 5.82608C15.4165 5.79127 15.4149 5.76106 15.4149 5.73341C15.4124 5.64842 15.4119 5.62333 15.4759 5.58237C15.5066 5.58237 15.6008 5.61667 15.6817 5.65763C15.6904 5.66275 15.8926 5.77642 16.0764 5.76823C16.1921 5.78359 16.3201 5.91517 16.3642 5.96893C16.3683 5.97303 16.7594 6.38365 17.3104 7.10403C17.4153 7.24074 17.8008 7.75427 17.9063 7.89763C18.0824 8.13725 18.3492 8.49975 18.6359 8.8904C19.1326 9.56675 19.6896 10.325 19.9425 10.666C20.0265 10.7792 20.1489 10.856 20.2871 10.8826L20.3808 10.9005C20.4167 10.9072 20.4525 10.9108 20.4883 10.9108C20.6542 10.9108 20.8114 10.8381 20.9153 10.709L20.921 10.7019C20.9957 10.6071 21.0367 10.4858 21.0367 10.3609V10.2754H21.0357ZM21.4765 4.20253L21.3674 4.09347C21.3357 4.06173 21.2912 4.04279 21.2461 4.04637C21.201 4.04842 21.1585 4.0689 21.1294 4.10371L20.4433 4.91523C20.4238 4.93827 20.3962 4.95261 20.3665 4.95568L20.1223 4.98026C20.091 4.98333 20.0598 4.9736 20.0357 4.95363L19.6456 4.62339C19.6205 4.60189 19.6056 4.57117 19.6046 4.5384L19.599 4.34282C19.598 4.31466 19.6077 4.28701 19.6256 4.26551L20.2928 3.46167C20.3429 3.40125 20.3424 3.3137 20.2917 3.25328L20.2247 3.17392C20.1806 3.12221 20.1079 3.10327 20.0444 3.12733C19.8867 3.18723 19.4894 3.34237 19.2022 3.49802C18.7962 3.71715 18.5141 4.0648 18.4654 4.40528C18.4296 4.65463 18.4444 5.06576 18.4567 5.28848C18.4613 5.37603 18.4419 5.46359 18.3994 5.54192C18.3472 5.6392 18.255 5.79485 18.1147 5.98224C18.043 6.08106 17.998 6.11741 17.936 6.19063L18.6907 7.07792C18.8725 6.86595 19.0317 6.70519 19.1704 6.55005C19.4239 6.26794 19.5027 6.26538 19.7137 6.25821C19.8437 6.2536 20.0219 6.24797 20.304 6.17731C21.0741 5.9848 21.3178 5.15127 21.328 5.1144L21.5195 4.35715C21.5333 4.30237 21.5169 4.24298 21.477 4.20304L21.4765 4.20253ZM9.63496 9.48842C9.55202 9.812 9.52488 10.3634 9.10402 10.3793C9.0692 10.5662 9.23355 10.6363 9.38255 10.5764C9.53051 10.5083 9.60066 10.6302 9.65032 10.7515C9.87867 10.7848 10.2166 10.6752 10.2294 10.4049C9.8884 10.2083 9.78293 9.83453 9.63445 9.48842H9.63496Z", + "fill": "#1C3C3C" + }, + "children": [] + } + ] + } + ] + } + ] + } + ] + }, + "name": "LangsmithIcon" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/LangsmithIcon.tsx b/web/app/components/base/icons/src/public/tracing/LangsmithIcon.tsx new file mode 100644 index 00000000000000..5565f24f519994 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangsmithIcon.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './LangsmithIcon.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'LangsmithIcon' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/LangsmithIconBig.json b/web/app/components/base/icons/src/public/tracing/LangsmithIconBig.json new file mode 100644 index 00000000000000..4aa76acc8d55d4 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangsmithIconBig.json @@ -0,0 +1,188 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "124", + "height": "20", + "viewBox": "0 0 124 20", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Clip path group" + }, + "children": [ + { + "type": "element", + "name": "mask", + "attributes": { + "id": "mask0_20135_18175", + "style": "mask-type:luminance", + "maskUnits": "userSpaceOnUse", + "x": "0", + "y": "0", + "width": "124", + "height": "20" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "a" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector", + "d": "M123.825 0.399902H0.200195V19.5999H123.825V0.399902Z", + "fill": "white" + }, + "children": [] + } + ] + } + ] + }, + { + "type": "element", + "name": "g", + "attributes": { + "mask": "url(#mask0_20135_18175)" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "Group" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_2", + "d": "M45.54 4.18408V15.827H53.561V14.069H47.361V4.18408H45.54Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_3", + "d": "M57.8358 6.94629C56.0878 6.94629 54.7807 7.76575 54.25 9.19423C54.2162 9.28562 54.1141 9.56133 54.1141 9.56133L55.6124 10.5305L55.8159 9.99986C56.1631 9.09515 56.8051 8.67352 57.8358 8.67352C58.8664 8.67352 59.4563 9.17349 59.4455 10.1581C59.4455 10.198 59.4424 10.3186 59.4424 10.3186C59.4424 10.3186 58.0785 10.5398 57.5163 10.6588C55.1178 11.1657 54.1133 12.0811 54.1133 13.5787C54.1133 14.3767 54.5564 15.2407 55.3651 15.7253C55.8505 16.0156 56.4841 16.1254 57.1837 16.1254C57.6438 16.1254 58.0908 16.0571 58.5047 15.9311C59.4455 15.6185 59.7082 15.0041 59.7082 15.0041V15.8075H61.2664V10.0644C61.2664 8.11211 59.9839 6.94629 57.8358 6.94629ZM59.4517 13.0749C59.4517 13.6786 58.7942 14.5288 57.2629 14.5288C56.8305 14.5288 56.524 14.4143 56.3197 14.2438C56.0463 14.0157 55.9565 13.6878 55.9941 13.3983C56.0102 13.2723 56.0863 13.0012 56.3681 12.7662C56.6561 12.5258 57.1653 12.3538 57.9517 12.1825C58.5984 12.042 59.4524 11.8868 59.4524 11.8868V13.0757L59.4517 13.0749Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_4", + "d": "M67.0275 6.94657C66.8109 6.94657 66.5997 6.96193 66.3946 6.99034C64.9992 7.20001 64.5906 7.90887 64.5906 7.90887L64.5921 7.20001H62.8457V15.8093H64.6666V11.0339C64.6666 9.41108 65.8501 8.67226 66.9499 8.67226C68.1388 8.67226 68.7163 9.31124 68.7163 10.6268V15.8093H70.5372V10.3765C70.5372 8.25985 69.1925 6.9458 67.0282 6.9458L67.0275 6.94657Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_5", + "d": "M78.1373 7.19359V8.08063C78.1373 8.08063 77.6911 6.94629 75.6611 6.94629C73.139 6.94629 71.5723 8.68658 71.5723 11.489C71.5723 13.0703 72.0776 14.3152 72.9693 15.1017C73.6628 15.713 74.589 16.0264 75.6918 16.0479C76.4591 16.0624 76.9559 15.8536 77.2664 15.6562C77.8623 15.2768 78.0835 14.9166 78.0835 14.9166C78.0835 14.9166 78.0582 15.1984 78.0121 15.5801C77.9791 15.8566 77.9169 16.0509 77.9169 16.0509C77.6396 17.0378 76.8285 17.6084 75.6457 17.6084C74.463 17.6084 73.7465 17.2191 73.6044 16.4518L71.8342 16.9802C72.1398 18.4548 73.5238 19.3349 75.5359 19.3349C76.9037 19.3349 77.976 18.9632 78.7233 18.229C79.4767 17.4886 79.8591 16.4219 79.8591 15.0579V7.19282H78.1373V7.19359ZM78.0229 11.5666C78.0229 13.29 77.1811 14.3191 75.7709 14.3191C74.2603 14.3191 73.394 13.2869 73.394 11.4882C73.394 9.68959 74.2603 8.67275 75.7709 8.67275C77.1473 8.67275 78.0098 9.69726 78.0229 11.3469V11.5666Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_6", + "d": "M90.532 14.0495C90.7777 13.6033 90.9022 13.0772 90.9022 12.4851C90.9022 11.893 90.7969 11.4383 90.5888 11.0704C90.3807 10.701 90.1119 10.3992 89.7909 10.1727C89.4675 9.94455 89.1258 9.76484 88.7771 9.63735C88.4269 9.50987 88.1051 9.40695 87.8217 9.33246L85.7427 8.75262C85.4801 8.68273 85.2174 8.59441 84.9609 8.48919C84.7021 8.38321 84.4817 8.23652 84.3073 8.05298C84.1292 7.86635 84.0385 7.62367 84.0385 7.32952C84.0385 7.02079 84.1437 6.74661 84.3503 6.51467C84.5554 6.28504 84.8288 6.10687 85.1637 5.98475C85.4962 5.86264 85.8625 5.80351 86.2527 5.80888C86.6536 5.81963 87.0368 5.90181 87.3909 6.05387C87.7464 6.20594 88.0521 6.43019 88.2994 6.7205C88.5398 7.00312 88.7064 7.34719 88.7955 7.74424L90.8054 7.3948C90.6341 6.70667 90.3423 6.10994 89.9368 5.62149C89.5236 5.12383 89.0029 4.73829 88.3885 4.4764C87.7733 4.21375 87.0629 4.07781 86.2765 4.07243C85.5023 4.06706 84.7858 4.19071 84.1522 4.44031C83.5201 4.68914 83.011 5.06853 82.6385 5.56773C82.2668 6.06616 82.0778 6.69131 82.0778 7.42552C82.0778 7.92779 82.1615 8.35403 82.3274 8.69349C82.4933 9.03371 82.7099 9.31634 82.9702 9.53522C83.2321 9.75487 83.5132 9.92843 83.8066 10.0529C84.1023 10.178 84.3811 10.2794 84.636 10.3531L87.6328 11.2394C87.8493 11.3047 88.0429 11.383 88.208 11.4721C88.3747 11.562 88.5129 11.6633 88.6197 11.7732C88.7272 11.8838 88.8101 12.0113 88.8654 12.1526C88.9207 12.2939 88.9484 12.449 88.9484 12.6134C88.9484 12.9812 88.8301 13.2969 88.5958 13.5496C88.3647 13.7999 88.0598 13.9935 87.6904 14.1232C87.3225 14.253 86.9254 14.3191 86.5092 14.3191C85.8065 14.3191 85.1767 14.1271 84.6383 13.7485C84.1077 13.3752 83.749 12.8422 83.5724 12.1648L81.6309 12.4598C81.7507 13.1909 82.0264 13.8322 82.4503 14.3652C82.8819 14.9081 83.441 15.3313 84.1107 15.6224C84.782 15.9142 85.5484 16.0624 86.3871 16.0624C86.9769 16.0624 87.5491 15.9872 88.089 15.8382C88.6273 15.69 89.1127 15.4642 89.5313 15.167C89.9491 14.8713 90.2855 14.4942 90.5304 14.048L90.532 14.0495Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_7", + "d": "M100.071 8.84108C100.322 8.69747 100.611 8.62451 100.928 8.62451C101.441 8.62451 101.855 8.79654 102.156 9.13676C102.457 9.47545 102.61 9.94931 102.61 10.5476V15.7462H104.474V10.0607C104.474 9.14368 104.218 8.39334 103.715 7.83116C103.212 7.27052 102.477 6.9856 101.532 6.9856C100.961 6.9856 100.436 7.11308 99.9714 7.36422C99.536 7.6 99.1789 7.9287 98.9116 8.34035L98.8763 8.39488L98.8455 8.33804C98.6343 7.9479 98.3348 7.62918 97.9547 7.3911C97.5253 7.1223 96.9831 6.9856 96.3442 6.9856C95.7628 6.9856 95.2306 7.11462 94.7636 7.36806C94.405 7.56236 94.0985 7.81657 93.8528 8.12224L93.7844 8.20748V7.2014H92.1455V15.7462H94.0263V10.4762C94.0263 9.93164 94.1799 9.48236 94.4833 9.14137C94.7874 8.79884 95.1968 8.62528 95.7006 8.62528C96.2044 8.62528 96.636 8.79884 96.9378 9.14137C97.2381 9.48236 97.3902 9.9639 97.3902 10.5714V15.7462H99.2464V10.4762C99.2464 10.0937 99.3209 9.75884 99.4684 9.48006C99.6166 9.20051 99.8194 8.98624 100.071 8.84185L100.071 8.84108Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_8", + "d": "M110.408 13.5589C110.418 13.9429 110.522 14.3254 110.717 14.694C110.938 15.0972 111.265 15.3967 111.689 15.5834C112.118 15.7707 112.61 15.8729 113.153 15.8859C113.689 15.899 114.243 15.8537 114.801 15.7515V14.201C114.276 14.2762 113.8 14.2962 113.387 14.2593C112.951 14.2209 112.63 14.0328 112.431 13.701C112.325 13.5305 112.269 13.307 112.26 13.0382C112.252 12.7748 112.248 12.466 112.248 12.1189V8.56844H114.801V7.12307H112.248V4.19775H110.392V7.12307H108.812V8.56844H110.392V12.2318C110.392 12.7249 110.397 13.1718 110.408 13.5597V13.5589Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_9", + "d": "M120.316 6.93339C120.116 6.93339 119.922 6.94645 119.733 6.97103C118.359 7.1853 117.955 7.88495 117.955 7.88495V7.67989H117.955V4.1709H116.134V15.7977H117.955V11.0222C117.955 9.38869 119.138 8.64527 120.238 8.64527C121.427 8.64527 122.004 9.28424 122.004 10.5998V15.7977H123.825V10.3495C123.825 8.27509 122.448 6.93416 120.316 6.93416L120.316 6.93339Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_10", + "d": "M107.589 7.19922H105.777V15.8162H107.589V7.19922Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_11", + "d": "M106.682 6.55761C107.334 6.55761 107.863 6.02913 107.863 5.37719C107.863 4.72527 107.334 4.19678 106.682 4.19678C106.03 4.19678 105.502 4.72527 105.502 5.37719C105.502 6.02913 106.03 6.55761 106.682 6.55761Z", + "fill": "#1C3C3C" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_12", + "d": "M22.3912 15.1309C22.286 15.306 21.9696 15.3175 21.7 15.1555C21.5618 15.0725 21.455 14.9581 21.3997 14.8337C21.349 14.7208 21.3483 14.614 21.3966 14.5334C21.4519 14.4412 21.5664 14.3944 21.7015 14.3944C21.8229 14.3944 21.9611 14.432 22.0886 14.5088C22.3582 14.6709 22.4972 14.9558 22.392 15.1309H22.3912ZM37.9908 9.9999C37.9908 15.293 33.6839 19.5999 28.3908 19.5999H9.81289C4.51983 19.5999 0.212891 15.2937 0.212891 9.9999C0.212891 4.70608 4.51983 0.399902 9.81289 0.399902H28.3908C33.6846 0.399902 37.9908 4.70685 37.9908 9.9999ZM18.714 14.8145C18.8653 14.6309 18.1664 14.1141 18.0236 13.9244C17.7333 13.6095 17.7317 13.1564 17.5359 12.7885C17.0567 11.678 16.506 10.5759 15.7357 9.63587C14.9216 8.60752 13.9171 7.75657 13.0347 6.7912C12.3795 6.11766 12.2044 5.15843 11.6261 4.43421C10.829 3.25686 8.30838 2.93584 7.93897 4.59856C7.94051 4.65078 7.92438 4.68381 7.87906 4.71683C7.67477 4.86505 7.49276 5.03478 7.33992 5.23984C6.96591 5.76054 6.90831 6.64374 7.37525 7.11145C7.39061 6.86493 7.39906 6.63222 7.59413 6.45558C7.9551 6.76585 8.50037 6.87491 8.91893 6.64374C9.8436 7.96393 9.6132 9.79024 10.3474 11.2126C10.5502 11.549 10.7545 11.8923 11.0148 12.1872C11.226 12.5159 11.9556 12.9037 11.9986 13.2078C12.0063 13.7301 11.9449 14.3007 12.2874 14.7377C12.4487 15.0649 12.0524 15.3936 11.7329 15.3529C11.3182 15.4097 10.8121 15.0741 10.4488 15.2807C10.3205 15.4197 10.0694 15.2661 9.9588 15.4588C9.9204 15.5587 9.71304 15.6992 9.83669 15.7952C9.97416 15.6908 10.1017 15.5817 10.2867 15.6439C10.2591 15.7945 10.3781 15.816 10.4726 15.8597C10.4695 15.9619 10.4096 16.0663 10.488 16.1531C10.5793 16.061 10.6339 15.9304 10.779 15.892C11.2613 16.5348 11.7521 15.2415 12.7958 15.8236C12.5838 15.8137 12.3957 15.8398 12.2528 16.0141C12.2175 16.0533 12.1875 16.0994 12.2497 16.15C12.8127 15.7868 12.8096 16.2745 13.1752 16.1247C13.4563 15.978 13.7358 15.7945 14.0699 15.8467C13.745 15.9404 13.732 16.2015 13.5415 16.4219C13.5093 16.4557 13.4939 16.4941 13.5315 16.5502C14.2058 16.4933 14.2611 16.2691 14.8057 15.9941C15.2119 15.7461 15.6167 16.3474 15.9684 16.0049C16.046 15.9304 16.152 15.9557 16.248 15.9458C16.1251 15.2907 14.7742 16.0656 14.7957 15.187C15.2304 14.8913 15.1305 14.3253 15.1597 13.8683C15.6597 14.1456 16.2157 14.3068 16.7057 14.5718C16.953 14.9712 17.3408 15.4988 17.8577 15.4642C17.8715 15.4243 17.8838 15.389 17.8984 15.3483C18.0551 15.3751 18.2563 15.4788 18.3423 15.2807C18.5765 15.5257 18.9206 15.5134 19.227 15.4504C19.4536 15.2661 18.8008 15.0034 18.7132 14.8137L18.714 14.8145ZM25.722 11.7187L24.6944 10.3317C23.7974 11.3577 23.1984 11.8577 23.1876 11.8669C23.1822 11.8723 22.6101 12.4291 22.0886 12.9068C21.5771 13.3753 21.1731 13.7462 20.9673 14.1517C20.9105 14.2638 20.7868 14.677 20.9604 15.0902C21.094 15.4097 21.3667 15.637 21.7714 15.766C21.8928 15.8044 22.0087 15.8213 22.1193 15.8213C22.8482 15.8213 23.3266 15.0902 23.3297 15.0841C23.3358 15.0756 23.9556 14.1901 24.7106 13.0719C24.9617 12.7002 25.2489 12.307 25.722 11.7179V11.7187ZM30.5535 14.9128C30.5535 14.7085 30.479 14.5111 30.3438 14.3583L30.2163 14.2139C29.446 13.3407 27.4684 11.0989 26.6989 10.228C25.7328 9.13437 24.6522 7.74045 24.5623 7.62371L24.4325 7.35491V6.88182C24.4325 6.70825 24.398 6.53853 24.3312 6.37878L24.0562 5.72598C24.0524 5.71677 24.0508 5.70601 24.0524 5.69603L24.0631 5.60541C24.0647 5.59081 24.0716 5.57776 24.0831 5.56777C24.2974 5.37885 25.0946 4.76598 26.3287 4.81744C26.49 4.82435 26.5184 4.73603 26.523 4.6984C26.5453 4.51715 26.1314 4.30365 25.7458 4.22454C25.2159 4.11625 23.8074 3.82902 22.6807 4.56861L22.6723 4.57475C21.9442 5.18301 21.359 5.64765 21.3529 5.65225L21.3398 5.66531C21.3314 5.67529 21.1255 5.92182 21.1755 6.23593C21.2077 6.44022 21.1017 6.51318 21.0956 6.51702C21.0894 6.52086 20.9451 6.61149 20.7961 6.50934C20.6156 6.37417 20.3022 6.60611 20.2385 6.6568L19.7654 7.06384L19.7562 7.07305C19.7477 7.08304 19.545 7.32035 19.8161 7.70128C20.0503 8.03075 20.1333 8.14057 20.3368 8.39401C20.5434 8.65053 20.9159 8.97539 20.9358 8.99229C20.9451 8.99997 21.1724 9.172 21.4857 8.93238C21.743 8.73501 21.9496 8.55683 21.9496 8.55683C21.9665 8.54301 22.1155 8.42013 22.1224 8.23888C22.1247 8.18665 22.1224 8.14134 22.1224 8.09987C22.1186 7.97238 22.1178 7.93475 22.2138 7.87331C22.2599 7.87331 22.4012 7.92477 22.5225 7.98621C22.5356 7.99389 22.8389 8.16438 23.1147 8.15209C23.2882 8.17513 23.4802 8.37251 23.5463 8.45315C23.5524 8.45929 24.1392 9.07523 24.9655 10.1558C25.123 10.3609 25.7013 11.1312 25.8595 11.3462C26.1237 11.7056 26.5238 12.2494 26.9539 12.8354C27.6988 13.8499 28.5344 14.9873 28.9138 15.4988C29.0398 15.6685 29.2233 15.7837 29.4307 15.8236L29.5712 15.8505C29.625 15.8605 29.6787 15.8659 29.7325 15.8659C29.9813 15.8659 30.2171 15.7568 30.373 15.5633L30.3815 15.5525C30.4936 15.4105 30.555 15.2284 30.555 15.0411V14.9128H30.5535ZM31.2147 5.80355L31.0512 5.63997C31.0035 5.59235 30.9367 5.56393 30.8691 5.56931C30.8016 5.57238 30.7378 5.6031 30.694 5.65533L29.6649 6.87261C29.6357 6.90717 29.5943 6.92867 29.5497 6.93328L29.1834 6.97014C29.1365 6.97475 29.0897 6.96016 29.0536 6.93021L28.4684 6.43485C28.4307 6.40259 28.4085 6.35651 28.4069 6.30736L28.3985 6.01398C28.397 5.97174 28.4115 5.93027 28.4384 5.89801L29.4391 4.69225C29.5144 4.60163 29.5136 4.4703 29.4376 4.37968L29.337 4.26064C29.2709 4.18307 29.1619 4.15465 29.0667 4.19075C28.8301 4.28061 28.2341 4.51331 27.8033 4.74678C27.1943 5.07549 26.7711 5.59696 26.6981 6.10768C26.6444 6.48169 26.6666 7.0984 26.6851 7.43248C26.692 7.56381 26.6628 7.69513 26.5991 7.81264C26.5207 7.95856 26.3825 8.19203 26.1721 8.47312C26.0645 8.62134 25.997 8.67587 25.904 8.78569L27.0361 10.1166C27.3087 9.79869 27.5475 9.55753 27.7557 9.32483C28.1358 8.90166 28.2541 8.89782 28.5705 8.88707C28.7656 8.88016 29.0329 8.87171 29.456 8.76573C30.6111 8.47696 30.9767 7.22665 30.992 7.17136L31.2793 6.03549C31.3 5.95331 31.2754 5.86422 31.2155 5.80432L31.2147 5.80355ZM13.4524 13.7324C13.328 14.2178 13.2873 15.0449 12.656 15.0687C12.6038 15.349 12.8503 15.4542 13.0738 15.3644C13.2958 15.2622 13.401 15.445 13.4755 15.627C13.818 15.677 14.3249 15.5126 14.3441 15.1071C13.8326 14.8122 13.6744 14.2516 13.4517 13.7324H13.4524Z", + "fill": "#1C3C3C" + }, + "children": [] + } + ] + } + ] + } + ] + } + ] + }, + "name": "LangsmithIconBig" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/LangsmithIconBig.tsx b/web/app/components/base/icons/src/public/tracing/LangsmithIconBig.tsx new file mode 100644 index 00000000000000..0a0f2e0d053cbf --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/LangsmithIconBig.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './LangsmithIconBig.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'LangsmithIconBig' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/TracingIcon.json b/web/app/components/base/icons/src/public/tracing/TracingIcon.json new file mode 100644 index 00000000000000..508b555b0f3841 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/TracingIcon.json @@ -0,0 +1,47 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "20", + "height": "20", + "viewBox": "0 0 20 20", + "fill": "none", + "xmlns": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "g", + "attributes": { + "id": "analytics-fill" + }, + "children": [ + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector", + "opacity": "0.6", + "d": "M5 2.5C3.61929 2.5 2.5 3.61929 2.5 5V9.16667H6.15164C6.78293 9.16667 7.36003 9.52333 7.64235 10.088L8.33333 11.4699L10.9213 6.29399C11.0625 6.01167 11.351 5.83333 11.6667 5.83333C11.9823 5.83333 12.2708 6.01167 12.412 6.29399L13.8483 9.16667H17.5V5C17.5 3.61929 16.3807 2.5 15 2.5H5Z", + "fill": "white" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "id": "Vector_2", + "d": "M2.5 14.9999C2.5 16.3807 3.61929 17.4999 5 17.4999H15C16.3807 17.4999 17.5 16.3807 17.5 14.9999V10.8333H13.8483C13.2171 10.8333 12.64 10.4766 12.3577 9.91195L11.6667 8.53003L9.07867 13.7059C8.9375 13.9883 8.649 14.1666 8.33333 14.1666C8.01769 14.1666 7.72913 13.9883 7.58798 13.7059L6.15164 10.8333H2.5V14.9999Z", + "fill": "white" + }, + "children": [] + } + ] + } + ] + }, + "name": "TracingIcon" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/TracingIcon.tsx b/web/app/components/base/icons/src/public/tracing/TracingIcon.tsx new file mode 100644 index 00000000000000..e24b2d05f68254 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/TracingIcon.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './TracingIcon.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'TracingIcon' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/index.ts b/web/app/components/base/icons/src/public/tracing/index.ts new file mode 100644 index 00000000000000..9cedf9cec39fb0 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/index.ts @@ -0,0 +1,5 @@ +export { default as LangfuseIconBig } from './LangfuseIconBig' +export { default as LangfuseIcon } from './LangfuseIcon' +export { default as LangsmithIconBig } from './LangsmithIconBig' +export { default as LangsmithIcon } from './LangsmithIcon' +export { default as TracingIcon } from './TracingIcon' From 17a9b8dc5be0ba391666b2e2638835c208d155e8 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 18:06:57 +0800 Subject: [PATCH 158/273] feat: 18n and config btn --- .../[appId]/overview/tracing/panel.tsx | 13 ++++++++++--- web/i18n/en-US/app.ts | 5 +++++ web/i18n/zh-Hans/app.ts | 5 +++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index cf6b6435f3e161..c6982b79fd7855 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -8,7 +8,9 @@ import type { TracingTool } from './type' import TracingIcon from './tracing-icon' import Button from '@/app/components/base/button' import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' +import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' +const I18N_PREFIX = 'app.tracing' const ConfigBtn = ({ className, }: { @@ -17,7 +19,12 @@ const ConfigBtn = ({ const { t } = useTranslation() return ( - + ) } @@ -36,9 +43,9 @@ const Panel: FC = () => {
-
Tracing app performance
+
{t(`${I18N_PREFIX}.title`)}
- Configuring a Third-Party LLMOps provider and tracing app performance. + {t(`${I18N_PREFIX}.description`)}
diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index c180443a6dfb5c..235b013b89257c 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -85,6 +85,11 @@ const translation = { workflow: 'Workflow', completion: 'Completion', }, + tracing: { + title: 'Tracing app performance', + description: 'Configuring a Third-Party LLMOps provider and tracing app performance.', + config: 'Config', + }, } export default translation diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index dd603951f282b9..0364a189df7c46 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -84,6 +84,11 @@ const translation = { workflow: '工作流', completion: '文本生成', }, + tracing: { + title: '追踪应用性能', + description: '配置第三方 LLMOps 提供商并跟踪应用程序性能。', + config: '配置', + }, } export default translation From b6a3f646cb3c0fe0ebcef041cb646beadfd91407 Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 12 Jun 2024 18:37:18 +0800 Subject: [PATCH 159/273] chore: fold btn --- .../[appId]/overview/tracing/panel.tsx | 15 ++++------ .../overview/tracing/toggle-fold-btn.tsx | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index c6982b79fd7855..05e921e0a93d44 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -1,15 +1,14 @@ 'use client' import type { FC } from 'react' -import React from 'react' +import React, { useState } from 'react' import { useTranslation } from 'react-i18next' -import { useBoolean } from 'ahooks' import cn from 'classnames' import type { TracingTool } from './type' import TracingIcon from './tracing-icon' +import ToggleExpandBtn from './toggle-fold-btn' import Button from '@/app/components/base/button' import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' - const I18N_PREFIX = 'app.tracing' const ConfigBtn = ({ className, @@ -20,7 +19,7 @@ const ConfigBtn = ({ return (
-
+
- +
) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx new file mode 100644 index 00000000000000..b0d9a0258435bc --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx @@ -0,0 +1,30 @@ +'use client' +import { ChevronDoubleDownIcon } from '@heroicons/react/20/solid' +import type { FC } from 'react' +import React from 'react' + +type Props = { + isFold: boolean + onFoldChange: (isFold: boolean) => void +} + +const ToggleFoldBtn: FC = ({ + isFold, + onFoldChange, +}) => { + return ( +
+ {isFold && ( +
+ +
+ )} + {!isFold && ( +
+ +
+ )} +
+ ) +} +export default React.memo(ToggleFoldBtn) From 0c6b71b04f59dd71a81452949c7c1e540499df91 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Wed, 12 Jun 2024 20:30:33 +0800 Subject: [PATCH 160/273] note node --- .../(appDetailLayout)/[appId]/note/page.tsx | 24 -------- .../components/workflow/candidate-node.tsx | 14 ++++- web/app/components/workflow/constants.ts | 1 + .../workflow/hooks/use-checklist.ts | 36 +++++++----- .../workflow/hooks/use-nodes-interactions.ts | 17 +++++- .../components/workflow/hooks/use-workflow.ts | 9 ++- web/app/components/workflow/index.tsx | 8 ++- web/app/components/workflow/nodes/index.tsx | 28 ++++++--- .../workflow/note-node/constants.ts | 43 +++++++++++++- .../components/workflow/note-node/hooks.ts | 16 +++-- .../components/workflow/note-node/index.tsx | 57 +++++++++++++----- .../note-node/note-editor/context.tsx | 3 + .../workflow/note-node/note-editor/editor.tsx | 4 +- .../note-editor/toolbar/color-picker.tsx | 58 +++++++++---------- .../note-node/note-editor/toolbar/index.tsx | 8 ++- .../components/workflow/note-node/types.ts | 15 +++++ .../components/workflow/operator/control.tsx | 13 ++++- web/app/components/workflow/operator/hooks.ts | 35 +++++++++++ web/app/components/workflow/utils.ts | 10 ++-- 19 files changed, 283 insertions(+), 116 deletions(-) delete mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx create mode 100644 web/app/components/workflow/operator/hooks.ts diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx deleted file mode 100644 index 54cdd2ac94c562..00000000000000 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/note/page.tsx +++ /dev/null @@ -1,24 +0,0 @@ -'use client' - -import { - NoteEditor, - NoteEditorContextProvider, - NoteEditorToolbar, -} from '@/app/components/workflow/note-node/note-editor' - -const Page = () => { - return ( -
-
- - -
- -
- - -
-
- ) -} -export default Page diff --git a/web/app/components/workflow/candidate-node.tsx b/web/app/components/workflow/candidate-node.tsx index 1987e6227bea53..e7ea24b4fd989a 100644 --- a/web/app/components/workflow/candidate-node.tsx +++ b/web/app/components/workflow/candidate-node.tsx @@ -12,7 +12,10 @@ import { useStore, useWorkflowStore, } from './store' +import { CUSTOM_NODE } from './constants' import CustomNode from './nodes' +import CustomNoteNode from './note-node' +import { CUSTOM_NOTE_NODE } from './note-node/constants' const CandidateNode = () => { const store = useStoreApi() @@ -73,7 +76,16 @@ const CandidateNode = () => { transformOrigin: '0 0', }} > - + { + candidateNode.type === CUSTOM_NODE && ( + + ) + } + { + candidateNode.type === CUSTOM_NOTE_NODE && ( + + ) + }
) } diff --git a/web/app/components/workflow/constants.ts b/web/app/components/workflow/constants.ts index 918649a26bbe3b..a6f313e98e4434 100644 --- a/web/app/components/workflow/constants.ts +++ b/web/app/components/workflow/constants.ts @@ -391,3 +391,4 @@ export const PARAMETER_EXTRACTOR_COMMON_STRUCT: Var[] = [ ] export const WORKFLOW_DATA_UPDATE = 'WORKFLOW_DATA_UPDATE' +export const CUSTOM_NODE = 'custom' diff --git a/web/app/components/workflow/hooks/use-checklist.ts b/web/app/components/workflow/hooks/use-checklist.ts index 818505d0fe444d..142f96ed2a15b9 100644 --- a/web/app/components/workflow/hooks/use-checklist.ts +++ b/web/app/components/workflow/hooks/use-checklist.ts @@ -14,7 +14,10 @@ import { getToolCheckParams, getValidTreeNodes, } from '../utils' -import { MAX_TREE_DEEPTH } from '../constants' +import { + CUSTOM_NODE, + MAX_TREE_DEEPTH, +} from '../constants' import type { ToolNodeType } from '../nodes/tool/types' import { useIsChatMode } from './use-workflow' import { useNodesExtraData } from './use-nodes-data' @@ -33,7 +36,7 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { const needWarningNodes = useMemo(() => { const list = [] - const { validNodes } = getValidTreeNodes(nodes, edges) + const { validNodes } = getValidTreeNodes(nodes.filter(node => node.type === CUSTOM_NODE), edges) for (let i = 0; i < nodes.length; i++) { const node = nodes[i] @@ -53,17 +56,20 @@ export const useChecklist = (nodes: Node[], edges: Edge[]) => { if (provider_type === CollectionType.workflow) toolIcon = workflowTools.find(tool => tool.id === node.data.provider_id)?.icon } - const { errorMessage } = nodesExtraData[node.data.type].checkValid(node.data, t, moreDataForCheckValid) - - if (errorMessage || !validNodes.find(n => n.id === node.id)) { - list.push({ - id: node.id, - type: node.data.type, - title: node.data.title, - toolIcon, - unConnected: !validNodes.find(n => n.id === node.id), - errorMessage, - }) + + if (node.type === CUSTOM_NODE) { + const { errorMessage } = nodesExtraData[node.data.type].checkValid(node.data, t, moreDataForCheckValid) + + if (errorMessage || !validNodes.find(n => n.id === node.id)) { + list.push({ + id: node.id, + type: node.data.type, + title: node.data.title, + toolIcon, + unConnected: !validNodes.find(n => n.id === node.id), + errorMessage, + }) + } } } @@ -107,11 +113,11 @@ export const useChecklistBeforePublish = () => { getNodes, edges, } = store.getState() - const nodes = getNodes() + const nodes = getNodes().filter(node => node.type === CUSTOM_NODE) const { validNodes, maxDepth, - } = getValidTreeNodes(nodes, edges) + } = getValidTreeNodes(nodes.filter(node => node.type === CUSTOM_NODE), edges) if (maxDepth > MAX_TREE_DEEPTH) { notify({ type: 'error', message: t('workflow.common.maxTreeDepth', { depth: MAX_TREE_DEEPTH }) }) diff --git a/web/app/components/workflow/hooks/use-nodes-interactions.ts b/web/app/components/workflow/hooks/use-nodes-interactions.ts index 64cba1c790dfcb..7dabe723f0a124 100644 --- a/web/app/components/workflow/hooks/use-nodes-interactions.ts +++ b/web/app/components/workflow/hooks/use-nodes-interactions.ts @@ -38,6 +38,7 @@ import { getNodesConnectedSourceOrTargetHandleIdsMap, getTopLeftNodePosition, } from '../utils' +import { CUSTOM_NOTE_NODE } from '../note-node/constants' import type { IterationNodeType } from '../nodes/iteration/types' import type { VariableAssignerNodeType } from '../nodes/variable-assigner/types' import { useNodeIterationInteractions } from '../nodes/iteration/use-interactions' @@ -71,7 +72,7 @@ export const useNodesInteractions = () => { if (getNodesReadOnly()) return - if (node.data.isIterationStart) + if (node.data.isIterationStart || node.type === CUSTOM_NOTE_NODE) return dragNodeStartPosition.current = { x: node.position.x, y: node.position.y } @@ -143,6 +144,9 @@ export const useNodesInteractions = () => { if (getNodesReadOnly()) return + if (node.type === CUSTOM_NOTE_NODE) + return + const { getNodes, setNodes, @@ -193,10 +197,13 @@ export const useNodesInteractions = () => { setEdges(newEdges) }, [store, workflowStore, getNodesReadOnly]) - const handleNodeLeave = useCallback(() => { + const handleNodeLeave = useCallback((_, node) => { if (getNodesReadOnly()) return + if (node.type === CUSTOM_NOTE_NODE) + return + const { setEnteringNodePayload, } = workflowStore.getState() @@ -298,6 +305,9 @@ export const useNodesInteractions = () => { if (targetNode?.data.isIterationStart) return + if (sourceNode?.type === CUSTOM_NOTE_NODE || targetNode?.type === CUSTOM_NOTE_NODE) + return + const needDeleteEdges = edges.filter((edge) => { if ( (edge.source === source && edge.sourceHandle === sourceHandle) @@ -361,6 +371,9 @@ export const useNodesInteractions = () => { const { getNodes } = store.getState() const node = getNodes().find(n => n.id === nodeId)! + if (node.type === CUSTOM_NOTE_NODE) + return + if (node.data.type === BlockEnum.VariableAggregator || node.data.type === BlockEnum.VariableAssigner) { if (handleType === 'target') return diff --git a/web/app/components/workflow/hooks/use-workflow.ts b/web/app/components/workflow/hooks/use-workflow.ts index d7759eb998d2c3..fc995ed976dad8 100644 --- a/web/app/components/workflow/hooks/use-workflow.ts +++ b/web/app/components/workflow/hooks/use-workflow.ts @@ -34,8 +34,10 @@ import { useWorkflowStore, } from '../store' import { + CUSTOM_NODE, SUPPORT_OUTPUT_VARS_NODE, } from '../constants' +import { CUSTOM_NOTE_NODE } from '../note-node/constants' import { findUsedVarNodes, getNodeOutputVars, updateNodeVars } from '../nodes/_base/components/variable/utils' import { useNodesExtraData } from './use-nodes-data' import { useWorkflowTemplate } from './use-workflow-template' @@ -88,7 +90,7 @@ export const useWorkflow = () => { const rankMap = {} as Record nodes.forEach((node) => { - if (!node.parentId) { + if (!node.parentId && node.type === CUSTOM_NODE) { const rank = layout.node(node.id).rank! if (!rankMap[rank]) { @@ -103,7 +105,7 @@ export const useWorkflow = () => { const newNodes = produce(nodes, (draft) => { draft.forEach((node) => { - if (!node.parentId) { + if (!node.parentId && node.type === CUSTOM_NODE) { const nodeWithPosition = layout.node(node.id) node.position = { @@ -345,6 +347,9 @@ export const useWorkflow = () => { if (targetNode.data.isIterationStart) return false + if (sourceNode.type === CUSTOM_NOTE_NODE || targetNode.type === CUSTOM_NOTE_NODE) + return false + if (sourceNode && targetNode) { const sourceNodeAvailableNextNodes = nodesExtraData[sourceNode.data.type].availableNextNodes const targetNodeAvailablePrevNodes = [...nodesExtraData[targetNode.data.type].availablePrevNodes, BlockEnum.Start] diff --git a/web/app/components/workflow/index.tsx b/web/app/components/workflow/index.tsx index e9ae8dd84fc8c1..23807e36ffe903 100644 --- a/web/app/components/workflow/index.tsx +++ b/web/app/components/workflow/index.tsx @@ -46,6 +46,8 @@ import { } from './hooks' import Header from './header' import CustomNode from './nodes' +import CustomNoteNode from './note-node' +import { CUSTOM_NOTE_NODE } from './note-node/constants' import Operator from './operator' import CustomEdge from './custom-edge' import CustomConnectionLine from './custom-connection-line' @@ -66,6 +68,7 @@ import { initialNodes, } from './utils' import { + CUSTOM_NODE, ITERATION_CHILDREN_Z_INDEX, WORKFLOW_DATA_UPDATE, } from './constants' @@ -76,10 +79,11 @@ import { useEventEmitterContextContext } from '@/context/event-emitter' import Confirm from '@/app/components/base/confirm/common' const nodeTypes = { - custom: CustomNode, + [CUSTOM_NODE]: CustomNode, + [CUSTOM_NOTE_NODE]: CustomNoteNode, } const edgeTypes = { - custom: CustomEdge, + [CUSTOM_NODE]: CustomEdge, } type WorkflowProps = { diff --git a/web/app/components/workflow/nodes/index.tsx b/web/app/components/workflow/nodes/index.tsx index a79651af7007fe..bebc140414fd9b 100644 --- a/web/app/components/workflow/nodes/index.tsx +++ b/web/app/components/workflow/nodes/index.tsx @@ -1,6 +1,10 @@ -import { memo } from 'react' +import { + memo, + useMemo, +} from 'react' import type { NodeProps } from 'reactflow' import type { Node } from '../types' +import { CUSTOM_NODE } from '../constants' import { NodeComponentMap, PanelComponentMap, @@ -23,14 +27,24 @@ const CustomNode = (props: NodeProps) => { CustomNode.displayName = 'CustomNode' export const Panel = memo((props: Node) => { + const nodeClass = props.type const nodeData = props.data - const PanelComponent = PanelComponentMap[nodeData.type] + const PanelComponent = useMemo(() => { + if (nodeClass === CUSTOM_NODE) + return PanelComponentMap[nodeData.type] - return ( - - - - ) + return () => null + }, [nodeClass, nodeData.type]) + + if (nodeClass === CUSTOM_NODE) { + return ( + + + + ) + } + + return null }) Panel.displayName = 'Panel' diff --git a/web/app/components/workflow/note-node/constants.ts b/web/app/components/workflow/note-node/constants.ts index a0f8cb03e5e9df..30b32e011911ea 100644 --- a/web/app/components/workflow/note-node/constants.ts +++ b/web/app/components/workflow/note-node/constants.ts @@ -1 +1,42 @@ -export const NOTE_NODE_TYPE = 'custom-note' +import { NoteTheme } from './types' + +export const CUSTOM_NOTE_NODE = 'custom-note' + +export const THEME_MAP: Record = { + [NoteTheme.blue]: { + outer: '#2E90FA', + title: '#D1E9FF', + bg: '#EFF8FF', + border: '#84CAFF', + }, + [NoteTheme.cyan]: { + outer: '#06AED4', + title: 'CFF9FE', + bg: '#ECFDFF', + border: '#67E3F9', + }, + [NoteTheme.green]: { + outer: '#16B364', + title: '#D3F8DF', + bg: '#EDFCF2', + border: '#73E2A3', + }, + [NoteTheme.yellow]: { + outer: '#EAAA08', + title: '#FEF7C3', + bg: '#FEFBE8', + border: '#FDE272', + }, + [NoteTheme.pink]: { + outer: '#EE46BC', + title: '#FCE7F6', + bg: '#FDF2FA', + border: '#FAA7E0', + }, + [NoteTheme.violet]: { + outer: '#875BF7', + title: '#ECE9FE', + bg: '#F5F3FF', + border: '#C3B5FD', + }, +} diff --git a/web/app/components/workflow/note-node/hooks.ts b/web/app/components/workflow/note-node/hooks.ts index 55f6a95b2b8073..0e998748885b02 100644 --- a/web/app/components/workflow/note-node/hooks.ts +++ b/web/app/components/workflow/note-node/hooks.ts @@ -1,17 +1,21 @@ import { useCallback } from 'react' import type { EditorState } from 'lexical' +import { useNodeDataUpdate } from '../hooks' +import type { NoteTheme } from './types' -export const useNote = () => { - const handleColorChange = useCallback(() => { +export const useNote = (id: string) => { + const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate() - }, []) + const handleThemeChange = useCallback((theme: NoteTheme) => { + handleNodeDataUpdateWithSyncDraft({ id, data: { theme } }) + }, [handleNodeDataUpdateWithSyncDraft, id]) const handleEditorChange = useCallback((editorState: EditorState) => { - - }, []) + handleNodeDataUpdateWithSyncDraft({ id, data: { text: JSON.stringify(editorState.toJSON()) } }) + }, [handleNodeDataUpdateWithSyncDraft, id]) return { - handleColorChange, + handleThemeChange, handleEditorChange, } } diff --git a/web/app/components/workflow/note-node/index.tsx b/web/app/components/workflow/note-node/index.tsx index 7be4b584180976..83263e43b0f945 100644 --- a/web/app/components/workflow/note-node/index.tsx +++ b/web/app/components/workflow/note-node/index.tsx @@ -1,28 +1,59 @@ import { memo } from 'react' +import cn from 'classnames' import type { NodeProps } from 'reactflow' import { NoteEditor, NoteEditorContextProvider, NoteEditorToolbar, } from './note-editor' +import { THEME_MAP } from './constants' +import { useNote } from './hooks' +import type { NoteNodeType } from './types' const NoteNode = ({ id, - type, data, -}: NodeProps) => { +}: NodeProps) => { + const theme = data.theme + const { + handleThemeChange, + handleEditorChange, + } = useNote(id) + return ( - -
-
-
- -
-
- -
-
- +
+ + <> +
+ { + data.selected && ( +
+ +
+ ) + } +
+
+ +
+
+ + +
) } diff --git a/web/app/components/workflow/note-node/note-editor/context.tsx b/web/app/components/workflow/note-node/note-editor/context.tsx index f41ae4e06dfbf2..d0ac4352e67ff0 100644 --- a/web/app/components/workflow/note-node/note-editor/context.tsx +++ b/web/app/components/workflow/note-node/note-editor/context.tsx @@ -18,9 +18,11 @@ type NoteEditorStore = ReturnType const NoteEditorContext = createContext(null) type NoteEditorContextProviderProps = { + value?: string children: JSX.Element | string | (JSX.Element | string)[] } export const NoteEditorContextProvider = memo(({ + value, children, }: NoteEditorContextProviderProps) => { const storeRef = useRef() @@ -35,6 +37,7 @@ export const NoteEditorContextProvider = memo(({ ListNode, ListItemNode, ], + editorState: value || null, onError: (error: Error) => { throw error }, diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index f8ad9239894b58..6787545bb242ee 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -16,7 +16,7 @@ import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin' import { OnChangePlugin } from '@lexical/react/LexicalOnChangePlugin' import LinkEditorPlugin from './plugins/link-editor-plugin' import FormatDetectorPlugin from './plugins/format-detector-plugin' -import TreeView from '@/app/components/base/prompt-editor/plugins/tree-view' +// import TreeView from '@/app/components/base/prompt-editor/plugins/tree-view' import Placeholder from '@/app/components/base/prompt-editor/plugins/placeholder' type EditorProps = { @@ -58,7 +58,7 @@ const Editor = ({ - + {/* */}
) } diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx index 570b209dc35d7f..7f7649d5316914 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/color-picker.tsx @@ -3,60 +3,55 @@ import { useState, } from 'react' import cn from 'classnames' +import { NoteTheme } from '../../types' +import { THEME_MAP } from '../../constants' import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' -export const COLOR_MAP = { - blue: '#D1E9FF', - cyan: '#CFF9FE', - green: '#D3F8DF', - yellow: '#FEF7C3', - pink: '#FCE7F6', - violet: '#ECE9FE', -} as Record export const COLOR_LIST = [ { - key: 'blue', - inner: COLOR_MAP.blue, - outer: '#2E90FA', + key: NoteTheme.blue, + inner: THEME_MAP[NoteTheme.blue].title, + outer: THEME_MAP[NoteTheme.blue].outer, }, { - key: 'cyan', - inner: COLOR_MAP.cyan, - outer: '#06AED4', + key: NoteTheme.cyan, + inner: THEME_MAP[NoteTheme.cyan].title, + outer: THEME_MAP[NoteTheme.cyan].outer, }, { - key: 'green', - inner: COLOR_MAP.green, - outer: '#16B364', + key: NoteTheme.green, + inner: THEME_MAP[NoteTheme.green].title, + outer: THEME_MAP[NoteTheme.green].outer, }, { - key: 'yellow', - inner: COLOR_MAP.yellow, - outer: '#EAAA08', + key: NoteTheme.yellow, + inner: THEME_MAP[NoteTheme.yellow].title, + outer: THEME_MAP[NoteTheme.yellow].outer, }, { - key: 'pink', - inner: COLOR_MAP.pink, - outer: '#EE46BC', + key: NoteTheme.pink, + inner: THEME_MAP[NoteTheme.pink].title, + outer: THEME_MAP[NoteTheme.pink].outer, }, { - key: 'violet', - inner: COLOR_MAP.violet, - outer: '#875BF7', + key: NoteTheme.violet, + inner: THEME_MAP[NoteTheme.violet].title, + outer: THEME_MAP[NoteTheme.violet].outer, }, ] export type ColorPickerProps = { - onColorChange?: (color: string) => void + theme: NoteTheme + onThemeChange: (theme: NoteTheme) => void } const ColorPicker = ({ - onColorChange, + theme, + onThemeChange, }: ColorPickerProps) => { - const [color, setColor] = useState('blue') const [open, setOpen] = useState(false) return ( @@ -73,7 +68,7 @@ const ColorPicker = ({ )}>
@@ -85,8 +80,7 @@ const ColorPicker = ({ key={color.key} className='group relative flex items-center justify-center w-8 h-8 rounded-md cursor-pointer' onClick={() => { - setColor(color.key) - onColorChange?.(color.key) + onThemeChange(color.key) setOpen(false) }} > diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx index cd8046fe1db85c..6a36b3a033ba67 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx @@ -8,11 +8,15 @@ import Operator from './operator' type ToolbarProps = ColorPickerProps const Toolbar = ({ - onColorChange, + theme, + onThemeChange, }: ToolbarProps) => { return (
- + diff --git a/web/app/components/workflow/note-node/types.ts b/web/app/components/workflow/note-node/types.ts index e69de29bb2d1d6..4ea51572c13930 100644 --- a/web/app/components/workflow/note-node/types.ts +++ b/web/app/components/workflow/note-node/types.ts @@ -0,0 +1,15 @@ +import type { CommonNodeType } from '../types' + +export enum NoteTheme { + blue = 'blue', + cyan = 'cyan', + green = 'green', + yellow = 'yellow', + pink = 'pink', + violet = 'violet', +} + +export type NoteNodeType = CommonNodeType & { + text: string + theme: NoteTheme +} diff --git a/web/app/components/workflow/operator/control.tsx b/web/app/components/workflow/operator/control.tsx index e5dc9b81707d6c..e512feef32883d 100644 --- a/web/app/components/workflow/operator/control.tsx +++ b/web/app/components/workflow/operator/control.tsx @@ -1,4 +1,8 @@ -import { memo, useCallback } from 'react' +import type { MouseEvent } from 'react' +import { + memo, + useCallback, +} from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' import { useKeyPress } from 'ahooks' @@ -11,6 +15,7 @@ import { isEventTargetInputArea } from '../utils' import { useStore } from '../store' import AddBlock from './add-block' import TipPopup from './tip-popup' +import { useOperator } from './hooks' import { Cursor02C, Hand02, @@ -27,6 +32,7 @@ const Control = () => { const controlMode = useStore(s => s.controlMode) const setControlMode = useStore(s => s.setControlMode) const { handleLayout } = useWorkflow() + const { handleAddNote } = useOperator() const { nodesReadOnly, getNodesReadOnly, @@ -76,11 +82,12 @@ const Control = () => { handleLayout() } - const addNote = () => { + const addNote = (e: MouseEvent) => { if (getNodesReadOnly()) return - console.log('add note') + e.stopPropagation() + handleAddNote() } return ( diff --git a/web/app/components/workflow/operator/hooks.ts b/web/app/components/workflow/operator/hooks.ts new file mode 100644 index 00000000000000..6986bbd4a0077b --- /dev/null +++ b/web/app/components/workflow/operator/hooks.ts @@ -0,0 +1,35 @@ +import { useCallback } from 'react' +import { generateNewNode } from '../utils' +import { useWorkflowStore } from '../store' +import type { NoteNodeType } from '../note-node/types' +import { CUSTOM_NOTE_NODE } from '../note-node/constants' +import { NoteTheme } from '../note-node/types' + +export const useOperator = () => { + const workflowStore = useWorkflowStore() + + const handleAddNote = useCallback(() => { + const newNode = generateNewNode({ + type: CUSTOM_NOTE_NODE, + data: { + title: '', + desc: '', + type: '' as any, + text: '', + theme: NoteTheme.blue, + _isCandidate: true, + } as NoteNodeType, + position: { + x: 0, + y: 0, + }, + }) + workflowStore.setState({ + candidateNode: newNode, + }) + }, [workflowStore]) + + return { + handleAddNote, + } +} diff --git a/web/app/components/workflow/utils.ts b/web/app/components/workflow/utils.ts index 60e9f69ddf4515..4ad9c6591c8631 100644 --- a/web/app/components/workflow/utils.ts +++ b/web/app/components/workflow/utils.ts @@ -17,6 +17,7 @@ import type { } from './types' import { BlockEnum } from './types' import { + CUSTOM_NODE, ITERATION_NODE_Z_INDEX, NODE_WIDTH_X_OFFSET, START_INITIAL_POSITION, @@ -105,7 +106,8 @@ export const initialNodes = (originNodes: Node[], originEdges: Edge[]) => { }, {} as Record) return nodes.map((node) => { - node.type = 'custom' + if (!node.type) + node.type = CUSTOM_NODE const connectedEdges = getConnectedEdges([node], edges) node.data._connectedSourceHandleIds = connectedEdges.filter(edge => edge.source === node.id).map(edge => edge.sourceHandle || 'source') @@ -189,7 +191,7 @@ export const initialEdges = (originEdges: Edge[], originNodes: Node[]) => { export const getLayoutByDagre = (originNodes: Node[], originEdges: Edge[]) => { const dagreGraph = new dagre.graphlib.Graph() dagreGraph.setDefaultEdgeLabel(() => ({})) - const nodes = cloneDeep(originNodes).filter(node => !node.parentId) + const nodes = cloneDeep(originNodes).filter(node => !node.parentId && node.type === CUSTOM_NODE) const edges = cloneDeep(originEdges).filter(edge => !edge.data?.isInIteration) dagreGraph.setGraph({ rankdir: 'LR', @@ -280,10 +282,10 @@ export const getNodesConnectedSourceOrTargetHandleIdsMap = (changes: ConnectedSo return nodesConnectedSourceOrTargetHandleIdsMap } -export const generateNewNode = ({ data, position, id, zIndex, ...rest }: Omit & { id?: string }) => { +export const generateNewNode = ({ data, position, id, zIndex, type, ...rest }: Omit & { id?: string }) => { return { id: id || `${Date.now()}`, - type: 'custom', + type: type || CUSTOM_NODE, data, position, targetPosition: Position.Left, From 1f60ae9a394e55f59e0d86759616cc2d49b22e4e Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Wed, 12 Jun 2024 23:36:09 +0800 Subject: [PATCH 161/273] fix firecrawl issue --- api/controllers/console/datasets/error.py | 6 ++++++ api/controllers/console/datasets/website.py | 11 ++++++++-- .../rag/extractor/firecrawl/firecrawl_app.py | 20 ++++++++++++++----- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/api/controllers/console/datasets/error.py b/api/controllers/console/datasets/error.py index 29142b80e627f3..3fb190a53725b1 100644 --- a/api/controllers/console/datasets/error.py +++ b/api/controllers/console/datasets/error.py @@ -71,3 +71,9 @@ class InvalidMetadataError(BaseHTTPException): error_code = 'invalid_metadata' description = "The metadata content is incorrect. Please check and verify." code = 400 + + +class WebsiteCrawlError(BaseHTTPException): + error_code = 'crawl_failed' + description = "{message}" + code = 500 diff --git a/api/controllers/console/datasets/website.py b/api/controllers/console/datasets/website.py index 6caf715aa31958..bbd91256f1c29c 100644 --- a/api/controllers/console/datasets/website.py +++ b/api/controllers/console/datasets/website.py @@ -1,6 +1,7 @@ from flask_restful import Resource, reqparse from controllers.console import api +from controllers.console.datasets.error import WebsiteCrawlError from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required from libs.login import login_required @@ -21,7 +22,10 @@ def post(self): args = parser.parse_args() WebsiteService.document_create_args_validate(args) # crawl url - result = WebsiteService.crawl_url(args) + try: + result = WebsiteService.crawl_url(args) + except Exception as e: + raise WebsiteCrawlError(str(e)) return result, 200 @@ -34,7 +38,10 @@ def get(self, job_id: str): parser.add_argument('provider', type=str, choices=['firecrawl'], required=True, location='args') args = parser.parse_args() # get crawl status - result = WebsiteService.get_crawl_status(job_id, args['provider']) + try: + result = WebsiteService.get_crawl_status(job_id, args['provider']) + except Exception as e: + raise WebsiteCrawlError(str(e)) return result, 200 diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 7bb18bfc845a99..b6447fd237a609 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -1,7 +1,10 @@ +import json import time import requests +from extensions.ext_storage import storage + class FirecrawlApp: def __init__(self, api_key=None, base_url=None): @@ -68,6 +71,9 @@ def check_crawl_status(self, job_id) -> dict: 'data': [] } else: + total = crawl_status_response.get('total', 0) + if total == 0: + raise Exception(f'Failed to check crawl status. Error: No page found') data = crawl_status_response.get('data', []) url_data_list = [] for item in data: @@ -78,6 +84,11 @@ def check_crawl_status(self, job_id) -> dict: 'markdown': item.get('markdown') } url_data_list.append(url_data) + # if url_data_list: + # file_key = 'website_files/' + job_id + '.txt' + # if storage.exists(file_key): + # storage.delete(file_key) + # storage.save(file_key, json.dumps(url_data_list).encode('utf-8')) return { 'status': 'completed', 'total': crawl_status_response.get('total'), @@ -112,8 +123,7 @@ def _get_request(self, url, headers, retries=3, backoff_factor=0.5): return response def _handle_error(self, response, action): - if response.status_code in [402, 409, 500]: - error_message = response.json().get('error', 'Unknown error occurred') - raise Exception(f'Failed to {action}. Status code: {response.status_code}. Error: {error_message}') - else: - raise Exception(f'Unexpected error occurred while trying to {action}. Status code: {response.status_code}') + error_message = response.json().get('error', 'Unknown error occurred') + raise Exception(f'Failed to {action}. Status code: {response.status_code}. Error: {error_message}') + + From e2212bdfb4fc8f4155bc0e4a3ccad7a428bcb0c5 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 13 Jun 2024 12:32:25 +0800 Subject: [PATCH 162/273] fix firecrawl issue --- .../rag/extractor/firecrawl/firecrawl_app.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index b6447fd237a609..82064390e056fe 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -63,14 +63,7 @@ def check_crawl_status(self, job_id) -> dict: response = self._get_request(f'{self.base_url}/v0/crawl/status/{job_id}', headers) if response.status_code == 200: crawl_status_response = response.json() - if crawl_status_response.get('status') != 'completed': - return { - 'status': crawl_status_response.get('status'), - 'total': crawl_status_response.get('total'), - 'current': crawl_status_response.get('current'), - 'data': [] - } - else: + if crawl_status_response.get('status') == 'completed': total = crawl_status_response.get('total', 0) if total == 0: raise Exception(f'Failed to check crawl status. Error: No page found') @@ -95,6 +88,15 @@ def check_crawl_status(self, job_id) -> dict: 'current': crawl_status_response.get('current'), 'data': url_data_list } + + else: + return { + 'status': crawl_status_response.get('status'), + 'total': crawl_status_response.get('total'), + 'current': crawl_status_response.get('current'), + 'data': [] + } + else: self._handle_error(response, 'check crawl status') From c402fff0aba95e21cdfc41949678a5a8f7655c17 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 13 Jun 2024 12:33:25 +0800 Subject: [PATCH 163/273] feat: unfold panel --- .../[appId]/overview/tracing/panel.tsx | 12 ++++++++++-- .../[appId]/overview/tracing/toggle-fold-btn.tsx | 10 +++++++--- .../[appId]/overview/tracing/tracing-icon.tsx | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 05e921e0a93d44..e790f1a61da1be 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -10,6 +10,7 @@ import Button from '@/app/components/base/button' import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' const I18N_PREFIX = 'app.tracing' + const ConfigBtn = ({ className, }: { @@ -32,7 +33,7 @@ const Panel: FC = () => { const inUseTracingTool: TracingTool | undefined = undefined const hasConfiguredTracing = !!inUseTracingTool - const [isFold, setFold] = useState(false) + const [isFold, setFold] = useState(true) if (!isFold) { return ( @@ -60,7 +61,14 @@ const Panel: FC = () => { } return ( -
+
+ +
{t(`${I18N_PREFIX}.title`)}
+
+ +
+
+
) } diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx index b0d9a0258435bc..7a0fc827f14704 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx @@ -1,7 +1,7 @@ 'use client' import { ChevronDoubleDownIcon } from '@heroicons/react/20/solid' import type { FC } from 'react' -import React from 'react' +import React, { useCallback } from 'react' type Props = { isFold: boolean @@ -12,15 +12,19 @@ const ToggleFoldBtn: FC = ({ isFold, onFoldChange, }) => { + const handleFoldChange = useCallback((e: React.MouseEvent) => { + e.stopPropagation() + onFoldChange(!isFold) + }, [isFold, onFoldChange]) return ( -
+
{isFold && (
)} {!isFold && ( -
+
)} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx index 21553b4e88bdc4..6eb324d923a89f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/tracing-icon.tsx @@ -11,7 +11,7 @@ type Props = { const sizeClassMap = { lg: 'w-9 h-9 p-2 rounded-[10px]', - md: 'w-4 h-4 p-1 rounded-lg', + md: 'w-6 h-6 p-1 rounded-lg', } const TracingIcon: FC = ({ From 626b6d08a8aed824d5414a9889bb39c01d0d0548 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 13 Jun 2024 12:50:16 +0800 Subject: [PATCH 164/273] feat: split config button --- .../overview/tracing/config-button.tsx | 64 +++++++++++++++++++ .../[appId]/overview/tracing/panel.tsx | 40 ++++++------ 2 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx new file mode 100644 index 00000000000000..457441569a1371 --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -0,0 +1,64 @@ +'use client' +import type { FC } from 'react' +import React, { useCallback, useState } from 'react' +import { useTranslation } from 'react-i18next' +import cn from 'classnames' +import Button from '@/app/components/base/button' +import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' +import { + PortalToFollowElem, + PortalToFollowElemContent, + PortalToFollowElemTrigger, +} from '@/app/components/base/portal-to-follow-elem' + +const I18N_PREFIX = 'app.tracing' + +type Props = { + className?: string + + hasConfigured: boolean + onConfigured?: () => void +} + +const ConfigBtn: FC = ({ + className, + hasConfigured, + onConfigured, +}) => { + const { t } = useTranslation() + const [open, setOpen] = useState(false) + const handleTrigger = useCallback(() => { + setOpen(v => !v) + }, [setOpen]) + + const triggerContent = hasConfigured + ? ( +
+ +
+ ) + : ( + + ) + + return ( + + + {triggerContent} + + + aaa + + + ) +} +export default React.memo(ConfigBtn) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index e790f1a61da1be..02e34fbe9034ed 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -2,31 +2,27 @@ import type { FC } from 'react' import React, { useState } from 'react' import { useTranslation } from 'react-i18next' -import cn from 'classnames' import type { TracingTool } from './type' import TracingIcon from './tracing-icon' import ToggleExpandBtn from './toggle-fold-btn' -import Button from '@/app/components/base/button' +import ConfigButton from './config-button' import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' -import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' const I18N_PREFIX = 'app.tracing' +// const ConfigBtn = ({ +// className, +// }: { +// className?: string +// }) => { -const ConfigBtn = ({ - className, -}: { - className?: string -}) => { - const { t } = useTranslation() - - return ( - - ) -} +// return ( +// +// ) +// } const Panel: FC = () => { const { t } = useTranslation() @@ -35,7 +31,7 @@ const Panel: FC = () => { const hasConfiguredTracing = !!inUseTracingTool const [isFold, setFold] = useState(true) - if (!isFold) { + if (!isFold && !hasConfiguredTracing) { return (
@@ -53,7 +49,7 @@ const Panel: FC = () => {
- +
@@ -65,7 +61,7 @@ const Panel: FC = () => {
{t(`${I18N_PREFIX}.title`)}
- +
From b5b28686e60cb73ae9a2d7d5ffd909f097683968 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 13 Jun 2024 12:53:00 +0800 Subject: [PATCH 165/273] chore: remove log --- web/app/components/datasets/create/website/firecrawl/index.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index 601bf986fff8c0..ba914669ce93d1 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -152,7 +152,6 @@ const FireCrawl: FC = ({ url, options: crawlOptions, }) as any - console.log(res) const jobId = res.job_id onJobIdChange(jobId) const { isError, data } = await waitForCrawlFinished(jobId) From 049dae04ed3b4312f6cb8c281b9372b6365a964b Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 13 Jun 2024 13:09:21 +0800 Subject: [PATCH 166/273] feat: popup struct --- .../overview/tracing/config-button.tsx | 10 +++++----- .../[appId]/overview/tracing/detail-popup.tsx | 0 .../[appId]/overview/tracing/panel.tsx | 19 +------------------ 3 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/detail-popup.tsx diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index 457441569a1371..86e1dca1bbada8 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -3,6 +3,7 @@ import type { FC } from 'react' import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' +import ConfigPopup from './config-popup' import Button from '@/app/components/base/button' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' import { @@ -10,7 +11,6 @@ import { PortalToFollowElemContent, PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' - const I18N_PREFIX = 'app.tracing' type Props = { @@ -26,20 +26,20 @@ const ConfigBtn: FC = ({ onConfigured, }) => { const { t } = useTranslation() - const [open, setOpen] = useState(false) + const [open, setOpen] = useState(true) const handleTrigger = useCallback(() => { setOpen(v => !v) }, [setOpen]) const triggerContent = hasConfigured ? ( -
+
) : ( -// ) -// } const Panel: FC = () => { const { t } = useTranslation() @@ -60,9 +45,7 @@ const Panel: FC = () => {
{t(`${I18N_PREFIX}.title`)}
-
- -
+
From bc3d07abe09ff89038d90fdfacd27667f87ac3fb Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Thu, 13 Jun 2024 15:27:37 +0800 Subject: [PATCH 167/273] note node --- .../workflow/hooks/use-nodes-interactions.ts | 4 ++ .../nodes/_base/components/node-resizer.tsx | 15 ++++-- .../workflow/note-node/constants.ts | 2 +- .../components/workflow/note-node/index.tsx | 50 ++++++++++++++++--- .../workflow/note-node/note-editor/editor.tsx | 6 +-- .../plugins/link-editor-plugin/component.tsx | 27 +++++----- .../note-node/note-editor/toolbar/command.tsx | 42 +++++++++++----- .../toolbar/font-size-selector.tsx | 40 ++++++++------- .../note-node/note-editor/toolbar/index.tsx | 12 ++++- .../note-editor/toolbar/operator.tsx | 50 +++++++++++++++---- .../components/workflow/operator/control.tsx | 2 +- web/app/components/workflow/operator/hooks.ts | 2 + .../components/workflow/panel-contextmenu.tsx | 12 +++++ web/i18n/en-US/workflow.ts | 16 ++++++ web/i18n/zh-Hans/workflow.ts | 16 ++++++ 15 files changed, 226 insertions(+), 70 deletions(-) diff --git a/web/app/components/workflow/hooks/use-nodes-interactions.ts b/web/app/components/workflow/hooks/use-nodes-interactions.ts index 7dabe723f0a124..c63d662e013271 100644 --- a/web/app/components/workflow/hooks/use-nodes-interactions.ts +++ b/web/app/components/workflow/hooks/use-nodes-interactions.ts @@ -988,6 +988,9 @@ export const useNodesInteractions = () => { }, [store]) const handleNodeContextMenu = useCallback((e: MouseEvent, node: Node) => { + if (node.type === CUSTOM_NOTE_NODE) + return + e.preventDefault() const container = document.querySelector('#workflow-container') const { x, y } = container!.getBoundingClientRect() @@ -1064,6 +1067,7 @@ export const useNodesInteractions = () => { const nodeType = nodeToPaste.data.type const newNode = generateNewNode({ + type: nodeToPaste.type, data: { ...NODES_INITIAL_DATA[nodeType], ...nodeToPaste.data, diff --git a/web/app/components/workflow/nodes/_base/components/node-resizer.tsx b/web/app/components/workflow/nodes/_base/components/node-resizer.tsx index 377f5f4c2d8d21..9de94845810887 100644 --- a/web/app/components/workflow/nodes/_base/components/node-resizer.tsx +++ b/web/app/components/workflow/nodes/_base/components/node-resizer.tsx @@ -19,10 +19,18 @@ const Icon = () => { type NodeResizerProps = { nodeId: string nodeData: CommonNodeType + icon?: JSX.Element + minWidth?: number + minHeight?: number + maxWidth?: number } const NodeResizer = ({ nodeId, nodeData, + icon = , + minWidth = 272, + minHeight = 176, + maxWidth, }: NodeResizerProps) => { const { handleNodeResize } = useNodesInteractions() @@ -39,10 +47,11 @@ const NodeResizer = ({ position='bottom-right' className='!border-none !bg-transparent' onResize={handleResize} - minWidth={272} - minHeight={176} + minWidth={minWidth} + minHeight={minHeight} + maxWidth={maxWidth} > -
+
{icon}
) diff --git a/web/app/components/workflow/note-node/constants.ts b/web/app/components/workflow/note-node/constants.ts index 30b32e011911ea..efd1e01b3c202a 100644 --- a/web/app/components/workflow/note-node/constants.ts +++ b/web/app/components/workflow/note-node/constants.ts @@ -11,7 +11,7 @@ export const THEME_MAP: Record { + return ( + + + + ) +} + const NoteNode = ({ id, data, }: NodeProps) => { + const { t } = useTranslation() const theme = data.theme const { handleThemeChange, handleEditorChange, } = useNote(id) + const { + handleNodesCopy, + handleNodesDuplicate, + handleNodeDelete, + } = useNodesInteractions() + + const handleDeleteNode = useCallback(() => { + handleNodeDelete(id) + }, [id, handleNodeDelete]) return (
<> -
+ } + minWidth={240} + maxWidth={640} + minHeight={88} + /> +
{ data.selected && (
) } -
+
- +
diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index 6787545bb242ee..973aabd67c2832 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -38,17 +38,17 @@ const Editor = ({ }, [onChange]) return ( -
+
} - placeholder={} + placeholder={} ErrorBoundary={LexicalErrorBoundary} /> diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx index 1ca7160eb29764..8ad017c8427d0e 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx @@ -75,7 +75,7 @@ const LinkEditorComponent = ({ className='mr-0.5 p-1 w-[196px] h-6 rounded-sm text-[13px] appearance-none outline-none' value={url} onChange={e => setUrl(e.target.value)} - placeholder='Enter URL...' + placeholder={t('workflow.nodes.note.editor.enterUrl') || ''} />
) } diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx index 593af4844b60d5..b805ade2db7ba2 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx @@ -2,7 +2,9 @@ import { memo, useState, } from 'react' +import { useTranslation } from 'react-i18next' import cn from 'classnames' +import ShortcutsName from '@/app/components/workflow/shortcuts-name' import { PortalToFollowElem, PortalToFollowElemContent, @@ -11,7 +13,17 @@ import { import { DotsHorizontal } from '@/app/components/base/icons/src/vender/line/general' import Switch from '@/app/components/base/switch' -const Operator = () => { +export type OperatorProps = { + onCopy: () => void + onDuplicate: () => void + onDelete: () => void +} +const Operator = ({ + onCopy, + onDelete, + onDuplicate, +}: OperatorProps) => { + const { t } = useTranslation() const [open, setOpen] = useState(false) return ( @@ -34,13 +46,25 @@ const Operator = () => {
-
-
Copy
-
C
+
{ + onCopy() + setOpen(false) + }} + > + {t('workflow.common.copy')} +
-
-
Duplicate
-
D
+
{ + onDuplicate() + setOpen(false) + }} + > + {t('workflow.common.duplicate')} +
@@ -52,9 +76,15 @@ const Operator = () => {
-
-
Delete
-
Backspace
+
{ + onDelete() + setOpen(false) + }} + > + {t('common.operation.delete')} +
diff --git a/web/app/components/workflow/operator/control.tsx b/web/app/components/workflow/operator/control.tsx index e512feef32883d..02d9b7cc6b7d99 100644 --- a/web/app/components/workflow/operator/control.tsx +++ b/web/app/components/workflow/operator/control.tsx @@ -93,7 +93,7 @@ const Control = () => { return (
- +
{ type: '' as any, text: '', theme: NoteTheme.blue, + width: 240, + height: 88, _isCandidate: true, } as NoteNodeType, position: { diff --git a/web/app/components/workflow/panel-contextmenu.tsx b/web/app/components/workflow/panel-contextmenu.tsx index eeae51c8d1aff4..a5e63fda4ec601 100644 --- a/web/app/components/workflow/panel-contextmenu.tsx +++ b/web/app/components/workflow/panel-contextmenu.tsx @@ -13,6 +13,7 @@ import { useWorkflowStartRun, } from './hooks' import AddBlock from './operator/add-block' +import { useOperator } from './operator/hooks' import { exportAppConfig } from '@/service/apps' import { useToastContext } from '@/app/components/base/toast' import { useStore as useAppStore } from '@/app/components/app/store' @@ -27,6 +28,7 @@ const PanelContextmenu = () => { const { handleNodesPaste } = useNodesInteractions() const { handlePaneContextmenuCancel } = usePanelInteractions() const { handleStartWorkflowRun } = useWorkflowStartRun() + const { handleAddNote } = useOperator() useClickAway(() => { handlePaneContextmenuCancel() @@ -78,6 +80,16 @@ const PanelContextmenu = () => { crossAxis: -4, }} /> +
{ + e.stopPropagation() + handleAddNote() + handlePaneContextmenuCancel() + }} + > + {t('workflow.nodes.note.addNote')} +
{ diff --git a/web/i18n/en-US/workflow.ts b/web/i18n/en-US/workflow.ts index 9ac975f8a22c2d..559429f0a86715 100644 --- a/web/i18n/en-US/workflow.ts +++ b/web/i18n/en-US/workflow.ts @@ -412,6 +412,22 @@ const translation = { iteration_other: '{{count}} Iterations', currentIteration: 'Current Iteration', }, + note: { + addNote: 'Add Note', + editor: { + placeholder: 'Write your note...', + small: 'Small', + medium: 'Medium', + large: 'Large', + bold: 'Bold', + strikethrough: 'Strikethrough', + link: 'Link', + openLink: 'Open', + unlink: 'Unlink', + enterUrl: 'Enter URL...', + bulletList: 'Bullet List', + }, + }, }, tracing: { stopBy: 'Stop by {{user}}', diff --git a/web/i18n/zh-Hans/workflow.ts b/web/i18n/zh-Hans/workflow.ts index 83b933bc89fcad..ceb665e877ebc7 100644 --- a/web/i18n/zh-Hans/workflow.ts +++ b/web/i18n/zh-Hans/workflow.ts @@ -412,6 +412,22 @@ const translation = { iteration_other: '{{count}}个迭代', currentIteration: '当前迭代', }, + note: { + addNote: '添加注释', + editor: { + placeholder: '输入注释...', + small: '小', + medium: '中', + large: '大', + bold: '加粗', + strikethrough: '删除线', + link: '链接', + openLink: '打开', + unlink: '取消链接', + enterUrl: '输入链接...', + bulletList: '列表', + }, + }, }, tracing: { stopBy: '由{{user}}终止', From 81ac5a7acf962bccc07d913613ee36a6c820b133 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 13 Jun 2024 17:52:17 +0800 Subject: [PATCH 168/273] feat: add error message --- .../create/website/firecrawl/index.tsx | 24 +++++++++++-------- web/i18n/en-US/dataset-creation.ts | 3 ++- web/i18n/zh-Hans/dataset-creation.ts | 3 ++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx index ba914669ce93d1..436af3f7cfd286 100644 --- a/web/app/components/datasets/create/website/firecrawl/index.tsx +++ b/web/app/components/datasets/create/website/firecrawl/index.tsx @@ -94,8 +94,8 @@ const FireCrawl: FC = ({ data: CrawlResultItem[] time_consuming: number | string } | undefined>(undefined) - const [crawlHasError, setCrawlHasError] = useState(false) - const showError = isCrawlFinished && crawlHasError + const [crawlErrorMessage, setCrawlErrorMessage] = useState('') + const showError = isCrawlFinished && crawlErrorMessage const waitForCrawlFinished = useCallback(async (jobId: string) => { try { @@ -113,6 +113,7 @@ const FireCrawl: FC = ({ // can't get the error message from the firecrawl api return { isError: true, + errorMessage: res.message, data: { data: [], }, @@ -126,9 +127,11 @@ const FireCrawl: FC = ({ await sleep(2500) return await waitForCrawlFinished(jobId) } - catch (e) { + catch (e: any) { + const errorBody = await e.json() return { isError: true, + errorMessage: errorBody.message, data: { data: [], }, @@ -145,7 +148,6 @@ const FireCrawl: FC = ({ }) return } - setCrawlHasError(false) setStep(Step.running) try { const res = await createFirecrawlTask({ @@ -154,22 +156,23 @@ const FireCrawl: FC = ({ }) as any const jobId = res.job_id onJobIdChange(jobId) - const { isError, data } = await waitForCrawlFinished(jobId) + const { isError, data, errorMessage } = await waitForCrawlFinished(jobId) if (isError) { - setCrawlHasError(true) + setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`)) } else { setCrawlResult(data) - setCrawlHasError(false) + setCrawlErrorMessage('') } } catch (e) { - setCrawlHasError(true) + setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`)!) + console.log(e) } finally { setStep(Step.finished) } - }, [checkValid, crawlOptions, onJobIdChange, waitForCrawlFinished]) + }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished]) return (
@@ -182,6 +185,7 @@ const FireCrawl: FC = ({ > + {!isInit && (
{isRunning @@ -191,7 +195,7 @@ const FireCrawl: FC = ({ totalNum={crawlResult?.total || parseFloat(crawlOptions.limit as string) || 0} />} {showError && ( - + )} {isCrawlFinished && !showError && Date: Thu, 13 Jun 2024 18:01:14 +0800 Subject: [PATCH 169/273] fix: step two detect dataset type error --- web/app/components/datasets/create/step-two/index.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index b62ed879ad522c..ad7bab9780e014 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -83,7 +83,7 @@ const StepTwo = ({ onSetting, datasetId, indexingType, - dataSourceType, + dataSourceType: inCreatePageDataSourceType, files, notionPages = [], websitePages = [], @@ -101,6 +101,8 @@ const StepTwo = ({ const isMobile = media === MediaType.mobile const { dataset: currentDataset, mutateDatasetRes } = useDatasetDetailContext() + const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type) + const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type const scrollRef = useRef(null) const [scrolled, setScrolled] = useState(false) const previewScrollRef = useRef(null) From f85d87e74aa4715042938e2ee1da7cfe08ad3652 Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 13 Jun 2024 18:02:46 +0800 Subject: [PATCH 170/273] missing file --- .../[appId]/overview/tracing/config-popup.tsx | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx new file mode 100644 index 00000000000000..38dd8167e087ba --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -0,0 +1,25 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import TracingIcon from './tracing-icon' +import Indicator from '@/app/components/header/indicator' +type Props = { + +} + +const ConfigPopup: FC = () => { + return ( +
+
+
+ +
Tracing
+
+
+ +
+
+
+ ) +} +export default React.memo(ConfigPopup) From b9635c93eea47f53cf5dcd2bd14598f2eab72436 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Thu, 13 Jun 2024 18:09:04 +0800 Subject: [PATCH 171/273] note node --- .../components/workflow/note-node/hooks.ts | 5 +++++ .../components/workflow/note-node/index.tsx | 21 +++++++++++++++++-- .../workflow/note-node/note-editor/editor.tsx | 11 +++------- .../plugins/link-editor-plugin/component.tsx | 5 +---- .../plugins/link-editor-plugin/index.tsx | 5 +++++ .../note-node/note-editor/toolbar/index.tsx | 4 ++++ .../note-editor/toolbar/operator.tsx | 12 +++++++++-- .../components/workflow/note-node/types.ts | 2 ++ web/app/components/workflow/operator/hooks.ts | 6 +++++- web/i18n/en-US/workflow.ts | 1 + web/i18n/zh-Hans/workflow.ts | 1 + 11 files changed, 56 insertions(+), 17 deletions(-) diff --git a/web/app/components/workflow/note-node/hooks.ts b/web/app/components/workflow/note-node/hooks.ts index 0e998748885b02..dcf69881302532 100644 --- a/web/app/components/workflow/note-node/hooks.ts +++ b/web/app/components/workflow/note-node/hooks.ts @@ -14,8 +14,13 @@ export const useNote = (id: string) => { handleNodeDataUpdateWithSyncDraft({ id, data: { text: JSON.stringify(editorState.toJSON()) } }) }, [handleNodeDataUpdateWithSyncDraft, id]) + const handleShowAuthorChange = useCallback((showAuthor: boolean) => { + handleNodeDataUpdateWithSyncDraft({ id, data: { showAuthor } }) + }, [handleNodeDataUpdateWithSyncDraft, id]) + return { handleThemeChange, handleEditorChange, + handleShowAuthorChange, } } diff --git a/web/app/components/workflow/note-node/index.tsx b/web/app/components/workflow/note-node/index.tsx index 6569acf62cdeb3..7873ee69bb6d52 100644 --- a/web/app/components/workflow/note-node/index.tsx +++ b/web/app/components/workflow/note-node/index.tsx @@ -1,4 +1,8 @@ -import { memo, useCallback } from 'react' +import { + memo, + useCallback, + useRef, +} from 'react' import cn from 'classnames' import { useTranslation } from 'react-i18next' import type { NodeProps } from 'reactflow' @@ -26,10 +30,12 @@ const NoteNode = ({ data, }: NodeProps) => { const { t } = useTranslation() + const ref = useRef(null) const theme = data.theme const { handleThemeChange, handleEditorChange, + handleShowAuthorChange, } = useNote(id) const { handleNodesCopy, @@ -52,6 +58,7 @@ const NoteNode = ({ width: data.width, height: data.height, }} + ref={ref} > <> @@ -73,20 +80,30 @@ const NoteNode = ({ onCopy={handleNodesCopy} onDuplicate={handleNodesDuplicate} onDelete={handleDeleteNode} + showAuthor={data.showAuthor} + onShowAuthorChange={handleShowAuthorChange} />
) }
+ { + data.showAuthor && ( +
+ {data.author} +
+ ) + }
diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index 973aabd67c2832..2b8446afd734a9 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -3,7 +3,6 @@ import { memo, useCallback, - useState, } from 'react' import type { EditorState } from 'lexical' import { RichTextPlugin } from '@lexical/react/LexicalRichTextPlugin' @@ -22,17 +21,13 @@ import Placeholder from '@/app/components/base/prompt-editor/plugins/placeholder type EditorProps = { placeholder?: string onChange?: (editorState: EditorState) => void + containerElement: HTMLDivElement | null } const Editor = ({ placeholder = 'write you note...', onChange, + containerElement, }: EditorProps) => { - const [containerElement, setContainerElement] = useState(null) - const onRef = (_containerElement: HTMLDivElement) => { - if (_containerElement !== null) - setContainerElement(_containerElement) - } - const handleEditorChange = useCallback((editorState: EditorState) => { onChange?.(editorState) }, [onChange]) @@ -41,7 +36,7 @@ const Editor = ({
+
{ @@ -61,7 +58,7 @@ const LinkEditorComponent = ({
{ useOpenLink() + const linkAnchorElement = useStore(s => s.linkAnchorElement) + + if (!linkAnchorElement) + return null return ( diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx index 1558444efa4a5e..3d37834b9373fc 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/index.tsx @@ -14,6 +14,8 @@ const Toolbar = ({ onCopy, onDuplicate, onDelete, + showAuthor, + onShowAuthorChange, }: ToolbarProps) => { return (
@@ -35,6 +37,8 @@ const Toolbar = ({ onCopy={onCopy} onDuplicate={onDuplicate} onDelete={onDelete} + showAuthor={showAuthor} + onShowAuthorChange={onShowAuthorChange} />
) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx index b805ade2db7ba2..9db850f607a383 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/operator.tsx @@ -17,11 +17,15 @@ export type OperatorProps = { onCopy: () => void onDuplicate: () => void onDelete: () => void + showAuthor: boolean + onShowAuthorChange: (showAuthor: boolean) => void } const Operator = ({ onCopy, onDelete, onDuplicate, + showAuthor, + onShowAuthorChange, }: OperatorProps) => { const { t } = useTranslation() const [open, setOpen] = useState(false) @@ -70,8 +74,12 @@ const Operator = ({
-
Show Author
- +
{t('workflow.nodes.note.editor.showAuthor')}
+
diff --git a/web/app/components/workflow/note-node/types.ts b/web/app/components/workflow/note-node/types.ts index 4ea51572c13930..ad68bd0f10867b 100644 --- a/web/app/components/workflow/note-node/types.ts +++ b/web/app/components/workflow/note-node/types.ts @@ -12,4 +12,6 @@ export enum NoteTheme { export type NoteNodeType = CommonNodeType & { text: string theme: NoteTheme + author: string + showAuthor: boolean } diff --git a/web/app/components/workflow/operator/hooks.ts b/web/app/components/workflow/operator/hooks.ts index 46a6dfb3596a3a..5b142114977f38 100644 --- a/web/app/components/workflow/operator/hooks.ts +++ b/web/app/components/workflow/operator/hooks.ts @@ -4,9 +4,11 @@ import { useWorkflowStore } from '../store' import type { NoteNodeType } from '../note-node/types' import { CUSTOM_NOTE_NODE } from '../note-node/constants' import { NoteTheme } from '../note-node/types' +import { useAppContext } from '@/context/app-context' export const useOperator = () => { const workflowStore = useWorkflowStore() + const { userProfile } = useAppContext() const handleAddNote = useCallback(() => { const newNode = generateNewNode({ @@ -17,6 +19,8 @@ export const useOperator = () => { type: '' as any, text: '', theme: NoteTheme.blue, + author: userProfile?.name || '', + showAuthor: true, width: 240, height: 88, _isCandidate: true, @@ -29,7 +33,7 @@ export const useOperator = () => { workflowStore.setState({ candidateNode: newNode, }) - }, [workflowStore]) + }, [workflowStore, userProfile]) return { handleAddNote, diff --git a/web/i18n/en-US/workflow.ts b/web/i18n/en-US/workflow.ts index 559429f0a86715..bd257cbdfece0b 100644 --- a/web/i18n/en-US/workflow.ts +++ b/web/i18n/en-US/workflow.ts @@ -426,6 +426,7 @@ const translation = { unlink: 'Unlink', enterUrl: 'Enter URL...', bulletList: 'Bullet List', + showAuthor: 'Show Author', }, }, }, diff --git a/web/i18n/zh-Hans/workflow.ts b/web/i18n/zh-Hans/workflow.ts index ceb665e877ebc7..45816d974edb83 100644 --- a/web/i18n/zh-Hans/workflow.ts +++ b/web/i18n/zh-Hans/workflow.ts @@ -426,6 +426,7 @@ const translation = { unlink: '取消链接', enterUrl: '输入链接...', bulletList: '列表', + showAuthor: '显示作者', }, }, }, From 5be69669ce4ada81140081dc3d38451d716816b0 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Thu, 13 Jun 2024 19:00:38 +0800 Subject: [PATCH 172/273] fix: note editor --- web/app/components/workflow/note-node/hooks.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/app/components/workflow/note-node/hooks.ts b/web/app/components/workflow/note-node/hooks.ts index dcf69881302532..f703e21848bd4c 100644 --- a/web/app/components/workflow/note-node/hooks.ts +++ b/web/app/components/workflow/note-node/hooks.ts @@ -11,7 +11,10 @@ export const useNote = (id: string) => { }, [handleNodeDataUpdateWithSyncDraft, id]) const handleEditorChange = useCallback((editorState: EditorState) => { - handleNodeDataUpdateWithSyncDraft({ id, data: { text: JSON.stringify(editorState.toJSON()) } }) + if (!editorState?.isEmpty()) + handleNodeDataUpdateWithSyncDraft({ id, data: { text: JSON.stringify(editorState.toJSON()) } }) + else + handleNodeDataUpdateWithSyncDraft({ id, data: { text: '' } }) }, [handleNodeDataUpdateWithSyncDraft, id]) const handleShowAuthorChange = useCallback((showAuthor: boolean) => { From 0515542a7eced5924d2c235dcc693b829407d17f Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 03:19:44 +0800 Subject: [PATCH 173/273] add aws s3 iam check --- api/config.py | 1 + api/extensions/storage/s3_storage.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/api/config.py b/api/config.py index 53cc6ce393b6ab..6bb6ba96a42454 100644 --- a/api/config.py +++ b/api/config.py @@ -226,6 +226,7 @@ def __init__(self): self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH') # S3 Storage settings + self.USE_AWS_MANAGED_IAM = get_env('USE_AWS_MANAGED_IAM') self.S3_ENDPOINT = get_env('S3_ENDPOINT') self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME') self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY') diff --git a/api/extensions/storage/s3_storage.py b/api/extensions/storage/s3_storage.py index 8aae68a740014a..ec47d3a87dbd35 100644 --- a/api/extensions/storage/s3_storage.py +++ b/api/extensions/storage/s3_storage.py @@ -16,14 +16,18 @@ def __init__(self, app: Flask): super().__init__(app) app_config = self.app.config self.bucket_name = app_config.get('S3_BUCKET_NAME') - self.client = boto3.client( - 's3', - aws_secret_access_key=app_config.get('S3_SECRET_KEY'), - aws_access_key_id=app_config.get('S3_ACCESS_KEY'), - endpoint_url=app_config.get('S3_ENDPOINT'), - region_name=app_config.get('S3_REGION'), - config=Config(s3={'addressing_style': app_config.get('S3_ADDRESS_STYLE')}) - ) + if app_config.get('USE_AWS_MANAGED_IAM'): + session = boto3.Session() + self.client = session.client('s3') + else: + self.client = boto3.client( + 's3', + aws_secret_access_key=app_config.get('S3_SECRET_KEY'), + aws_access_key_id=app_config.get('S3_ACCESS_KEY'), + endpoint_url=app_config.get('S3_ENDPOINT'), + region_name=app_config.get('S3_REGION'), + config=Config(s3={'addressing_style': app_config.get('S3_ADDRESS_STYLE')}) + ) def save(self, filename, data): self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data) From f718b17ea99ccb3f4a15b94a2d007c44ad786d2e Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 03:31:36 +0800 Subject: [PATCH 174/273] add aws s3 iam check --- api/.env.example | 1 + api/config.py | 2 +- docker/docker-compose.yaml | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api/.env.example b/api/.env.example index 571f5b168ea5a7..96aab320009b08 100644 --- a/api/.env.example +++ b/api/.env.example @@ -42,6 +42,7 @@ DB_DATABASE=dify # storage type: local, s3, azure-blob STORAGE_TYPE=local STORAGE_LOCAL_PATH=storage +USE_AWS_MANAGED_IAM=false S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com S3_BUCKET_NAME=your-bucket-name S3_ACCESS_KEY=your-access-key diff --git a/api/config.py b/api/config.py index 6bb6ba96a42454..d8ebe71d633dac 100644 --- a/api/config.py +++ b/api/config.py @@ -226,7 +226,7 @@ def __init__(self): self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH') # S3 Storage settings - self.USE_AWS_MANAGED_IAM = get_env('USE_AWS_MANAGED_IAM') + self.USE_AWS_MANAGED_IAM = get_bool_env('USE_AWS_MANAGED_IAM') self.S3_ENDPOINT = get_env('S3_ENDPOINT') self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME') self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY') diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 5c67406bcb5eeb..1e6917614cea50 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -81,6 +81,7 @@ services: # only available when STORAGE_TYPE is `local`. STORAGE_LOCAL_PATH: storage # The S3 storage configurations, only available when STORAGE_TYPE is `s3`. + USE_AWS_MANAGED_IAM: 'false' S3_ENDPOINT: 'https://xxx.r2.cloudflarestorage.com' S3_BUCKET_NAME: 'difyai' S3_ACCESS_KEY: 'ak-difyai' @@ -236,6 +237,7 @@ services: STORAGE_TYPE: local STORAGE_LOCAL_PATH: storage # The S3 storage configurations, only available when STORAGE_TYPE is `s3`. + USE_AWS_MANAGED_IAM: 'false' S3_ENDPOINT: 'https://xxx.r2.cloudflarestorage.com' S3_BUCKET_NAME: 'difyai' S3_ACCESS_KEY: 'ak-difyai' From 6c194a6681e6d3ef15207432a489951d22306c67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:36:47 +0000 Subject: [PATCH 175/273] chore(deps): bump authlib from 1.2.0 to 1.3.1 in /api Bumps [authlib](https://github.com/lepture/authlib) from 1.2.0 to 1.3.1. - [Release notes](https://github.com/lepture/authlib/releases) - [Changelog](https://github.com/lepture/authlib/blob/master/docs/changelog.rst) - [Commits](https://github.com/lepture/authlib/compare/v1.2.0...v1.3.1) --- updated-dependencies: - dependency-name: authlib dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- api/poetry.lock | 12 ++++++------ api/pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index c630bdae59924e..8fabdc7f7d8f64 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -377,17 +377,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "authlib" -version = "1.2.0" +version = "1.3.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "Authlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:4ddf4fd6cfa75c9a460b361d4bd9dac71ffda0be879dbe4292a02e92349ad55a"}, - {file = "Authlib-1.2.0.tar.gz", hash = "sha256:4fa3e80883a5915ef9f5bc28630564bc4ed5b5af39812a3ff130ec76bd631e9d"}, + {file = "Authlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:d35800b973099bbadc49b42b256ecb80041ad56b7fe1216a362c7943c088f377"}, + {file = "authlib-1.3.1.tar.gz", hash = "sha256:7ae843f03c06c5c0debd63c9db91f9fda64fa62a42a77419fa15fbb7e7a58917"}, ] [package.dependencies] -cryptography = ">=3.2" +cryptography = "*" [[package]] name = "azure-core" @@ -8935,4 +8935,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "e2ce8246ee3113b7c7d080ef52e0998ab71ffe76dbe85b324803ed4e369882c4" +content-hash = "a8683613366c50518046395d46bcad094748176d285b409120445c2dbf09967b" diff --git a/api/pyproject.toml b/api/pyproject.toml index 277b1e703c7f3e..9f361ca6e58aac 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -105,7 +105,7 @@ tiktoken = "~0.7.0" psycopg2-binary = "~2.9.6" pycryptodome = "3.19.1" python-dotenv = "1.0.0" -authlib = "1.2.0" +authlib = "1.3.1" boto3 = "1.28.17" cachetools = "~5.3.0" weaviate-client = "~3.21.0" From 0bcb28e6a35c2e4950cc8bd6be31c198efa63a85 Mon Sep 17 00:00:00 2001 From: takatost Date: Fri, 14 Jun 2024 03:41:19 +0800 Subject: [PATCH 176/273] update requirements.txt --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index 8aeb41c868bd6d..c883bd1e54e71f 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -14,7 +14,7 @@ tiktoken~=0.7.0 psycopg2-binary~=2.9.6 pycryptodome==3.19.1 python-dotenv==1.0.0 -Authlib==1.2.0 +Authlib==1.3.1 boto3==1.34.123 cachetools~=5.3.0 weaviate-client~=3.21.0 From ecb9b506e8be70643c90fa0ed7df983148520512 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Fri, 14 Jun 2024 09:56:07 +0800 Subject: [PATCH 177/273] fix --- .../workflow/note-node/note-editor/editor.tsx | 1 + .../plugins/format-detector-plugin/hooks.ts | 9 ++-- .../plugins/link-editor-plugin/component.tsx | 1 + .../plugins/link-editor-plugin/hooks.ts | 44 +++++++++---------- .../workflow/note-node/note-editor/store.ts | 20 ++++++--- .../toolbar/font-size-selector.tsx | 2 +- .../note-node/note-editor/toolbar/hooks.ts | 18 +++++++- 7 files changed, 60 insertions(+), 35 deletions(-) diff --git a/web/app/components/workflow/note-node/note-editor/editor.tsx b/web/app/components/workflow/note-node/note-editor/editor.tsx index 2b8446afd734a9..189cc78c42e54e 100644 --- a/web/app/components/workflow/note-node/note-editor/editor.tsx +++ b/web/app/components/workflow/note-node/note-editor/editor.tsx @@ -38,6 +38,7 @@ const Editor = ({ contentEditable={
diff --git a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts index da202f9841c931..02a9bbf050ed65 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/plugins/format-detector-plugin/hooks.ts @@ -36,10 +36,13 @@ export const useFormatDetector = () => { setSelectedIsBold(selection.hasFormat('bold')) setSelectedIsStrikeThrough(selection.hasFormat('strikethrough')) const parent = node.getParent() - if ($isLinkNode(parent) || $isLinkNode(node)) - setSelectedLinkUrl($isLinkNode(parent) ? parent.getURL() : (node as LinkNode).getURL()) - else + if ($isLinkNode(parent) || $isLinkNode(node)) { + const linkUrl = ($isLinkNode(parent) ? parent : node as LinkNode).getURL() + setSelectedLinkUrl(linkUrl) + } + else { setSelectedLinkUrl('') + } if ($isListItemNode(parent) || $isListItemNode(node)) setSelectedIsBullet(true) diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx index fbeeaf3ce64d73..101844c227ddd3 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx @@ -73,6 +73,7 @@ const LinkEditorComponent = ({ value={url} onChange={e => setUrl(e.target.value)} placeholder={t('workflow.nodes.note.editor.enterUrl') || ''} + // autoFocus />
diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts index 414bd49a900b04..e52bee20be6e50 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts @@ -4,6 +4,7 @@ import { useState, } from 'react' import { + $createParagraphNode, $getSelection, $isRangeSelection, $setSelection, @@ -14,6 +15,7 @@ import { import { $getSelectionStyleValueForProperty, $patchStyleText, + $setBlocksType, } from '@lexical/selection' import { INSERT_UNORDERED_LIST_COMMAND } from '@lexical/list' import { mergeRegister } from '@lexical/utils' @@ -53,8 +55,20 @@ export const useCommand = () => { }) } - if (type === 'bullet') - editor.dispatchCommand(INSERT_UNORDERED_LIST_COMMAND, undefined) + if (type === 'bullet') { + const { selectedIsBullet } = noteEditorStore.getState() + + if (selectedIsBullet) { + editor.update(() => { + const selection = $getSelection() + if ($isRangeSelection(selection)) + $setBlocksType(selection, () => $createParagraphNode()) + }) + } + else { + editor.dispatchCommand(INSERT_UNORDERED_LIST_COMMAND, undefined) + } + } }, [editor, noteEditorStore]) return { From d26f1a8d605615d6f2f265f978605e4c17aacd43 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 10:11:36 +0800 Subject: [PATCH 178/273] chore: change field loc --- .../data-source-website/config-firecrawl-modal.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx index 7fb60c389e4874..21277c8ec1d442 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx @@ -100,13 +100,6 @@ const ConfigFirecrawlModal: FC = ({
- = ({ onChange={handleConfigChange('api_key')} placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!} /> +
From 92f46313f717e9c6c0bf2934093528eb15ad0858 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Fri, 14 Jun 2024 10:52:43 +0800 Subject: [PATCH 179/273] fix --- .../components/workflow/note-node/index.tsx | 4 +- .../plugins/format-detector-plugin/hooks.ts | 3 ++ .../plugins/link-editor-plugin/component.tsx | 36 +++++++------ .../plugins/link-editor-plugin/hooks.ts | 52 +++++++------------ .../workflow/note-node/note-editor/store.ts | 4 ++ .../note-node/note-editor/toolbar/command.tsx | 8 +-- 6 files changed, 54 insertions(+), 53 deletions(-) diff --git a/web/app/components/workflow/note-node/index.tsx b/web/app/components/workflow/note-node/index.tsx index 7873ee69bb6d52..8686c217a87c58 100644 --- a/web/app/components/workflow/note-node/index.tsx +++ b/web/app/components/workflow/note-node/index.tsx @@ -20,7 +20,7 @@ import type { NoteNodeType } from './types' const Icon = () => { return ( - + ) } @@ -73,7 +73,7 @@ const NoteNode = ({
{ data.selected && ( -
+
{ setSelectedIsBold, setSelectedIsStrikeThrough, setSelectedLinkUrl, + setSelectedIsLink, setSelectedIsBullet, } = noteEditorStore.getState() setSelectedIsBold(selection.hasFormat('bold')) @@ -39,9 +40,11 @@ export const useFormatDetector = () => { if ($isLinkNode(parent) || $isLinkNode(node)) { const linkUrl = ($isLinkNode(parent) ? parent : node as LinkNode).getURL() setSelectedLinkUrl(linkUrl) + setSelectedIsLink(true) } else { setSelectedLinkUrl('') + setSelectedIsLink(false) } if ($isListItemNode(parent) || $isListItemNode(node)) diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx index 101844c227ddd3..fdb8cf22c41b12 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx @@ -46,6 +46,10 @@ const LinkEditorComponent = ({ ], }) + useEffect(() => { + setUrl(selectedLinkUrl) + }, [selectedLinkUrl]) + useEffect(() => { if (linkAnchorElement) refs.setReference(linkAnchorElement) @@ -92,34 +96,36 @@ const LinkEditorComponent = ({ { linkOperatorShow && ( <> - + +
setLinkOperatorShow(false)} > - + {t('common.operation.edit')}
- + {t('workflow.nodes.note.editor.unlink')}
diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts index fff53caa9db394..dadc4caade6ed1 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/hooks.ts @@ -3,8 +3,6 @@ import { useEffect, } from 'react' import { - $getSelection, - $isRangeSelection, CLICK_COMMAND, COMMAND_PRIORITY_LOW, } from 'lexical' @@ -12,13 +10,10 @@ import { mergeRegister, } from '@lexical/utils' import { - $isLinkNode, TOGGLE_LINK_COMMAND, } from '@lexical/link' -import type { LinkNode } from '@lexical/link' import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext' import { useNoteEditorStore } from '../../store' -import { getSelectedNode } from '../../utils' export const useOpenLink = () => { const [editor] = useLexicalComposerContext() @@ -29,36 +24,29 @@ export const useOpenLink = () => { editor.registerCommand( CLICK_COMMAND, (payload) => { - editor.getEditorState().read(() => { - const selection = $getSelection() + setTimeout(() => { + const { + selectedLinkUrl, + selectedIsLink, + setLinkAnchorElement, + setLinkOperatorShow, + } = noteEditorStore.getState() - if ($isRangeSelection(selection) && selection.isCollapsed()) { - const node = getSelectedNode(selection) - const parent = node.getParent() - - if ($isLinkNode(parent) || $isLinkNode(node)) { - const linkUrl = ((parent || node) as LinkNode).getURL() - if (payload.metaKey || payload.ctrlKey) { - window.open(linkUrl, '_blank') - return true - } - else { - const { - setLinkAnchorElement, - setLinkOperatorShow, - } = noteEditorStore.getState() - setLinkAnchorElement(true) - setLinkOperatorShow(true) - } + if (selectedIsLink) { + if ((payload.metaKey || payload.ctrlKey) && selectedLinkUrl) { + window.open(selectedLinkUrl, '_blank') + return true } - else { - const { - setLinkAnchorElement, - setLinkOperatorShow, - } = noteEditorStore.getState() - setLinkAnchorElement() + setLinkAnchorElement(true) + + if (selectedLinkUrl) + setLinkOperatorShow(true) + else setLinkOperatorShow(false) - } + } + else { + setLinkAnchorElement() + setLinkOperatorShow(false) } }) return false diff --git a/web/app/components/workflow/note-node/note-editor/store.ts b/web/app/components/workflow/note-node/note-editor/store.ts index 184339f389f146..25b3fc71ff0f92 100644 --- a/web/app/components/workflow/note-node/note-editor/store.ts +++ b/web/app/components/workflow/note-node/note-editor/store.ts @@ -16,6 +16,8 @@ type Shape = { setSelectedIsStrikeThrough: (selectedIsStrikeThrough: boolean) => void selectedLinkUrl: string setSelectedLinkUrl: (selectedLinkUrl: string) => void + selectedIsLink: boolean + setSelectedIsLink: (selectedIsLink: boolean) => void selectedIsBullet: boolean setSelectedIsBullet: (selectedIsBullet: boolean) => void } @@ -52,6 +54,8 @@ export const createNoteEditorStore = () => { setSelectedIsStrikeThrough: selectedIsStrikeThrough => set(() => ({ selectedIsStrikeThrough })), selectedLinkUrl: '', setSelectedLinkUrl: selectedLinkUrl => set(() => ({ selectedLinkUrl })), + selectedIsLink: false, + setSelectedIsLink: selectedIsLink => set(() => ({ selectedIsLink })), selectedIsBullet: false, setSelectedIsBullet: selectedIsBullet => set(() => ({ selectedIsBullet })), })) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx index 5d78067d1c3d68..dfc43f326ebee1 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx +++ b/web/app/components/workflow/note-node/note-editor/toolbar/command.tsx @@ -23,7 +23,7 @@ const Command = ({ const { t } = useTranslation() const selectedIsBold = useStore(s => s.selectedIsBold) const selectedIsStrikeThrough = useStore(s => s.selectedIsStrikeThrough) - const selectedLinkUrl = useStore(s => s.selectedLinkUrl) + const selectedIsLink = useStore(s => s.selectedIsLink) const selectedIsBullet = useStore(s => s.selectedIsBullet) const { handleCommand } = useCommand() @@ -34,11 +34,11 @@ const Command = ({ case 'strikethrough': return case 'link': - return + return case 'bullet': return } - }, [type, selectedIsBold, selectedIsStrikeThrough, selectedLinkUrl, selectedIsBullet]) + }, [type, selectedIsBold, selectedIsStrikeThrough, selectedIsLink, selectedIsBullet]) const tip = useMemo(() => { switch (type) { @@ -60,7 +60,7 @@ const Command = ({ 'flex items-center justify-center w-8 h-8 cursor-pointer rounded-md text-gray-500 hover:text-gray-800 hover:bg-black/5', type === 'bold' && selectedIsBold && 'bg-primary-50', type === 'strikethrough' && selectedIsStrikeThrough && 'bg-primary-50', - type === 'link' && selectedLinkUrl && 'bg-primary-50', + type === 'link' && selectedIsLink && 'bg-primary-50', type === 'bullet' && selectedIsBullet && 'bg-primary-50', )} onClick={() => handleCommand(type)} From 51539ceadb1d6e218fbbb912fc12c0d4ede5e94f Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 11:04:35 +0800 Subject: [PATCH 180/273] feat: tracing state --- .../overview/tracing/config-button.tsx | 11 ++++++-- .../[appId]/overview/tracing/config-popup.tsx | 28 ++++++++++++++----- .../[appId]/overview/tracing/panel.tsx | 15 ++++++++-- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index 86e1dca1bbada8..5107bb0168c955 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -3,6 +3,7 @@ import type { FC } from 'react' import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' +import type { PopupProps } from './config-popup' import ConfigPopup from './config-popup' import Button from '@/app/components/base/button' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' @@ -11,19 +12,20 @@ import { PortalToFollowElemContent, PortalToFollowElemTrigger, } from '@/app/components/base/portal-to-follow-elem' + const I18N_PREFIX = 'app.tracing' type Props = { className?: string - hasConfigured: boolean onConfigured?: () => void -} +} & PopupProps const ConfigBtn: FC = ({ className, hasConfigured, onConfigured, + ...popupProps }) => { const { t } = useTranslation() const [open, setOpen] = useState(true) @@ -51,12 +53,15 @@ const ConfigBtn: FC = ({ open={open} onOpenChange={setOpen} placement='bottom-end' + offset={{ + mainAxis: 12, + }} > {triggerContent} - + ) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 38dd8167e087ba..7ac31160c493eb 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -3,20 +3,34 @@ import type { FC } from 'react' import React from 'react' import TracingIcon from './tracing-icon' import Indicator from '@/app/components/header/indicator' -type Props = { - +import Switch from '@/app/components/base/switch' +export type PopupProps = { + enabled: boolean + onStatusChange?: (enabled: boolean) => void } -const ConfigPopup: FC = () => { +const ConfigPopup: FC = ({ + enabled, + onStatusChange, +}) => { return (
-
-
+
+
Tracing
-
- +
+ +
+ {enabled ? 'enabled' : 'disabled'} +
+
diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 6ac8fe835acc18..2d8054c1129bf0 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -16,6 +16,8 @@ const Panel: FC = () => { const hasConfiguredTracing = !!inUseTracingTool const [isFold, setFold] = useState(true) + const [enabled, setEnabled] = useState(false) + if (!isFold && !hasConfiguredTracing) { return (
@@ -34,7 +36,11 @@ const Panel: FC = () => {
- +
@@ -45,7 +51,12 @@ const Panel: FC = () => {
{t(`${I18N_PREFIX}.title`)}
- +
From 823b077e8a4613eb0b623ce418269a6ec8c338ff Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Fri, 14 Jun 2024 11:17:38 +0800 Subject: [PATCH 181/273] fix --- .../workflow/note-node/note-editor/store.ts | 20 +++++++------------ .../note-node/note-editor/toolbar/hooks.ts | 14 ++++++++----- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/web/app/components/workflow/note-node/note-editor/store.ts b/web/app/components/workflow/note-node/note-editor/store.ts index 25b3fc71ff0f92..d301d4ad19154e 100644 --- a/web/app/components/workflow/note-node/note-editor/store.ts +++ b/web/app/components/workflow/note-node/note-editor/store.ts @@ -27,20 +27,14 @@ export const createNoteEditorStore = () => { linkAnchorElement: null, setLinkAnchorElement: (open) => { if (open) { - const nativeSelection = window.getSelection() + setTimeout(() => { + const nativeSelection = window.getSelection() - if (nativeSelection?.focusNode) { - const parent = nativeSelection.focusNode.parentElement - set(() => ({ linkAnchorElement: parent })) - } - // setTimeout(() => { - // const nativeSelection = window.getSelection() - - // if (nativeSelection?.focusNode) { - // const parent = nativeSelection.focusNode.parentElement - // set(() => ({ linkAnchorElement: parent })) - // } - // }) + if (nativeSelection?.focusNode) { + const parent = nativeSelection.focusNode.parentElement + set(() => ({ linkAnchorElement: parent })) + } + }) } else { set(() => ({ linkAnchorElement: null })) diff --git a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts index e52bee20be6e50..f12e45d353b547 100644 --- a/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts +++ b/web/app/components/workflow/note-node/note-editor/toolbar/hooks.ts @@ -42,15 +42,19 @@ export const useCommand = () => { editor.update(() => { const selection = $getSelection() - if ($isRangeSelection(selection) && !selection.isCollapsed()) { + if ($isRangeSelection(selection)) { const node = getSelectedNode(selection) const parent = node.getParent() + const { setLinkAnchorElement } = noteEditorStore.getState() - if (!($isLinkNode(parent) && $isLinkNode(node))) + if ($isLinkNode(parent) || $isLinkNode(node)) { + editor.dispatchCommand(TOGGLE_LINK_COMMAND, null) + setLinkAnchorElement() + } + else { editor.dispatchCommand(TOGGLE_LINK_COMMAND, '') - - const { setLinkAnchorElement } = noteEditorStore.getState() - setLinkAnchorElement(true) + setLinkAnchorElement(true) + } } }) } From a725daa3de982d5604b6c3911a2fd21e2bec044c Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 11:44:27 +0800 Subject: [PATCH 182/273] feat: tracing des and i18n --- .../[appId]/overview/tracing/config-popup.tsx | 15 +++++++++++++-- web/i18n/en-US/app.ts | 4 ++++ web/i18n/zh-Hans/app.ts | 4 ++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 7ac31160c493eb..87b8fcbf4660cc 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -1,9 +1,12 @@ 'use client' import type { FC } from 'react' import React from 'react' +import { useTranslation } from 'react-i18next' import TracingIcon from './tracing-icon' import Indicator from '@/app/components/header/indicator' import Switch from '@/app/components/base/switch' +const I18N_PREFIX = 'app.tracing' + export type PopupProps = { enabled: boolean onStatusChange?: (enabled: boolean) => void @@ -13,17 +16,19 @@ const ConfigPopup: FC = ({ enabled, onStatusChange, }) => { + const { t } = useTranslation() + return (
-
Tracing
+
{t(`${I18N_PREFIX}.tracing`)}
- {enabled ? 'enabled' : 'disabled'} + {t(`${I18N_PREFIX}.${enabled ? 'enabled' : 'disabled'}`)}
= ({ />
+ +
+ {t(`${I18N_PREFIX}.tracingDescription`)} +
+
+
) } diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index 235b013b89257c..f4be44a5cb4a33 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -89,6 +89,10 @@ const translation = { title: 'Tracing app performance', description: 'Configuring a Third-Party LLMOps provider and tracing app performance.', config: 'Config', + tracing: 'Tracing', + disabled: 'Disabled', + enabled: 'Enabled', + tracingDescription: 'Capture the full context of app execution, including LLM calls, context, prompts, HTTP requests, and more, to a third-party tracing platform.', }, } diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 0364a189df7c46..2ecb3f462cf310 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -88,6 +88,10 @@ const translation = { title: '追踪应用性能', description: '配置第三方 LLMOps 提供商并跟踪应用程序性能。', config: '配置', + tracing: '追踪', + disabled: '已禁用', + enabled: '已启用', + tracingDescription: '捕获应用程序执行的完整上下文，包括 LLM 调用、上下文、提示、HTTP 请求等，发送到第三方跟踪平台。', }, } From 8285fd3a4dc05eab567eca39da5b3d0390d635e9 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 13:02:34 +0800 Subject: [PATCH 183/273] add aws s3 iam check --- api/.env.example | 2 +- api/config.py | 3 ++- api/extensions/storage/s3_storage.py | 2 +- docker/docker-compose.yaml | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/.env.example b/api/.env.example index 96aab320009b08..42ed40e4c0e25a 100644 --- a/api/.env.example +++ b/api/.env.example @@ -42,7 +42,7 @@ DB_DATABASE=dify # storage type: local, s3, azure-blob STORAGE_TYPE=local STORAGE_LOCAL_PATH=storage -USE_AWS_MANAGED_IAM=false +S3_USE_AWS_MANAGED_IAM=false S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com S3_BUCKET_NAME=your-bucket-name S3_ACCESS_KEY=your-access-key diff --git a/api/config.py b/api/config.py index d8ebe71d633dac..0c2c8d7cf679f2 100644 --- a/api/config.py +++ b/api/config.py @@ -24,6 +24,7 @@ 'APP_WEB_URL': 'https://udify.app', 'FILES_URL': '', 'FILES_ACCESS_TIMEOUT': 300, + 'S3_USE_AWS_MANAGED_IAM': 'False', 'S3_ADDRESS_STYLE': 'auto', 'STORAGE_TYPE': 'local', 'STORAGE_LOCAL_PATH': 'storage', @@ -226,7 +227,7 @@ def __init__(self): self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH') # S3 Storage settings - self.USE_AWS_MANAGED_IAM = get_bool_env('USE_AWS_MANAGED_IAM') + self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM') self.S3_ENDPOINT = get_env('S3_ENDPOINT') self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME') self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY') diff --git a/api/extensions/storage/s3_storage.py b/api/extensions/storage/s3_storage.py index ec47d3a87dbd35..787596fa791d4a 100644 --- a/api/extensions/storage/s3_storage.py +++ b/api/extensions/storage/s3_storage.py @@ -16,7 +16,7 @@ def __init__(self, app: Flask): super().__init__(app) app_config = self.app.config self.bucket_name = app_config.get('S3_BUCKET_NAME') - if app_config.get('USE_AWS_MANAGED_IAM'): + if app_config.get('S3_USE_AWS_MANAGED_IAM'): session = boto3.Session() self.client = session.client('s3') else: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 1e6917614cea50..f68e2aec3f153f 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -81,7 +81,7 @@ services: # only available when STORAGE_TYPE is `local`. STORAGE_LOCAL_PATH: storage # The S3 storage configurations, only available when STORAGE_TYPE is `s3`. - USE_AWS_MANAGED_IAM: 'false' + S3_USE_AWS_MANAGED_IAM: 'false' S3_ENDPOINT: 'https://xxx.r2.cloudflarestorage.com' S3_BUCKET_NAME: 'difyai' S3_ACCESS_KEY: 'ak-difyai' @@ -237,7 +237,7 @@ services: STORAGE_TYPE: local STORAGE_LOCAL_PATH: storage # The S3 storage configurations, only available when STORAGE_TYPE is `s3`. - USE_AWS_MANAGED_IAM: 'false' + S3_USE_AWS_MANAGED_IAM: 'false' S3_ENDPOINT: 'https://xxx.r2.cloudflarestorage.com' S3_BUCKET_NAME: 'difyai' S3_ACCESS_KEY: 'ak-difyai' From 2d9742cafbd555990a2a862026c42cc1b885d372 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 14:25:53 +0800 Subject: [PATCH 184/273] feat: tracing panel --- .../[appId]/overview/tracing/config-popup.tsx | 12 +++++- .../[appId]/overview/tracing/panel.tsx | 4 +- .../overview/tracing/provider-panel.tsx | 42 +++++++++++++++++++ .../[appId]/overview/tracing/type.ts | 2 +- web/i18n/en-US/app.ts | 7 ++++ web/i18n/zh-Hans/app.ts | 7 ++++ 6 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 87b8fcbf4660cc..16e6cb7d91975a 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -3,8 +3,11 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' import TracingIcon from './tracing-icon' +import ProviderPanel from './provider-panel' +import { TracingProvider } from './type' import Indicator from '@/app/components/header/indicator' import Switch from '@/app/components/base/switch' + const I18N_PREFIX = 'app.tracing' export type PopupProps = { @@ -42,7 +45,14 @@ const ConfigPopup: FC = ({
{t(`${I18N_PREFIX}.tracingDescription`)}
-
+
+
+
{t(`${I18N_PREFIX}.configProviderTitle`)}
+
+ + +
+
) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 2d8054c1129bf0..e85d8db0064a2b 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -2,7 +2,7 @@ import type { FC } from 'react' import React, { useState } from 'react' import { useTranslation } from 'react-i18next' -import type { TracingTool } from './type' +import type { TracingProvider } from './type' import TracingIcon from './tracing-icon' import ToggleExpandBtn from './toggle-fold-btn' import ConfigButton from './config-button' @@ -12,7 +12,7 @@ const I18N_PREFIX = 'app.tracing' const Panel: FC = () => { const { t } = useTranslation() - const inUseTracingTool: TracingTool | undefined = undefined + const inUseTracingTool: TracingProvider | undefined = undefined const hasConfiguredTracing = !!inUseTracingTool const [isFold, setFold] = useState(true) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx new file mode 100644 index 00000000000000..75f98001506050 --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx @@ -0,0 +1,42 @@ +'use client' +import type { FC } from 'react' +import React from 'react' +import { useTranslation } from 'react-i18next' +import { TracingProvider } from './type' +import { LangfuseIconBig, LangsmithIconBig } from '@/app/components/base/icons/src/public/tracing' +import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' +const I18N_PREFIX = 'app.tracing' + +type Props = { + type: TracingProvider +} + +const getIcon = (type: TracingProvider) => { + return ({ + [TracingProvider.langSmith]: LangsmithIconBig, + [TracingProvider.langfuse]: LangfuseIconBig, + })[type] +} + +const ProviderPanel: FC = ({ + type, +}) => { + const { t } = useTranslation() + + const Icon = getIcon(type) + return ( +
+
+ +
+ +
{t(`${I18N_PREFIX}.config`)}
+
+
+
+ {t(`${I18N_PREFIX}.${type}.description`)} +
+
+ ) +} +export default React.memo(ProviderPanel) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts index 1bb571a9997b51..432a0cb3e2fc9f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts @@ -1,4 +1,4 @@ -export enum TracingTool { +export enum TracingProvider { langSmith = 'langSmith', langfuse = 'langfuse', } diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index f4be44a5cb4a33..5f4d0b652f088b 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -93,6 +93,13 @@ const translation = { disabled: 'Disabled', enabled: 'Enabled', tracingDescription: 'Capture the full context of app execution, including LLM calls, context, prompts, HTTP requests, and more, to a third-party tracing platform.', + configProviderTitle: 'Config provider to enable tracing', + langSmith: { + description: 'An all-in-one developer platform for every step of the LLM-powered application lifecycle.', + }, + langfuse: { + description: 'Traces, evals, prompt management and metrics to debug and improve your LLM application.', + }, }, } diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 2ecb3f462cf310..ddc0d50565cf44 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -92,6 +92,13 @@ const translation = { disabled: '已禁用', enabled: '已启用', tracingDescription: '捕获应用程序执行的完整上下文，包括 LLM 调用、上下文、提示、HTTP 请求等，发送到第三方跟踪平台。', + configProviderTitle: '配置提供商以启用追踪', + langSmith: { + description: '一个全方位的开发者平台，适用于 LLM 驱动应用程序生命周期的每个步骤。', + }, + langfuse: { + description: '跟踪、评估、提示管理和指标，以调试和改进您的 LLM 应用程序。', + }, }, } From ea0930c37dd8ce9ca97f189090a374a79e9e4875 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 14:35:00 +0800 Subject: [PATCH 185/273] feat: fold btn tooltip --- .../overview/tracing/config-button.tsx | 2 +- .../overview/tracing/toggle-fold-btn.tsx | 34 +++++++++++++------ web/i18n/en-US/app.ts | 2 ++ web/i18n/zh-Hans/app.ts | 2 ++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index 5107bb0168c955..99634163314cfc 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -28,7 +28,7 @@ const ConfigBtn: FC = ({ ...popupProps }) => { const { t } = useTranslation() - const [open, setOpen] = useState(true) + const [open, setOpen] = useState(false) const handleTrigger = useCallback(() => { setOpen(v => !v) }, [setOpen]) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx index 7a0fc827f14704..9119deede879b2 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/toggle-fold-btn.tsx @@ -1,7 +1,11 @@ 'use client' import { ChevronDoubleDownIcon } from '@heroicons/react/20/solid' import type { FC } from 'react' +import { useTranslation } from 'react-i18next' import React, { useCallback } from 'react' +import TooltipPlus from '@/app/components/base/tooltip-plus' + +const I18N_PREFIX = 'app.tracing' type Props = { isFold: boolean @@ -12,22 +16,30 @@ const ToggleFoldBtn: FC = ({ isFold, onFoldChange, }) => { + const { t } = useTranslation() + const handleFoldChange = useCallback((e: React.MouseEvent) => { e.stopPropagation() onFoldChange(!isFold) }, [isFold, onFoldChange]) return ( -
- {isFold && ( -
- -
- )} - {!isFold && ( -
- -
- )} + // text-[0px] to hide spacing between tooltip elements +
+ + {isFold && ( +
+ +
+ )} + {!isFold && ( +
+ +
+ )} +
) } diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index 5f4d0b652f088b..a0b7253e2e95c8 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -89,6 +89,8 @@ const translation = { title: 'Tracing app performance', description: 'Configuring a Third-Party LLMOps provider and tracing app performance.', config: 'Config', + collapse: 'Collapse', + expand: 'Expand', tracing: 'Tracing', disabled: 'Disabled', enabled: 'Enabled', diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index ddc0d50565cf44..8b8bb1bd2cc25e 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -88,6 +88,8 @@ const translation = { title: '追踪应用性能', description: '配置第三方 LLMOps 提供商并跟踪应用程序性能。', config: '配置', + collapse: '折叠', + expand: '展开', tracing: '追踪', disabled: '已禁用', enabled: '已启用', From 545f142f3bcba8eef06d6b08407344b92122d502 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Fri, 14 Jun 2024 15:06:59 +0800 Subject: [PATCH 186/273] fix --- web/app/components/workflow/candidate-node.tsx | 5 +++++ .../workflow/hooks/use-node-data-update.ts | 3 ++- web/app/components/workflow/note-node/hooks.ts | 2 +- web/app/components/workflow/note-node/index.tsx | 11 ++++++++++- .../workflow/note-node/note-editor/context.tsx | 12 ++++++++++-- .../plugins/link-editor-plugin/component.tsx | 10 ++++++++-- .../note-editor/plugins/link-editor-plugin/hooks.ts | 11 ++++++++++- .../workflow/note-node/note-editor/utils.ts | 2 ++ web/i18n/en-US/workflow.ts | 1 + web/i18n/zh-Hans/workflow.ts | 1 + 10 files changed, 50 insertions(+), 8 deletions(-) diff --git a/web/app/components/workflow/candidate-node.tsx b/web/app/components/workflow/candidate-node.tsx index e7ea24b4fd989a..8528e61daaaab4 100644 --- a/web/app/components/workflow/candidate-node.tsx +++ b/web/app/components/workflow/candidate-node.tsx @@ -12,6 +12,7 @@ import { useStore, useWorkflowStore, } from './store' +import { useNodesInteractions } from './hooks' import { CUSTOM_NODE } from './constants' import CustomNode from './nodes' import CustomNoteNode from './note-node' @@ -24,6 +25,7 @@ const CandidateNode = () => { const candidateNode = useStore(s => s.candidateNode) const mousePosition = useStore(s => s.mousePosition) const { zoom } = useViewport() + const { handleNodeSelect } = useNodesInteractions() useEventListener('click', (e) => { const { candidateNode, mousePosition } = workflowStore.getState() @@ -52,6 +54,9 @@ const CandidateNode = () => { }) setNodes(newNodes) workflowStore.setState({ candidateNode: undefined }) + + if (candidateNode.type === CUSTOM_NOTE_NODE) + handleNodeSelect(candidateNode.id) } }) diff --git a/web/app/components/workflow/hooks/use-node-data-update.ts b/web/app/components/workflow/hooks/use-node-data-update.ts index 4cb4d6b26bcf94..c59c858184120d 100644 --- a/web/app/components/workflow/hooks/use-node-data-update.ts +++ b/web/app/components/workflow/hooks/use-node-data-update.ts @@ -22,7 +22,8 @@ export const useNodeDataUpdate = () => { const newNodes = produce(getNodes(), (draft) => { const currentNode = draft.find(node => node.id === id)! - currentNode.data = { ...currentNode?.data, ...data } + if (currentNode) + currentNode.data = { ...currentNode.data, ...data } }) setNodes(newNodes) }, [store]) diff --git a/web/app/components/workflow/note-node/hooks.ts b/web/app/components/workflow/note-node/hooks.ts index f703e21848bd4c..7606951726cfea 100644 --- a/web/app/components/workflow/note-node/hooks.ts +++ b/web/app/components/workflow/note-node/hooks.ts @@ -12,7 +12,7 @@ export const useNote = (id: string) => { const handleEditorChange = useCallback((editorState: EditorState) => { if (!editorState?.isEmpty()) - handleNodeDataUpdateWithSyncDraft({ id, data: { text: JSON.stringify(editorState.toJSON()) } }) + handleNodeDataUpdateWithSyncDraft({ id, data: { text: JSON.stringify(editorState) } }) else handleNodeDataUpdateWithSyncDraft({ id, data: { text: '' } }) }, [handleNodeDataUpdateWithSyncDraft, id]) diff --git a/web/app/components/workflow/note-node/index.tsx b/web/app/components/workflow/note-node/index.tsx index 8686c217a87c58..747ec2518ddcfb 100644 --- a/web/app/components/workflow/note-node/index.tsx +++ b/web/app/components/workflow/note-node/index.tsx @@ -5,9 +5,13 @@ import { } from 'react' import cn from 'classnames' import { useTranslation } from 'react-i18next' +import { useClickAway } from 'ahooks' import type { NodeProps } from 'reactflow' import NodeResizer from '../nodes/_base/components/node-resizer' -import { useNodesInteractions } from '../hooks' +import { + useNodeDataUpdate, + useNodesInteractions, +} from '../hooks' import { NoteEditor, NoteEditorContextProvider, @@ -42,11 +46,16 @@ const NoteNode = ({ handleNodesDuplicate, handleNodeDelete, } = useNodesInteractions() + const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate() const handleDeleteNode = useCallback(() => { handleNodeDelete(id) }, [id, handleNodeDelete]) + useClickAway(() => { + handleNodeDataUpdateWithSyncDraft({ id, data: { selected: false } }) + }, ref) + return (
const NoteEditorContext = createContext(null) type NoteEditorContextProviderProps = { - value?: string + value: string children: JSX.Element | string | (JSX.Element | string)[] } export const NoteEditorContextProvider = memo(({ @@ -30,6 +30,14 @@ export const NoteEditorContextProvider = memo(({ if (!storeRef.current) storeRef.current = createNoteEditorStore() + let initialValue = null + try { + initialValue = JSON.parse(value) + } + catch (e) { + + } + const initialConfig = { namespace: 'note-editor', nodes: [ @@ -37,7 +45,7 @@ export const NoteEditorContextProvider = memo(({ ListNode, ListItemNode, ], - editorState: value || null, + editorState: !initialValue?.root.children.length ? null : JSON.stringify(initialValue), onError: (error: Error) => { throw error }, diff --git a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx index fdb8cf22c41b12..82398ded6b687d 100644 --- a/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx +++ b/web/app/components/workflow/note-node/note-editor/plugins/link-editor-plugin/component.tsx @@ -11,6 +11,7 @@ import { useFloating, } from '@floating-ui/react' import { useTranslation } from 'react-i18next' +import { useClickAway } from 'ahooks' import cn from 'classnames' import { useStore } from '../../store' import { useLink } from './hooks' @@ -35,6 +36,7 @@ const LinkEditorComponent = ({ const selectedLinkUrl = useStore(s => s.selectedLinkUrl) const linkAnchorElement = useStore(s => s.linkAnchorElement) const linkOperatorShow = useStore(s => s.linkOperatorShow) + const setLinkAnchorElement = useStore(s => s.setLinkAnchorElement) const setLinkOperatorShow = useStore(s => s.setLinkOperatorShow) const [url, setUrl] = useState(selectedLinkUrl) const { refs, floatingStyles, elements } = useFloating({ @@ -46,6 +48,10 @@ const LinkEditorComponent = ({ ], }) + useClickAway(() => { + setLinkAnchorElement() + }, linkAnchorElement) + useEffect(() => { setUrl(selectedLinkUrl) }, [selectedLinkUrl]) @@ -62,7 +68,7 @@ const LinkEditorComponent = ({
setUrl(e.target.value)} placeholder={t('workflow.nodes.note.editor.enterUrl') || ''} - // autoFocus + autoFocus /> + +
+ +
+
+
+
+ + {t('common.modelProvider.encrypted.front')} + + PKCS1_OAEP + + {t('common.modelProvider.encrypted.back')} +
+
+
+
+ + + ) +} +export default React.memo(ProviderConfigModal) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx index 75f98001506050..7c11b2c5f4b0a3 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React from 'react' +import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' import { TracingProvider } from './type' import { LangfuseIconBig, LangsmithIconBig } from '@/app/components/base/icons/src/public/tracing' @@ -9,6 +9,8 @@ const I18N_PREFIX = 'app.tracing' type Props = { type: TracingProvider + onConfig: () => void + onChoose: () => void } const getIcon = (type: TracingProvider) => { @@ -20,15 +22,27 @@ const getIcon = (type: TracingProvider) => { const ProviderPanel: FC = ({ type, + onConfig, + onChoose, }) => { const { t } = useTranslation() - const Icon = getIcon(type) + + const handleConfigBtnClick = useCallback((e: React.MouseEvent) => { + e.stopPropagation() + onConfig() + }, [onConfig]) return ( -
+
-
+
{t(`${I18N_PREFIX}.config`)}
From de7167edffff9cace7112ac23291303fe69b77ec Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Fri, 14 Jun 2024 16:58:37 +0800 Subject: [PATCH 194/273] fix --- web/app/components/workflow/note-node/index.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/web/app/components/workflow/note-node/index.tsx b/web/app/components/workflow/note-node/index.tsx index 67946a5f59ec20..850c6b730a67f6 100644 --- a/web/app/components/workflow/note-node/index.tsx +++ b/web/app/components/workflow/note-node/index.tsx @@ -12,6 +12,7 @@ import { useNodeDataUpdate, useNodesInteractions, } from '../hooks' +import { useStore } from '../store' import { NoteEditor, NoteEditorContextProvider, @@ -34,6 +35,7 @@ const NoteNode = ({ data, }: NodeProps) => { const { t } = useTranslation() + const controlPromptEditorRerenderKey = useStore(s => s.controlPromptEditorRerenderKey) const ref = useRef(null) const theme = data.theme const { @@ -69,7 +71,10 @@ const NoteNode = ({ }} ref={ref} > - + <> Date: Fri, 14 Jun 2024 17:02:42 +0800 Subject: [PATCH 195/273] add crawl to s3 service --- .../rag/extractor/firecrawl/firecrawl_app.py | 10 +++---- api/services/website_service.py | 29 ++++++++++++------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 82064390e056fe..99bd7fb0fd322e 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -77,11 +77,11 @@ def check_crawl_status(self, job_id) -> dict: 'markdown': item.get('markdown') } url_data_list.append(url_data) - # if url_data_list: - # file_key = 'website_files/' + job_id + '.txt' - # if storage.exists(file_key): - # storage.delete(file_key) - # storage.save(file_key, json.dumps(url_data_list).encode('utf-8')) + if url_data_list: + file_key = 'website_files/' + job_id + '.txt' + if storage.exists(file_key): + storage.delete(file_key) + storage.save(file_key, json.dumps(url_data_list).encode('utf-8')) return { 'status': 'completed', 'total': crawl_status_response.get('total'), diff --git a/api/services/website_service.py b/api/services/website_service.py index a1758d5bbd574c..334cf96cc2fc99 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -9,6 +9,7 @@ from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp from extensions.ext_database import db from extensions.ext_redis import redis_client +from extensions.ext_storage import storage from models.source import DataSourceApiKeyAuthBinding from services.auth.api_key_auth_service import ApiKeyAuthService @@ -124,17 +125,23 @@ def get_crawl_url_data(cls, job_id: str, provider: str, url: str, tenant_id: str 'website', provider) if provider == 'firecrawl': - # decrypt api_key - api_key = encrypter.decrypt_token( - tenant_id=tenant_id, - token=credentials.get('config').get('api_key') - ) - firecrawl_app = FirecrawlApp(api_key=api_key, - base_url=credentials.get('config').get('base_url', None)) - result = firecrawl_app.check_crawl_status(job_id) - if result.get('status') != 'completed': - raise ValueError('Crawl job is not completed') - data = result.get('data') + file_key = 'website_files/' + job_id + '.txt' + if storage.exists(file_key): + data = storage.load_once(file_key) + if data: + data = json.loads(data.decode('utf-8')) + else: + # decrypt api_key + api_key = encrypter.decrypt_token( + tenant_id=tenant_id, + token=credentials.get('config').get('api_key') + ) + firecrawl_app = FirecrawlApp(api_key=api_key, + base_url=credentials.get('config').get('base_url', None)) + result = firecrawl_app.check_crawl_status(job_id) + if result.get('status') != 'completed': + raise ValueError('Crawl job is not completed') + data = result.get('data') if data: for item in data: if item.get('source_url') == url: From 231b8f6e4ac236101443f3a9f8c3e26bb1aab6c8 Mon Sep 17 00:00:00 2001 From: StyleZhang Date: Fri, 14 Jun 2024 17:26:46 +0800 Subject: [PATCH 196/273] fix: note editor italic --- .../components/workflow/note-node/note-editor/theme/index.ts | 1 + .../components/workflow/note-node/note-editor/theme/theme.css | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/web/app/components/workflow/note-node/note-editor/theme/index.ts b/web/app/components/workflow/note-node/note-editor/theme/index.ts index 42069d2359bbdd..5cb8dec37f450a 100644 --- a/web/app/components/workflow/note-node/note-editor/theme/index.ts +++ b/web/app/components/workflow/note-node/note-editor/theme/index.ts @@ -10,6 +10,7 @@ const theme: EditorThemeClasses = { }, link: 'note-editor-theme_link', text: { + italic: 'note-editor-theme_text-italic', strikethrough: 'note-editor-theme_text-strikethrough', }, } diff --git a/web/app/components/workflow/note-node/note-editor/theme/theme.css b/web/app/components/workflow/note-node/note-editor/theme/theme.css index 8b04d85bb5a944..8c42757c30ba1b 100644 --- a/web/app/components/workflow/note-node/note-editor/theme/theme.css +++ b/web/app/components/workflow/note-node/note-editor/theme/theme.css @@ -21,4 +21,8 @@ .note-editor-theme_text-strikethrough { text-decoration: line-through; +} + +.note-editor-theme_text-italic { + font-style: italic; } \ No newline at end of file From 8cf55c1583524477e06ab906ea218da2d1a068ad Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 17:51:04 +0800 Subject: [PATCH 197/273] temp --- web/i18n/en-US/app.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index 1b971f17613ccb..c8b7b9acd67e1c 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -102,6 +102,16 @@ const translation = { langfuse: { description: 'Traces, evals, prompt management and metrics to debug and improve your LLM application.', }, + configProvider: { + title: 'Config', + placeholder: 'Enter your {key}', + project: 'Project', + endpoint: 'Endpoint', + publicKey: 'Public Key', + secretKey: 'Secret Key', + viewDocsLink: 'View LangSmith docs', + saveAndEnable: 'Save and Enable', + }, }, } From 240c7326fc478b30e9cca174bd56d43448463df1 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 18:06:50 +0800 Subject: [PATCH 198/273] add crawl to s3 service --- api/services/website_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/services/website_service.py b/api/services/website_service.py index 334cf96cc2fc99..afe3614b1bb631 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -49,7 +49,6 @@ def crawl_url(cls, args: dict) -> dict: "includes": [], "excludes": [], "generateImgAltText": True, - "maxDepth": 1, "limit": 1, 'returnOnlyUrls': False, 'pageOptions': { @@ -66,7 +65,6 @@ def crawl_url(cls, args: dict) -> dict: "includes": includes if includes else [], "excludes": excludes if excludes else [], "generateImgAltText": True, - "maxDepth": options.get('max_depth', 1), "limit": options.get('limit', 1), 'returnOnlyUrls': False, 'pageOptions': { @@ -75,6 +73,8 @@ def crawl_url(cls, args: dict) -> dict: } } } + if options.get('max_depth'): + params['crawlerOptions']['maxDepth'] = options.get('max_depth') job_id = firecrawl_app.crawl_url(url, params) website_crawl_time_cache_key = f'website_crawl_{job_id}' time = str(datetime.datetime.now().timestamp()) From d842ea16b4a4af1849d1b91d96547808e3716ae6 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 18:21:40 +0800 Subject: [PATCH 199/273] add crawl to s3 service --- api/services/website_service.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/services/website_service.py b/api/services/website_service.py index afe3614b1bb631..58ff0b012118a7 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -1,16 +1,12 @@ import datetime import json -from typing import Any from flask_login import current_user -from werkzeug.exceptions import NotFound from core.helper import encrypter from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp -from extensions.ext_database import db from extensions.ext_redis import redis_client from extensions.ext_storage import storage -from models.source import DataSourceApiKeyAuthBinding from services.auth.api_key_auth_service import ApiKeyAuthService From da83e241520892f13efd284e67cea290d2f12a56 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 18:21:58 +0800 Subject: [PATCH 200/273] chore: max depth optional --- web/app/components/datasets/create/index.tsx | 2 +- .../create/website/firecrawl/base/field.tsx | 11 +++++++++++ .../datasets/create/website/firecrawl/index.tsx | 14 +++++++------- .../datasets/create/website/firecrawl/options.tsx | 2 +- web/i18n/en-US/dataset-creation.ts | 1 + web/i18n/zh-Hans/dataset-creation.ts | 1 + 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index ccb8e0668d9c4c..12c6284d882c5b 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -25,7 +25,7 @@ const DEFAULT_CRAWL_OPTIONS: CrawlOptions = { includes: '', excludes: '', limit: 10, - max_depth: 2, + max_depth: '', } const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx index 9cf978d0af191d..6d31b2a4d97d5a 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx @@ -3,6 +3,8 @@ import type { FC } from 'react' import React from 'react' import cn from 'classnames' import Input from './input' +import TooltipPlus from '@/app/components/base/tooltip-plus' +import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general' type Props = { className?: string @@ -13,6 +15,7 @@ type Props = { isRequired?: boolean placeholder?: string isNumber?: boolean + tooltip?: string } const Field: FC = ({ @@ -24,12 +27,20 @@ const Field: FC = ({ isRequired = false, placeholder = '', isNumber = false, + tooltip, }) => { return (
{label}
{isRequired && *} + {tooltip && ( + {tooltip}
+ }> + + + )}
= ({ }) } - if (!errorMsg && (crawlOptions.max_depth === null || crawlOptions.max_depth === undefined || crawlOptions.max_depth === '')) { - errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, { - field: t(`${I18N_PREFIX}.maxDepth`), - }) - } - return { isValid: !errorMsg, errorMsg, @@ -150,9 +144,15 @@ const FireCrawl: FC = ({ } setStep(Step.running) try { + const passToServerCrawlOptions: any = { + ...crawlOptions, + } + if (crawlOptions.max_depth === '') + delete passToServerCrawlOptions.max_depth + const res = await createFirecrawlTask({ url, - options: crawlOptions, + options: passToServerCrawlOptions, }) as any const jobId = res.job_id onJobIdChange(jobId) diff --git a/web/app/components/datasets/create/website/firecrawl/options.tsx b/web/app/components/datasets/create/website/firecrawl/options.tsx index b882febd9d3c25..a06671105163a9 100644 --- a/web/app/components/datasets/create/website/firecrawl/options.tsx +++ b/web/app/components/datasets/create/website/firecrawl/options.tsx @@ -52,7 +52,7 @@ const Options: FC = ({ value={payload.max_depth} onChange={handleChange('max_depth')} isNumber - isRequired + tooltip={t(`${I18N_PREFIX}.maxDepthTooltip`)!} />
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index 9ae5c6054b20e2..ff884f1c14c451 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -78,6 +78,7 @@ const translation = { resetAll: 'Reset All', scrapTimeInfo: 'Scraped {{total}} pages in total within {{time}}s', preview: 'Preview', + maxDepthTooltip: 'Maximum depth to crawl. Depth 1 is the base URL, depth 2 includes the base URL and its direct children, and so on.', }, }, stepTwo: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 9541d88fdb8778..27a070535f29a3 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -78,6 +78,7 @@ const translation = { resetAll: '重置全部', scrapTimeInfo: '总共在 {{time}}秒 内抓取了 {{total}} 个页面', preview: '预览', + maxDepthTooltip: '最大抓取深度。深度 1 表示 Base URL，深度 2 表示 Base URL及其直接子页面，依此类推。', }, }, stepTwo: { From a85d2204fe2dcf7d0c4db8535343f0ca18f5d46c Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 18:23:52 +0800 Subject: [PATCH 201/273] feat: change doc link --- web/app/components/datasets/create/website/firecrawl/header.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/website/firecrawl/header.tsx b/web/app/components/datasets/create/website/firecrawl/header.tsx index ae46f3f210cedf..c06b8161202c2a 100644 --- a/web/app/components/datasets/create/website/firecrawl/header.tsx +++ b/web/app/components/datasets/create/website/firecrawl/header.tsx @@ -29,7 +29,7 @@ const Header: FC = ({
From 233c9e928a09642a1dc4ff8f1a471f6f79358d0d Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 18:36:41 +0800 Subject: [PATCH 202/273] add crawl to s3 service --- api/core/rag/extractor/firecrawl/firecrawl_app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 7ec4beb7c8e292..fcab0f382034d1 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -1,7 +1,10 @@ +import json import time import requests +from extensions.ext_storage import storage + class FirecrawlApp: def __init__(self, api_key=None, base_url=None): From bea2557f0b0bbea4ea5574bceec657e0a6f658d4 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 19:02:55 +0800 Subject: [PATCH 203/273] lint --- api/controllers/console/__init__.py | 2 +- api/controllers/console/auth/data_source_bearer_auth.py | 3 ++- api/core/rag/extractor/firecrawl/firecrawl_app.py | 2 +- .../versions/7b45942e39bb_add_api_key_auth_binding.py | 4 ++-- api/services/dataset_service.py | 1 - 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 12dd975e728f75..29eac070a08fcb 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -29,7 +29,7 @@ ) # Import auth controllers -from .auth import activate, data_source_oauth, login, oauth, data_source_bearer_auth +from .auth import activate, data_source_bearer_auth, data_source_oauth, login, oauth # Import billing controllers from .billing import billing diff --git a/api/controllers/console/auth/data_source_bearer_auth.py b/api/controllers/console/auth/data_source_bearer_auth.py index f93e9180ad5ba6..81678f61fcbf02 100644 --- a/api/controllers/console/auth/data_source_bearer_auth.py +++ b/api/controllers/console/auth/data_source_bearer_auth.py @@ -1,11 +1,12 @@ from flask_login import current_user from flask_restful import Resource, reqparse from werkzeug.exceptions import Forbidden -from controllers.console.auth.error import ApiKeyAuthFailedError from controllers.console import api +from controllers.console.auth.error import ApiKeyAuthFailedError from libs.login import login_required from services.auth.api_key_auth_service import ApiKeyAuthService + from ..setup import setup_required from ..wraps import account_initialization_required diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 99bd7fb0fd322e..fcab0f382034d1 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -66,7 +66,7 @@ def check_crawl_status(self, job_id) -> dict: if crawl_status_response.get('status') == 'completed': total = crawl_status_response.get('total', 0) if total == 0: - raise Exception(f'Failed to check crawl status. Error: No page found') + raise Exception('Failed to check crawl status. Error: No page found') data = crawl_status_response.get('data', []) url_data_list = [] for item in data: diff --git a/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py b/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py index 30992e00c9c01b..f63bad93457d30 100644 --- a/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py +++ b/api/migrations/versions/7b45942e39bb_add_api_key_auth_binding.py @@ -5,10 +5,10 @@ Create Date: 2024-05-14 07:31:29.702766 """ +import sqlalchemy as sa from alembic import op + import models as models -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = '7b45942e39bb' diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index fb3c791f680c8b..c1ac8400064b58 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -51,7 +51,6 @@ from tasks.sync_website_document_indexing_task import sync_website_document_indexing_task - class DatasetService: @staticmethod From 66a73f778ecbdf0305d72ebcee7be7eb501cbefa Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 19:07:44 +0800 Subject: [PATCH 204/273] lint --- api/core/rag/extractor/firecrawl/firecrawl_app.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index fcab0f382034d1..af6b5689366904 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -70,13 +70,14 @@ def check_crawl_status(self, job_id) -> dict: data = crawl_status_response.get('data', []) url_data_list = [] for item in data: - url_data = { - 'title': item.get('metadata').get('title'), - 'description': item.get('metadata').get('description'), - 'source_url': item.get('metadata').get('sourceURL'), - 'markdown': item.get('markdown') - } - url_data_list.append(url_data) + if isinstance(item, dict) and 'metadata' in item and 'markdown' in item: + url_data = { + 'title': item.get('metadata').get('title'), + 'description': item.get('metadata').get('description'), + 'source_url': item.get('metadata').get('sourceURL'), + 'markdown': item.get('markdown') + } + url_data_list.append(url_data) if url_data_list: file_key = 'website_files/' + job_id + '.txt' if storage.exists(file_key): From 867f17be566660cf8ce9a8cabbbdd832c70c5b26 Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Fri, 14 Jun 2024 19:59:57 +0800 Subject: [PATCH 205/273] lint --- api/services/website_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/services/website_service.py b/api/services/website_service.py index 58ff0b012118a7..c166b01237b6c4 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -54,8 +54,8 @@ def crawl_url(cls, args: dict) -> dict: } } else: - includes = ','.join(options.get('includes')) if options.get('includes') else [] - excludes = ','.join(options.get('excludes')) if options.get('excludes') else [] + includes = options.get('includes').split(',') if options.get('includes') else [] + excludes = options.get('excludes').split(',') if options.get('excludes') else [] params = { 'crawlerOptions': { "includes": includes if includes else [], From 6be340cdc5a824f63346cb1b9e98845fcf3f3a44 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 14 Jun 2024 20:48:33 +0800 Subject: [PATCH 206/273] fix: setting firecrawl error --- .../datasets/documents/detail/settings/index.tsx | 10 ++++++++++ web/models/datasets.ts | 2 ++ 2 files changed, 12 insertions(+) diff --git a/web/app/components/datasets/documents/detail/settings/index.tsx b/web/app/components/datasets/documents/detail/settings/index.tsx index cab0c5d4007c21..def00ad37b1063 100644 --- a/web/app/components/datasets/documents/detail/settings/index.tsx +++ b/web/app/components/datasets/documents/detail/settings/index.tsx @@ -73,6 +73,16 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => { datasetId={datasetId} dataSourceType={documentDetail.data_source_type} notionPages={[currentPage]} + websitePages={[ + { + title: documentDetail.name, + source_url: documentDetail.data_source_info?.url, + markdown: '', + description: '', + }, + ]} + fireCrawlJobId={documentDetail.data_source_info?.job_id} + crawlOptions={documentDetail.data_source_info} indexingType={indexingTechnique || ''} isSetting documentDetail={documentDetail} diff --git a/web/models/datasets.ts b/web/models/datasets.ts index d417ec4943b807..a28798ba6870ef 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -165,6 +165,8 @@ export type DataSourceInfo = { extension: string } notion_page_icon?: string + job_id: string + url: string } export type InitialDocumentDetail = { From 901670f19d943ad333650ad98f44c56065e306b5 Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 17 Jun 2024 14:50:02 +0800 Subject: [PATCH 207/273] chore: upgrade next to 14.1.1 --- web/package.json | 4 +- web/yarn.lock | 140 ++++++++++++++++++++++++----------------------- 2 files changed, 75 insertions(+), 69 deletions(-) diff --git a/web/package.json b/web/package.json index a95b3a1b57fe0d..832ad6ef444b9f 100644 --- a/web/package.json +++ b/web/package.json @@ -51,7 +51,7 @@ "lodash-es": "^4.17.21", "mermaid": "10.4.0", "negotiator": "^0.6.3", - "next": "^14.0.4", + "next": "^14.1.1", "next-nprogress-bar": "^2.3.8", "qrcode.react": "^3.1.0", "qs": "^6.11.1", @@ -137,4 +137,4 @@ "@types/react": "~18.2.0", "@types/react-dom": "~18.2.0" } -} +} \ No newline at end of file diff --git a/web/yarn.lock b/web/yarn.lock index d8aa078e6a3b14..6e90ef839f62db 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -668,10 +668,10 @@ dependencies: "@monaco-editor/loader" "^1.4.0" -"@next/env@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/env/-/env-14.1.0.tgz" - integrity sha512-Py8zIo+02ht82brwwhTg36iogzFqGLPXlRGKQw5s+qP/kMNc4MAyDeEwBKDijk6zTIbegEgu8Qy7C1LboslQAw== +"@next/env@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/env/-/env-14.2.4.tgz#5546813dc4f809884a37d257b254a5ce1b0248d7" + integrity sha512-3EtkY5VDkuV2+lNmKlbkibIJxcO4oIHEhBWne6PaAp+76J9KoSsGvNikp6ivzAT8dhhBMYrm6op2pS1ApG0Hzg== "@next/eslint-plugin-next@14.1.0": version "14.1.0" @@ -687,50 +687,50 @@ dependencies: source-map "^0.7.0" -"@next/swc-darwin-arm64@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.0.tgz" - integrity sha512-nUDn7TOGcIeyQni6lZHfzNoo9S0euXnu0jhsbMOmMJUBfgsnESdjN97kM7cBqQxZa8L/bM9om/S5/1dzCrW6wQ== - -"@next/swc-darwin-x64@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz#0863a22feae1540e83c249384b539069fef054e9" - integrity sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g== - -"@next/swc-linux-arm64-gnu@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz#893da533d3fce4aec7116fe772d4f9b95232423c" - integrity sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ== - -"@next/swc-linux-arm64-musl@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz#d81ddcf95916310b8b0e4ad32b637406564244c0" - integrity sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g== - -"@next/swc-linux-x64-gnu@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz#18967f100ec19938354332dcb0268393cbacf581" - integrity sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ== - -"@next/swc-linux-x64-musl@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz#77077cd4ba8dda8f349dc7ceb6230e68ee3293cf" - integrity sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg== - -"@next/swc-win32-arm64-msvc@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz#5f0b8cf955644104621e6d7cc923cad3a4c5365a" - integrity sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ== - -"@next/swc-win32-ia32-msvc@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz#21f4de1293ac5e5a168a412b139db5d3420a89d0" - integrity sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw== - -"@next/swc-win32-x64-msvc@14.1.0": - version "14.1.0" - resolved "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz#e561fb330466d41807123d932b365cf3d33ceba2" - integrity sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg== +"@next/swc-darwin-arm64@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.4.tgz#da9f04c34a3d5f0b8401ed745768420e4a604036" + integrity sha512-AH3mO4JlFUqsYcwFUHb1wAKlebHU/Hv2u2kb1pAuRanDZ7pD/A/KPD98RHZmwsJpdHQwfEc/06mgpSzwrJYnNg== + +"@next/swc-darwin-x64@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.4.tgz#46dedb29ec5503bf171a72a3ecb8aac6e738e9d6" + integrity sha512-QVadW73sWIO6E2VroyUjuAxhWLZWEpiFqHdZdoQ/AMpN9YWGuHV8t2rChr0ahy+irKX5mlDU7OY68k3n4tAZTg== + +"@next/swc-linux-arm64-gnu@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.4.tgz#c9697ab9eb422bd1d7ffd0eb0779cc2aefa9d4a1" + integrity sha512-KT6GUrb3oyCfcfJ+WliXuJnD6pCpZiosx2X3k66HLR+DMoilRb76LpWPGb4tZprawTtcnyrv75ElD6VncVamUQ== + +"@next/swc-linux-arm64-musl@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.4.tgz#cbbceb2008571c743b5a310a488d2e166d200a75" + integrity sha512-Alv8/XGSs/ytwQcbCHwze1HmiIkIVhDHYLjczSVrf0Wi2MvKn/blt7+S6FJitj3yTlMwMxII1gIJ9WepI4aZ/A== + +"@next/swc-linux-x64-gnu@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.4.tgz#d79184223f857bacffb92f643cb2943a43632568" + integrity sha512-ze0ShQDBPCqxLImzw4sCdfnB3lRmN3qGMB2GWDRlq5Wqy4G36pxtNOo2usu/Nm9+V2Rh/QQnrRc2l94kYFXO6Q== + +"@next/swc-linux-x64-musl@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.4.tgz#6b6c3e5ac02ca5e63394d280ec8ee607491902df" + integrity sha512-8dwC0UJoc6fC7PX70csdaznVMNr16hQrTDAMPvLPloazlcaWfdPogq+UpZX6Drqb1OBlwowz8iG7WR0Tzk/diQ== + +"@next/swc-win32-arm64-msvc@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.4.tgz#dbad3906e870dba84c5883d9d4c4838472e0697f" + integrity sha512-jxyg67NbEWkDyvM+O8UDbPAyYRZqGLQDTPwvrBBeOSyVWW/jFQkQKQ70JDqDSYg1ZDdl+E3nkbFbq8xM8E9x8A== + +"@next/swc-win32-ia32-msvc@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.4.tgz#6074529b91ba49132922ce89a2e16d25d2ec235d" + integrity sha512-twrmN753hjXRdcrZmZttb/m5xaCBFa48Dt3FbeEItpJArxriYDunWxJn+QFXdJ3hPkm4u7CKxncVvnmgQMY1ag== + +"@next/swc-win32-x64-msvc@14.2.4": + version "14.2.4" + resolved "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.4.tgz#e65a1c6539a671f97bb86d5183d6e3a1733c29c7" + integrity sha512-tkLrjBzqFTP8DVrAAQmZelEahfR9OxWpFR++vAI9FBhCiIxtwHwBHC23SBHCTURBtwB4kc/x44imVOnkKGNVGg== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -910,11 +910,17 @@ "@sentry/types" "7.54.0" tslib "^1.9.3" -"@swc/helpers@0.5.2": - version "0.5.2" - resolved "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz" - integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== +"@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== + +"@swc/helpers@0.5.5": + version "0.5.5" + resolved "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0" + integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A== dependencies: + "@swc/counter" "^0.1.3" tslib "^2.4.0" "@tailwindcss/line-clamp@^0.4.4": @@ -5229,28 +5235,28 @@ next-nprogress-bar@^2.3.8: dependencies: nprogress "^0.2.0" -next@^14.0.4: - version "14.1.0" - resolved "https://registry.npmjs.org/next/-/next-14.1.0.tgz" - integrity sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q== +next@^14.1.1: + version "14.2.4" + resolved "https://registry.npmjs.org/next/-/next-14.2.4.tgz#ef66c39c71e2d8ad0a3caa0383c8933f4663e4d1" + integrity sha512-R8/V7vugY+822rsQGQCjoLhMuC9oFj9SOi4Cl4b2wjDrseD0LRZ10W7R6Czo4w9ZznVSshKjuIomsRjvm9EKJQ== dependencies: - "@next/env" "14.1.0" - "@swc/helpers" "0.5.2" + "@next/env" "14.2.4" + "@swc/helpers" "0.5.5" busboy "1.6.0" caniuse-lite "^1.0.30001579" graceful-fs "^4.2.11" postcss "8.4.31" styled-jsx "5.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "14.1.0" - "@next/swc-darwin-x64" "14.1.0" - "@next/swc-linux-arm64-gnu" "14.1.0" - "@next/swc-linux-arm64-musl" "14.1.0" - "@next/swc-linux-x64-gnu" "14.1.0" - "@next/swc-linux-x64-musl" "14.1.0" - "@next/swc-win32-arm64-msvc" "14.1.0" - "@next/swc-win32-ia32-msvc" "14.1.0" - "@next/swc-win32-x64-msvc" "14.1.0" + "@next/swc-darwin-arm64" "14.2.4" + "@next/swc-darwin-x64" "14.2.4" + "@next/swc-linux-arm64-gnu" "14.2.4" + "@next/swc-linux-arm64-musl" "14.2.4" + "@next/swc-linux-x64-gnu" "14.2.4" + "@next/swc-linux-x64-musl" "14.2.4" + "@next/swc-win32-arm64-msvc" "14.2.4" + "@next/swc-win32-ia32-msvc" "14.2.4" + "@next/swc-win32-x64-msvc" "14.2.4" node-releases@^2.0.12: version "2.0.12" From 69533fb4af4a2126c14d682a7f3ff04ae95b672c Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 17 Jun 2024 17:40:22 +0800 Subject: [PATCH 208/273] feat: dialog i18n --- .../overview/tracing/config-button.tsx | 2 +- .../[appId]/overview/tracing/config-popup.tsx | 2 +- .../tracing/provider-config-modal.tsx | 20 +++++++++---------- web/i18n/en-US/app.ts | 10 +++++----- web/i18n/zh-Hans/app.ts | 10 ++++++++++ 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index 99634163314cfc..5107bb0168c955 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -28,7 +28,7 @@ const ConfigBtn: FC = ({ ...popupProps }) => { const { t } = useTranslation() - const [open, setOpen] = useState(false) + const [open, setOpen] = useState(true) const handleTrigger = useCallback(() => { setOpen(v => !v) }, [setOpen]) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 84e83f9e5e57fa..629b161e020fe6 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -23,7 +23,7 @@ const ConfigPopup: FC = ({ }) => { const { t } = useTranslation() - const [currentProvider, setCurrentProvider] = useState(TracingProvider.langSmith) + const [currentProvider, setCurrentProvider] = useState(TracingProvider.langfuse) const [isShowConfigModal, { setTrue: showConfigModal, setFalse: hideConfigModal, diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index ed7646c36b1503..3cc9303c197d4f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -18,7 +18,7 @@ type Props = { onSaved: () => void } -const I18N_PREFIX = 'app.tracing.config' +const I18N_PREFIX = 'app.tracing.configProvider' const ProviderConfigModal: FC = ({ type, @@ -49,7 +49,7 @@ const ProviderConfigModal: FC = ({
-
{t(`${I18N_PREFIX}.configFirecrawl`)}
+
{t(`${I18N_PREFIX}.title`)}{t(`app.tracing.${type}.title`)}
@@ -61,15 +61,15 @@ const ProviderConfigModal: FC = ({ isRequired value={config.api_key} onChange={handleConfigChange('api_key')} - placeholder={'Enter your API Key'} + placeholder={t(`${I18N_PREFIX}.placeholder`, { key: 'API Key' })!} /> = ({ {type === TracingProvider.langfuse && ( <> = ({
- {t(`${I18N_PREFIX}.getApiKeyLinkText`)} + {t(`${I18N_PREFIX}.viewDocsLink`, { key: t(`app.tracing.${type}.title`) })}
diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index c8b7b9acd67e1c..c9ace3f7ef9408 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -97,20 +97,20 @@ const translation = { tracingDescription: 'Capture the full context of app execution, including LLM calls, context, prompts, HTTP requests, and more, to a third-party tracing platform.', configProviderTitle: 'Config provider to enable tracing', langSmith: { + title: 'LangSmith', description: 'An all-in-one developer platform for every step of the LLM-powered application lifecycle.', }, langfuse: { + title: 'Langfuse', description: 'Traces, evals, prompt management and metrics to debug and improve your LLM application.', }, configProvider: { - title: 'Config', - placeholder: 'Enter your {key}', + title: 'Config ', + placeholder: 'Enter your {{key}}', project: 'Project', - endpoint: 'Endpoint', publicKey: 'Public Key', secretKey: 'Secret Key', - viewDocsLink: 'View LangSmith docs', - saveAndEnable: 'Save and Enable', + viewDocsLink: 'View {{key}} docs', }, }, } diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 8b8bb1bd2cc25e..e03e4afa59e989 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -96,11 +96,21 @@ const translation = { tracingDescription: '捕获应用程序执行的完整上下文，包括 LLM 调用、上下文、提示、HTTP 请求等，发送到第三方跟踪平台。', configProviderTitle: '配置提供商以启用追踪', langSmith: { + title: 'LangSmith', description: '一个全方位的开发者平台，适用于 LLM 驱动应用程序生命周期的每个步骤。', }, langfuse: { + title: 'Langfuse', description: '跟踪、评估、提示管理和指标，以调试和改进您的 LLM 应用程序。', }, + configProvider: { + title: '配置 ', + placeholder: '输入你的{{key}}', + project: '项目', + publicKey: '公钥', + secretKey: '密钥', + viewDocsLink: '查看 {{key}} 的文档', + }, }, } From c856e743b07dc18ad43ebaf25d20e1d889d4231e Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 17 Jun 2024 18:33:22 +0800 Subject: [PATCH 209/273] feat: remove dialog i18n --- .../[appId]/overview/tracing/config.ts | 6 + .../tracing/provider-config-modal.tsx | 255 +++++++++++------- web/i18n/en-US/app.ts | 2 + web/i18n/zh-Hans/app.ts | 2 + 4 files changed, 161 insertions(+), 104 deletions(-) create mode 100644 web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config.ts diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config.ts new file mode 100644 index 00000000000000..bbc1f0f2209980 --- /dev/null +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config.ts @@ -0,0 +1,6 @@ +import { TracingProvider } from './type' + +export const docURL = { + [TracingProvider.langSmith]: 'https://docs.smith.langchain.com/', + [TracingProvider.langfuse]: 'https://docs.langfuse.com', +} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 3cc9303c197d4f..15dcad6eb9e916 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -2,8 +2,10 @@ import type { FC } from 'react' import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' +import { useBoolean } from 'ahooks' import Field from './field' import { TracingProvider } from './type' +import { docURL } from './config' import { PortalToFollowElem, PortalToFollowElemContent, @@ -11,9 +13,11 @@ import { import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security' import Button from '@/app/components/base/button' import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general' +import ConfirmUi from '@/app/components/base/confirm' type Props = { type: TracingProvider payload?: any + onRemove?: () => void onCancel: () => void onSaved: () => void } @@ -22,12 +26,24 @@ const I18N_PREFIX = 'app.tracing.configProvider' const ProviderConfigModal: FC = ({ type, + payload, + onRemove, onCancel, onSaved, }) => { const { t } = useTranslation() + const isEdit = !!payload const [isSaving, setIsSaving] = useState(false) const [config, setConfig] = useState>({}) + const [isShowRemoveConfirm, { + setTrue: showRemoveConfirm, + setFalse: hideRemoveConfirm, + }] = useBoolean(true) + + const handleRemove = useCallback(async () => { + hideRemoveConfirm() + onRemove?.() + }, [onRemove, hideRemoveConfirm]) const handleConfigChange = useCallback((key: string) => { return () => { @@ -43,114 +59,145 @@ const ProviderConfigModal: FC = ({ }, [isSaving, onSaved]) return ( - - -
-
-
-
-
{t(`${I18N_PREFIX}.title`)}{t(`app.tracing.${type}.title`)}
-
+ <> + {!isShowRemoveConfirm + ? ( + + +
+
+
+
+
{t(`${I18N_PREFIX}.title`)}{t(`app.tracing.${type}.title`)}
+
-
- {type === TracingProvider.langSmith && ( - <> - - - - - )} - {type === TracingProvider.langfuse && ( - <> - - - - - )} +
+ {type === TracingProvider.langSmith && ( + <> + + + + + )} + {type === TracingProvider.langfuse && ( + <> + + + + + )} -
-
- - {t(`${I18N_PREFIX}.viewDocsLink`, { key: t(`app.tracing.${type}.title`) })} - - -
- - -
+
+
+ + {t(`${I18N_PREFIX}.viewDocsLink`, { key: t(`app.tracing.${type}.title`) })} + + +
+ {!isEdit && ( + <> + +
+ + )} + + +
+
+
+
+
+ + {t('common.modelProvider.encrypted.front')} + + PKCS1_OAEP + + {t('common.modelProvider.encrypted.back')} +
+
+
-
-
-
- - {t('common.modelProvider.encrypted.front')} - - PKCS1_OAEP - - {t('common.modelProvider.encrypted.back')} -
-
-
-
- - + + + ) + : ( + + )} + ) } export default React.memo(ProviderConfigModal) diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index c9ace3f7ef9408..6b5a1df70fc2ef 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -111,6 +111,8 @@ const translation = { publicKey: 'Public Key', secretKey: 'Secret Key', viewDocsLink: 'View {{key}} docs', + removeConfirmTitle: 'Remove {{key}} configuration?', + removeConfirmContent: 'The current configuration is in use, removing it will turn off the Tracing feature.', }, }, } diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index e03e4afa59e989..b59785014c61e5 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -110,6 +110,8 @@ const translation = { publicKey: '公钥', secretKey: '密钥', viewDocsLink: '查看 {{key}} 的文档', + removeConfirmTitle: '删除 {{key}} 配置?', + removeConfirmContent: '当前配置正在使用中，删除它将关闭追踪功能。', }, }, } From 48760e85bc70d1ead2c6aafccd71aa548f88aa9f Mon Sep 17 00:00:00 2001 From: Joel Date: Tue, 18 Jun 2024 10:41:24 +0800 Subject: [PATCH 210/273] feat: page header to comp --- .../[appId]/overview/page.tsx | 12 +- .../overview/tracing/config-button.tsx | 2 +- .../[appId]/overview/tracing/panel.tsx | 117 +++++++++++------- .../tracing/provider-config-modal.tsx | 2 +- 4 files changed, 72 insertions(+), 61 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx index 0cc9175b54e8ff..137c2c36ee750f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/page.tsx @@ -2,7 +2,6 @@ import React from 'react' import ChartView from './chartView' import CardView from './cardView' import TracingPanel from './tracing/panel' -import { getLocaleOnServer, useTranslation as translate } from '@/i18n/server' import ApikeyInfoPanel from '@/app/components/app/overview/apikey-info-panel' export type IDevelopProps = { @@ -12,19 +11,10 @@ export type IDevelopProps = { const Overview = async ({ params: { appId }, }: IDevelopProps) => { - const locale = getLocaleOnServer() - /* - rename useTranslation to avoid lint error - please check: https://github.com/i18next/next-13-app-dir-i18next-example/issues/24 - */ - const { t } = await translate(locale, 'app-overview') return (
- -
- {t('overview.title')} -
+
diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index 5107bb0168c955..99634163314cfc 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -28,7 +28,7 @@ const ConfigBtn: FC = ({ ...popupProps }) => { const { t } = useTranslation() - const [open, setOpen] = useState(true) + const [open, setOpen] = useState(false) const handleTrigger = useCallback(() => { setOpen(v => !v) }, [setOpen]) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 7baa3f1432437e..f595b5646a336b 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -2,6 +2,7 @@ import type { FC } from 'react' import React, { useState } from 'react' import { useTranslation } from 'react-i18next' +import cn from 'classnames' import { TracingProvider } from './type' import TracingIcon from './tracing-icon' import ToggleExpandBtn from './toggle-fold-btn' @@ -11,81 +12,101 @@ import Indicator from '@/app/components/header/indicator' const I18N_PREFIX = 'app.tracing' +const Title = ({ + className, +}: { + className?: string +}) => { + const { t } = useTranslation() + + return ( +
+ {t('appOverview.overview.title')} +
+ ) +} const Panel: FC = () => { const { t } = useTranslation() const inUseTracingProvider: TracingProvider | undefined = undefined const InUseProviderIcon = inUseTracingProvider === TracingProvider.langSmith ? LangsmithIcon : LangfuseIcon const hasConfiguredTracing = !!inUseTracingProvider - const [isFold, setFold] = useState(true) + const [isFold, setFold] = useState(false) const [enabled, setEnabled] = useState(false) if (!isFold && !hasConfiguredTracing) { return ( -
-
- -
-
{t(`${I18N_PREFIX}.title`)}
-
- {t(`${I18N_PREFIX}.description`)} -
- - +
+ + <div className='mt-2 flex justify-between p-3 pr-4 items-center bg-white border-[0.5px] border-black/8 rounded-xl shadow-md'> + <div className='flex space-x-2'> + <TracingIcon size='lg' className='m-1' /> + <div> + <div className='mb-0.5 leading-6 text-base font-semibold text-gray-900'>{t(`${I18N_PREFIX}.title`)}</div> + <div className='flex justify-between leading-4 text-xs font-normal text-gray-500'> + <span className='mr-2'>{t(`${I18N_PREFIX}.description`)}</span> + <div className='flex space-x-3'> + <LangsmithIcon className='h-4' /> + <LangfuseIcon className='h-4' /> + </div> </div> </div> </div> - </div> - <div className='flex items-center space-x-1'> - <ConfigButton - hasConfigured={false} - enabled={enabled} - onStatusChange={setEnabled} - /> - <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> + <div className='flex items-center space-x-1'> + <ConfigButton + hasConfigured={false} + enabled={enabled} + onStatusChange={setEnabled} + /> + <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> + </div> </div> </div> ) } return ( - <div className='inline-flex items-center p-2 rounded-xl border-[0.5px] border-gray-200 shadow-xs hover:bg-gray-100'> - {!hasConfiguredTracing - ? <> - <TracingIcon size='md' className='mr-2' /> - <div className='leading-5 text-sm font-semibold text-gray-700'>{t(`${I18N_PREFIX}.title`)}</div> - </> - : <InUseProviderIcon className='ml-1 h-4' />} + <div className='mb-3 flex justify-between items-center'> + <Title /> + <div className='flex items-center p-2 rounded-xl border-[0.5px] border-gray-200 shadow-xs hover:bg-gray-100'> + {!hasConfiguredTracing + ? <> + <TracingIcon size='md' className='mr-2' /> + <div className='leading-5 text-sm font-semibold text-gray-700'>{t(`${I18N_PREFIX}.title`)}</div> + </> + : <InUseProviderIcon className='ml-1 h-4' />} - {hasConfiguredTracing && ( - <div className='ml-4 mr-1 flex items-center'> - <Indicator color={enabled ? 'green' : 'gray'} /> - <div className='ml-1.5 text-xs font-semibold text-gray-500 uppercase'> - {t(`${I18N_PREFIX}.${enabled ? 'enabled' : 'disabled'}`)} + {hasConfiguredTracing && ( + <div className='ml-4 mr-1 flex items-center'> + <Indicator color={enabled ? 'green' : 'gray'} /> + <div className='ml-1.5 text-xs font-semibold text-gray-500 uppercase'> + {t(`${I18N_PREFIX}.${enabled ? 'enabled' : 'disabled'}`)} + </div> </div> - </div> - )} + )} - {hasConfiguredTracing && ( - <div className='ml-2 w-px h-3.5 bg-gray-200'></div> - )} + {hasConfiguredTracing && ( + <div className='ml-2 w-px h-3.5 bg-gray-200'></div> + )} - <ConfigButton - hasConfigured - className='ml-2' - enabled={enabled} - onStatusChange={setEnabled} - /> - {!hasConfiguredTracing && ( - <> - <div className='mx-2 w-px h-3.5 bg-gray-200'></div> - <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> - </> - )} + <ConfigButton + hasConfigured + className='ml-2' + enabled={enabled} + onStatusChange={setEnabled} + /> + {!hasConfiguredTracing && ( + <> + <div className='mx-2 w-px h-3.5 bg-gray-200'></div> + <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> + </> + )} + </div> </div> + ) } export default React.memo(Panel) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 15dcad6eb9e916..fd3088712e8279 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -38,7 +38,7 @@ const ProviderConfigModal: FC<Props> = ({ const [isShowRemoveConfirm, { setTrue: showRemoveConfirm, setFalse: hideRemoveConfirm, - }] = useBoolean(true) + }] = useBoolean(false) const handleRemove = useCallback(async () => { hideRemoveConfirm() From d14bc6be1d1c817ea0bcafaa733ec415dc4394cc Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 11:54:54 +0800 Subject: [PATCH 211/273] feat: detail config --- .../[appId]/overview/tracing/config-popup.tsx | 2 +- .../[appId]/overview/tracing/panel.tsx | 19 ++++++++++++++++++- .../[appId]/overview/tracing/type.ts | 12 ++++++++++++ web/models/app.ts | 6 ++++++ web/service/apps.ts | 15 ++++++++++++++- 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 629b161e020fe6..0ff481238d45fc 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -27,7 +27,7 @@ const ConfigPopup: FC<PopupProps> = ({ const [isShowConfigModal, { setTrue: showConfigModal, setFalse: hideConfigModal, - }] = useBoolean(true) + }] = useBoolean(false) const handleOnConfig = useCallback((provider: TracingProvider) => { return () => { setCurrentProvider(provider) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index f595b5646a336b..81c80b411a414c 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -1,14 +1,17 @@ 'use client' import type { FC } from 'react' -import React, { useState } from 'react' +import React, { useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' +import { usePathname } from 'next/navigation' import { TracingProvider } from './type' import TracingIcon from './tracing-icon' import ToggleExpandBtn from './toggle-fold-btn' import ConfigButton from './config-button' import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' import Indicator from '@/app/components/header/indicator' +import { fetchTracingConfig } from '@/service/apps' +import type { TracingConfig } from '@/models/app' const I18N_PREFIX = 'app.tracing' @@ -27,14 +30,28 @@ const Title = ({ } const Panel: FC = () => { const { t } = useTranslation() + const pathname = usePathname() + const matched = pathname.match(/\/app\/([^/]+)/) + const appId = (matched?.length && matched[1]) ? matched[1] : '' const inUseTracingProvider: TracingProvider | undefined = undefined + const [tracingConfig, setTracingConfig] = useState<TracingConfig | null>(null) + const InUseProviderIcon = inUseTracingProvider === TracingProvider.langSmith ? LangsmithIcon : LangfuseIcon const hasConfiguredTracing = !!inUseTracingProvider const [isFold, setFold] = useState(false) const [enabled, setEnabled] = useState(false) + useEffect(() => { + (async () => { + const tracingConfig = await fetchTracingConfig({ appId }) + setTracingConfig(tracingConfig) + // debugger + })() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + if (!isFold && !hasConfiguredTracing) { return ( <div className='mb-3'> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts index 432a0cb3e2fc9f..69299d1c3e0b50 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts @@ -2,3 +2,15 @@ export enum TracingProvider { langSmith = 'langSmith', langfuse = 'langfuse', } + +export type LangSmitConfig = { + api_key: string + project: string + endpoint: string +} + +export type LangFuseConfig = { + public_key: string + secret_key: string + host: string +} diff --git a/web/models/app.ts b/web/models/app.ts index ac8f502fd23892..3862684c05291f 100644 --- a/web/models/app.ts +++ b/web/models/app.ts @@ -1,3 +1,4 @@ +import type { LangFuseConfig, LangSmitConfig, TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' import type { App, AppTemplate, SiteConfig } from '@/types/app' /* export type App = { @@ -129,3 +130,8 @@ export type AppVoicesListResponse = [{ name: string value: string }] + +export type TracingConfig = { + tracing_provider: TracingProvider + tracing_config: LangSmitConfig | LangFuseConfig +} diff --git a/web/service/apps.ts b/web/service/apps.ts index b20739898e4946..3dd87ba228484d 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -1,6 +1,6 @@ import type { Fetcher } from 'swr' import { del, get, post, put } from './base' -import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' +import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, TracingConfig, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' import type { CommonResponse } from '@/models/common' import type { AppMode, ModelConfig } from '@/types/app' @@ -121,3 +121,16 @@ export const generationIntroduction: Fetcher<GenerationIntroductionResponse, { u export const fetchAppVoices: Fetcher<AppVoicesListResponse, { appId: string; language?: string }> = ({ appId, language }) => { return get<AppVoicesListResponse>(`apps/${appId}/text-to-audio/voices?language=${language}`) } + +// Tracing +export const fetchTracingConfig: Fetcher<TracingConfig, { appId: string }> = ({ appId }) => { + // return get(`/apps/${appId}/tracing-config`) + return Promise.resolve({ + tracing_provider: 'langSmith', + tracing_config: { + api_key: '123132*********************21', + endpoint: 'https://api.langsmith.ai', + project: 'test', + }, + }) +} From ee039fb7b43bec815d5c323e4aa2ddb454f1202c Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Mon, 17 Jun 2024 00:26:43 +0800 Subject: [PATCH 212/273] feat: add llm ops tracing --- api/controllers/console/__init__.py | 1 + api/controllers/console/app/error.py | 12 + api/controllers/console/app/ops_trace.py | 79 ++ api/core/agent/cot_agent_runner.py | 24 +- api/core/agent/fc_agent_runner.py | 13 +- api/core/app/app_config/entities.py | 10 +- .../app/apps/advanced_chat/app_generator.py | 26 +- api/core/app/apps/advanced_chat/app_runner.py | 18 +- .../advanced_chat/generate_task_pipeline.py | 74 +- api/core/app/apps/agent_chat/app_generator.py | 21 +- api/core/app/apps/agent_chat/app_runner.py | 17 +- api/core/app/apps/base_app_runner.py | 17 +- api/core/app/apps/chat/app_generator.py | 24 +- api/core/app/apps/chat/app_runner.py | 4 +- api/core/app/apps/completion/app_generator.py | 19 +- api/core/app/apps/completion/app_runner.py | 4 +- .../app/apps/message_based_app_generator.py | 38 +- api/core/app/apps/workflow/app_generator.py | 46 +- .../apps/workflow/generate_task_pipeline.py | 20 +- .../easy_ui_based_generate_task_pipeline.py | 31 +- .../task_pipeline/workflow_cycle_manage.py | 82 +- .../agent_tool_callback_handler.py | 19 + api/core/llm_generator/llm_generator.py | 52 +- api/core/moderation/input_moderation.py | 40 +- api/core/rag/retrieval/dataset_retrieval.py | 141 ++-- api/core/tools/tool/workflow_tool.py | 10 +- api/core/tools/tool_engine.py | 33 +- api/core/workflow/nodes/tool/tool_node.py | 10 +- ...9b_update_appmodelconfig_and_add_table_.py | 49 ++ api/models/model.py | 36 +- api/poetry.lock | 42 +- api/pyproject.toml | 2 + api/services/app_generate_service.py | 3 +- api/services/conversation_service.py | 2 +- api/services/message_service.py | 27 +- api/services/ops_trace/base_trace_instance.py | 31 + api/services/ops_trace/langfuse_trace.py | 712 ++++++++++++++++++ api/services/ops_trace/langsmith_trace.py | 545 ++++++++++++++ api/services/ops_trace/ops_trace_service.py | 321 ++++++++ api/services/ops_trace/trace_queue_manager.py | 133 ++++ api/services/ops_trace/utils.py | 28 + 41 files changed, 2572 insertions(+), 244 deletions(-) create mode 100644 api/controllers/console/app/ops_trace.py create mode 100644 api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py create mode 100644 api/services/ops_trace/base_trace_instance.py create mode 100644 api/services/ops_trace/langfuse_trace.py create mode 100644 api/services/ops_trace/langsmith_trace.py create mode 100644 api/services/ops_trace/ops_trace_service.py create mode 100644 api/services/ops_trace/trace_queue_manager.py create mode 100644 api/services/ops_trace/utils.py diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 29eac070a08fcb..8c67fef95f5f4c 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -20,6 +20,7 @@ generator, message, model_config, + ops_trace, site, statistic, workflow, diff --git a/api/controllers/console/app/error.py b/api/controllers/console/app/error.py index fbe42fbd2a7135..cd7919b5e426bd 100644 --- a/api/controllers/console/app/error.py +++ b/api/controllers/console/app/error.py @@ -97,3 +97,15 @@ class DraftWorkflowNotSync(BaseHTTPException): error_code = 'draft_workflow_not_sync' description = "Workflow graph might have been modified, please refresh and resubmit." code = 400 + + +class TracingConfigNotExist(BaseHTTPException): + error_code = 'trace_config_not_exist' + description = "Trace config not exist." + code = 400 + + +class TracingConfigIsExist(BaseHTTPException): + error_code = 'trace_config_is_exist' + description = "Trace config is exist." + code = 400 diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py new file mode 100644 index 00000000000000..c4b3641b14ffec --- /dev/null +++ b/api/controllers/console/app/ops_trace.py @@ -0,0 +1,79 @@ +from flask_restful import Resource, reqparse + +from controllers.console import api +from controllers.console.app.error import TracingConfigIsExist, TracingConfigNotExist +from controllers.console.setup import setup_required +from controllers.console.wraps import account_initialization_required +from libs.login import login_required +from services.ops_trace.ops_trace_service import OpsTraceService + + +class TraceAppConfigApi(Resource): + """ + Manage trace app configurations + """ + + @setup_required + @login_required + @account_initialization_required + def get(self, app_id): + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='args') + args = parser.parse_args() + + try: + trace_config = OpsTraceService.get_tracing_app_config( + app_id=app_id, tracing_provider=args['tracing_provider'] + ) + if not trace_config: + raise TracingConfigNotExist() + return trace_config + except Exception as e: + raise e + + @setup_required + @login_required + @account_initialization_required + def post(self, app_id): + """Create a new trace app configuration""" + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='json') + parser.add_argument('tracing_config', type=dict, required=True, location='json') + args = parser.parse_args() + + try: + result = OpsTraceService.create_tracing_app_config( + app_id=app_id, + tracing_provider=args['tracing_provider'], + tracing_config=args['tracing_config'] + ) + if not result: + raise TracingConfigIsExist() + return {"result": "success"} + except Exception as e: + raise e + + @setup_required + @login_required + @account_initialization_required + def put(self, app_id): + """Update an existing trace app configuration""" + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='json') + parser.add_argument('tracing_config', type=dict, required=True, location='json') + args = parser.parse_args() + + try: + result = OpsTraceService.update_tracing_app_config( + app_id=app_id, + tracing_provider=args['tracing_provider'], + tracing_config=args['tracing_config'] + ) + if not result: + raise TracingConfigNotExist() + return {"result": "success"} + except Exception as e: + raise e + + +api.add_resource(TraceAppConfigApi, '/apps/<uuid:app_id>/trace-config') diff --git a/api/core/agent/cot_agent_runner.py b/api/core/agent/cot_agent_runner.py index 54aa0c9906421b..dfe35e8aa1825f 100644 --- a/api/core/agent/cot_agent_runner.py +++ b/api/core/agent/cot_agent_runner.py @@ -1,7 +1,7 @@ import json from abc import ABC, abstractmethod from collections.abc import Generator -from typing import Union +from typing import Union, Optional from core.agent.base_agent_runner import BaseAgentRunner from core.agent.entities import AgentScratchpadUnit @@ -20,6 +20,7 @@ from core.tools.tool.tool import Tool from core.tools.tool_engine import ToolEngine from models.model import Message +from services.ops_trace.base_trace_instance import BaseTraceInstance class CotAgentRunner(BaseAgentRunner, ABC): @@ -32,9 +33,9 @@ class CotAgentRunner(BaseAgentRunner, ABC): _prompt_messages_tools: list[PromptMessage] = None def run(self, message: Message, - query: str, - inputs: dict[str, str], - ) -> Union[Generator, LLMResult]: + query: str, + inputs: dict[str, str], + ) -> Union[Generator, LLMResult]: """ Run Cot agent application """ @@ -185,7 +186,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): messages_ids=[], llm_usage=usage_dict['usage'] ) - + if not scratchpad.is_final(): self.queue_manager.publish(QueueAgentThoughtEvent( agent_thought_id=agent_thought.id @@ -211,7 +212,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): function_call_state = True # action is tool call, invoke tool tool_invoke_response, tool_invoke_meta = self._handle_invoke_action( - action=scratchpad.action, + action=scratchpad.action, tool_instances=tool_instances, message_file_ids=message_file_ids ) @@ -259,12 +260,12 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): # save agent thought self.save_agent_thought( - agent_thought=agent_thought, + agent_thought=agent_thought, tool_name='', tool_input={}, tool_invoke_meta={}, thought=final_answer, - observation={}, + observation={}, answer=final_answer, messages_ids=[] ) @@ -284,7 +285,9 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, tool_instances: dict[str, Tool], - message_file_ids: list[str]) -> tuple[str, ToolInvokeMeta]: + message_file_ids: list[str], + tracing_instance: Optional[BaseTraceInstance] = None + ) -> tuple[str, ToolInvokeMeta]: """ handle invoke action :param action: action @@ -314,7 +317,8 @@ def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, tenant_id=self.tenant_id, message=self.message, invoke_from=self.application_generate_entity.invoke_from, - agent_tool_callback=self.agent_callback + agent_tool_callback=self.agent_callback, + tracing_instance=tracing_instance, ) # publish files diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index d7b063eb92ec55..e64722d22ca58d 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -20,7 +20,9 @@ from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform from core.tools.entities.tool_entities import ToolInvokeMeta from core.tools.tool_engine import ToolEngine -from models.model import Message +from extensions.ext_database import db +from models.model import AppModelConfig, Message +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -50,6 +52,14 @@ def run(self, } final_answer = '' + # get tracing instance + app_id = app_config.app_id + app_model_config_id = app_config.app_model_config_id + app_model_config = db.session.query(AppModelConfig).filter_by(id=app_model_config_id).first() + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id, app_model_config=app_model_config + ) + def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): if not final_llm_usage_dict['usage']: final_llm_usage_dict['usage'] = usage @@ -243,6 +253,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): message=self.message, invoke_from=self.application_generate_entity.invoke_from, agent_tool_callback=self.agent_callback, + tracing_instance=tracing_instance ) # publish files for message_file, save_as in message_files: diff --git a/api/core/app/app_config/entities.py b/api/core/app/app_config/entities.py index d6b6d894166d7e..6b58df617d7825 100644 --- a/api/core/app/app_config/entities.py +++ b/api/core/app/app_config/entities.py @@ -183,6 +183,14 @@ class TextToSpeechEntity(BaseModel): language: Optional[str] = None +class TracingConfigEntity(BaseModel): + """ + Tracing Config Entity. + """ + enabled: bool + tracing_provider: str + + class FileExtraConfig(BaseModel): """ File Upload Entity. @@ -199,7 +207,7 @@ class AppAdditionalFeatures(BaseModel): more_like_this: bool = False speech_to_text: bool = False text_to_speech: Optional[TextToSpeechEntity] = None - + trace_config: Optional[TracingConfigEntity] = None class AppConfig(BaseModel): """ diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 3b1ee3578dea3c..a06ae902fed5db 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -3,7 +3,7 @@ import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Any, Optional, Union from flask import Flask, current_app from pydantic import ValidationError @@ -29,13 +29,15 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): - def generate(self, app_model: App, - workflow: Workflow, - user: Union[Account, EndUser], - args: dict, - invoke_from: InvokeFrom, - stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: bool = True, + tracing_instance: Optional[Any] = None + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -45,6 +47,7 @@ def generate(self, app_model: App, :param args: request args :param invoke_from: invoke from source :param stream: is stream + :param tracing_instance: tracing instance """ if not args.get('query'): raise ValueError('query is required') @@ -105,7 +108,8 @@ def generate(self, app_model: App, invoke_from=invoke_from, application_generate_entity=application_generate_entity, conversation=conversation, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) def single_iteration_generate(self, app_model: App, @@ -227,7 +231,7 @@ def _generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, ) return AdvancedChatAppGenerateResponseConverter.convert( @@ -326,7 +330,7 @@ def _handle_advanced_chat_response(self, application_generate_entity: AdvancedCh ) try: - return generate_task_pipeline.process() + return generate_task_pipeline.process(workflow) except ValueError as e: if e.args[0] == "I/O operation on closed file.": # ignore this error raise GenerateTaskStoppedException() diff --git a/api/core/app/apps/advanced_chat/app_runner.py b/api/core/app/apps/advanced_chat/app_runner.py index de3632894de2ed..96e9319dda58d2 100644 --- a/api/core/app/apps/advanced_chat/app_runner.py +++ b/api/core/app/apps/advanced_chat/app_runner.py @@ -70,7 +70,8 @@ def run(self, application_generate_entity: AdvancedChatAppGenerateEntity, app_record=app_record, app_generate_entity=application_generate_entity, inputs=inputs, - query=query + query=query, + message_id=message.id ): return @@ -156,11 +157,14 @@ def get_workflow(self, app_model: App, workflow_id: str) -> Optional[Workflow]: # return workflow return workflow - def handle_input_moderation(self, queue_manager: AppQueueManager, - app_record: App, - app_generate_entity: AdvancedChatAppGenerateEntity, - inputs: dict, - query: str) -> bool: + def handle_input_moderation( + self, queue_manager: AppQueueManager, + app_record: App, + app_generate_entity: AdvancedChatAppGenerateEntity, + inputs: dict, + query: str, + message_id: str + ) -> bool: """ Handle input moderation :param queue_manager: application queue manager @@ -168,6 +172,7 @@ def handle_input_moderation(self, queue_manager: AppQueueManager, :param app_generate_entity: application generate entity :param inputs: inputs :param query: query + :param message_id: message id :return: """ try: @@ -178,6 +183,7 @@ def handle_input_moderation(self, queue_manager: AppQueueManager, app_generate_entity=app_generate_entity, inputs=inputs, query=query, + message_id=message_id, ) except ModerationException as e: self._stream_output( diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 7c70afc2ae393c..207e962b376efd 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -54,6 +54,7 @@ WorkflowNodeExecution, WorkflowRunStatus, ) +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -69,13 +70,15 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc _workflow_system_variables: dict[SystemVariable, Any] _iteration_nested_relations: dict[str, list[str]] - def __init__(self, application_generate_entity: AdvancedChatAppGenerateEntity, - workflow: Workflow, - queue_manager: AppQueueManager, - conversation: Conversation, - message: Message, - user: Union[Account, EndUser], - stream: bool) -> None: + def __init__( + self, application_generate_entity: AdvancedChatAppGenerateEntity, + workflow: Workflow, + queue_manager: AppQueueManager, + conversation: Conversation, + message: Message, + user: Union[Account, EndUser], + stream: bool + ) -> None: """ Initialize AdvancedChatAppGenerateTaskPipeline. :param application_generate_entity: application generate entity @@ -111,7 +114,10 @@ def __init__(self, application_generate_entity: AdvancedChatAppGenerateEntity, self._stream_generate_routes = self._get_stream_generate_routes() self._conversation_name_generate_thread = None - def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]: + def process( + self, + workflow: Optional[Workflow] = None + ) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]: """ Process generate task pipeline. :return: @@ -126,14 +132,14 @@ def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStrea self._application_generate_entity.query ) - generator = self._process_stream_response() + generator = self._process_stream_response(workflow) if self._stream: return self._to_stream_response(generator) else: return self._to_blocking_response(generator) def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) \ - -> ChatbotAppBlockingResponse: + -> ChatbotAppBlockingResponse: """ Process blocking response. :return: @@ -164,7 +170,7 @@ def _to_blocking_response(self, generator: Generator[StreamResponse, None, None] raise Exception('Queue listening stopped unexpectedly.') def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) \ - -> Generator[ChatbotAppStreamResponse, None, None]: + -> Generator[ChatbotAppStreamResponse, None, None]: """ To stream response. :return: @@ -177,11 +183,13 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) stream_response=stream_response ) - def _process_stream_response(self) -> Generator[StreamResponse, None, None]: + def _process_stream_response(self, workflow: Optional[Workflow] = None) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: """ + app_id = self._conversation.app_id + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) for message in self._queue_manager.listen(): event = message.event @@ -249,7 +257,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event) self._handle_iteration_operation(event) elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent): - workflow_run = self._handle_workflow_finished(event) + workflow_run = self._handle_workflow_finished(event, tracing_instance) if workflow_run: yield self._workflow_finish_to_stream_response( task_id=self._application_generate_entity.task_id, @@ -292,7 +300,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: continue if not self._is_stream_out_support( - event=event + event=event ): continue @@ -361,7 +369,7 @@ def _message_end_to_stream_response(self) -> MessageEndStreamResponse: id=self._message.id, **extras ) - + def _get_stream_generate_routes(self) -> dict[str, ChatflowStreamGenerateRoute]: """ Get stream generate routes. @@ -391,9 +399,9 @@ def _get_stream_generate_routes(self) -> dict[str, ChatflowStreamGenerateRoute]: ) return stream_generate_routes - + def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \ - -> list[str]: + -> list[str]: """ Get answer start at node id. :param graph: graph @@ -414,14 +422,14 @@ def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \ target_node = next((node for node in nodes if node.get('id') == target_node_id), None) if not target_node: return [] - + node_iteration_id = target_node.get('data', {}).get('iteration_id') # get iteration start node id for node in nodes: if node.get('id') == node_iteration_id: if node.get('data', {}).get('start_node_id') == target_node_id: return [target_node_id] - + return [] start_node_ids = [] @@ -457,7 +465,7 @@ def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \ start_node_ids.extend(sub_start_node_ids) return start_node_ids - + def _get_iteration_nested_relations(self, graph: dict) -> dict[str, list[str]]: """ Get iteration nested relations. @@ -466,18 +474,18 @@ def _get_iteration_nested_relations(self, graph: dict) -> dict[str, list[str]]: """ nodes = graph.get('nodes') - iteration_ids = [node.get('id') for node in nodes + iteration_ids = [node.get('id') for node in nodes if node.get('data', {}).get('type') in [ NodeType.ITERATION.value, NodeType.LOOP.value, - ]] + ]] return { iteration_id: [ node.get('id') for node in nodes if node.get('data', {}).get('iteration_id') == iteration_id ] for iteration_id in iteration_ids } - + def _generate_stream_outputs_when_node_started(self) -> Generator: """ Generate stream outputs. @@ -485,8 +493,8 @@ def _generate_stream_outputs_when_node_started(self) -> Generator: """ if self._task_state.current_stream_generate_state: route_chunks = self._task_state.current_stream_generate_state.generate_route[ - self._task_state.current_stream_generate_state.current_route_position: - ] + self._task_state.current_stream_generate_state.current_route_position: + ] for route_chunk in route_chunks: if route_chunk.type == 'text': @@ -506,7 +514,8 @@ def _generate_stream_outputs_when_node_started(self) -> Generator: # all route chunks are generated if self._task_state.current_stream_generate_state.current_route_position == len( - self._task_state.current_stream_generate_state.generate_route): + self._task_state.current_stream_generate_state.generate_route + ): self._task_state.current_stream_generate_state = None def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: @@ -519,7 +528,7 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: route_chunks = self._task_state.current_stream_generate_state.generate_route[ self._task_state.current_stream_generate_state.current_route_position:] - + for route_chunk in route_chunks: if route_chunk.type == 'text': route_chunk = cast(TextGenerateRouteChunk, route_chunk) @@ -551,7 +560,8 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: value = iteration_state.current_index elif value_selector[1] == 'item': value = iterator_selector[iteration_state.current_index] if iteration_state.current_index < len( - iterator_selector) else None + iterator_selector + ) else None else: # check chunk node id is before current node id or equal to current node id if route_chunk_node_id not in self._task_state.ran_node_execution_infos: @@ -562,14 +572,15 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: # get route chunk node execution info route_chunk_node_execution_info = self._task_state.ran_node_execution_infos[route_chunk_node_id] if (route_chunk_node_execution_info.node_type == NodeType.LLM - and latest_node_execution_info.node_type == NodeType.LLM): + and latest_node_execution_info.node_type == NodeType.LLM): # only LLM support chunk stream output self._task_state.current_stream_generate_state.current_route_position += 1 continue # get route chunk node execution route_chunk_node_execution = db.session.query(WorkflowNodeExecution).filter( - WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id).first() + WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id + ).first() outputs = route_chunk_node_execution.outputs_dict @@ -631,7 +642,8 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: # all route chunks are generated if self._task_state.current_stream_generate_state.current_route_position == len( - self._task_state.current_stream_generate_state.generate_route): + self._task_state.current_stream_generate_state.generate_route + ): self._task_state.current_stream_generate_state = None def _is_stream_out_support(self, event: QueueTextChunkEvent) -> bool: diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index 407fb931ecb9bd..ca8fbe138efa37 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -3,7 +3,7 @@ import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Optional, Union from flask import Flask, current_app from pydantic import ValidationError @@ -22,6 +22,7 @@ from extensions.ext_database import db from models.account import Account from models.model import App, EndUser +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -70,6 +71,12 @@ def generate(self, app_model: App, conversation=conversation ) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # validate override model config override_model_config_dict = None if args.get('model_config'): @@ -142,6 +149,7 @@ def generate(self, app_model: App, 'queue_manager': queue_manager, 'conversation_id': conversation.id, 'message_id': message.id, + 'tracing_instance': tracing_instance, }) worker_thread.start() @@ -153,7 +161,8 @@ def generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return AgentChatAppGenerateResponseConverter.convert( @@ -165,7 +174,9 @@ def _generate_worker(self, flask_app: Flask, application_generate_entity: AgentChatAppGenerateEntity, queue_manager: AppQueueManager, conversation_id: str, - message_id: str) -> None: + message_id: str, + tracing_instance: Optional[Any] = None + ) -> None: """ Generate worker in a new thread. :param flask_app: Flask app @@ -173,6 +184,7 @@ def _generate_worker(self, flask_app: Flask, :param queue_manager: queue manager :param conversation_id: conversation ID :param message_id: message ID + :param tracing_instance: tracing instance :return: """ with flask_app.app_context(): @@ -187,7 +199,8 @@ def _generate_worker(self, flask_app: Flask, application_generate_entity=application_generate_entity, queue_manager=queue_manager, conversation=conversation, - message=message + message=message, + tracing_instance=tracing_instance ) except GenerateTaskStoppedException: pass diff --git a/api/core/app/apps/agent_chat/app_runner.py b/api/core/app/apps/agent_chat/app_runner.py index d6367300de26e3..a942522996ca00 100644 --- a/api/core/app/apps/agent_chat/app_runner.py +++ b/api/core/app/apps/agent_chat/app_runner.py @@ -1,5 +1,5 @@ import logging -from typing import cast +from typing import Any, Optional, cast from core.agent.cot_chat_agent_runner import CotChatAgentRunner from core.agent.cot_completion_agent_runner import CotCompletionAgentRunner @@ -28,16 +28,21 @@ class AgentChatAppRunner(AppRunner): """ Agent Application Runner """ - def run(self, application_generate_entity: AgentChatAppGenerateEntity, - queue_manager: AppQueueManager, - conversation: Conversation, - message: Message) -> None: + + def run( + self, application_generate_entity: AgentChatAppGenerateEntity, + queue_manager: AppQueueManager, + conversation: Conversation, + message: Message, + tracing_instance: Optional[Any] = None + ) -> None: """ Run assistant application :param application_generate_entity: application generate entity :param queue_manager: application queue manager :param conversation: conversation :param message: message + :param tracing_instance: tracing instance :return: """ app_config = application_generate_entity.app_config @@ -100,6 +105,7 @@ def run(self, application_generate_entity: AgentChatAppGenerateEntity, app_generate_entity=application_generate_entity, inputs=inputs, query=query, + message_id=message.id ) except ModerationException as e: self.direct_output( @@ -241,6 +247,7 @@ def run(self, application_generate_entity: AgentChatAppGenerateEntity, message=message, query=query, inputs=inputs, + tracing_instance=tracing_instance, ) # handle invoke result diff --git a/api/core/app/apps/base_app_runner.py b/api/core/app/apps/base_app_runner.py index 53f457cb116c02..1ccc9597cee3a9 100644 --- a/api/core/app/apps/base_app_runner.py +++ b/api/core/app/apps/base_app_runner.py @@ -338,11 +338,14 @@ def _handle_invoke_result_stream(self, invoke_result: Generator, ), PublishFrom.APPLICATION_MANAGER ) - def moderation_for_inputs(self, app_id: str, - tenant_id: str, - app_generate_entity: AppGenerateEntity, - inputs: dict, - query: str) -> tuple[bool, dict, str]: + def moderation_for_inputs( + self, app_id: str, + tenant_id: str, + app_generate_entity: AppGenerateEntity, + inputs: dict, + query: str, + message_id: str, + ) -> tuple[bool, dict, str]: """ Process sensitive_word_avoidance. :param app_id: app id @@ -350,6 +353,7 @@ def moderation_for_inputs(self, app_id: str, :param app_generate_entity: app generate entity :param inputs: inputs :param query: query + :param message_id: message id :return: """ moderation_feature = InputModeration() @@ -358,7 +362,8 @@ def moderation_for_inputs(self, app_id: str, tenant_id=tenant_id, app_config=app_generate_entity.app_config, inputs=inputs, - query=query if query else '' + query=query if query else '', + message_id=message_id, ) def check_hosting_moderation(self, application_generate_entity: EasyUIBasedAppGenerateEntity, diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index 505ada09db5c90..e333e3458cbbc8 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -22,17 +22,19 @@ from extensions.ext_database import db from models.account import Account from models.model import App, EndUser +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) class ChatAppGenerator(MessageBasedAppGenerator): - def generate(self, app_model: App, - user: Union[Account, EndUser], - args: Any, - invoke_from: InvokeFrom, - stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: Any, + invoke_from: InvokeFrom, + stream: bool = True, + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -41,6 +43,7 @@ def generate(self, app_model: App, :param args: request args :param invoke_from: invoke from source :param stream: is stream + :param tracing_instance: tracing instance """ if not args.get('query'): raise ValueError('query is required') @@ -121,6 +124,12 @@ def generate(self, app_model: App, message ) = self._init_generate_records(application_generate_entity, conversation) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # init queue manager queue_manager = MessageBasedAppQueueManager( task_id=application_generate_entity.task_id, @@ -149,7 +158,8 @@ def generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return ChatAppGenerateResponseConverter.convert( diff --git a/api/core/app/apps/chat/app_runner.py b/api/core/app/apps/chat/app_runner.py index 7d243d0726724e..0a029af86a1ce1 100644 --- a/api/core/app/apps/chat/app_runner.py +++ b/api/core/app/apps/chat/app_runner.py @@ -96,6 +96,7 @@ def run(self, application_generate_entity: ChatAppGenerateEntity, app_generate_entity=application_generate_entity, inputs=inputs, query=query, + message_id=message.id ) except ModerationException as e: self.direct_output( @@ -165,7 +166,8 @@ def run(self, application_generate_entity: ChatAppGenerateEntity, invoke_from=application_generate_entity.invoke_from, show_retrieve_source=app_config.additional_features.show_retrieve_source, hit_callback=hit_callback, - memory=memory + memory=memory, + message_id=message.id, ) # reorganize all inputs and template to prompt messages diff --git a/api/core/app/apps/completion/app_generator.py b/api/core/app/apps/completion/app_generator.py index 52d907b5353143..fcf00e685594dd 100644 --- a/api/core/app/apps/completion/app_generator.py +++ b/api/core/app/apps/completion/app_generator.py @@ -24,6 +24,7 @@ from models.model import App, EndUser, Message from services.errors.app import MoreLikeThisDisabledError from services.errors.message import MessageNotExistsError +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -114,6 +115,12 @@ def generate(self, app_model: App, message ) = self._init_generate_records(application_generate_entity) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # init queue manager queue_manager = MessageBasedAppQueueManager( task_id=application_generate_entity.task_id, @@ -141,7 +148,8 @@ def generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return CompletionAppGenerateResponseConverter.convert( @@ -273,6 +281,12 @@ def generate_more_like_this(self, app_model: App, message ) = self._init_generate_records(application_generate_entity) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # init queue manager queue_manager = MessageBasedAppQueueManager( task_id=application_generate_entity.task_id, @@ -300,7 +314,8 @@ def generate_more_like_this(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return CompletionAppGenerateResponseConverter.convert( diff --git a/api/core/app/apps/completion/app_runner.py b/api/core/app/apps/completion/app_runner.py index a3a9945bc0436b..2e701320148408 100644 --- a/api/core/app/apps/completion/app_runner.py +++ b/api/core/app/apps/completion/app_runner.py @@ -77,6 +77,7 @@ def run(self, application_generate_entity: CompletionAppGenerateEntity, app_generate_entity=application_generate_entity, inputs=inputs, query=query, + message_id=message.id ) except ModerationException as e: self.direct_output( @@ -124,7 +125,8 @@ def run(self, application_generate_entity: CompletionAppGenerateEntity, query=query, invoke_from=application_generate_entity.invoke_from, show_retrieve_source=app_config.additional_features.show_retrieve_source, - hit_callback=hit_callback + hit_callback=hit_callback, + message_id=message.id ) # reorganize all inputs and template to prompt messages diff --git a/api/core/app/apps/message_based_app_generator.py b/api/core/app/apps/message_based_app_generator.py index 6acf5da8df4d2a..11763d1c6cde53 100644 --- a/api/core/app/apps/message_based_app_generator.py +++ b/api/core/app/apps/message_based_app_generator.py @@ -1,7 +1,7 @@ import json import logging from collections.abc import Generator -from typing import Optional, Union +from typing import Any, Optional, Union from sqlalchemy import and_ @@ -35,22 +35,24 @@ class MessageBasedAppGenerator(BaseAppGenerator): - def _handle_response(self, application_generate_entity: Union[ - ChatAppGenerateEntity, - CompletionAppGenerateEntity, - AgentChatAppGenerateEntity, - AdvancedChatAppGenerateEntity - ], - queue_manager: AppQueueManager, - conversation: Conversation, - message: Message, - user: Union[Account, EndUser], - stream: bool = False) \ - -> Union[ - ChatbotAppBlockingResponse, - CompletionAppBlockingResponse, - Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None] - ]: + def _handle_response( + self, application_generate_entity: Union[ + ChatAppGenerateEntity, + CompletionAppGenerateEntity, + AgentChatAppGenerateEntity, + AdvancedChatAppGenerateEntity + ], + queue_manager: AppQueueManager, + conversation: Conversation, + message: Message, + user: Union[Account, EndUser], + stream: bool = False, + tracing_instance: Optional[Any] = None + ) -> Union[ + ChatbotAppBlockingResponse, + CompletionAppBlockingResponse, + Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None] + ]: """ Handle response. :param application_generate_entity: application generate entity @@ -72,7 +74,7 @@ def _handle_response(self, application_generate_entity: Union[ ) try: - return generate_task_pipeline.process() + return generate_task_pipeline.process(tracing_instance) except ValueError as e: if e.args[0] == "I/O operation on closed file.": # ignore this error raise GenerateTaskStoppedException() diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index c4324978d81fc3..829ccc8cab2b5a 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -3,7 +3,7 @@ import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Any, Optional, Union from flask import Flask, current_app from pydantic import ValidationError @@ -29,14 +29,16 @@ class WorkflowAppGenerator(BaseAppGenerator): - def generate(self, app_model: App, - workflow: Workflow, - user: Union[Account, EndUser], - args: dict, - invoke_from: InvokeFrom, - stream: bool = True, - call_depth: int = 0) \ - -> Union[dict, Generator[dict, None, None]]: + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: bool = True, + call_depth: int = 0, + tracing_instance: Optional[Any] = None + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -46,6 +48,8 @@ def generate(self, app_model: App, :param args: request args :param invoke_from: invoke from source :param stream: is stream + :param call_depth: call depth + :param tracing_instance: ops tracing instance """ inputs = args['inputs'] @@ -87,17 +91,18 @@ def generate(self, app_model: App, application_generate_entity=application_generate_entity, invoke_from=invoke_from, stream=stream, - call_depth=call_depth + call_depth=call_depth, ) - def _generate(self, app_model: App, - workflow: Workflow, - user: Union[Account, EndUser], - application_generate_entity: WorkflowAppGenerateEntity, - invoke_from: InvokeFrom, - stream: bool = True, - call_depth: int = 0) \ - -> Union[dict, Generator[dict, None, None]]: + def _generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + application_generate_entity: WorkflowAppGenerateEntity, + invoke_from: InvokeFrom, + stream: bool = True, + call_depth: int = 0 + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -131,7 +136,7 @@ def _generate(self, app_model: App, workflow=workflow, queue_manager=queue_manager, user=user, - stream=stream + stream=stream, ) return WorkflowAppGenerateResponseConverter.convert( @@ -271,9 +276,10 @@ def _handle_response(self, application_generate_entity: WorkflowAppGenerateEntit user=user, stream=stream ) + app_id = application_generate_entity.app_config.app_id try: - return generate_task_pipeline.process() + return generate_task_pipeline.process(app_id, workflow) except ValueError as e: if e.args[0] == "I/O operation on closed file.": # ignore this error raise GenerateTaskStoppedException() diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 8d961e0993b96f..944c3736a140d1 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -1,6 +1,6 @@ import logging from collections.abc import Generator -from typing import Any, Union +from typing import Any, Optional, Union from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import ( @@ -48,6 +48,7 @@ WorkflowNodeExecution, WorkflowRun, ) +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -95,7 +96,11 @@ def __init__(self, application_generate_entity: WorkflowAppGenerateEntity, self._stream_generate_nodes = self._get_stream_generate_nodes() self._iteration_nested_relations = self._get_iteration_nested_relations(self._workflow.graph_dict) - def process(self) -> Union[WorkflowAppBlockingResponse, Generator[WorkflowAppStreamResponse, None, None]]: + def process( + self, + app_id: Optional[str] = None, + workflow: Optional[Workflow] = None, + ) -> Union[WorkflowAppBlockingResponse, Generator[WorkflowAppStreamResponse, None, None]]: """ Process generate task pipeline. :return: @@ -104,7 +109,7 @@ def process(self) -> Union[WorkflowAppBlockingResponse, Generator[WorkflowAppStr db.session.refresh(self._user) db.session.close() - generator = self._process_stream_response() + generator = self._process_stream_response(app_id, workflow) if self._stream: return self._to_stream_response(generator) else: @@ -158,11 +163,16 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) stream_response=stream_response ) - def _process_stream_response(self) -> Generator[StreamResponse, None, None]: + def _process_stream_response( + self, + app_id: Optional[str] = None, + workflow: Optional[Workflow] = None, + ) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: """ + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) for message in self._queue_manager.listen(): event = message.event @@ -215,7 +225,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event) self._handle_iteration_operation(event) elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent): - workflow_run = self._handle_workflow_finished(event) + workflow_run = self._handle_workflow_finished(event, tracing_instance) # save workflow app log self._save_workflow_app_log(workflow_run) diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py index ccb684d84b0c8f..da0731df3eff1f 100644 --- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py @@ -2,7 +2,7 @@ import logging import time from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom from core.app.entities.app_invoke_entities import ( @@ -50,6 +50,7 @@ from extensions.ext_database import db from models.account import Account from models.model import AppMode, Conversation, EndUser, Message, MessageAgentThought +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName logger = logging.getLogger(__name__) @@ -100,7 +101,10 @@ def __init__(self, application_generate_entity: Union[ self._conversation_name_generate_thread = None - def process(self) -> Union[ + def process( + self, + tracing_instance: Optional[Any] = None + ) -> Union[ ChatbotAppBlockingResponse, CompletionAppBlockingResponse, Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None] @@ -120,7 +124,7 @@ def process(self) -> Union[ self._application_generate_entity.query ) - generator = self._process_stream_response() + generator = self._process_stream_response(tracing_instance) if self._stream: return self._to_stream_response(generator) else: @@ -197,7 +201,9 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) stream_response=stream_response ) - def _process_stream_response(self) -> Generator[StreamResponse, None, None]: + def _process_stream_response( + self, tracing_instance: Optional[Any] = None + ) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: @@ -224,7 +230,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: yield self._message_replace_to_stream_response(answer=output_moderation_answer) # Save message - self._save_message() + self._save_message(tracing_instance) yield self._message_end_to_stream_response() elif isinstance(event, QueueRetrieverResourcesEvent): @@ -269,7 +275,9 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: if self._conversation_name_generate_thread: self._conversation_name_generate_thread.join() - def _save_message(self) -> None: + def _save_message( + self, tracing_instance: Optional[Any] = None, + ) -> None: """ Save message. :return: @@ -300,6 +308,17 @@ def _save_message(self) -> None: db.session.commit() + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.MESSAGE_TRACE, + conversation_id=self._conversation.id, + message_id=self._message.id + ) + ) + message_was_created.send( self._message, application_generate_entity=self._application_generate_entity, diff --git a/api/core/app/task_pipeline/workflow_cycle_manage.py b/api/core/app/task_pipeline/workflow_cycle_manage.py index 978a318279165f..5c0ffbe07bdd5b 100644 --- a/api/core/app/task_pipeline/workflow_cycle_manage.py +++ b/api/core/app/task_pipeline/workflow_cycle_manage.py @@ -1,7 +1,7 @@ import json import time from datetime import datetime, timezone -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.queue_entities import ( @@ -39,6 +39,8 @@ WorkflowRunStatus, WorkflowRunTriggeredFrom, ) +from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName class WorkflowCycleManage(WorkflowIterationCycleManage): @@ -94,11 +96,15 @@ def _init_workflow_run(self, workflow: Workflow, return workflow_run - def _workflow_run_success(self, workflow_run: WorkflowRun, - start_at: float, - total_tokens: int, - total_steps: int, - outputs: Optional[str] = None) -> WorkflowRun: + def _workflow_run_success( + self, workflow_run: WorkflowRun, + start_at: float, + total_tokens: int, + total_steps: int, + outputs: Optional[str] = None, + conversation_id: Optional[str] = None, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> WorkflowRun: """ Workflow run success :param workflow_run: workflow run @@ -106,6 +112,8 @@ def _workflow_run_success(self, workflow_run: WorkflowRun, :param total_tokens: total tokens :param total_steps: total steps :param outputs: outputs + :param conversation_id: conversation id + :param tracing_instance: tracing instance :return: """ workflow_run.status = WorkflowRunStatus.SUCCEEDED.value @@ -119,14 +127,29 @@ def _workflow_run_success(self, workflow_run: WorkflowRun, db.session.refresh(workflow_run) db.session.close() + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.WORKFLOW_TRACE, + workflow_run=workflow_run, + conversation_id=conversation_id, + ) + ) + return workflow_run - def _workflow_run_failed(self, workflow_run: WorkflowRun, - start_at: float, - total_tokens: int, - total_steps: int, - status: WorkflowRunStatus, - error: str) -> WorkflowRun: + def _workflow_run_failed( + self, workflow_run: WorkflowRun, + start_at: float, + total_tokens: int, + total_steps: int, + status: WorkflowRunStatus, + error: str, + conversation_id: Optional[str] = None, + tracing_instance: Optional[Any] = None + ) -> WorkflowRun: """ Workflow run failed :param workflow_run: workflow run @@ -148,6 +171,17 @@ def _workflow_run_failed(self, workflow_run: WorkflowRun, db.session.refresh(workflow_run) db.session.close() + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.WORKFLOW_TRACE, + workflow_run=workflow_run, + conversation_id=conversation_id, + ) + ) + return workflow_run def _init_node_execution_from_workflow_run(self, workflow_run: WorkflowRun, @@ -440,9 +474,9 @@ def _handle_node_finished(self, event: QueueNodeSucceededEvent | QueueNodeFailed current_node_execution = self._task_state.ran_node_execution_infos[event.node_id] workflow_node_execution = db.session.query(WorkflowNodeExecution).filter( WorkflowNodeExecution.id == current_node_execution.workflow_node_execution_id).first() - + execution_metadata = event.execution_metadata if isinstance(event, QueueNodeSucceededEvent) else None - + if self._iteration_state and self._iteration_state.current_iterations: if not execution_metadata: execution_metadata = {} @@ -470,7 +504,7 @@ def _handle_node_finished(self, event: QueueNodeSucceededEvent | QueueNodeFailed if execution_metadata and execution_metadata.get(NodeRunMetadataKey.TOTAL_TOKENS): self._task_state.total_tokens += ( int(execution_metadata.get(NodeRunMetadataKey.TOTAL_TOKENS))) - + if self._iteration_state: for iteration_node_id in self._iteration_state.current_iterations: data = self._iteration_state.current_iterations[iteration_node_id] @@ -496,13 +530,16 @@ def _handle_node_finished(self, event: QueueNodeSucceededEvent | QueueNodeFailed return workflow_node_execution - def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent) \ - -> Optional[WorkflowRun]: + def _handle_workflow_finished( + self, event: QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> Optional[WorkflowRun]: workflow_run = db.session.query(WorkflowRun).filter( WorkflowRun.id == self._task_state.workflow_run_id).first() if not workflow_run: return None + conversation_id = self._application_generate_entity.inputs.get('sys.conversation_id') if isinstance(event, QueueStopEvent): workflow_run = self._workflow_run_failed( workflow_run=workflow_run, @@ -510,7 +547,8 @@ def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceed total_tokens=self._task_state.total_tokens, total_steps=self._task_state.total_steps, status=WorkflowRunStatus.STOPPED, - error='Workflow stopped.' + error='Workflow stopped.', + conversation_id=conversation_id, ) latest_node_execution_info = self._task_state.latest_node_execution_info @@ -531,7 +569,9 @@ def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceed total_tokens=self._task_state.total_tokens, total_steps=self._task_state.total_steps, status=WorkflowRunStatus.FAILED, - error=event.error + error=event.error, + conversation_id=conversation_id, + tracing_instance=tracing_instance, ) else: if self._task_state.latest_node_execution_info: @@ -546,7 +586,9 @@ def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceed start_at=self._task_state.start_at, total_tokens=self._task_state.total_tokens, total_steps=self._task_state.total_steps, - outputs=outputs + outputs=outputs, + conversation_id=conversation_id, + tracing_instance=tracing_instance, ) self._task_state.workflow_run_id = workflow_run.id diff --git a/api/core/callback_handler/agent_tool_callback_handler.py b/api/core/callback_handler/agent_tool_callback_handler.py index ac5076cd012d0d..d70161f64c1ebf 100644 --- a/api/core/callback_handler/agent_tool_callback_handler.py +++ b/api/core/callback_handler/agent_tool_callback_handler.py @@ -3,6 +3,8 @@ from pydantic import BaseModel +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName + _TEXT_COLOR_MAPPING = { "blue": "36;1", "yellow": "33;1", @@ -51,6 +53,9 @@ def on_tool_end( tool_name: str, tool_inputs: dict[str, Any], tool_outputs: str, + message_id: Optional[str] = None, + timer: Optional[Any] = None, + tracing_instance: Optional[Any] = None, ) -> None: """If not the final action, print out observation.""" print_text("\n[on_tool_end]\n", color=self.color) @@ -59,6 +64,20 @@ def on_tool_end( print_text("Outputs: " + str(tool_outputs)[:1000] + "\n", color=self.color) print_text("\n") + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.TOOL_TRACE, + message_id=message_id, + tool_name=tool_name, + tool_inputs=tool_inputs, + tool_outputs=tool_outputs, + timer=timer, + ) + ) + def on_tool_error( self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any ) -> None: diff --git a/api/core/llm_generator/llm_generator.py b/api/core/llm_generator/llm_generator.py index 14de8649c637e7..c705cc754286c7 100644 --- a/api/core/llm_generator/llm_generator.py +++ b/api/core/llm_generator/llm_generator.py @@ -1,5 +1,6 @@ import json import logging +from typing import Optional from core.llm_generator.output_parser.errors import OutputParserException from core.llm_generator.output_parser.rule_config_generator import RuleConfigGeneratorOutputParser @@ -10,11 +11,16 @@ from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError from core.prompt.utils.prompt_template_parser import PromptTemplateParser +from extensions.ext_database import db +from models.model import Conversation +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName +from services.ops_trace.utils import measure_time class LLMGenerator: @classmethod - def generate_conversation_name(cls, tenant_id: str, query): + def generate_conversation_name(cls, tenant_id: str, query, conversation_id: Optional[str] = None): prompt = CONVERSATION_TITLE_PROMPT if len(query) > 2000: @@ -29,18 +35,19 @@ def generate_conversation_name(cls, tenant_id: str, query): tenant_id=tenant_id, model_type=ModelType.LLM, ) - prompts = [UserPromptMessage(content=prompt)] - response = model_instance.invoke_llm( - prompt_messages=prompts, - model_parameters={ - "max_tokens": 100, - "temperature": 1 - }, - stream=False - ) - answer = response.message.content + with measure_time() as timer: + response = model_instance.invoke_llm( + prompt_messages=prompts, + model_parameters={ + "max_tokens": 100, + "temperature": 1 + }, + stream=False + ) + + answer = response.message.content result_dict = json.loads(answer) answer = result_dict['Your Output'] name = answer.strip() @@ -48,6 +55,29 @@ def generate_conversation_name(cls, tenant_id: str, query): if len(name) > 75: name = name[:75] + '...' + # get tracing instance + conversation_data: Conversation = db.query(Conversation).filter(Conversation.id == conversation_id).first() + app_id = conversation_data.app_id + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id=conversation_data.message_id) + + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id, app_model_config=app_model_config + ) + + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.CONVERSATION_TRACE, + conversation_id=conversation_id, + generate_conversation_name=name, + inputs=prompt, + timer=timer, + tenant_id=tenant_id, + ) + ) + return name @classmethod diff --git a/api/core/moderation/input_moderation.py b/api/core/moderation/input_moderation.py index 8fbc0c2d5003f6..0d915f74fed64d 100644 --- a/api/core/moderation/input_moderation.py +++ b/api/core/moderation/input_moderation.py @@ -3,16 +3,21 @@ from core.app.app_config.entities import AppConfig from core.moderation.base import ModerationAction, ModerationException from core.moderation.factory import ModerationFactory +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.utils import measure_time logger = logging.getLogger(__name__) class InputModeration: - def check(self, app_id: str, - tenant_id: str, - app_config: AppConfig, - inputs: dict, - query: str) -> tuple[bool, dict, str]: + def check( + self, app_id: str, + tenant_id: str, + app_config: AppConfig, + inputs: dict, + query: str, + message_id: str, + ) -> tuple[bool, dict, str]: """ Process sensitive_word_avoidance. :param app_id: app id @@ -20,6 +25,7 @@ def check(self, app_id: str, :param app_config: app config :param inputs: inputs :param query: query + :param message_id: message id :return: """ if not app_config.sensitive_word_avoidance: @@ -35,8 +41,30 @@ def check(self, app_id: str, config=sensitive_word_avoidance_config.config ) - moderation_result = moderation_factory.moderation_for_inputs(inputs, query) + with measure_time() as timer: + moderation_result = moderation_factory.moderation_for_inputs(inputs, query) + from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName + + # get tracing instance + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id, app_model_config=app_model_config + ) + + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.MODERATION_TRACE, + message_id=message_id, + moderation_result=moderation_result, + inputs=inputs, + timer=timer + ) + ) + if not moderation_result.flagged: return False, inputs, query diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index b42a441a3f12ad..878587e09cb7af 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -23,6 +23,9 @@ from extensions.ext_database import db from models.dataset import Dataset, DatasetQuery, DocumentSegment from models.dataset import Document as DatasetDocument +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName +from services.ops_trace.utils import measure_time default_retrieval_model = { 'search_method': 'semantic_search', @@ -37,14 +40,17 @@ class DatasetRetrieval: - def retrieve(self, app_id: str, user_id: str, tenant_id: str, - model_config: ModelConfigWithCredentialsEntity, - config: DatasetEntity, - query: str, - invoke_from: InvokeFrom, - show_retrieve_source: bool, - hit_callback: DatasetIndexToolCallbackHandler, - memory: Optional[TokenBufferMemory] = None) -> Optional[str]: + def retrieve( + self, app_id: str, user_id: str, tenant_id: str, + model_config: ModelConfigWithCredentialsEntity, + config: DatasetEntity, + query: str, + invoke_from: InvokeFrom, + show_retrieve_source: bool, + hit_callback: DatasetIndexToolCallbackHandler, + message_id: str, + memory: Optional[TokenBufferMemory] = None, + ) -> Optional[str]: """ Retrieve dataset. :param app_id: app_id @@ -56,6 +62,7 @@ def retrieve(self, app_id: str, user_id: str, tenant_id: str, :param invoke_from: invoke from :param show_retrieve_source: show retrieve source :param hit_callback: hit callback + :param message_id: message id :param memory: memory :return: """ @@ -112,15 +119,20 @@ def retrieve(self, app_id: str, user_id: str, tenant_id: str, all_documents = [] user_from = 'account' if invoke_from in [InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER] else 'end_user' if retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE: - all_documents = self.single_retrieve(app_id, tenant_id, user_id, user_from, available_datasets, query, - model_instance, - model_config, planning_strategy) + all_documents = self.single_retrieve( + app_id, tenant_id, user_id, user_from, available_datasets, query, + model_instance, + model_config, planning_strategy, message_id + ) elif retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE: - all_documents = self.multiple_retrieve(app_id, tenant_id, user_id, user_from, - available_datasets, query, retrieve_config.top_k, - retrieve_config.score_threshold, - retrieve_config.reranking_model.get('reranking_provider_name'), - retrieve_config.reranking_model.get('reranking_model_name')) + all_documents = self.multiple_retrieve( + app_id, tenant_id, user_id, user_from, + available_datasets, query, retrieve_config.top_k, + retrieve_config.score_threshold, + retrieve_config.reranking_model.get('reranking_provider_name'), + retrieve_config.reranking_model.get('reranking_model_name'), + message_id, + ) document_score_list = {} for item in all_documents: @@ -188,16 +200,18 @@ def retrieve(self, app_id: str, user_id: str, tenant_id: str, return str("\n".join(document_context_list)) return '' - def single_retrieve(self, app_id: str, - tenant_id: str, - user_id: str, - user_from: str, - available_datasets: list, - query: str, - model_instance: ModelInstance, - model_config: ModelConfigWithCredentialsEntity, - planning_strategy: PlanningStrategy, - ): + def single_retrieve( + self, app_id: str, + tenant_id: str, + user_id: str, + user_from: str, + available_datasets: list, + query: str, + model_instance: ModelInstance, + model_config: ModelConfigWithCredentialsEntity, + planning_strategy: PlanningStrategy, + message_id: Optional[str] = None, + ): tools = [] for dataset in available_datasets: description = dataset.description @@ -250,27 +264,35 @@ def single_retrieve(self, app_id: str, if score_threshold_enabled: score_threshold = retrieval_model_config.get("score_threshold") - results = RetrievalService.retrieve(retrival_method=retrival_method, dataset_id=dataset.id, - query=query, - top_k=top_k, score_threshold=score_threshold, - reranking_model=reranking_model) + with measure_time() as timer: + results = RetrievalService.retrieve( + retrival_method=retrival_method, dataset_id=dataset.id, + query=query, + top_k=top_k, score_threshold=score_threshold, + reranking_model=reranking_model + ) self._on_query(query, [dataset_id], app_id, user_from, user_id) + if results: - self._on_retrival_end(results) + self._on_retrival_end(results, message_id, timer) + return results return [] - def multiple_retrieve(self, - app_id: str, - tenant_id: str, - user_id: str, - user_from: str, - available_datasets: list, - query: str, - top_k: int, - score_threshold: float, - reranking_provider_name: str, - reranking_model_name: str): + def multiple_retrieve( + self, + app_id: str, + tenant_id: str, + user_id: str, + user_from: str, + available_datasets: list, + query: str, + top_k: int, + score_threshold: float, + reranking_provider_name: str, + reranking_model_name: str, + message_id: Optional[str] = None, + ): threads = [] all_documents = [] dataset_ids = [dataset.id for dataset in available_datasets] @@ -296,15 +318,23 @@ def multiple_retrieve(self, ) rerank_runner = RerankRunner(rerank_model_instance) - all_documents = rerank_runner.run(query, all_documents, - score_threshold, - top_k) + + with measure_time() as timer: + all_documents = rerank_runner.run( + query, all_documents, + score_threshold, + top_k + ) self._on_query(query, dataset_ids, app_id, user_from, user_id) + if all_documents: - self._on_retrival_end(all_documents) + self._on_retrival_end(all_documents, message_id, timer) + return all_documents - def _on_retrival_end(self, documents: list[Document]) -> None: + def _on_retrival_end( + self, documents: list[Document], message_id: Optional[str] = None, timer: Optional[dict] = None + ) -> None: """Handle retrival end.""" for document in documents: query = db.session.query(DocumentSegment).filter( @@ -323,6 +353,23 @@ def _on_retrival_end(self, documents: list[Document]) -> None: db.session.commit() + # get tracing instance + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model_config.app_id, app_model_config=app_model_config + ) + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.DATASET_RETRIEVAL_TRACE, + message_id=message_id, + documents=documents, + timer=timer + ) + ) + def _on_query(self, query: str, dataset_ids: list[str], app_id: str, user_from: str, user_id: str) -> None: """ Handle query. diff --git a/api/core/tools/tool/workflow_tool.py b/api/core/tools/tool/workflow_tool.py index 122b663f943be3..2c362d80c47831 100644 --- a/api/core/tools/tool/workflow_tool.py +++ b/api/core/tools/tool/workflow_tool.py @@ -1,7 +1,7 @@ import json import logging from copy import deepcopy -from typing import Any, Union +from typing import Any, Optional, Union from core.file.file_obj import FileTransferMethod, FileVar from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolProviderType @@ -31,9 +31,10 @@ def tool_provider_type(self) -> ToolProviderType: :return: the tool provider type """ return ToolProviderType.WORKFLOW - - def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \ - -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + def _invoke( + self, user_id: str, tool_parameters: dict[str, Any], tracing_instance: Optional[Any] = None + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: """ invoke the tool """ @@ -56,6 +57,7 @@ def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \ invoke_from=self.runtime.invoke_from, stream=False, call_depth=self.workflow_call_depth + 1, + tracing_instance=tracing_instance, ) data = result.get('data', {}) diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py index 16fe9051e3b34a..bf96461cc1447e 100644 --- a/api/core/tools/tool_engine.py +++ b/api/core/tools/tool_engine.py @@ -1,7 +1,7 @@ from copy import deepcopy from datetime import datetime, timezone from mimetypes import guess_type -from typing import Union +from typing import Optional, Union from yarl import URL @@ -24,6 +24,7 @@ from core.tools.utils.message_transformer import ToolFileMessageTransformer from extensions.ext_database import db from models.model import Message, MessageFile +from services.ops_trace.base_trace_instance import BaseTraceInstance class ToolEngine: @@ -31,10 +32,12 @@ class ToolEngine: Tool runtime engine take care of the tool executions. """ @staticmethod - def agent_invoke(tool: Tool, tool_parameters: Union[str, dict], - user_id: str, tenant_id: str, message: Message, invoke_from: InvokeFrom, - agent_tool_callback: DifyAgentCallbackHandler) \ - -> tuple[str, list[tuple[MessageFile, bool]], ToolInvokeMeta]: + def agent_invoke( + tool: Tool, tool_parameters: Union[str, dict], + user_id: str, tenant_id: str, message: Message, invoke_from: InvokeFrom, + agent_tool_callback: DifyAgentCallbackHandler, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> tuple[str, list[tuple[MessageFile, bool]], ToolInvokeMeta]: """ Agent invokes the tool with the given arguments. """ @@ -82,9 +85,11 @@ def agent_invoke(tool: Tool, tool_parameters: Union[str, dict], # hit the callback handler agent_tool_callback.on_tool_end( - tool_name=tool.identity.name, - tool_inputs=tool_parameters, - tool_outputs=plain_text + tool_name=tool.identity.name, + tool_inputs=tool_parameters, + tool_outputs=plain_text, + message_id=message.id, + tracing_instance=tracing_instance, ) # transform tool invoke message to get LLM friendly message @@ -120,8 +125,9 @@ def agent_invoke(tool: Tool, tool_parameters: Union[str, dict], def workflow_invoke(tool: Tool, tool_parameters: dict, user_id: str, workflow_id: str, workflow_tool_callback: DifyWorkflowCallbackHandler, - workflow_call_depth: int) \ - -> list[ToolInvokeMessage]: + workflow_call_depth: int, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> list[ToolInvokeMessage]: """ Workflow invokes the tool with the given arguments. """ @@ -139,9 +145,10 @@ def workflow_invoke(tool: Tool, tool_parameters: dict, # hit the callback handler workflow_tool_callback.on_tool_end( - tool_name=tool.identity.name, - tool_inputs=tool_parameters, - tool_outputs=response + tool_name=tool.identity.name, + tool_inputs=tool_parameters, + tool_outputs=response, + tracing_instance=tracing_instance, ) return response diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 2a472fc8d2cbf4..42d8648594ad5f 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -13,7 +13,9 @@ from core.workflow.nodes.base_node import BaseNode from core.workflow.nodes.tool.entities import ToolNodeData from core.workflow.utils.variable_template_parser import VariableTemplateParser -from models.workflow import WorkflowNodeExecutionStatus +from extensions.ext_database import db +from models.workflow import Workflow, WorkflowNodeExecutionStatus +from services.ops_trace.ops_trace_service import OpsTraceService class ToolNode(BaseNode): @@ -54,6 +56,11 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: # get parameters parameters = self._generate_parameters(variable_pool, node_data, tool_runtime) + # get tracing instance + workflow: Workflow = db.session.query(Workflow).filter(Workflow.id == self.workflow_id).first() + app_id = workflow.app_id + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + try: messages = ToolEngine.workflow_invoke( tool=tool_runtime, @@ -62,6 +69,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: workflow_id=self.workflow_id, workflow_tool_callback=DifyWorkflowCallbackHandler(), workflow_call_depth=self.workflow_call_depth, + tracing_instance=tracing_instance ) except Exception as e: return NodeRunResult( diff --git a/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py new file mode 100644 index 00000000000000..cffd36a0cd3ee8 --- /dev/null +++ b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py @@ -0,0 +1,49 @@ +"""update AppModelConfig and add table TracingAppConfig + +Revision ID: 04c602f5dc9b +Revises: 4e99a8df00ff +Create Date: 2024-06-12 07:49:07.666510 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '04c602f5dc9b' +down_revision = '4e99a8df00ff' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tracing_app_configs', + sa.Column('id', models.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('app_id', models.StringUUID(), nullable=False), + sa.Column('tracing_provider', sa.String(length=255), nullable=True), + sa.Column('tracing_config', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.add_column(sa.Column('trace_config', sa.Text(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.drop_column('trace_config') + + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + op.drop_table('tracing_app_configs') + # ### end Alembic commands ### diff --git a/api/models/model.py b/api/models/model.py index 657db5a5c2e274..d290776f2f6186 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -6,7 +6,7 @@ from flask import current_app, request from flask_login import UserMixin -from sqlalchemy import Float, text +from sqlalchemy import Float, func, text from core.file.tool_file_parser import ToolFileParser from core.file.upload_file_parser import UploadFileParser @@ -233,6 +233,7 @@ class AppModelConfig(db.Model): dataset_configs = db.Column(db.Text) external_data_tools = db.Column(db.Text) file_upload = db.Column(db.Text) + trace_config = db.Column(db.Text) @property def app(self): @@ -1327,3 +1328,36 @@ class TagBinding(db.Model): target_id = db.Column(StringUUID, nullable=True) created_by = db.Column(StringUUID, nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + + +class TracingAppConfig(db.Model): + __tablename__ = 'tracing_app_configs' + __table_args__ = ( + db.PrimaryKeyConstraint('id', name='tracing_app_config_pkey'), + db.Index('tracing_app_config_app_id_idx', 'app_id'), + ) + + id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) + app_id = db.Column(StringUUID, nullable=False) + tracing_provider = db.Column(db.String(255), nullable=True) + tracing_config = db.Column(db.JSON, nullable=True) + created_at = db.Column(db.DateTime, nullable=False, server_default=func.now()) + updated_at = db.Column(db.DateTime, nullable=False, server_default=func.now(), onupdate=func.now()) + + @property + def tracing_config_dict(self): + return self.tracing_config if self.tracing_config else {} + + @property + def tracing_config_str(self): + return json.dumps(self.tracing_config_dict) + + def to_dict(self): + return { + 'id': self.id, + 'app_id': self.app_id, + 'tracing_provider': self.tracing_provider, + 'tracing_config': self.tracing_config_dict, + "created_at": self.created_at.__str__() if self.created_at else None, + 'updated_at': self.updated_at.__str__() if self.updated_at else None, + } diff --git a/api/poetry.lock b/api/poetry.lock index 88d05038b705a8..862f452e9378c6 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -3884,6 +3884,46 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "langfuse" +version = "2.36.1" +description = "A client library for accessing langfuse" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langfuse-2.36.1-py3-none-any.whl", hash = "sha256:2cba244024bdf63aa3295bffada6b257497f9d18a3ca09026595470f46e09e5b"}, + {file = "langfuse-2.36.1.tar.gz", hash = "sha256:5117038d35d4629cfa37e8d89986132c926c52c05631cdfef4f9d54bc05591d8"}, +] + +[package.dependencies] +backoff = ">=1.10.0" +httpx = ">=0.15.4,<1.0" +idna = ">=3.7,<4.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1.10.7,<3.0" +wrapt = ">=1.14,<2.0" + +[package.extras] +langchain = ["langchain (>=0.0.309)"] +llama-index = ["llama-index (>=0.10.12,<2.0.0)"] +openai = ["openai (>=0.27.8)"] + +[[package]] +name = "langsmith" +version = "0.1.77" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.77-py3-none-any.whl", hash = "sha256:2202cc21b1ed7e7b9e5d2af2694be28898afa048c09fdf09f620cbd9301755ae"}, + {file = "langsmith-0.1.77.tar.gz", hash = "sha256:4ace09077a9a4e412afeb4b517ca68e7de7b07f36e4792dc8236ac5207c0c0c7"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = ">=1,<3" +requests = ">=2,<3" + [[package]] name = "llvmlite" version = "0.42.0" @@ -8921,4 +8961,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "7f9fada276f5050d1418e973f830c690807fc7f37e05c5f2e31319522daf0323" +content-hash = "e967aa4b61dc7c40f2f50eb325038da1dc0ff633d8f778e7a7560bdabce744dc" diff --git a/api/pyproject.toml b/api/pyproject.toml index 4f6f0651b382e8..9017da423617c2 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -187,6 +187,8 @@ tcvectordb = "1.3.2" chromadb = "~0.5.0" tenacity = "~8.3.0" cos-python-sdk-v5 = "1.9.30" +langfuse = "^2.36.1" +langsmith = "^0.1.77" [tool.poetry.group.dev] optional = true diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index f73a6dcbb686b1..f73a88fdd11451 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -18,7 +18,8 @@ def generate(cls, app_model: App, user: Union[Account, EndUser], args: Any, invoke_from: InvokeFrom, - streaming: bool = True) -> Union[dict, Generator[dict, None, None]]: + streaming: bool = True, + ) -> Union[dict, Generator[dict, None, None]]: """ App Content Generate :param app_model: app model diff --git a/api/services/conversation_service.py b/api/services/conversation_service.py index 5c2fb83b7249e5..44a264087cbd69 100644 --- a/api/services/conversation_service.py +++ b/api/services/conversation_service.py @@ -96,7 +96,7 @@ def auto_generate_name(cls, app_model: App, conversation: Conversation): # generate conversation name try: - name = LLMGenerator.generate_conversation_name(app_model.tenant_id, message.query) + name = LLMGenerator.generate_conversation_name(app_model.tenant_id, message.query, conversation.id) conversation.name = name except: pass diff --git a/api/services/message_service.py b/api/services/message_service.py index e826dcc6bf1455..49555e55588a77 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -19,6 +19,9 @@ MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError, ) +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName +from services.ops_trace.utils import measure_time from services.workflow_service import WorkflowService @@ -262,9 +265,27 @@ def get_suggested_questions_after_answer(cls, app_model: App, user: Optional[Uni message_limit=3, ) - questions = LLMGenerator.generate_suggested_questions_after_answer( - tenant_id=app_model.tenant_id, - histories=histories + with measure_time() as timer: + questions = LLMGenerator.generate_suggested_questions_after_answer( + tenant_id=app_model.tenant_id, + histories=histories + ) + + # get tracing instance + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model_config.app_id, app_model_config=app_model_config ) + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.SUGGESTED_QUESTION_TRACE, + message_id=message_id, + suggested_question=questions, + timer=timer + ) + ) return questions diff --git a/api/services/ops_trace/base_trace_instance.py b/api/services/ops_trace/base_trace_instance.py new file mode 100644 index 00000000000000..d785c09b75cd79 --- /dev/null +++ b/api/services/ops_trace/base_trace_instance.py @@ -0,0 +1,31 @@ +from abc import ABC, abstractmethod + + +class BaseTraceInstance(ABC): + @abstractmethod + def __init__(self): + ... + + @abstractmethod + def message_trace(self, **kwargs): + return kwargs + + @abstractmethod + def moderation_trace(self, **kwargs): + return kwargs + + @abstractmethod + def suggested_question_trace(self, **kwargs): + return kwargs + + @abstractmethod + def dataset_retrieval_trace(self, **kwargs): + return kwargs + + @abstractmethod + def tool_trace(self, **kwargs): + return kwargs + + @abstractmethod + def generate_name_trace(self, **kwargs): + return kwargs diff --git a/api/services/ops_trace/langfuse_trace.py b/api/services/ops_trace/langfuse_trace.py new file mode 100644 index 00000000000000..af6badbd96e626 --- /dev/null +++ b/api/services/ops_trace/langfuse_trace.py @@ -0,0 +1,712 @@ +import json +import os +from datetime import datetime, timedelta +from enum import Enum +from typing import Any, Optional, Union + +from langfuse import Langfuse +from pydantic import BaseModel, Field, field_validator +from pydantic_core.core_schema import ValidationInfo + +from core.moderation.base import ModerationInputsResult +from extensions.ext_database import db +from models.dataset import Document +from models.model import Message, MessageAgentThought, MessageFile +from models.workflow import WorkflowNodeExecution, WorkflowRun +from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.utils import filter_none_values + + +def validate_input_output(v, field_name): + """ + Validate input output + :param v: + :param field_name: + :return: + """ + if v == {} or v is None: + return v + if isinstance(v, str): + return {field_name: v} + elif isinstance(v, list): + if len(v) > 0 and isinstance(v[0], dict): + return {"message": v} + else: + return {field_name: v} + return v + + +class LevelEnum(str, Enum): + DEBUG = "DEBUG" + WARNING = "WARNING" + ERROR = "ERROR" + DEFAULT = "DEFAULT" + + +class LangfuseTrace(BaseModel): + """ + Langfuse trace model + """ + id: Optional[str] = Field( + default=None, + description="The id of the trace can be set, defaults to a random id. Used to link traces to external systems " + "or when creating a distributed trace. Traces are upserted on id.", + ) + name: Optional[str] = Field( + default=None, + description="Identifier of the trace. Useful for sorting/filtering in the UI.", + ) + input: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The input of the trace. Can be any JSON object." + ) + output: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The output of the trace. Can be any JSON object." + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="Additional metadata of the trace. Can be any JSON object. Metadata is merged when being updated " + "via the API.", + ) + user_id: Optional[str] = Field( + default=None, + description="The id of the user that triggered the execution. Used to provide user-level analytics.", + ) + session_id: Optional[str] = Field( + default=None, + description="Used to group multiple traces into a session in Langfuse. Use your own session/thread identifier.", + ) + version: Optional[str] = Field( + default=None, + description="The version of the trace type. Used to understand how changes to the trace type affect metrics. " + "Useful in debugging.", + ) + release: Optional[str] = Field( + default=None, + description="The release identifier of the current deployment. Used to understand how changes of different " + "deployments affect metrics. Useful in debugging.", + ) + tags: Optional[list[str]] = Field( + default=None, + description="Tags are used to categorize or label traces. Traces can be filtered by tags in the UI and GET " + "API. Tags can also be changed in the UI. Tags are merged and never deleted via the API.", + ) + public: Optional[bool] = Field( + default=None, + description="You can make a trace public to share it via a public link. This allows others to view the trace " + "without needing to log in or be members of your Langfuse project.", + ) + + @field_validator("input", "output") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + return validate_input_output(v, field_name) + + +class LangfuseSpan(BaseModel): + """ + Langfuse span model + """ + id: Optional[str] = Field( + default=None, + description="The id of the span can be set, otherwise a random id is generated. Spans are upserted on id.", + ) + session_id: Optional[str] = Field( + default=None, + description="Used to group multiple spans into a session in Langfuse. Use your own session/thread identifier.", + ) + trace_id: Optional[str] = Field( + default=None, + description="The id of the trace the span belongs to. Used to link spans to traces.", + ) + user_id: Optional[str] = Field( + default=None, + description="The id of the user that triggered the execution. Used to provide user-level analytics.", + ) + start_time: Optional[datetime | str] = Field( + default_factory=datetime.now, + description="The time at which the span started, defaults to the current time.", + ) + end_time: Optional[datetime | str] = Field( + default=None, + description="The time at which the span ended. Automatically set by span.end().", + ) + name: Optional[str] = Field( + default=None, + description="Identifier of the span. Useful for sorting/filtering in the UI.", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="Additional metadata of the span. Can be any JSON object. Metadata is merged when being updated " + "via the API.", + ) + level: Optional[str] = Field( + default=None, + description="The level of the span. Can be DEBUG, DEFAULT, WARNING or ERROR. Used for sorting/filtering of " + "traces with elevated error levels and for highlighting in the UI.", + ) + status_message: Optional[str] = Field( + default=None, + description="The status message of the span. Additional field for context of the event. E.g. the error " + "message of an error event.", + ) + input: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The input of the span. Can be any JSON object." + ) + output: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The output of the span. Can be any JSON object." + ) + version: Optional[str] = Field( + default=None, + description="The version of the span type. Used to understand how changes to the span type affect metrics. " + "Useful in debugging.", + ) + + @field_validator("input", "output") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + return validate_input_output(v, field_name) + + +class UnitEnum(str, Enum): + CHARACTERS = "CHARACTERS" + TOKENS = "TOKENS" + SECONDS = "SECONDS" + MILLISECONDS = "MILLISECONDS" + IMAGES = "IMAGES" + + +class GenerationUsage(BaseModel): + promptTokens: Optional[int] = None + completionTokens: Optional[int] = None + totalTokens: Optional[int] = None + input: Optional[int] = None + output: Optional[int] = None + total: Optional[int] = None + unit: Optional[UnitEnum] = None + inputCost: Optional[float] = None + outputCost: Optional[float] = None + totalCost: Optional[float] = None + + +class LangfuseGeneration(BaseModel): + id: Optional[str] = Field( + default=None, + description="The id of the generation can be set, defaults to random id.", + ) + trace_id: Optional[str] = Field( + default=None, + description="The id of the trace the generation belongs to. Used to link generations to traces.", + ) + parent_observation_id: Optional[str] = Field( + default=None, + description="The id of the observation the generation belongs to. Used to link generations to observations.", + ) + name: Optional[str] = Field( + default=None, + description="Identifier of the generation. Useful for sorting/filtering in the UI.", + ) + start_time: Optional[datetime | str] = Field( + default_factory=datetime.now, + description="The time at which the generation started, defaults to the current time.", + ) + completion_start_time: Optional[datetime | str] = Field( + default=None, + description="The time at which the completion started (streaming). Set it to get latency analytics broken " + "down into time until completion started and completion duration.", + ) + end_time: Optional[datetime | str] = Field( + default=None, + description="The time at which the generation ended. Automatically set by generation.end().", + ) + model: Optional[str] = Field( + default=None, description="The name of the model used for the generation." + ) + model_parameters: Optional[dict[str, Any]] = Field( + default=None, + description="The parameters of the model used for the generation; can be any key-value pairs.", + ) + input: Optional[Any] = Field( + default=None, + description="The prompt used for the generation. Can be any string or JSON object.", + ) + output: Optional[Any] = Field( + default=None, + description="The completion generated by the model. Can be any string or JSON object.", + ) + usage: Optional[GenerationUsage] = Field( + default=None, + description="The usage object supports the OpenAi structure with tokens and a more generic version with " + "detailed costs and units.", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="Additional metadata of the generation. Can be any JSON object. Metadata is merged when being " + "updated via the API.", + ) + level: Optional[LevelEnum] = Field( + default=None, + description="The level of the generation. Can be DEBUG, DEFAULT, WARNING or ERROR. Used for sorting/filtering " + "of traces with elevated error levels and for highlighting in the UI.", + ) + status_message: Optional[str] = Field( + default=None, + description="The status message of the generation. Additional field for context of the event. E.g. the error " + "message of an error event.", + ) + version: Optional[str] = Field( + default=None, + description="The version of the generation type. Used to understand how changes to the span type affect " + "metrics. Useful in debugging.", + ) + + @field_validator("input", "output") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + return validate_input_output(v, field_name) + + +class LangFuseDataTrace(BaseTraceInstance): + def __init__( + self, + langfuse_client_public_key: str = None, + langfuse_client_secret_key: str = None, + langfuse_client_host: str = "https://cloud.langfuse.com", + ): + super().__init__() + self.langfuse_client = Langfuse( + public_key=langfuse_client_public_key, + secret_key=langfuse_client_secret_key, + host=langfuse_client_host, + ) + self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001") + + def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): + conversion_id = kwargs.get("conversation_id") + workflow_id = workflow_run.workflow_id + tenant_id = workflow_run.tenant_id + workflow_run_id = workflow_run.id + workflow_run_elapsed_time = workflow_run.elapsed_time + workflow_run_status = workflow_run.status + workflow_run_inputs = ( + json.loads(workflow_run.inputs) if workflow_run.inputs else {} + ) + workflow_run_outputs = ( + json.loads(workflow_run.outputs) if workflow_run.outputs else {} + ) + workflow_run_version = workflow_run.version + error = workflow_run.error if workflow_run.error else "" + + total_tokens = workflow_run.total_tokens + + file_list = workflow_run_inputs.get("sys.file") if workflow_run_inputs.get("sys.file") else [] + query = workflow_run_inputs.get("query") or workflow_run_inputs.get("sys.query") or "" + + metadata = { + "workflow_id": workflow_id, + "conversation_id": conversion_id, + "workflow_run_id": workflow_run_id, + "tenant_id": tenant_id, + "elapsed_time": workflow_run_elapsed_time, + "status": workflow_run_status, + "version": workflow_run_version, + "total_tokens": total_tokens, + "file_list": file_list, + } + + trace_data = LangfuseTrace( + id=workflow_run_id, + name=f"workflow_{workflow_run_id}", + user_id=tenant_id, + input=query, + output=workflow_run_outputs, + metadata=metadata, + session_id=conversion_id, + tags=["workflow"], + ) + + self.add_trace(langfuse_trace_data=trace_data) + + # through workflow_run_id get all_nodes_execution + workflow_nodes_executions = ( + db.session.query(WorkflowNodeExecution) + .filter(WorkflowNodeExecution.workflow_run_id == workflow_run_id) + .order_by(WorkflowNodeExecution.created_at) + .all() + ) + + for node_execution in workflow_nodes_executions: + node_execution_id = node_execution.id + tenant_id = node_execution.tenant_id + app_id = node_execution.app_id + node_name = node_execution.title + node_type = node_execution.node_type + status = node_execution.status + inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} + outputs = ( + json.loads(node_execution.outputs) if node_execution.outputs else {} + ) + created_at = node_execution.created_at if node_execution.created_at else datetime.now() + finished_at = node_execution.finished_at if node_execution.finished_at else datetime.now() + metadata = json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} + metadata.update( + { + "workflow_run_id": workflow_run_id, + "node_execution_id": node_execution_id, + "tenant_id": tenant_id, + "app_id": app_id, + "node_name": node_name, + "node_type": node_type, + "status": status, + } + ) + + process_data = json.loads(node_execution.process_data) if node_execution.process_data else {} + if process_data and process_data.get("model_mode") == "chat": + # add generation + node_total_tokens = json.loads(node_execution.execution_metadata).get("total_tokens") + generation_usage = GenerationUsage( + totalTokens=node_total_tokens, + ) + + langfuse_generation_data = LangfuseGeneration( + name=f"{node_name}_{node_execution_id}", + trace_id=workflow_run_id, + start_time=created_at, + end_time=finished_at, + input=inputs, + output=outputs, + metadata=metadata, + level=LevelEnum.DEFAULT if status == 'succeeded' else LevelEnum.ERROR, + status_message=error if error else "", + usage=generation_usage, + ) + + self.add_generation(langfuse_generation_data) + + # add span + span_data = LangfuseSpan( + name=f"{node_name}_{node_execution_id}", + input=inputs, + output=outputs, + trace_id=workflow_run_id, + start_time=created_at, + end_time=finished_at, + metadata=metadata, + level=LevelEnum.DEFAULT if status == 'succeeded' else LevelEnum.ERROR, + status_message=error if error else "", + ) + + self.add_span(langfuse_span_data=span_data) + + def message_trace(self, message_id: str, conversation_id: str, **kwargs): + message_data = kwargs.get("message_data") + conversation_mode = kwargs.get("conversation_mode") + message_tokens = message_data.message_tokens + answer_tokens = message_data.answer_tokens + total_tokens = message_tokens + answer_tokens + error = message_data.error if message_data.error else "" + input = message_data.message + file_list = input[0].get("files", []) + provider_response_latency = message_data.provider_response_latency + created_at = message_data.created_at + end_time = created_at + timedelta(seconds=provider_response_latency) + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else "" + file_list.append(file_url) + + metadata = { + "conversation_id": conversation_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + + trace_data = LangfuseTrace( + id=message_id, + user_id=message_data.from_end_user_id if message_data.from_end_user_id else message_data.from_account_id, + name=f"message_{message_id}", + input={ + "message": input, + "files": file_list, + "message_tokens": message_tokens, + "answer_tokens": answer_tokens, + "total_tokens": total_tokens, + "error": error, + "provider_response_latency": provider_response_latency, + "created_at": created_at, + }, + output=message_data.answer, + metadata=metadata, + session_id=conversation_id, + tags=["message", str(conversation_mode)], + ) + self.add_trace(langfuse_trace_data=trace_data) + + # start add span + generation_usage = GenerationUsage( + totalTokens=total_tokens, + input=message_tokens, + output=answer_tokens, + total=total_tokens, + unit=UnitEnum.TOKENS, + ) + + langfuse_generation_data = LangfuseGeneration( + name=f"generation_{message_id}", + trace_id=message_id, + start_time=created_at, + end_time=end_time, + model=message_data.model_id, + input=input, + output=message_data.answer, + metadata=metadata, + level=LevelEnum.DEFAULT if message_data.status != 'error' else LevelEnum.ERROR, + status_message=message_data.error if message_data.error else "", + usage=generation_usage, + ) + + self.add_generation(langfuse_generation_data) + + def moderation_trace(self, message_id: str, moderation_result: ModerationInputsResult, **kwargs): + inputs = kwargs.get("inputs") + message_data = kwargs.get("message_data") + flagged = moderation_result.flagged + action = moderation_result.action + preset_response = moderation_result.preset_response + query = moderation_result.query + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "message_id": message_id, + "action": action, + "preset_response": preset_response, + "query": query, + } + + span_data = LangfuseSpan( + name="moderation", + input=inputs, + output={ + "action": action, + "flagged": flagged, + "preset_response": preset_response, + "inputs": inputs, + }, + trace_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.created_at, + metadata=metadata, + ) + + self.add_span(langfuse_span_data=span_data) + + def suggested_question_trace(self, message_id: str, suggested_question: str, **kwargs): + message_data = kwargs.get("message_data") + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + inputs = message_data.query + + metadata = { + "message_id": message_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + + span_data = LangfuseSpan( + name="suggested_question", + input=inputs, + output=suggested_question, + trace_id=message_id, + start_time=start_time, + end_time=end_time, + metadata=metadata, + level=LevelEnum.DEFAULT if message_data.status != 'error' else LevelEnum.ERROR, + status_message=message_data.error if message_data.error else "", + ) + + self.add_span(langfuse_span_data=span_data) + + def dataset_retrieval_trace(self, message_id: str, documents: list[Document], **kwargs): + message_data = kwargs.get("message_data") + inputs = message_data.query if message_data.query else message_data.inputs + metadata = { + "message_id": message_id, + "documents": documents + } + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + dataset_retrieval_span_data = LangfuseSpan( + name="dataset_retrieval", + input=inputs, + output={"documents": documents}, + trace_id=message_id, + start_time=start_time, + end_time=end_time, + metadata=metadata, + ) + + self.add_span(langfuse_span_data=dataset_retrieval_span_data) + + def tool_trace(self, message_id: str, tool_name: str, tool_inputs: dict[str, Any], tool_outputs: str, **kwargs): + message_data: Message = kwargs.get("message_data") + created_time = message_data.created_at + end_time = message_data.updated_at + tool_config = {} + time_cost = 0 + error = None + tool_parameters = {} + + agent_thoughts: list[MessageAgentThought] = message_data.agent_thoughts + for agent_thought in agent_thoughts: + if tool_name in agent_thought.tools: + created_time = agent_thought.created_at + tool_meta_data = agent_thought.tool_meta.get(tool_name, {}) + tool_config = tool_meta_data.get('tool_config', {}) + time_cost = tool_meta_data.get('time_cost', 0) + end_time = created_time + timedelta(seconds=time_cost) + error = tool_meta_data.get('error', "") + tool_parameters = tool_meta_data.get('tool_parameters', {}) + + metadata = { + "message_id": message_id, + "tool_name": tool_name, + "tool_inputs": tool_inputs, + "tool_outputs": tool_outputs, + "tool_config": tool_config, + "time_cost": time_cost, + "error": error, + "tool_parameters": tool_parameters, + } + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + if message_file_data: + message_file_id = message_file_data.id if message_file_data else None + type = message_file_data.type + created_by_role = message_file_data.created_by_role + created_user_id = message_file_data.created_by + + metadata.update( + { + "message_file_id": message_file_id, + "created_by_role": created_by_role, + "created_user_id": created_user_id, + "type": type, + } + ) + + tool_span_data = LangfuseSpan( + name=tool_name, + input=tool_inputs, + output=tool_outputs, + trace_id=message_id, + start_time=created_time, + end_time=end_time, + metadata=metadata, + level=LevelEnum.DEFAULT if error == "" else LevelEnum.ERROR, + status_message=error, + ) + + self.add_span(langfuse_span_data=tool_span_data) + + def generate_name_trace(self, conversation_id: str, inputs: str, generate_conversation_name: str, **kwargs): + timer = kwargs.get("timer") + tenant_id = kwargs.get("tenant_id") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "conversation_id": conversation_id, + } + + name_generation_trace_data = LangfuseTrace( + name="generate_name", + input=inputs, + output=generate_conversation_name, + user_id=tenant_id, + metadata=metadata, + session_id=conversation_id, + ) + + self.add_trace(langfuse_trace_data=name_generation_trace_data) + + name_generation_span_data = LangfuseSpan( + name="generate_name", + input=inputs, + output=generate_conversation_name, + trace_id=conversation_id, + start_time=start_time, + end_time=end_time, + metadata=metadata, + ) + self.add_span(langfuse_span_data=name_generation_span_data) + + def add_trace(self, langfuse_trace_data: Optional[LangfuseTrace] = None): + format_trace_data = ( + filter_none_values(langfuse_trace_data.model_dump()) if langfuse_trace_data else {} + ) + try: + self.langfuse_client.trace(**format_trace_data) + print("LangFuse Trace created successfully") + except Exception as e: + raise f"LangFuse Failed to create trace: {str(e)}" + + def add_span(self, langfuse_span_data: Optional[LangfuseSpan] = None): + format_span_data = ( + filter_none_values(langfuse_span_data.model_dump()) if langfuse_span_data else {} + ) + try: + self.langfuse_client.span(**format_span_data) + print("LangFuse Span created successfully") + except Exception as e: + raise f"LangFuse Failed to create span: {str(e)}" + + def update_span(self, span, langfuse_span_data: Optional[LangfuseSpan] = None): + format_span_data = ( + filter_none_values(langfuse_span_data.model_dump()) if langfuse_span_data else {} + ) + + span.end(**format_span_data) + + def add_generation( + self, langfuse_generation_data: Optional[LangfuseGeneration] = None + ): + format_generation_data = ( + filter_none_values(langfuse_generation_data.model_dump()) + if langfuse_generation_data + else {} + ) + try: + self.langfuse_client.generation(**format_generation_data) + print("LangFuse Generation created successfully") + except Exception as e: + raise f"LangFuse Failed to create generation: {str(e)}" + + def update_generation( + self, generation, langfuse_generation_data: Optional[LangfuseGeneration] = None + ): + format_generation_data = ( + filter_none_values(langfuse_generation_data.model_dump()) + if langfuse_generation_data + else {} + ) + + generation.end(**format_generation_data) diff --git a/api/services/ops_trace/langsmith_trace.py b/api/services/ops_trace/langsmith_trace.py new file mode 100644 index 00000000000000..ef0afdfeb91681 --- /dev/null +++ b/api/services/ops_trace/langsmith_trace.py @@ -0,0 +1,545 @@ +import json +import os +from datetime import datetime, timedelta +from enum import Enum +from typing import Any, Optional, Union + +from langsmith import Client +from pydantic import BaseModel, Field, field_validator +from pydantic_core.core_schema import ValidationInfo + +from core.moderation.base import ModerationInputsResult +from extensions.ext_database import db +from models.dataset import Document +from models.model import Message, MessageAgentThought, MessageFile +from models.workflow import WorkflowNodeExecution, WorkflowRun +from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.utils import filter_none_values + + +class LangSmithRunType(str, Enum): + tool = "tool" + chain = "chain" + llm = "llm" + retriever = "retriever" + embedding = "embedding" + prompt = "prompt" + parser = "parser" + + +class LangSmithTokenUsage(BaseModel): + input_tokens: Optional[int] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + +class LangSmithMultiModel(BaseModel): + file_list: Optional[list[str]] = Field(None, description="List of files") + + +class LangSmithRunModel(LangSmithTokenUsage, LangSmithMultiModel): + name: Optional[str] = Field(..., description="Name of the run") + inputs: Optional[Union[str, dict[str, Any], list, None]] = Field(None, description="Inputs of the run") + outputs: Optional[Union[str, dict[str, Any], list, None]] = Field(None, description="Outputs of the run") + run_type: LangSmithRunType = Field(..., description="Type of the run") + start_time: Optional[datetime | str] = Field(None, description="Start time of the run") + end_time: Optional[datetime | str] = Field(None, description="End time of the run") + extra: Optional[dict[str, Any]] = Field( + None, description="Extra information of the run" + ) + error: Optional[str] = Field(None, description="Error message of the run") + serialized: Optional[dict[str, Any]] = Field( + None, description="Serialized data of the run" + ) + parent_run_id: Optional[str] = Field(None, description="Parent run ID") + events: Optional[list[dict[str, Any]]] = Field( + None, description="Events associated with the run" + ) + tags: Optional[list[str]] = Field(None, description="Tags associated with the run") + trace_id: Optional[str] = Field( + None, description="Trace ID associated with the run" + ) + dotted_order: Optional[str] = Field(None, description="Dotted order of the run") + id: Optional[str] = Field(None, description="ID of the run") + session_id: Optional[str] = Field( + None, description="Session ID associated with the run" + ) + session_name: Optional[str] = Field( + None, description="Session name associated with the run" + ) + reference_example_id: Optional[str] = Field( + None, description="Reference example ID associated with the run" + ) + input_attachments: Optional[dict[str, Any]] = Field( + None, description="Input attachments of the run" + ) + output_attachments: Optional[dict[str, Any]] = Field( + None, description="Output attachments of the run" + ) + + @field_validator("inputs", "outputs") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + values = info.data + if v == {} or v is None: + return v + usage_metadata = { + "input_tokens": values.get('input_tokens', 0), + "output_tokens": values.get('output_tokens', 0), + "total_tokens": values.get('total_tokens', 0), + } + file_list = values.get("file_list", []) + if isinstance(v, str): + return { + field_name: v, + "file_list": file_list, + "usage_metadata": usage_metadata, + } + elif isinstance(v, list): + if len(v) > 0 and isinstance(v[0], dict): + data = { + "message": v, + "usage_metadata": usage_metadata, + "file_list": file_list, + } + return data + else: + return {field_name: v} + if isinstance(v, dict): + v["usage_metadata"] = usage_metadata + v["file_list"] = file_list + return v + return v + + @field_validator("start_time", "end_time") + def format_time(cls, v, info: ValidationInfo): + if not isinstance(v, datetime): + raise ValueError(f"{info.field_name} must be a datetime object") + else: + return v.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +class LangSmithRunUpdateModel(BaseModel): + run_id: str = Field(..., description="ID of the run") + trace_id: Optional[str] = Field( + None, description="Trace ID associated with the run" + ) + dotted_order: Optional[str] = Field(None, description="Dotted order of the run") + parent_run_id: Optional[str] = Field(None, description="Parent run ID") + end_time: Optional[datetime | str] = Field(None, description="End time of the run") + error: Optional[str] = Field(None, description="Error message of the run") + inputs: Optional[dict[str, Any]] = Field(None, description="Inputs of the run") + outputs: Optional[dict[str, Any]] = Field(None, description="Outputs of the run") + events: Optional[list[dict[str, Any]]] = Field( + None, description="Events associated with the run" + ) + tags: Optional[list[str]] = Field(None, description="Tags associated with the run") + extra: Optional[dict[str, Any]] = Field( + None, description="Extra information of the run" + ) + input_attachments: Optional[dict[str, Any]] = Field( + None, description="Input attachments of the run" + ) + output_attachments: Optional[dict[str, Any]] = Field( + None, description="Output attachments of the run" + ) + + +class LangSmithDataTrace(BaseTraceInstance): + def __init__( + self, + langsmith_key: str = None, + project_name: str = None, + endpoint: str = "https://api.smith.langchain.com" + ): + super().__init__() + self.langsmith_key = langsmith_key + self.project_name = project_name + self.project_id = None + self.langsmith_client = Client( + api_key=langsmith_key, api_url=endpoint + ) + self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001") + + def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): + conversion_id = kwargs.get("conversation_id") + workflow_id = workflow_run.workflow_id + tenant_id = workflow_run.tenant_id + workflow_run_id = workflow_run.id + workflow_run_created_at = workflow_run.created_at + workflow_run_finished_at = workflow_run.finished_at + workflow_run_elapsed_time = workflow_run.elapsed_time + workflow_run_status = workflow_run.status + workflow_run_inputs = ( + json.loads(workflow_run.inputs) if workflow_run.inputs else {} + ) + workflow_run_outputs = ( + json.loads(workflow_run.outputs) if workflow_run.outputs else {} + ) + workflow_run_version = workflow_run.version + error = workflow_run.error if workflow_run.error else "" + + total_tokens = workflow_run.total_tokens + + file_list = workflow_run_inputs.get("sys.file") if workflow_run_inputs.get("sys.file") else [] + query = workflow_run_inputs.get("query") or workflow_run_inputs.get("sys.query") or "" + + metadata = { + "workflow_id": workflow_id, + "conversation_id": conversion_id, + "workflow_run_id": workflow_run_id, + "tenant_id": tenant_id, + "elapsed_time": workflow_run_elapsed_time, + "status": workflow_run_status, + "version": workflow_run_version, + "total_tokens": total_tokens, + } + + langsmith_run = LangSmithRunModel( + file_list=file_list, + total_tokens=total_tokens, + id=workflow_run_id, + name=f"workflow_run_{workflow_run_id}", + inputs=query, + run_type=LangSmithRunType.tool, + start_time=workflow_run_created_at, + end_time=workflow_run_finished_at, + outputs=workflow_run_outputs, + extra={ + "metadata": metadata, + }, + error=error, + tags=["workflow"], + ) + + self.add_run(langsmith_run) + + # through workflow_run_id get all_nodes_execution + workflow_nodes_executions = ( + db.session.query(WorkflowNodeExecution) + .filter(WorkflowNodeExecution.workflow_run_id == workflow_run_id) + .order_by(WorkflowNodeExecution.created_at) + .all() + ) + + for node_execution in workflow_nodes_executions: + node_execution_id = node_execution.id + tenant_id = node_execution.tenant_id + app_id = node_execution.app_id + node_name = node_execution.title + node_type = node_execution.node_type + status = node_execution.status + inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} + outputs = ( + json.loads(node_execution.outputs) if node_execution.outputs else {} + ) + created_at = node_execution.created_at if node_execution.created_at else datetime.now() + finished_at = node_execution.finished_at if node_execution.finished_at else datetime.now() + execution_metadata = ( + json.loads(node_execution.execution_metadata) + if node_execution.execution_metadata + else {} + ) + node_total_tokens = execution_metadata.get("total_tokens", 0) + + metadata = json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} + metadata.update( + { + "node_execution_id": node_execution_id, + "tenant_id": tenant_id, + "app_id": app_id, + "app_name": node_name, + "node_type": node_type, + "status": status, + } + ) + + process_data = json.loads(node_execution.process_data) if node_execution.process_data else {} + if process_data and process_data.get("model_mode") == "chat": + run_type = LangSmithRunType.llm + elif node_type == "knowledge-retrieval": + run_type = LangSmithRunType.retriever + else: + run_type = LangSmithRunType.tool + + langsmith_run = LangSmithRunModel( + total_tokens=node_total_tokens, + name=f"{node_name}_{node_execution_id}", + inputs=inputs, + run_type=run_type, + start_time=created_at, + end_time=finished_at, + outputs=outputs, + file_list=file_list, + extra={ + "metadata": metadata, + }, + parent_run_id=workflow_run_id, + tags=["node_execution"], + ) + + self.add_run(langsmith_run) + + def message_trace(self, message_id: str, conversation_id: str, **kwargs): + message_data = kwargs.get("message_data") + conversation_mode = kwargs.get("conversation_mode") + message_tokens = message_data.message_tokens + answer_tokens = message_data.answer_tokens + total_tokens = message_tokens + answer_tokens + error = message_data.error if message_data.error else "" + inputs = message_data.message + file_list = inputs[0].get("files", []) + provider_response_latency = message_data.provider_response_latency + created_at = message_data.created_at + end_time = created_at + timedelta(seconds=provider_response_latency) + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else "" + file_list.append(file_url) + + metadata = { + "conversation_id": conversation_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + message_run = LangSmithRunModel( + input_tokens=message_tokens, + output_tokens=answer_tokens, + total_tokens=total_tokens, + id=message_id, + name=f"message_{message_id}", + inputs=inputs, + run_type=LangSmithRunType.llm, + start_time=created_at, + end_time=end_time, + outputs=message_data.answer, + extra={ + "metadata": metadata, + }, + tags=["message", str(conversation_mode)], + error=error, + file_list=file_list, + ) + self.add_run(message_run) + + def moderation_trace(self, message_id: str, moderation_result: ModerationInputsResult, **kwargs): + inputs = kwargs.get("inputs") + message_data = kwargs.get("message_data") + flagged = moderation_result.flagged + action = moderation_result.action + preset_response = moderation_result.preset_response + query = moderation_result.query + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "message_id": message_id, + "action": action, + "preset_response": preset_response, + "query": query, + } + + langsmith_run = LangSmithRunModel( + name="moderation", + inputs=inputs, + outputs={ + "action": action, + "flagged": flagged, + "preset_response": preset_response, + "inputs": inputs, + }, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["moderation"], + parent_run_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.updated_at, + ) + + self.add_run(langsmith_run) + + def suggested_question_trace(self, message_id: str, suggested_question: str, **kwargs): + message_data = kwargs.get("message_data") + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + inputs = message_data.query + + metadata = { + "message_id": message_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + + suggested_question_run = LangSmithRunModel( + name="suggested_question", + inputs=inputs, + outputs=suggested_question, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["suggested_question"], + parent_run_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.updated_at, + ) + + self.add_run(suggested_question_run) + + def dataset_retrieval_trace(self, message_id: str, documents: list[Document], **kwargs): + message_data = kwargs.get("message_data") + inputs = message_data.query if message_data.query else message_data.inputs + metadata = { + "message_id": message_id, + "documents": documents + } + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + dataset_retrieval_run = LangSmithRunModel( + name="dataset_retrieval", + inputs=inputs, + outputs={"documents": documents}, + run_type=LangSmithRunType.retriever, + extra={ + "metadata": metadata, + }, + tags=["dataset_retrieval"], + parent_run_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.updated_at, + ) + + self.add_run(dataset_retrieval_run) + + def tool_trace(self, message_id: str, tool_name: str, tool_inputs: dict[str, Any], tool_outputs: str, **kwargs): + message_data: Message = kwargs.get("message_data") + created_time = message_data.created_at + end_time = message_data.updated_at + tool_config = {} + time_cost = 0 + error = "" + tool_parameters = {} + file_url = "" + + agent_thoughts: list[MessageAgentThought] = message_data.agent_thoughts + for agent_thought in agent_thoughts: + if tool_name in agent_thought.tools: + created_time = agent_thought.created_at + tool_meta_data = agent_thought.tool_meta.get(tool_name, {}) + tool_config = tool_meta_data.get('tool_config', {}) + time_cost = tool_meta_data.get('time_cost', 0) + end_time = created_time + timedelta(seconds=time_cost) + error = tool_meta_data.get('error', "") + tool_parameters = tool_meta_data.get('tool_parameters', {}) + + metadata = { + "message_id": message_id, + "tool_name": tool_name, + "tool_inputs": tool_inputs, + "tool_outputs": tool_outputs, + "tool_config": tool_config, + "time_cost": time_cost, + "error": error, + "tool_parameters": tool_parameters, + } + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + if message_file_data: + message_file_id = message_file_data.id if message_file_data else None + type = message_file_data.type + created_by_role = message_file_data.created_by_role + created_user_id = message_file_data.created_by + file_url = f"{self.file_base_url}/{message_file_data.url}" + + metadata.update( + { + "message_file_id": message_file_id, + "created_by_role": created_by_role, + "created_user_id": created_user_id, + "type": type, + } + ) + + tool_run = LangSmithRunModel( + name=tool_name, + inputs=tool_inputs, + outputs=tool_outputs, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["tool", tool_name], + parent_run_id=message_id, + start_time=created_time, + end_time=end_time, + file_list=[file_url], + ) + + self.add_run(tool_run) + + def generate_name_trace(self, conversation_id: str, inputs: str, generate_conversation_name: str, **kwargs): + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "conversation_id": conversation_id, + } + + name_run = LangSmithRunModel( + name="generate_name", + inputs=inputs, + outputs=generate_conversation_name, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["generate_name"], + start_time=start_time or datetime.now(), + end_time=end_time or datetime.now(), + ) + + self.add_run(name_run) + + def add_run(self, run_data: LangSmithRunModel): + data = run_data.model_dump() + if self.project_id: + data["session_id"] = self.project_id + elif self.project_name: + data["session_name"] = self.project_name + + data = filter_none_values(data) + try: + self.langsmith_client.create_run(**data) + print("LangSmith Run created successfully.") + except Exception as e: + raise f"LangSmith Failed to create run: {str(e)}" + + def update_run(self, update_run_data: LangSmithRunUpdateModel): + data = update_run_data.model_dump() + data = filter_none_values(data) + try: + self.langsmith_client.update_run(**data) + print("LangSmith Run updated successfully.") + except Exception as e: + raise f"LangSmith Failed to update run: {str(e)}" diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py new file mode 100644 index 00000000000000..8072ca3948a830 --- /dev/null +++ b/api/services/ops_trace/ops_trace_service.py @@ -0,0 +1,321 @@ +import json +from enum import Enum +from typing import Optional + +from pydantic import BaseModel + +from core.app.app_config.entities import AppAdditionalFeatures +from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token +from extensions.ext_database import db +from models.model import App, AppModelConfig, Conversation, Message, TracingAppConfig +from models.workflow import Workflow +from services.ops_trace.langfuse_trace import LangFuseDataTrace +from services.ops_trace.langsmith_trace import LangSmithDataTrace + + +class TracingProviderEnum(Enum): + LANGFUSE = 'langfuse' + LANGSMITH = 'langSmith' + + +class LangfuseConfig(BaseModel): + """ + Model class for Langfuse tracing config. + """ + public_key: str + secret_key: str + host: str + + +class LangsmithConfig(BaseModel): + """ + Model class for Langsmith tracing config. + """ + api_key: str + project: str + endpoint: str + + +class OpsTraceService: + @classmethod + def get_tracing_app_config(cls, app_id: str, tracing_provider: str): + """ + Get tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :return: + """ + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config_data: + return None + + # decrypt_token and obfuscated_token + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + decrypt_tracing_config = cls.decrypt_tracing_config(tenant_id, tracing_provider, trace_config_data.tracing_config) + decrypt_tracing_config = cls.obfuscated_decrypt_token(tracing_provider, decrypt_tracing_config) + + trace_config_data.tracing_config = decrypt_tracing_config + + return trace_config_data.to_dict() + + @classmethod + def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_config: dict): + """ + Create tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + # check if trace config already exists + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if trace_config_data: + return None + + # get tenant id + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) + trace_config_data = TracingAppConfig( + app_id=app_id, + tracing_provider=tracing_provider, + tracing_config=tracing_config, + ) + db.session.add(trace_config_data) + db.session.commit() + + return trace_config_data.to_dict() + + @classmethod + def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_config: dict): + """ + Update tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + # check if trace config already exists + trace_config = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config: + return None + + # get tenant id + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) + + trace_config.tracing_config = tracing_config + db.session.commit() + + return trace_config.to_dict() + + @classmethod + def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): + """ + Encrypt tracing config + :param tenant_id: tenant id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + tracing_config = LangfuseConfig(**tracing_config) + encrypt_public_key = encrypt_token(tenant_id, tracing_config.public_key) + encrypt_secret_key = encrypt_token(tenant_id, tracing_config.secret_key) + tracing_config = LangfuseConfig( + public_key=encrypt_public_key, + secret_key=encrypt_secret_key, + host=tracing_config.host + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + tracing_config = LangsmithConfig(**tracing_config) + encrypt_api_key = encrypt_token(tenant_id, tracing_config.api_key) + tracing_config = LangsmithConfig( + api_key=encrypt_api_key, + project=tracing_config.project, + endpoint=tracing_config.endpoint + ) + + if isinstance(tracing_config, BaseModel): + return tracing_config.dict() + return tracing_config + + @classmethod + def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): + """ + Decrypt tracing config + :param tenant_id: tenant id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + tracing_config = LangfuseConfig(**tracing_config) + decrypt_public_key = decrypt_token(tenant_id, tracing_config.public_key) + decrypt_secret_key = decrypt_token(tenant_id, tracing_config.secret_key) + tracing_config = LangfuseConfig( + public_key=decrypt_public_key, + secret_key=decrypt_secret_key, + host=tracing_config.host + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + tracing_config = LangsmithConfig(**tracing_config) + decrypt_api_key = decrypt_token(tenant_id, tracing_config.api_key) + tracing_config = LangsmithConfig( + api_key=decrypt_api_key, + project=tracing_config.project, + endpoint=tracing_config.endpoint + ) + + if isinstance(tracing_config, BaseModel): + return tracing_config.dict() + return tracing_config + + @classmethod + def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config:dict): + """ + Decrypt tracing config + :param tracing_provider: tracing provider + :param decrypt_tracing_config: tracing config + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + decrypt_tracing_config = LangfuseConfig(**decrypt_tracing_config) + decrypt_public_key = decrypt_tracing_config.public_key + decrypt_secret_key = decrypt_tracing_config.secret_key + obfuscated_public_key = obfuscated_token(decrypt_public_key) + obfuscated_secret_key = obfuscated_token(decrypt_secret_key) + decrypt_tracing_config = LangfuseConfig( + public_key=obfuscated_public_key, + secret_key=obfuscated_secret_key, + host=decrypt_tracing_config.host + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + decrypt_tracing_config = LangsmithConfig(**decrypt_tracing_config) + decrypt_api_key = decrypt_tracing_config.api_key + obfuscated_api_key = obfuscated_token(decrypt_api_key) + decrypt_tracing_config = LangsmithConfig( + api_key=obfuscated_api_key, + project=decrypt_tracing_config.project, + endpoint=decrypt_tracing_config.endpoint + ) + + return decrypt_tracing_config.dict() + + @classmethod + def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): + """ + Get decrypted tracing config + :param app_id: app id + :param tracing_provider: tracing provider + :return: + """ + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config_data: + return None + + # decrypt_token + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + decrypt_tracing_config = cls.decrypt_tracing_config( + tenant_id, tracing_provider, trace_config_data.tracing_config + ) + + return decrypt_tracing_config + + @classmethod + def get_ops_trace_instance( + cls, + app_id: str, + workflow: Optional[Workflow] = None, + app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None + ): + """ + Get ops trace through model config + :param app_id: app_id + :param workflow: workflow + :param app_model_config: app_model_config + :return: + """ + tracing_instance = None + app_ops_trace_config = None + + # get trace configuration from available sources + if app_model_config is not None: + if isinstance(app_model_config, AppAdditionalFeatures): + app_ops_trace_config = app_model_config.trace_config + elif isinstance(app_model_config, AppModelConfig): + app_ops_trace_config = json.loads( + app_model_config.trace_config + ) if app_model_config.trace_config else None + elif workflow: + features_data = json.loads(workflow.features) + app_ops_trace_config = features_data.get('trace_config') if features_data else None + else: + # As a last resort, fetch from the database + trace_config_data = db.session.query(AppModelConfig.trace_config).filter( + AppModelConfig.app_id == app_id + ).order_by(AppModelConfig.updated_at.desc()).first() + if trace_config_data: + app_ops_trace_config = json.loads(trace_config_data.trace_config) + else: + raise ValueError('Trace config not found') + + if app_ops_trace_config is not None: + tracing_provider = app_ops_trace_config.get('tracing_provider') + else: + return None + + # decrypt_token + decrypt_trace_config = cls.get_decrypted_tracing_config(app_id, tracing_provider) + if app_ops_trace_config.get('enabled'): + tracing_provider = tracing_provider + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + langfuse_client_public_key = decrypt_trace_config.get('public_key') + langfuse_client_secret_key = decrypt_trace_config.get('secret_key') + langfuse_host = decrypt_trace_config.get('host') + tracing_instance = LangFuseDataTrace( + langfuse_client_public_key, + langfuse_client_secret_key, + langfuse_host, + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + langsmith_api_key = decrypt_trace_config.get('api_key') + langsmith_project = decrypt_trace_config.get('project') + langsmith_endpoint = decrypt_trace_config.get('endpoint') + print(langsmith_api_key, langsmith_project, langsmith_endpoint) + tracing_instance = LangSmithDataTrace( + langsmith_api_key, + langsmith_project, + langsmith_endpoint, + ) + + return tracing_instance + + return None + + @classmethod + def get_app_config_through_message_id(cls, message_id: str): + app_model_config = None + message_data = db.session.query(Message).filter(Message.id == message_id).first() + conversation_id = message_data.conversation_id + conversation_data = db.session.query(Conversation).filter(Conversation.id == conversation_id).first() + + if conversation_data.app_model_config_id: + app_model_config = db.session.query(AppModelConfig).filter( + AppModelConfig.id == conversation_data.app_model_config_id + ).first() + elif conversation_data.app_model_config_id is None and conversation_data.override_model_configs: + app_model_config = conversation_data.override_model_configs + + return app_model_config diff --git a/api/services/ops_trace/trace_queue_manager.py b/api/services/ops_trace/trace_queue_manager.py new file mode 100644 index 00000000000000..bb65d7e1a2cde0 --- /dev/null +++ b/api/services/ops_trace/trace_queue_manager.py @@ -0,0 +1,133 @@ +import queue +import threading +from enum import Enum + +from extensions.ext_database import db +from models.model import Conversation, MessageFile +from services.ops_trace.utils import get_message_data + + +class TraceTaskName(str, Enum): + CONVERSATION_TRACE = 'conversation_trace' + WORKFLOW_TRACE = 'workflow_trace' + MESSAGE_TRACE = 'message_trace' + MODERATION_TRACE = 'moderation_trace' + SUGGESTED_QUESTION_TRACE = 'suggested_question_trace' + DATASET_RETRIEVAL_TRACE = 'dataset_retrieval_trace' + TOOL_TRACE = 'tool_trace' + GENERATE_NAME_TRACE = 'generate_name_trace' + + +class TraceTask: + def __init__(self, trace_instance, trace_type, **kwargs): + self.trace_instance = trace_instance + self.trace_type = trace_type + self.kwargs = kwargs + + def execute(self): + method_name, processed_kwargs = self.preprocess() + method = getattr(self.trace_instance, method_name) + method(**processed_kwargs) + + def preprocess(self): + if self.trace_type == TraceTaskName.CONVERSATION_TRACE: + return TraceTaskName.CONVERSATION_TRACE, self.process_conversation_trace(**self.kwargs) + if self.trace_type == TraceTaskName.WORKFLOW_TRACE: + return TraceTaskName.WORKFLOW_TRACE, self.process_workflow_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.MESSAGE_TRACE: + return TraceTaskName.MESSAGE_TRACE, self.process_message_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.MODERATION_TRACE: + return TraceTaskName.MODERATION_TRACE, self.process_moderation_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.SUGGESTED_QUESTION_TRACE: + return TraceTaskName.SUGGESTED_QUESTION_TRACE, self.process_suggested_question_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.DATASET_RETRIEVAL_TRACE: + return TraceTaskName.DATASET_RETRIEVAL_TRACE, self.process_dataset_retrieval_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.TOOL_TRACE: + return TraceTaskName.TOOL_TRACE, self.process_tool_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.GENERATE_NAME_TRACE: + return TraceTaskName.GENERATE_NAME_TRACE, self.process_generate_name_trace(**self.kwargs) + else: + return '', {} + + # process methods for different trace types + def process_conversation_trace(self, **kwargs): + return kwargs + + def process_workflow_trace(self, **kwargs): + return kwargs + + def process_message_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + message_file_data = db.session.query(MessageFile).filter_by(message_id=message_id).first() + conversation_mode = db.session.query(Conversation.mode).filter_by(id=message_data.conversation_id).first() + conversation_mode = conversation_mode[0] + kwargs['message_data'] = message_data + kwargs['message_file_data'] = message_file_data + kwargs['conversation_mode'] = conversation_mode + return kwargs + + def process_moderation_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + kwargs['message_data'] = message_data + return kwargs + + def process_suggested_question_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + kwargs['message_data'] = message_data + return kwargs + + def process_dataset_retrieval_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + kwargs['message_data'] = message_data + return kwargs + + def process_tool_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + message_file_data = db.session.query(MessageFile).filter_by(message_id=message_id).first() + kwargs['message_data'] = message_data + kwargs['message_file_data'] = message_file_data + return kwargs + + def process_generate_name_trace(self, **kwargs): + return kwargs + + +class TraceQueueManager: + def __init__(self): + from app import app + self.app = app + self.queue = queue.Queue() + self.is_running = True + self.thread = threading.Thread(target=self.process_queue) + self.thread.start() + + def stop(self): + self.is_running = False + + def process_queue(self): + with self.app.app_context(): + while self.is_running: + try: + task = self.queue.get(timeout=1) + task.execute() + self.queue.task_done() + except queue.Empty: + self.stop() + + def add_trace_task(self, trace_task): + self.queue.put(trace_task) diff --git a/api/services/ops_trace/utils.py b/api/services/ops_trace/utils.py new file mode 100644 index 00000000000000..c556947380d9e5 --- /dev/null +++ b/api/services/ops_trace/utils.py @@ -0,0 +1,28 @@ +from contextlib import contextmanager +from datetime import datetime + +from extensions.ext_database import db +from models.model import Message + + +def filter_none_values(data: dict): + for key, value in data.items(): + if value is None: + continue + if isinstance(value, datetime): + data[key] = value.isoformat() + return {key: value for key, value in data.items() if value is not None} + + +def get_message_data(message_id): + return db.session.query(Message).filter(Message.id == message_id).first() + + +@contextmanager +def measure_time(): + timing_info = {'start': datetime.now(), 'end': None} + try: + yield timing_info + finally: + timing_info['end'] = datetime.now() + print(f"Execution time: {timing_info['end'] - timing_info['start']}") \ No newline at end of file From ebe8012678ba91a15e724d6a642353bed962c8d1 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Mon, 17 Jun 2024 10:33:57 +0800 Subject: [PATCH 213/273] feat: add remove tracing app --- api/controllers/console/app/ops_trace.py | 20 ++++++++++++++++++++ api/services/ops_trace/ops_trace_service.py | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index c4b3641b14ffec..d421a913ac09e5 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -75,5 +75,25 @@ def put(self, app_id): except Exception as e: raise e + @setup_required + @login_required + @account_initialization_required + def delete(self, app_id): + """Delete an existing trace app configuration""" + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='args') + args = parser.parse_args() + + try: + result = OpsTraceService.delete_tracing_app_config( + app_id=app_id, + tracing_provider=args['tracing_provider'] + ) + if not result: + raise TracingConfigNotExist() + return {"result": "success"} + except Exception as e: + raise e + api.add_resource(TraceAppConfigApi, '/apps/<uuid:app_id>/trace-config') diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 8072ca3948a830..0b604367d4c907 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -117,6 +117,26 @@ def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c return trace_config.to_dict() + @classmethod + def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): + """ + Delete tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :return: + """ + trace_config = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config: + return None + + db.session.delete(trace_config) + db.session.commit() + + return True + @classmethod def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): """ @@ -279,7 +299,6 @@ def get_ops_trace_instance( # decrypt_token decrypt_trace_config = cls.get_decrypted_tracing_config(app_id, tracing_provider) if app_ops_trace_config.get('enabled'): - tracing_provider = tracing_provider if tracing_provider == TracingProviderEnum.LANGFUSE.value: langfuse_client_public_key = decrypt_trace_config.get('public_key') langfuse_client_secret_key = decrypt_trace_config.get('secret_key') From 79867d37b7ffb3dccd13b39ea8830f4f47a46b4b Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Mon, 17 Jun 2024 18:09:45 +0800 Subject: [PATCH 214/273] feat: update trace table --- ...9b_update_appmodelconfig_and_add_table_.py | 2 +- .../versions/2a3aebbbf4bb_add_app_tracing.py | 39 +++++++++++ ...9_remove_app_model_config_trace_config_.py | 66 +++++++++++++++++++ api/models/model.py | 8 ++- api/services/ops_trace/ops_trace_service.py | 28 ++++---- 5 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py create mode 100644 api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py diff --git a/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py index cffd36a0cd3ee8..316a44edce0c55 100644 --- a/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py +++ b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = '04c602f5dc9b' -down_revision = '4e99a8df00ff' +down_revision = '7b45942e39bb' branch_labels = None depends_on = None diff --git a/api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py b/api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py new file mode 100644 index 00000000000000..09ef5e186cd089 --- /dev/null +++ b/api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py @@ -0,0 +1,39 @@ +"""add app tracing + +Revision ID: 2a3aebbbf4bb +Revises: c031d46af369 +Create Date: 2024-06-17 10:08:54.803701 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '2a3aebbbf4bb' +down_revision = 'c031d46af369' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('apps', schema=None) as batch_op: + batch_op.add_column(sa.Column('tracing', sa.Text(), nullable=True)) + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + with op.batch_alter_table('apps', schema=None) as batch_op: + batch_op.drop_column('tracing') + + # ### end Alembic commands ### diff --git a/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py new file mode 100644 index 00000000000000..8d69fa86eb8487 --- /dev/null +++ b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py @@ -0,0 +1,66 @@ +"""remove app model config trace config and rename trace app config + +Revision ID: c031d46af369 +Revises: 04c602f5dc9b +Create Date: 2024-06-17 10:01:00.255189 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +import models as models + +# revision identifiers, used by Alembic. +revision = 'c031d46af369' +down_revision = '04c602f5dc9b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('trace_app_config', + sa.Column('id', models.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('app_id', models.StringUUID(), nullable=False), + sa.Column('tracing_provider', sa.String(length=255), nullable=True), + sa.Column('tracing_config', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('is_active', sa.Boolean(), server_default=sa.text('true'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.drop_column('trace_config') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.add_column(sa.Column('trace_config', sa.TEXT(), autoincrement=False, nullable=True)) + + op.create_table('tracing_app_configs', + sa.Column('id', sa.UUID(), server_default=sa.text('uuid_generate_v4()'), autoincrement=False, nullable=False), + sa.Column('app_id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('tracing_provider', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('tracing_config', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + op.drop_table('trace_app_config') + # ### end Alembic commands ### diff --git a/api/models/model.py b/api/models/model.py index d290776f2f6186..8a6b324343e68c 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -73,6 +73,7 @@ class App(db.Model): is_demo = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) is_public = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) is_universal = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) + tracing = db.Column(db.Text, nullable=True) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) @@ -233,7 +234,6 @@ class AppModelConfig(db.Model): dataset_configs = db.Column(db.Text) external_data_tools = db.Column(db.Text) file_upload = db.Column(db.Text) - trace_config = db.Column(db.Text) @property def app(self): @@ -1330,8 +1330,8 @@ class TagBinding(db.Model): created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) -class TracingAppConfig(db.Model): - __tablename__ = 'tracing_app_configs' +class TraceAppConfig(db.Model): + __tablename__ = 'trace_app_config' __table_args__ = ( db.PrimaryKeyConstraint('id', name='tracing_app_config_pkey'), db.Index('tracing_app_config_app_id_idx', 'app_id'), @@ -1343,6 +1343,7 @@ class TracingAppConfig(db.Model): tracing_config = db.Column(db.JSON, nullable=True) created_at = db.Column(db.DateTime, nullable=False, server_default=func.now()) updated_at = db.Column(db.DateTime, nullable=False, server_default=func.now(), onupdate=func.now()) + is_active = db.Column(db.Boolean, nullable=False, server_default=db.text('true')) @property def tracing_config_dict(self): @@ -1358,6 +1359,7 @@ def to_dict(self): 'app_id': self.app_id, 'tracing_provider': self.tracing_provider, 'tracing_config': self.tracing_config_dict, + "is_active": self.is_active, "created_at": self.created_at.__str__() if self.created_at else None, 'updated_at': self.updated_at.__str__() if self.updated_at else None, } diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 0b604367d4c907..9cc88a10e4639d 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -7,7 +7,7 @@ from core.app.app_config.entities import AppAdditionalFeatures from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from extensions.ext_database import db -from models.model import App, AppModelConfig, Conversation, Message, TracingAppConfig +from models.model import App, AppModelConfig, Conversation, Message, TraceAppConfig from models.workflow import Workflow from services.ops_trace.langfuse_trace import LangFuseDataTrace from services.ops_trace.langsmith_trace import LangSmithDataTrace @@ -15,7 +15,7 @@ class TracingProviderEnum(Enum): LANGFUSE = 'langfuse' - LANGSMITH = 'langSmith' + LANGSMITH = 'langsmith' class LangfuseConfig(BaseModel): @@ -45,8 +45,8 @@ def get_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: @@ -71,8 +71,8 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :return: """ # check if trace config already exists - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if trace_config_data: @@ -81,7 +81,7 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c # get tenant id tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) - trace_config_data = TracingAppConfig( + trace_config_data = TraceAppConfig( app_id=app_id, tracing_provider=tracing_provider, tracing_config=tracing_config, @@ -101,8 +101,8 @@ def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :return: """ # check if trace config already exists - trace_config = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -125,8 +125,8 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -136,7 +136,7 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): db.session.commit() return True - + @classmethod def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): """ @@ -238,8 +238,8 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: From 32b555c4caa4c39185cdb149296b0162948bc94d Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Tue, 18 Jun 2024 12:34:59 +0800 Subject: [PATCH 215/273] feat: change table struct --- api/controllers/console/app/app.py | 45 +++++++++-- api/core/agent/cot_agent_runner.py | 39 ++++++---- api/core/agent/fc_agent_runner.py | 7 +- .../advanced_chat/generate_task_pipeline.py | 2 +- api/core/app/apps/agent_chat/app_generator.py | 1 - api/core/app/apps/chat/app_generator.py | 1 - api/core/app/apps/completion/app_generator.py | 6 +- .../apps/workflow/generate_task_pipeline.py | 2 +- api/core/llm_generator/llm_generator.py | 3 +- api/core/moderation/input_moderation.py | 3 +- api/core/rag/retrieval/dataset_retrieval.py | 5 +- api/core/workflow/nodes/tool/tool_node.py | 2 +- api/fields/app_fields.py | 1 + api/services/message_service.py | 4 +- api/services/ops_trace/ops_trace_service.py | 74 +++++++++++-------- 15 files changed, 123 insertions(+), 72 deletions(-) diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 082838334ae040..bc9a3c04300311 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -1,4 +1,3 @@ -import json import uuid from flask_login import current_user @@ -9,17 +8,14 @@ from controllers.console.app.wraps import get_app_model from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check -from core.tools.tool_manager import ToolManager -from core.tools.utils.configuration import ToolParameterConfigurationManager from fields.app_fields import ( app_detail_fields, app_detail_fields_with_site, app_pagination_fields, ) from libs.login import login_required -from models.model import App, AppMode, AppModelConfig from services.app_service import AppService -from services.tag_service import TagService +from services.ops_trace.ops_trace_service import OpsTraceService ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion'] @@ -286,6 +282,44 @@ def post(self, app_model): return app_model +class AppTraceApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, app_id): + """Get app trace""" + app_trace_config = OpsTraceService.get_app_tracing_config( + app_id=app_id + ) + if not app_trace_config: + raise BadRequest("Tracing config not found") + + return app_trace_config + + @setup_required + @login_required + @account_initialization_required + def post(self, app_id): + # add app trace + if not current_user.is_admin_or_owner: + raise Forbidden() + parser = reqparse.RequestParser() + parser.add_argument('enabled', type=bool, required=True, location='json') + parser.add_argument('tracing_provider', type=str, required=True, location='json') + args = parser.parse_args() + + try: + OpsTraceService.update_app_tracing_config( + app_id=app_id, + enabled=args['enabled'], + tracing_provider=args['tracing_provider'], + ) + except Exception as e: + raise e + + return {"result": "success"} + + api.add_resource(AppListApi, '/apps') api.add_resource(AppImportApi, '/apps/import') api.add_resource(AppApi, '/apps/<uuid:app_id>') @@ -295,3 +329,4 @@ def post(self, app_model): api.add_resource(AppIconApi, '/apps/<uuid:app_id>/icon') api.add_resource(AppSiteStatus, '/apps/<uuid:app_id>/site-enable') api.add_resource(AppApiStatus, '/apps/<uuid:app_id>/api-enable') +api.add_resource(AppTraceApi, '/apps/<uuid:app_id>/trace') diff --git a/api/core/agent/cot_agent_runner.py b/api/core/agent/cot_agent_runner.py index dfe35e8aa1825f..698c483287a220 100644 --- a/api/core/agent/cot_agent_runner.py +++ b/api/core/agent/cot_agent_runner.py @@ -1,7 +1,7 @@ import json from abc import ABC, abstractmethod from collections.abc import Generator -from typing import Union, Optional +from typing import Optional, Union from core.agent.base_agent_runner import BaseAgentRunner from core.agent.entities import AgentScratchpadUnit @@ -21,6 +21,7 @@ from core.tools.tool_engine import ToolEngine from models.model import Message from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.ops_trace_service import OpsTraceService class CotAgentRunner(BaseAgentRunner, ABC): @@ -32,7 +33,8 @@ class CotAgentRunner(BaseAgentRunner, ABC): _query: str = None _prompt_messages_tools: list[PromptMessage] = None - def run(self, message: Message, + def run( + self, message: Message, query: str, inputs: dict[str, str], ) -> Union[Generator, LLMResult]: @@ -43,6 +45,12 @@ def run(self, message: Message, self._repack_app_generate_entity(app_generate_entity) self._init_react_state(query) + # get tracing instance + app_id = self.app_config.app_id + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id + ) + # check model mode if 'Observation' not in app_generate_entity.model_conf.stop: if app_generate_entity.model_conf.provider not in self._ignore_observation_providers: @@ -214,7 +222,8 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): tool_invoke_response, tool_invoke_meta = self._handle_invoke_action( action=scratchpad.action, tool_instances=tool_instances, - message_file_ids=message_file_ids + message_file_ids=message_file_ids, + tracing_instance=tracing_instance, ) scratchpad.observation = tool_invoke_response scratchpad.agent_response = tool_invoke_response @@ -240,8 +249,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): # update prompt tool message for prompt_tool in self._prompt_messages_tools: - self.update_prompt_message_tool( - tool_instances[prompt_tool.name], prompt_tool) + self.update_prompt_message_tool(tool_instances[prompt_tool.name], prompt_tool) iteration_step += 1 @@ -278,12 +286,11 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): message=AssistantPromptMessage( content=final_answer ), - usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage( - ), + usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(), system_fingerprint='' )), PublishFrom.APPLICATION_MANAGER) - def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, + def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, tool_instances: dict[str, Tool], message_file_ids: list[str], tracing_instance: Optional[BaseTraceInstance] = None @@ -302,7 +309,7 @@ def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, if not tool_instance: answer = f"there is not a tool named {tool_call_name}" return answer, ToolInvokeMeta.error_instance(answer) - + if isinstance(tool_call_args, str): try: tool_call_args = json.loads(tool_call_args) @@ -324,8 +331,7 @@ def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, # publish files for message_file, save_as in message_files: if save_as: - self.variables_pool.set_file( - tool_name=tool_call_name, value=message_file.id, name=save_as) + self.variables_pool.set_file(tool_name=tool_call_name, value=message_file.id, name=save_as) # publish message file self.queue_manager.publish(QueueMessageFileEvent( @@ -356,7 +362,7 @@ def _fill_in_inputs_from_external_data_tools(self, instruction: str, inputs: dic continue return instruction - + def _init_react_state(self, query) -> None: """ init agent scratchpad @@ -364,7 +370,7 @@ def _init_react_state(self, query) -> None: self._query = query self._agent_scratchpad = [] self._historic_prompt_messages = self._organize_historic_prompt_messages() - + @abstractmethod def _organize_prompt_messages(self) -> list[PromptMessage]: """ @@ -396,6 +402,13 @@ def _organize_historic_prompt_messages(self, current_session_messages: list[Prom scratchpads: list[AgentScratchpadUnit] = [] current_scratchpad: AgentScratchpadUnit = None + self.history_prompt_messages = AgentHistoryPromptTransform( + model_config=self.model_config, + prompt_messages=current_session_messages or [], + history_messages=self.history_prompt_messages, + memory=self.memory + ).get_prompt() + for message in self.history_prompt_messages: if isinstance(message, AssistantPromptMessage): if not current_scratchpad: diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index e64722d22ca58d..3ddd43a771602e 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -20,8 +20,7 @@ from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform from core.tools.entities.tool_entities import ToolInvokeMeta from core.tools.tool_engine import ToolEngine -from extensions.ext_database import db -from models.model import AppModelConfig, Message +from models.model import Message from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -54,10 +53,8 @@ def run(self, # get tracing instance app_id = app_config.app_id - app_model_config_id = app_config.app_model_config_id - app_model_config = db.session.query(AppModelConfig).filter_by(id=app_model_config_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id, app_model_config=app_model_config + app_id=app_id ) def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 207e962b376efd..0d461528bdc737 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -189,7 +189,7 @@ def _process_stream_response(self, workflow: Optional[Workflow] = None) -> Gener :return: """ app_id = self._conversation.app_id - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) for message in self._queue_manager.listen(): event = message.event diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index ca8fbe138efa37..431a8b88a15d5f 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -74,7 +74,6 @@ def generate(self, app_model: App, # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( app_id=app_model.id, - app_model_config=app_model_config, ) # validate override model config diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index e333e3458cbbc8..2dde1331891cc7 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -127,7 +127,6 @@ def generate( # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( app_id=app_model.id, - app_model_config=app_model_config, ) # init queue manager diff --git a/api/core/app/apps/completion/app_generator.py b/api/core/app/apps/completion/app_generator.py index fcf00e685594dd..186a11557af565 100644 --- a/api/core/app/apps/completion/app_generator.py +++ b/api/core/app/apps/completion/app_generator.py @@ -117,8 +117,7 @@ def generate(self, app_model: App, # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model.id, - app_model_config=app_model_config, + app_id=app_model.id ) # init queue manager @@ -283,8 +282,7 @@ def generate_more_like_this(self, app_model: App, # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model.id, - app_model_config=app_model_config, + app_id=app_model.id ) # init queue manager diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 944c3736a140d1..54bd3fbf0db264 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -172,7 +172,7 @@ def _process_stream_response( Process stream response. :return: """ - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) for message in self._queue_manager.listen(): event = message.event diff --git a/api/core/llm_generator/llm_generator.py b/api/core/llm_generator/llm_generator.py index c705cc754286c7..4059f637c58e2a 100644 --- a/api/core/llm_generator/llm_generator.py +++ b/api/core/llm_generator/llm_generator.py @@ -58,10 +58,9 @@ def generate_conversation_name(cls, tenant_id: str, query, conversation_id: Opti # get tracing instance conversation_data: Conversation = db.query(Conversation).filter(Conversation.id == conversation_id).first() app_id = conversation_data.app_id - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id=conversation_data.message_id) tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id, app_model_config=app_model_config + app_id=app_id ) if tracing_instance: diff --git a/api/core/moderation/input_moderation.py b/api/core/moderation/input_moderation.py index 0d915f74fed64d..c835c97eda0e34 100644 --- a/api/core/moderation/input_moderation.py +++ b/api/core/moderation/input_moderation.py @@ -47,9 +47,8 @@ def check( from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName # get tracing instance - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id, app_model_config=app_model_config + app_id=app_id ) if tracing_instance: diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index 878587e09cb7af..7d7645b8e68fab 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -23,6 +23,7 @@ from extensions.ext_database import db from models.dataset import Dataset, DatasetQuery, DocumentSegment from models.dataset import Document as DatasetDocument +from models.model import Message from services.ops_trace.ops_trace_service import OpsTraceService from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName from services.ops_trace.utils import measure_time @@ -354,9 +355,9 @@ def _on_retrival_end( db.session.commit() # get tracing instance - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + app_id = db.session.query(Message.app_id).filter(Message.id == message_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model_config.app_id, app_model_config=app_model_config + app_id=app_id ) if tracing_instance: trace_manager = TraceQueueManager() diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 42d8648594ad5f..45a2e812197f06 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -59,7 +59,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: # get tracing instance workflow: Workflow = db.session.query(Workflow).filter(Workflow.id == self.workflow_id).first() app_id = workflow.app_id - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) try: messages = ToolEngine.workflow_invoke( diff --git a/api/fields/app_fields.py b/api/fields/app_fields.py index 212c3e7f179862..1a0a1b815d6325 100644 --- a/api/fields/app_fields.py +++ b/api/fields/app_fields.py @@ -50,6 +50,7 @@ 'enable_site': fields.Boolean, 'enable_api': fields.Boolean, 'model_config': fields.Nested(model_config_fields, attribute='app_model_config', allow_null=True), + 'tracing': fields.Raw, 'created_at': TimestampField } diff --git a/api/services/message_service.py b/api/services/message_service.py index 49555e55588a77..e43d244d58bdbd 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -272,9 +272,9 @@ def get_suggested_questions_after_answer(cls, app_model: App, user: Optional[Uni ) # get tracing instance - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + app_id = db.session.query(Message.app_id).filter(Message.id == message_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model_config.app_id, app_model_config=app_model_config + app_id=app_id, ) if tracing_instance: trace_manager = TraceQueueManager() diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 9cc88a10e4639d..e5cf1b57d63db9 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -27,7 +27,7 @@ class LangfuseConfig(BaseModel): host: str -class LangsmithConfig(BaseModel): +class LangSmithConfig(BaseModel): """ Model class for Langsmith tracing config. """ @@ -156,9 +156,9 @@ def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c host=tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangsmithConfig(**tracing_config) + tracing_config = LangSmithConfig(**tracing_config) encrypt_api_key = encrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangsmithConfig( + tracing_config = LangSmithConfig( api_key=encrypt_api_key, project=tracing_config.project, endpoint=tracing_config.endpoint @@ -187,9 +187,9 @@ def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c host=tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangsmithConfig(**tracing_config) + tracing_config = LangSmithConfig(**tracing_config) decrypt_api_key = decrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangsmithConfig( + tracing_config = LangSmithConfig( api_key=decrypt_api_key, project=tracing_config.project, endpoint=tracing_config.endpoint @@ -219,10 +219,10 @@ def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config: host=decrypt_tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - decrypt_tracing_config = LangsmithConfig(**decrypt_tracing_config) + decrypt_tracing_config = LangSmithConfig(**decrypt_tracing_config) decrypt_api_key = decrypt_tracing_config.api_key obfuscated_api_key = obfuscated_token(decrypt_api_key) - decrypt_tracing_config = LangsmithConfig( + decrypt_tracing_config = LangSmithConfig( api_key=obfuscated_api_key, project=decrypt_tracing_config.project, endpoint=decrypt_tracing_config.endpoint @@ -256,9 +256,9 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): @classmethod def get_ops_trace_instance( cls, - app_id: str, + app_id, workflow: Optional[Workflow] = None, - app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None + app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None, ): """ Get ops trace through model config @@ -268,29 +268,10 @@ def get_ops_trace_instance( :return: """ tracing_instance = None - app_ops_trace_config = None - - # get trace configuration from available sources - if app_model_config is not None: - if isinstance(app_model_config, AppAdditionalFeatures): - app_ops_trace_config = app_model_config.trace_config - elif isinstance(app_model_config, AppModelConfig): - app_ops_trace_config = json.loads( - app_model_config.trace_config - ) if app_model_config.trace_config else None - elif workflow: - features_data = json.loads(workflow.features) - app_ops_trace_config = features_data.get('trace_config') if features_data else None - else: - # As a last resort, fetch from the database - trace_config_data = db.session.query(AppModelConfig.trace_config).filter( - AppModelConfig.app_id == app_id - ).order_by(AppModelConfig.updated_at.desc()).first() - if trace_config_data: - app_ops_trace_config = json.loads(trace_config_data.trace_config) - else: - raise ValueError('Trace config not found') - + app: App = db.session.query(App).filter( + App.id == app_id + ).first() + app_ops_trace_config = json.loads(app.tracing) if app.tracing else None if app_ops_trace_config is not None: tracing_provider = app_ops_trace_config.get('tracing_provider') else: @@ -338,3 +319,32 @@ def get_app_config_through_message_id(cls, message_id: str): app_model_config = conversation_data.override_model_configs return app_model_config + + @classmethod + def update_app_tracing_config(cls, app_id: str, enabled: bool, tracing_provider: str): + """ + Update app tracing config + :param app_id: app id + :param enabled: enabled + :param tracing_provider: tracing provider + :return: + """ + app_config: App = db.session.query(App).filter(App.id == app_id).first() + app_config.tracing = json.dumps( + { + "enabled": enabled, + "tracing_provider": tracing_provider, + } + ) + db.session.commit() + + @classmethod + def get_app_tracing_config(cls, app_id: str): + """ + Get app tracing config + :param app_id: app id + :return: + """ + app: App = db.session.query(App).filter(App.id == app_id).first() + app_trace_config = json.loads(app.tracing) + return app_trace_config From 0ec3b99fb2f76e769093b16ac06532e65916274b Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Tue, 18 Jun 2024 14:17:18 +0800 Subject: [PATCH 216/273] feat: change TraceAppConfigApi request type patch --- api/controllers/console/app/ops_trace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index d421a913ac09e5..1316fb7cdbc630 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -56,7 +56,7 @@ def post(self, app_id): @setup_required @login_required @account_initialization_required - def put(self, app_id): + def patch(self, app_id): """Update an existing trace app configuration""" parser = reqparse.RequestParser() parser.add_argument('tracing_provider', type=str, required=True, location='json') From f32f587829f699ce34464de9f52f51af30c32778 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 14:57:21 +0800 Subject: [PATCH 217/273] feat: tracing status enabled --- .../[appId]/overview/tracing/panel.tsx | 56 ++++++++++++++----- .../[appId]/overview/tracing/type.ts | 2 +- web/i18n/en-US/app.ts | 2 +- web/i18n/zh-Hans/app.ts | 2 +- web/models/app.ts | 5 ++ web/service/apps.ts | 40 +++++++++++-- 6 files changed, 84 insertions(+), 23 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 81c80b411a414c..65e33f538af317 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -4,14 +4,16 @@ import React, { useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' import { usePathname } from 'next/navigation' +import { useBoolean } from 'ahooks' import { TracingProvider } from './type' import TracingIcon from './tracing-icon' import ToggleExpandBtn from './toggle-fold-btn' import ConfigButton from './config-button' import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' import Indicator from '@/app/components/header/indicator' -import { fetchTracingConfig } from '@/service/apps' -import type { TracingConfig } from '@/models/app' +import { fetchTracingConfig as doFetchTracingConfig, fetchTracingStatus, updateTracingStatus } from '@/service/apps' +import type { TracingConfig, TracingStatus } from '@/models/app' +import Toast from '@/app/components/base/toast' const I18N_PREFIX = 'app.tracing' @@ -34,24 +36,52 @@ const Panel: FC = () => { const matched = pathname.match(/\/app\/([^/]+)/) const appId = (matched?.length && matched[1]) ? matched[1] : '' - const inUseTracingProvider: TracingProvider | undefined = undefined - const [tracingConfig, setTracingConfig] = useState<TracingConfig | null>(null) + const [isLoaded, { + setTrue: setLoaded, + }] = useBoolean(false) + const [tracingStatus, setTracingStatus] = useState<TracingStatus | null>(null) + const enabled = tracingStatus?.enabled || false + const handleTracingStatusChange = async (tracingStatus: TracingStatus) => { + await updateTracingStatus({ appId, body: tracingStatus }) + setTracingStatus(tracingStatus) + Toast.notify({ + type: 'success', + message: t('common.api.success'), + }) + } + const handleTracingEnabledChange = (enabled: boolean) => { + handleTracingStatusChange({ + tracing_provider: tracingStatus?.tracing_provider || null, + enabled, + }) + } + const inUseTracingProvider: TracingProvider | null = tracingStatus?.tracing_provider || null const InUseProviderIcon = inUseTracingProvider === TracingProvider.langSmith ? LangsmithIcon : LangfuseIcon - const hasConfiguredTracing = !!inUseTracingProvider - const [isFold, setFold] = useState(false) - const [enabled, setEnabled] = useState(false) + const [langSmithConfig, setLangSmithConfig] = useState<TracingConfig | null>(null) + const [langFuseConfig, setLangFuseConfig] = useState<TracingConfig | null>(null) + const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig) + + const fetchTracingConfig = async () => { + const langSmithConfig = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) + setLangSmithConfig(langSmithConfig) + const langFuseConfig = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) + setLangFuseConfig(langFuseConfig) + } useEffect(() => { (async () => { - const tracingConfig = await fetchTracingConfig({ appId }) - setTracingConfig(tracingConfig) - // debugger + const tracingStatus = await fetchTracingStatus({ appId }) + setTracingStatus(tracingStatus) + await fetchTracingConfig() + setLoaded() })() // eslint-disable-next-line react-hooks/exhaustive-deps }, []) + const [isFold, setFold] = useState(false) + if (!isFold && !hasConfiguredTracing) { return ( <div className='mb-3'> @@ -75,7 +105,7 @@ const Panel: FC = () => { <ConfigButton hasConfigured={false} enabled={enabled} - onStatusChange={setEnabled} + onStatusChange={handleTracingEnabledChange} /> <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> </div> @@ -112,7 +142,7 @@ const Panel: FC = () => { hasConfigured className='ml-2' enabled={enabled} - onStatusChange={setEnabled} + onStatusChange={handleTracingEnabledChange} /> {!hasConfiguredTracing && ( <> @@ -120,10 +150,8 @@ const Panel: FC = () => { <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> </> )} - </div> </div> - ) } export default React.memo(Panel) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts index 69299d1c3e0b50..4e89703ccb539f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts @@ -1,5 +1,5 @@ export enum TracingProvider { - langSmith = 'langSmith', + langSmith = 'langsmith', langfuse = 'langfuse', } diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index 6b5a1df70fc2ef..e86450d9850fc6 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -96,7 +96,7 @@ const translation = { enabled: 'In Service', tracingDescription: 'Capture the full context of app execution, including LLM calls, context, prompts, HTTP requests, and more, to a third-party tracing platform.', configProviderTitle: 'Config provider to enable tracing', - langSmith: { + langsmith: { title: 'LangSmith', description: 'An all-in-one developer platform for every step of the LLM-powered application lifecycle.', }, diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index b59785014c61e5..0010784c732c9a 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -95,7 +95,7 @@ const translation = { enabled: '已启用', tracingDescription: '捕获应用程序执行的完整上下文，包括 LLM 调用、上下文、提示、HTTP 请求等，发送到第三方跟踪平台。', configProviderTitle: '配置提供商以启用追踪', - langSmith: { + langsmith: { title: 'LangSmith', description: '一个全方位的开发者平台，适用于 LLM 驱动应用程序生命周期的每个步骤。', }, diff --git a/web/models/app.ts b/web/models/app.ts index 3862684c05291f..f49fc18c76c9c9 100644 --- a/web/models/app.ts +++ b/web/models/app.ts @@ -131,6 +131,11 @@ export type AppVoicesListResponse = [{ value: string }] +export type TracingStatus = { + enabled: boolean + tracing_provider: TracingProvider | null +} + export type TracingConfig = { tracing_provider: TracingProvider tracing_config: LangSmitConfig | LangFuseConfig diff --git a/web/service/apps.ts b/web/service/apps.ts index 3dd87ba228484d..ca08292266f128 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -1,8 +1,9 @@ import type { Fetcher } from 'swr' import { del, get, post, put } from './base' -import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, TracingConfig, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' +import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, TracingConfig, TracingStatus, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' import type { CommonResponse } from '@/models/common' import type { AppMode, ModelConfig } from '@/types/app' +import { TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' export const fetchAppList: Fetcher<AppListResponse, { url: string; params?: Record<string, any> }> = ({ url, params }) => { return get<AppListResponse>(url, { params }) @@ -123,14 +124,41 @@ export const fetchAppVoices: Fetcher<AppVoicesListResponse, { appId: string; lan } // Tracing -export const fetchTracingConfig: Fetcher<TracingConfig, { appId: string }> = ({ appId }) => { +export const fetchTracingStatus: Fetcher<TracingStatus, { appId: string }> = ({ appId }) => { // return get(`/apps/${appId}/tracing-config`) return Promise.resolve({ - tracing_provider: 'langSmith', + enabled: true, + tracing_provider: 'langsmith', + }) +} + +export const updateTracingStatus: Fetcher<CommonResponse, { appId: string; body: Record<string, any> }> = ({ appId, body }) => { + return post(`/apps/${appId}/tracing-config`, { body }) + // return Promise.resolve({ result: 'success' }) +} + +export const fetchTracingConfig: Fetcher<TracingConfig, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { + // return get(`/apps/${appId}/tracing-config`, { + // params: { + // tracing_provider: provider, + // } + // }) + if (provider === TracingProvider.langSmith) { + return Promise.resolve({ + tracing_provider: 'langsmith', + tracing_config: { + api_key: '123132*********************21', + endpoint: 'https://api.langsmith.ai', + project: 'test', + }, + }) + } + return Promise.resolve({ + tracing_provider: 'langfuse', tracing_config: { - api_key: '123132*********************21', - endpoint: 'https://api.langsmith.ai', - project: 'test', + public_key: '123132*********************21', + secret_key: '888877*********55', + host: 'https://api.langfuse.ai', }, }) } From 8e6ffd3b74696355d9ada35e58d1b66b8bea13d2 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 15:33:24 +0800 Subject: [PATCH 218/273] feat: create and add config --- .../[appId]/overview/tracing/config-popup.tsx | 16 ++- .../tracing/provider-config-modal.tsx | 103 +++++++++++++++--- .../[appId]/overview/tracing/type.ts | 2 +- web/models/app.ts | 4 +- web/service/apps.ts | 18 ++- 5 files changed, 121 insertions(+), 22 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 0ff481238d45fc..52fd22e7d0d2df 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -5,6 +5,7 @@ import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import TracingIcon from './tracing-icon' import ProviderPanel from './provider-panel' +import type { LangFuseConfig, LangSmithConfig } from './type' import { TracingProvider } from './type' import ProviderConfigModal from './provider-config-modal' import Indicator from '@/app/components/header/indicator' @@ -14,12 +15,20 @@ const I18N_PREFIX = 'app.tracing' export type PopupProps = { enabled: boolean - onStatusChange?: (enabled: boolean) => void + onStatusChange: (enabled: boolean) => void + onChooseProvider: (provider: TracingProvider) => void + langSmithConfig: LangSmithConfig | null + langFuseConfig: LangFuseConfig | null + onConfigUpdated: () => void } const ConfigPopup: FC<PopupProps> = ({ enabled, onStatusChange, + onChooseProvider, + langSmithConfig, + langFuseConfig, + onConfigUpdated, }) => { const { t } = useTranslation() @@ -37,9 +46,9 @@ const ConfigPopup: FC<PopupProps> = ({ const handleOnChoose = useCallback((provider: TracingProvider) => { return () => { - console.log(provider) + onChooseProvider(provider) } - }, []) + }, [onChooseProvider]) return ( <div className='w-[420px] p-4 rounded-2xl bg-white border-[0.5px] border-black/5 shadow-lg'> @@ -83,6 +92,7 @@ const ConfigPopup: FC<PopupProps> = ({ {isShowConfigModal && ( <ProviderConfigModal type={currentProvider!} + payload={currentProvider === TracingProvider.langSmith ? langSmithConfig : langFuseConfig} onCancel={hideConfigModal} onSaved={hideConfigModal} /> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index fd3088712e8279..d329e7358e43ee 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -4,6 +4,7 @@ import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import Field from './field' +import type { LangFuseConfig, LangSmithConfig } from './type' import { TracingProvider } from './type' import { docURL } from './config' import { @@ -14,9 +15,13 @@ import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security' import Button from '@/app/components/base/button' import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general' import ConfirmUi from '@/app/components/base/confirm' +import { addTracingConfig, updateTracingConfig } from '@/service/apps' +import Toast from '@/app/components/base/toast' + type Props = { + appId: string type: TracingProvider - payload?: any + payload?: LangSmithConfig | LangFuseConfig | null onRemove?: () => void onCancel: () => void onSaved: () => void @@ -24,7 +29,20 @@ type Props = { const I18N_PREFIX = 'app.tracing.configProvider' +const langSmithConfigTemplate = { + api_key: '', + project: '', + endpoint: '', +} + +const langFuseConfigTemplate = { + public_key: '', + secret_key: '', + host: '', +} + const ProviderConfigModal: FC<Props> = ({ + appId, type, payload, onRemove, @@ -34,7 +52,15 @@ const ProviderConfigModal: FC<Props> = ({ const { t } = useTranslation() const isEdit = !!payload const [isSaving, setIsSaving] = useState(false) - const [config, setConfig] = useState<Record<string, string>>({}) + const [config, setConfig] = useState<LangSmithConfig | LangFuseConfig>((() => { + if (isEdit) + return payload + + if (type === TracingProvider.langSmith) + return langSmithConfigTemplate + + return langFuseConfigTemplate + })()) const [isShowRemoveConfirm, { setTrue: showRemoveConfirm, setFalse: hideRemoveConfirm, @@ -46,17 +72,64 @@ const ProviderConfigModal: FC<Props> = ({ }, [onRemove, hideRemoveConfirm]) const handleConfigChange = useCallback((key: string) => { - return () => { + return (value: string) => { + setConfig({ + ...config, + [key]: value, + }) + } + }, [config]) + const checkValid = useCallback(() => { + let errorMessage = '' + if (type === TracingProvider.langSmith) { + const postData = config as LangSmithConfig + if (!postData.api_key) + errorMessage = t('common.errorMsg.fieldRequired', { field: 'API Key' }) + if (!errorMessage && !postData.project) + errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.project`) }) } - }, []) + if (type === TracingProvider.langfuse) { + const postData = config as LangFuseConfig + if (!postData.public_key) + errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.publicKey`) }) + if (!errorMessage && !postData.secret_key) + errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.secretKey`) }) + } + + return errorMessage + }, [config, t, type]) const handleSave = useCallback(async () => { if (isSaving) return - - onSaved() - }, [isSaving, onSaved]) + const errorMessage = checkValid() + if (errorMessage) { + Toast.notify({ + type: 'error', + message: errorMessage, + }) + return + } + const action = isEdit ? updateTracingConfig : addTracingConfig + try { + await action({ + appId, + body: { + tracing_provider: type, + tracing_config: payload as LangSmithConfig | LangFuseConfig, + }, + }) + Toast.notify({ + type: 'success', + message: t('common.api.success'), + }) + onSaved() + } + finally { + onSaved() + } + }, [appId, checkValid, isEdit, isSaving, onSaved, payload, t, type]) return ( <> @@ -78,7 +151,7 @@ const ProviderConfigModal: FC<Props> = ({ label='API Key' labelClassName='!text-sm' isRequired - value={config.api_key} + value={(config as LangSmithConfig).api_key} onChange={handleConfigChange('api_key')} placeholder={t(`${I18N_PREFIX}.placeholder`, { key: 'API Key' })!} /> @@ -86,14 +159,14 @@ const ProviderConfigModal: FC<Props> = ({ label={t(`${I18N_PREFIX}.project`)!} labelClassName='!text-sm' isRequired - value={config.base_url} + value={(config as LangSmithConfig).project} onChange={handleConfigChange('project')} placeholder={t(`${I18N_PREFIX}.placeholder`, { key: t(`${I18N_PREFIX}.project`) })!} /> <Field label='Endpoint' labelClassName='!text-sm' - value={config.base_url} + value={(config as LangSmithConfig).endpoint} onChange={handleConfigChange('endpoint')} placeholder={'https://api.smith.langchain.com'} /> @@ -105,23 +178,23 @@ const ProviderConfigModal: FC<Props> = ({ label={t(`${I18N_PREFIX}.publicKey`)!} labelClassName='!text-sm' isRequired - value={config.api_key} + value={(config as LangFuseConfig).public_key} onChange={handleConfigChange('public_key')} placeholder={t(`${I18N_PREFIX}.placeholder`, { key: t(`${I18N_PREFIX}.publicKey`) })!} /> <Field label={t(`${I18N_PREFIX}.secretKey`)!} labelClassName='!text-sm' - value={config.base_url} + value={(config as LangFuseConfig).secret_key} isRequired - onChange={handleConfigChange('base_url')} + onChange={handleConfigChange('secret_key')} placeholder={t(`${I18N_PREFIX}.placeholder`, { key: t(`${I18N_PREFIX}.secretKey`) })!} /> <Field label='Host' labelClassName='!text-sm' - value={config.base_url} - onChange={handleConfigChange('base_url')} + value={(config as LangFuseConfig).host} + onChange={handleConfigChange('host')} placeholder='https://cloud.langfuse.com' /> </> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts index 4e89703ccb539f..e07cf37c9d26a4 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts @@ -3,7 +3,7 @@ export enum TracingProvider { langfuse = 'langfuse', } -export type LangSmitConfig = { +export type LangSmithConfig = { api_key: string project: string endpoint: string diff --git a/web/models/app.ts b/web/models/app.ts index f49fc18c76c9c9..80d121c7a3ec38 100644 --- a/web/models/app.ts +++ b/web/models/app.ts @@ -1,4 +1,4 @@ -import type { LangFuseConfig, LangSmitConfig, TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' +import type { LangFuseConfig, LangSmithConfig, TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' import type { App, AppTemplate, SiteConfig } from '@/types/app' /* export type App = { @@ -138,5 +138,5 @@ export type TracingStatus = { export type TracingConfig = { tracing_provider: TracingProvider - tracing_config: LangSmitConfig | LangFuseConfig + tracing_config: LangSmithConfig | LangFuseConfig } diff --git a/web/service/apps.ts b/web/service/apps.ts index ca08292266f128..a54a9a0b69d9b6 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -1,5 +1,5 @@ import type { Fetcher } from 'swr' -import { del, get, post, put } from './base' +import { del, get, patch, post, put } from './base' import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, TracingConfig, TracingStatus, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' import type { CommonResponse } from '@/models/common' import type { AppMode, ModelConfig } from '@/types/app' @@ -162,3 +162,19 @@ export const fetchTracingConfig: Fetcher<TracingConfig, { appId: string; provide }, }) } + +export const addTracingConfig: Fetcher<CommonResponse, { appId: string; body: TracingConfig }> = ({ appId, body }) => { + return post(`/apps/${appId}/tracing-config`, { body }) +} + +export const updateTracingConfig: Fetcher<CommonResponse, { appId: string; body: TracingConfig }> = ({ appId, body }) => { + return patch(`/apps/${appId}/tracing-config`, { body }) +} + +export const removeTracingConfig: Fetcher<CommonResponse, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { + return del(`/apps/${appId}/tracing-config`, { + body: { + tracing_provider: provider, + }, + }) +} From aa2bdd0eb4cde3ddaa80c890f1d0b30ec48b1e1c Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 16:19:48 +0800 Subject: [PATCH 219/273] feat: configged sync to panel --- .../overview/tracing/config-button.tsx | 2 - .../[appId]/overview/tracing/config-popup.tsx | 23 +++++++- .../[appId]/overview/tracing/panel.tsx | 54 ++++++++++++++++--- .../tracing/provider-config-modal.tsx | 30 +++++++---- web/service/apps.ts | 54 +++++++++---------- 5 files changed, 116 insertions(+), 47 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index 99634163314cfc..fff7ad33b70ea6 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -18,13 +18,11 @@ const I18N_PREFIX = 'app.tracing' type Props = { className?: string hasConfigured: boolean - onConfigured?: () => void } & PopupProps const ConfigBtn: FC<Props> = ({ className, hasConfigured, - onConfigured, ...popupProps }) => { const { t } = useTranslation() diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 52fd22e7d0d2df..28af5bef1d6828 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -14,21 +14,25 @@ import Switch from '@/app/components/base/switch' const I18N_PREFIX = 'app.tracing' export type PopupProps = { + appId: string enabled: boolean onStatusChange: (enabled: boolean) => void onChooseProvider: (provider: TracingProvider) => void langSmithConfig: LangSmithConfig | null langFuseConfig: LangFuseConfig | null - onConfigUpdated: () => void + onConfigUpdated: (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig) => void + onConfigRemoved: (provider: TracingProvider) => void } const ConfigPopup: FC<PopupProps> = ({ + appId, enabled, onStatusChange, onChooseProvider, langSmithConfig, langFuseConfig, onConfigUpdated, + onConfigRemoved, }) => { const { t } = useTranslation() @@ -50,6 +54,18 @@ const ConfigPopup: FC<PopupProps> = ({ } }, [onChooseProvider]) + const handleConfigUpdated = useCallback(() => { + return (payload: LangSmithConfig | LangFuseConfig) => { + onConfigUpdated(currentProvider!, payload) + hideConfigModal() + } + }, [currentProvider, hideConfigModal, onConfigUpdated]) + + const handleConfigRemoved = useCallback(() => { + onConfigRemoved(currentProvider!) + hideConfigModal() + }, [currentProvider, hideConfigModal, onConfigRemoved]) + return ( <div className='w-[420px] p-4 rounded-2xl bg-white border-[0.5px] border-black/5 shadow-lg'> <div className='flex justify-between items-center'> @@ -91,10 +107,13 @@ const ConfigPopup: FC<PopupProps> = ({ </div> {isShowConfigModal && ( <ProviderConfigModal + appId={appId} type={currentProvider!} payload={currentProvider === TracingProvider.langSmith ? langSmithConfig : langFuseConfig} onCancel={hideConfigModal} - onSaved={hideConfigModal} + onSaved={handleConfigUpdated} + onChosen={onChooseProvider} + onRemoved={handleConfigRemoved} /> )} </div> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 65e33f538af317..b2026fe82f7e0a 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -5,6 +5,7 @@ import { useTranslation } from 'react-i18next' import cn from 'classnames' import { usePathname } from 'next/navigation' import { useBoolean } from 'ahooks' +import type { LangFuseConfig, LangSmithConfig } from './type' import { TracingProvider } from './type' import TracingIcon from './tracing-icon' import ToggleExpandBtn from './toggle-fold-btn' @@ -12,7 +13,7 @@ import ConfigButton from './config-button' import { LangfuseIcon, LangsmithIcon } from '@/app/components/base/icons/src/public/tracing' import Indicator from '@/app/components/header/indicator' import { fetchTracingConfig as doFetchTracingConfig, fetchTracingStatus, updateTracingStatus } from '@/service/apps' -import type { TracingConfig, TracingStatus } from '@/models/app' +import type { TracingStatus } from '@/models/app' import Toast from '@/app/components/base/toast' const I18N_PREFIX = 'app.tracing' @@ -50,24 +51,45 @@ const Panel: FC = () => { message: t('common.api.success'), }) } + const handleTracingEnabledChange = (enabled: boolean) => { handleTracingStatusChange({ tracing_provider: tracingStatus?.tracing_provider || null, enabled, }) } + const handleChooseProvider = (provider: TracingProvider) => { + handleTracingStatusChange({ + tracing_provider: provider, + enabled, + }) + } const inUseTracingProvider: TracingProvider | null = tracingStatus?.tracing_provider || null const InUseProviderIcon = inUseTracingProvider === TracingProvider.langSmith ? LangsmithIcon : LangfuseIcon - const [langSmithConfig, setLangSmithConfig] = useState<TracingConfig | null>(null) - const [langFuseConfig, setLangFuseConfig] = useState<TracingConfig | null>(null) + const [langSmithConfig, setLangSmithConfig] = useState<LangSmithConfig | null>(null) + const [langFuseConfig, setLangFuseConfig] = useState<LangFuseConfig | null>(null) const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig) const fetchTracingConfig = async () => { - const langSmithConfig = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) - setLangSmithConfig(langSmithConfig) - const langFuseConfig = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) - setLangFuseConfig(langFuseConfig) + const { tracing_config: langSmithConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) + setLangSmithConfig(langSmithConfig as LangSmithConfig) + const { tracing_config: langFuseConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) + setLangFuseConfig(langFuseConfig as LangFuseConfig) + } + + const handleTracingConfigUpdated = (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig) => { + if (provider === TracingProvider.langSmith) + setLangSmithConfig(payload as LangSmithConfig) + else + setLangFuseConfig(payload as LangFuseConfig) + } + + const handleTracingConfigRemoved = (provider: TracingProvider) => { + if (provider === TracingProvider.langSmith) + setLangSmithConfig(null) + else + setLangFuseConfig(null) } useEffect(() => { @@ -82,6 +104,12 @@ const Panel: FC = () => { const [isFold, setFold] = useState(false) + if (!isLoaded) { + return <div className='mb-3'> + <Title /> + </div> + } + if (!isFold && !hasConfiguredTracing) { return ( <div className='mb-3'> @@ -103,9 +131,15 @@ const Panel: FC = () => { <div className='flex items-center space-x-1'> <ConfigButton + appId={appId} hasConfigured={false} enabled={enabled} onStatusChange={handleTracingEnabledChange} + onChooseProvider={handleChooseProvider} + langSmithConfig={langSmithConfig} + langFuseConfig={langFuseConfig} + onConfigUpdated={handleTracingConfigUpdated} + onConfigRemoved={handleTracingConfigRemoved} /> <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> </div> @@ -139,10 +173,16 @@ const Panel: FC = () => { )} <ConfigButton + appId={appId} hasConfigured className='ml-2' enabled={enabled} onStatusChange={handleTracingEnabledChange} + onChooseProvider={handleChooseProvider} + langSmithConfig={langSmithConfig} + langFuseConfig={langFuseConfig} + onConfigUpdated={handleTracingConfigUpdated} + onConfigRemoved={handleTracingConfigRemoved} /> {!hasConfiguredTracing && ( <> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index d329e7358e43ee..7e24ae5c4f66db 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -15,16 +15,17 @@ import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security' import Button from '@/app/components/base/button' import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general' import ConfirmUi from '@/app/components/base/confirm' -import { addTracingConfig, updateTracingConfig } from '@/service/apps' +import { addTracingConfig, removeTracingConfig, updateTracingConfig } from '@/service/apps' import Toast from '@/app/components/base/toast' type Props = { appId: string type: TracingProvider payload?: LangSmithConfig | LangFuseConfig | null - onRemove?: () => void + onRemoved: () => void onCancel: () => void - onSaved: () => void + onSaved: (payload: LangSmithConfig | LangFuseConfig) => void + onChosen: (provider: TracingProvider) => void } const I18N_PREFIX = 'app.tracing.configProvider' @@ -45,9 +46,10 @@ const ProviderConfigModal: FC<Props> = ({ appId, type, payload, - onRemove, + onRemoved, onCancel, onSaved, + onChosen, }) => { const { t } = useTranslation() const isEdit = !!payload @@ -68,8 +70,16 @@ const ProviderConfigModal: FC<Props> = ({ const handleRemove = useCallback(async () => { hideRemoveConfirm() - onRemove?.() - }, [onRemove, hideRemoveConfirm]) + await removeTracingConfig({ + appId, + provider: type, + }) + Toast.notify({ + type: 'success', + message: t('common.api.remove'), + }) + onRemoved() + }, [onRemoved, hideRemoveConfirm]) const handleConfigChange = useCallback((key: string) => { return (value: string) => { @@ -124,12 +134,14 @@ const ProviderConfigModal: FC<Props> = ({ type: 'success', message: t('common.api.success'), }) - onSaved() + onSaved(config) + if (!isEdit) + onChosen(type) } finally { - onSaved() + setIsSaving(true) } - }, [appId, checkValid, isEdit, isSaving, onSaved, payload, t, type]) + }, [appId, checkValid, config, isEdit, isSaving, onChosen, onSaved, payload, t, type]) return ( <> diff --git a/web/service/apps.ts b/web/service/apps.ts index a54a9a0b69d9b6..159962566c26ef 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -3,7 +3,7 @@ import { del, get, patch, post, put } from './base' import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, TracingConfig, TracingStatus, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' import type { CommonResponse } from '@/models/common' import type { AppMode, ModelConfig } from '@/types/app' -import { TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' +import type { TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' export const fetchAppList: Fetcher<AppListResponse, { url: string; params?: Record<string, any> }> = ({ url, params }) => { return get<AppListResponse>(url, { params }) @@ -125,11 +125,11 @@ export const fetchAppVoices: Fetcher<AppVoicesListResponse, { appId: string; lan // Tracing export const fetchTracingStatus: Fetcher<TracingStatus, { appId: string }> = ({ appId }) => { - // return get(`/apps/${appId}/tracing-config`) - return Promise.resolve({ - enabled: true, - tracing_provider: 'langsmith', - }) + return get(`/apps/${appId}/tracing-config`) + // return Promise.resolve({ + // enabled: true, + // tracing_provider: 'langsmith', + // }) } export const updateTracingStatus: Fetcher<CommonResponse, { appId: string; body: Record<string, any> }> = ({ appId, body }) => { @@ -138,29 +138,29 @@ export const updateTracingStatus: Fetcher<CommonResponse, { appId: string; body: } export const fetchTracingConfig: Fetcher<TracingConfig, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { - // return get(`/apps/${appId}/tracing-config`, { - // params: { - // tracing_provider: provider, - // } - // }) - if (provider === TracingProvider.langSmith) { - return Promise.resolve({ - tracing_provider: 'langsmith', - tracing_config: { - api_key: '123132*********************21', - endpoint: 'https://api.langsmith.ai', - project: 'test', - }, - }) - } - return Promise.resolve({ - tracing_provider: 'langfuse', - tracing_config: { - public_key: '123132*********************21', - secret_key: '888877*********55', - host: 'https://api.langfuse.ai', + return get(`/apps/${appId}/tracing-config`, { + params: { + tracing_provider: provider, }, }) + // if (provider === TracingProvider.langSmith) { + // return Promise.resolve({ + // tracing_provider: 'langsmith', + // tracing_config: { + // api_key: '123132*********************21', + // endpoint: 'https://api.langsmith.ai', + // project: 'test', + // }, + // }) + // } + // return Promise.resolve({ + // tracing_provider: 'langfuse', + // tracing_config: { + // public_key: '123132*********************21', + // secret_key: '888877*********55', + // host: 'https://api.langfuse.ai', + // }, + // }) } export const addTracingConfig: Fetcher<CommonResponse, { appId: string; body: TracingConfig }> = ({ appId, body }) => { From 3560b2c50acf5f86e17c4f41ad0b8fdc32ba9220 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 17:11:23 +0800 Subject: [PATCH 220/273] feat: handle config actrive --- .../[appId]/overview/tracing/config-popup.tsx | 7 ++- .../[appId]/overview/tracing/panel.tsx | 9 ++- .../tracing/provider-config-modal.tsx | 8 +-- .../overview/tracing/provider-panel.tsx | 28 +++++++-- web/service/apps.ts | 57 ++++++++++--------- 5 files changed, 69 insertions(+), 40 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 28af5bef1d6828..4c43c89ebc514e 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -17,6 +17,7 @@ export type PopupProps = { appId: string enabled: boolean onStatusChange: (enabled: boolean) => void + chosenProvider: TracingProvider | null onChooseProvider: (provider: TracingProvider) => void langSmithConfig: LangSmithConfig | null langFuseConfig: LangFuseConfig | null @@ -28,6 +29,7 @@ const ConfigPopup: FC<PopupProps> = ({ appId, enabled, onStatusChange, + chosenProvider, onChooseProvider, langSmithConfig, langFuseConfig, @@ -97,10 +99,13 @@ const ConfigPopup: FC<PopupProps> = ({ <ProviderPanel type={TracingProvider.langSmith} onConfig={handleOnConfig(TracingProvider.langSmith)} + isChosen={chosenProvider === TracingProvider.langSmith} onChoose={handleOnChoose(TracingProvider.langSmith)} /> - <ProviderPanel type={TracingProvider.langfuse} + <ProviderPanel + type={TracingProvider.langfuse} onConfig={handleOnConfig(TracingProvider.langfuse)} + isChosen={chosenProvider === TracingProvider.langfuse} onChoose={handleOnChoose(TracingProvider.langfuse)} /> </div> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index b2026fe82f7e0a..f022b02784f11c 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -72,10 +72,13 @@ const Panel: FC = () => { const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig) const fetchTracingConfig = async () => { + // TODO: not configured will return what const { tracing_config: langSmithConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) - setLangSmithConfig(langSmithConfig as LangSmithConfig) + if (langSmithConfig) + setLangSmithConfig(langSmithConfig as LangSmithConfig) const { tracing_config: langFuseConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) - setLangFuseConfig(langFuseConfig as LangFuseConfig) + if (langFuseConfig) + setLangFuseConfig(langFuseConfig as LangFuseConfig) } const handleTracingConfigUpdated = (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig) => { @@ -135,6 +138,7 @@ const Panel: FC = () => { hasConfigured={false} enabled={enabled} onStatusChange={handleTracingEnabledChange} + chosenProvider={inUseTracingProvider} onChooseProvider={handleChooseProvider} langSmithConfig={langSmithConfig} langFuseConfig={langFuseConfig} @@ -178,6 +182,7 @@ const Panel: FC = () => { className='ml-2' enabled={enabled} onStatusChange={handleTracingEnabledChange} + chosenProvider={inUseTracingProvider} onChooseProvider={handleChooseProvider} langSmithConfig={langSmithConfig} langFuseConfig={langFuseConfig} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 7e24ae5c4f66db..61855fbe8052b1 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -79,7 +79,7 @@ const ProviderConfigModal: FC<Props> = ({ message: t('common.api.remove'), }) onRemoved() - }, [onRemoved, hideRemoveConfirm]) + }, [hideRemoveConfirm, appId, type, t, onRemoved]) const handleConfigChange = useCallback((key: string) => { return (value: string) => { @@ -127,7 +127,7 @@ const ProviderConfigModal: FC<Props> = ({ appId, body: { tracing_provider: type, - tracing_config: payload as LangSmithConfig | LangFuseConfig, + tracing_config: config, }, }) Toast.notify({ @@ -141,7 +141,7 @@ const ProviderConfigModal: FC<Props> = ({ finally { setIsSaving(true) } - }, [appId, checkValid, config, isEdit, isSaving, onChosen, onSaved, payload, t, type]) + }, [appId, checkValid, config, isEdit, isSaving, onChosen, onSaved, t, type]) return ( <> @@ -223,7 +223,7 @@ const ProviderConfigModal: FC<Props> = ({ <LinkExternal02 className='w-3 h-3' /> </a> <div className='flex items-center'> - {!isEdit && ( + {isEdit && ( <> <Button className='h-9 text-sm font-medium text-gray-700' diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx index 7c11b2c5f4b0a3..9727ac35da2d10 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx @@ -2,15 +2,19 @@ import type { FC } from 'react' import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' +import cn from 'classnames' import { TracingProvider } from './type' import { LangfuseIconBig, LangsmithIconBig } from '@/app/components/base/icons/src/public/tracing' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' + const I18N_PREFIX = 'app.tracing' type Props = { type: TracingProvider - onConfig: () => void + isChosen: boolean onChoose: () => void + hasConfigured: boolean + onConfig: () => void } const getIcon = (type: TracingProvider) => { @@ -22,8 +26,10 @@ const getIcon = (type: TracingProvider) => { const ProviderPanel: FC<Props> = ({ type, - onConfig, + isChosen, onChoose, + hasConfigured, + onConfig, }) => { const { t } = useTranslation() const Icon = getIcon(type) @@ -32,13 +38,23 @@ const ProviderPanel: FC<Props> = ({ e.stopPropagation() onConfig() }, [onConfig]) + + const handleChosen = useCallback((e: React.MouseEvent) => { + e.stopPropagation() + if (isChosen || !hasConfigured) + return + onChoose() + }, [hasConfigured, isChosen, onChoose]) return ( <div - className='px-4 py-3 rounded-xl bg-gray-100' - onClick={onChoose} + className={cn(isChosen ? 'border-primary-400' : 'border-transparent', !isChosen && hasConfigured && 'cursor-pointer', 'px-4 py-3 rounded-xl border-[1.5px] bg-gray-100')} + onClick={handleChosen} > - <div className='flex justify-between items-center space-x-1'> - <Icon className='h-6' /> + <div className={'flex justify-between items-center space-x-1'}> + <div className='flex'> + <Icon className='h-6' /> + + </div> <div className='flex px-2 items-center h-6 bg-white rounded-md border-[0.5px] border-gray-200 shadow-xs cursor-pointer text-gray-700 space-x-1' onClick={handleConfigBtnClick} diff --git a/web/service/apps.ts b/web/service/apps.ts index 159962566c26ef..2dca0e2daff0e1 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -3,7 +3,7 @@ import { del, get, patch, post, put } from './base' import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, TracingConfig, TracingStatus, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' import type { CommonResponse } from '@/models/common' import type { AppMode, ModelConfig } from '@/types/app' -import type { TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' +import { TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' export const fetchAppList: Fetcher<AppListResponse, { url: string; params?: Record<string, any> }> = ({ url, params }) => { return get<AppListResponse>(url, { params }) @@ -125,11 +125,11 @@ export const fetchAppVoices: Fetcher<AppVoicesListResponse, { appId: string; lan // Tracing export const fetchTracingStatus: Fetcher<TracingStatus, { appId: string }> = ({ appId }) => { - return get(`/apps/${appId}/tracing-config`) - // return Promise.resolve({ - // enabled: true, - // tracing_provider: 'langsmith', - // }) + // return get(`/apps/${appId}/tracing-config`) + return Promise.resolve({ + enabled: false, + tracing_provider: 'langfuse', + }) } export const updateTracingStatus: Fetcher<CommonResponse, { appId: string; body: Record<string, any> }> = ({ appId, body }) => { @@ -138,29 +138,32 @@ export const updateTracingStatus: Fetcher<CommonResponse, { appId: string; body: } export const fetchTracingConfig: Fetcher<TracingConfig, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { - return get(`/apps/${appId}/tracing-config`, { - params: { - tracing_provider: provider, - }, - }) - // if (provider === TracingProvider.langSmith) { - // return Promise.resolve({ - // tracing_provider: 'langsmith', - // tracing_config: { - // api_key: '123132*********************21', - // endpoint: 'https://api.langsmith.ai', - // project: 'test', - // }, - // }) - // } - // return Promise.resolve({ - // tracing_provider: 'langfuse', - // tracing_config: { - // public_key: '123132*********************21', - // secret_key: '888877*********55', - // host: 'https://api.langfuse.ai', + // return get(`/apps/${appId}/tracing-config`, { + // params: { + // tracing_provider: provider, // }, // }) + return Promise.resolve({ + tracing_config: null, + }) + if (provider === TracingProvider.langSmith) { + return Promise.resolve({ + tracing_provider: 'langsmith', + tracing_config: { + api_key: '123132*********************21', + endpoint: 'https://api.langsmith.ai', + project: 'test', + }, + }) + } + return Promise.resolve({ + tracing_provider: 'langfuse', + tracing_config: { + public_key: '123132*********************21', + secret_key: '888877*********55', + host: 'https://api.langfuse.ai', + }, + }) } export const addTracingConfig: Fetcher<CommonResponse, { appId: string; body: TracingConfig }> = ({ appId, body }) => { From c4a611cc0953e78a70cb723400b53be01a251145 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 17:18:48 +0800 Subject: [PATCH 221/273] feat: in use label --- .../[appId]/overview/tracing/provider-panel.tsx | 4 ++-- web/i18n/en-US/app.ts | 1 + web/i18n/zh-Hans/app.ts | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx index 9727ac35da2d10..a4759ef0ab536d 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx @@ -51,9 +51,9 @@ const ProviderPanel: FC<Props> = ({ onClick={handleChosen} > <div className={'flex justify-between items-center space-x-1'}> - <div className='flex'> + <div className='flex items-center'> <Icon className='h-6' /> - + {isChosen && <div className='ml-1 flex items-center h-4 px-1 rounded-[4px] border border-primary-500 leading-4 text-xs font-medium text-primary-500 uppercase '>{t(`${I18N_PREFIX}.inUse`)}</div>} </div> <div className='flex px-2 items-center h-6 bg-white rounded-md border-[0.5px] border-gray-200 shadow-xs cursor-pointer text-gray-700 space-x-1' diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index e86450d9850fc6..d5498325c68529 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -104,6 +104,7 @@ const translation = { title: 'Langfuse', description: 'Traces, evals, prompt management and metrics to debug and improve your LLM application.', }, + inUse: 'In use', configProvider: { title: 'Config ', placeholder: 'Enter your {{key}}', diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 0010784c732c9a..68dcdf2d58b34b 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -103,6 +103,7 @@ const translation = { title: 'Langfuse', description: '跟踪、评估、提示管理和指标，以调试和改进您的 LLM 应用程序。', }, + inUse: '使用中', configProvider: { title: '配置 ', placeholder: '输入你的{{key}}', From 2953be57db5f8b10d7b8fe71668897c285cc0b6d Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 17:57:25 +0800 Subject: [PATCH 222/273] feat: different choose status --- .../[appId]/overview/tracing/config-popup.tsx | 60 +++++++++++---- .../[appId]/overview/tracing/panel.tsx | 9 +-- web/i18n/en-US/app.ts | 6 +- web/i18n/zh-Hans/app.ts | 6 +- web/service/apps.ts | 74 ++++++++++--------- 5 files changed, 97 insertions(+), 58 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 4c43c89ebc514e..81372cfb569529 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -68,6 +68,28 @@ const ConfigPopup: FC<PopupProps> = ({ hideConfigModal() }, [currentProvider, hideConfigModal, onConfigRemoved]) + const providerAllConfigured = langSmithConfig && langFuseConfig + const providerAllNotConfigured = !langSmithConfig && !langFuseConfig + const langSmithPanel = ( + <ProviderPanel + type={TracingProvider.langSmith} + hasConfigured={!!langSmithConfig} + onConfig={handleOnConfig(TracingProvider.langSmith)} + isChosen={chosenProvider === TracingProvider.langSmith} + onChoose={handleOnChoose(TracingProvider.langSmith)} + /> + ) + + const langfusePanel = ( + <ProviderPanel + type={TracingProvider.langfuse} + hasConfigured={!!langFuseConfig} + onConfig={handleOnConfig(TracingProvider.langfuse)} + isChosen={chosenProvider === TracingProvider.langfuse} + onChoose={handleOnChoose(TracingProvider.langfuse)} + /> + ) + return ( <div className='w-[420px] p-4 rounded-2xl bg-white border-[0.5px] border-black/5 shadow-lg'> <div className='flex justify-between items-center'> @@ -94,21 +116,29 @@ const ConfigPopup: FC<PopupProps> = ({ </div> <div className='mt-3 h-px bg-gray-100'></div> <div className='mt-3'> - <div className='leading-4 text-xs font-medium text-gray-500 uppercase'>{t(`${I18N_PREFIX}.configProviderTitle`)}</div> - <div className='mt-2 space-y-2'> - <ProviderPanel - type={TracingProvider.langSmith} - onConfig={handleOnConfig(TracingProvider.langSmith)} - isChosen={chosenProvider === TracingProvider.langSmith} - onChoose={handleOnChoose(TracingProvider.langSmith)} - /> - <ProviderPanel - type={TracingProvider.langfuse} - onConfig={handleOnConfig(TracingProvider.langfuse)} - isChosen={chosenProvider === TracingProvider.langfuse} - onChoose={handleOnChoose(TracingProvider.langfuse)} - /> - </div> + {(providerAllConfigured || providerAllNotConfigured) + ? ( + <> + <div className='leading-4 text-xs font-medium text-gray-500 uppercase'>{t(`${I18N_PREFIX}.configProviderTitle.${providerAllConfigured ? 'configured' : 'notConfigured'}`)}</div> + <div className='mt-2 space-y-2'> + {langSmithPanel} + {langfusePanel} + </div> + </> + ) + : ( + <> + <div className='leading-4 text-xs font-medium text-gray-500 uppercase'>{t(`${I18N_PREFIX}.configProviderTitle.configured`)}</div> + <div className='mt-2'> + {langSmithConfig ? langSmithPanel : langfusePanel} + </div> + <div className='mt-3 leading-4 text-xs font-medium text-gray-500 uppercase'>{t(`${I18N_PREFIX}.configProviderTitle.moreProvider`)}</div> + <div className='mt-2'> + {!langSmithConfig ? langSmithPanel : langfusePanel} + </div> + </> + )} + </div> {isShowConfigModal && ( <ProviderConfigModal diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index f022b02784f11c..48e6d56daa5e0e 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -72,12 +72,11 @@ const Panel: FC = () => { const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig) const fetchTracingConfig = async () => { - // TODO: not configured will return what - const { tracing_config: langSmithConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) - if (langSmithConfig) + const { tracing_config: langSmithConfig, has_not_configured: langSmithHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) + if (!langSmithHasNotConfig) setLangSmithConfig(langSmithConfig as LangSmithConfig) - const { tracing_config: langFuseConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) - if (langFuseConfig) + const { tracing_config: langFuseConfig, has_not_configured: langFuseHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) + if (!langFuseHasNotConfig) setLangFuseConfig(langFuseConfig as LangFuseConfig) } diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index d5498325c68529..a975f91ea27806 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -95,7 +95,11 @@ const translation = { disabled: 'Disabled', enabled: 'In Service', tracingDescription: 'Capture the full context of app execution, including LLM calls, context, prompts, HTTP requests, and more, to a third-party tracing platform.', - configProviderTitle: 'Config provider to enable tracing', + configProviderTitle: { + configured: 'Configured', + notConfigured: 'Config provider to enable tracing', + moreProvider: 'More Provider', + }, langsmith: { title: 'LangSmith', description: 'An all-in-one developer platform for every step of the LLM-powered application lifecycle.', diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 68dcdf2d58b34b..641f8774f395f1 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -94,7 +94,11 @@ const translation = { disabled: '已禁用', enabled: '已启用', tracingDescription: '捕获应用程序执行的完整上下文，包括 LLM 调用、上下文、提示、HTTP 请求等，发送到第三方跟踪平台。', - configProviderTitle: '配置提供商以启用追踪', + configProviderTitle: { + configured: '已配置', + notConfigured: '配置提供商以启用追踪', + moreProvider: '更多提供商', + }, langsmith: { title: 'LangSmith', description: '一个全方位的开发者平台，适用于 LLM 驱动应用程序生命周期的每个步骤。', diff --git a/web/service/apps.ts b/web/service/apps.ts index 2dca0e2daff0e1..96904958ed45e3 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -3,7 +3,7 @@ import { del, get, patch, post, put } from './base' import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUsersResponse, AppDetailResponse, AppListResponse, AppStatisticsResponse, AppTemplatesResponse, AppTokenCostsResponse, AppVoicesListResponse, CreateApiKeyResponse, GenerationIntroductionResponse, TracingConfig, TracingStatus, UpdateAppModelConfigResponse, UpdateAppSiteCodeResponse, UpdateOpenAIKeyResponse, ValidateOpenAIKeyResponse, WorkflowDailyConversationsResponse } from '@/models/app' import type { CommonResponse } from '@/models/common' import type { AppMode, ModelConfig } from '@/types/app' -import { TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' +import type { TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' export const fetchAppList: Fetcher<AppListResponse, { url: string; params?: Record<string, any> }> = ({ url, params }) => { return get<AppListResponse>(url, { params }) @@ -125,57 +125,59 @@ export const fetchAppVoices: Fetcher<AppVoicesListResponse, { appId: string; lan // Tracing export const fetchTracingStatus: Fetcher<TracingStatus, { appId: string }> = ({ appId }) => { - // return get(`/apps/${appId}/tracing-config`) - return Promise.resolve({ - enabled: false, - tracing_provider: 'langfuse', - }) + return get(`/apps/${appId}/trace`) + // return Promise.resolve({ + // enabled: false, + // tracing_provider: 'langfuse', + // }) } export const updateTracingStatus: Fetcher<CommonResponse, { appId: string; body: Record<string, any> }> = ({ appId, body }) => { - return post(`/apps/${appId}/tracing-config`, { body }) - // return Promise.resolve({ result: 'success' }) + return post(`/apps/${appId}/trace`, { body }) } -export const fetchTracingConfig: Fetcher<TracingConfig, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { - // return get(`/apps/${appId}/tracing-config`, { - // params: { - // tracing_provider: provider, - // }, - // }) - return Promise.resolve({ - tracing_config: null, - }) - if (provider === TracingProvider.langSmith) { - return Promise.resolve({ - tracing_provider: 'langsmith', - tracing_config: { - api_key: '123132*********************21', - endpoint: 'https://api.langsmith.ai', - project: 'test', - }, - }) - } - return Promise.resolve({ - tracing_provider: 'langfuse', - tracing_config: { - public_key: '123132*********************21', - secret_key: '888877*********55', - host: 'https://api.langfuse.ai', +export const fetchTracingConfig: Fetcher<TracingConfig & { has_not_configured: true }, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { + return get(`/apps/${appId}/trace-config`, { + params: { + tracing_provider: provider, }, }) + // if (provider === TracingProvider.langSmith) { + // // return Promise.resolve({ + // // has_not_configured: true, + // // }) + // return Promise.resolve({ + // tracing_provider: 'langsmith', + // tracing_config: { + // api_key: '123132*********************21', + // endpoint: 'https://api.langsmith.ai', + // project: 'test', + // }, + // }) + // } + // return Promise.resolve({ + // has_not_configured: true, + // }) + // return Promise.resolve({ + // tracing_provider: 'langfuse', + // tracing_config: { + // public_key: '123132*********************21', + // secret_key: '888877*********55', + // host: 'https://api.langfuse.ai', + // }, + // }) } export const addTracingConfig: Fetcher<CommonResponse, { appId: string; body: TracingConfig }> = ({ appId, body }) => { - return post(`/apps/${appId}/tracing-config`, { body }) + return post(`/apps/${appId}/trace-config`, { body }) } export const updateTracingConfig: Fetcher<CommonResponse, { appId: string; body: TracingConfig }> = ({ appId, body }) => { - return patch(`/apps/${appId}/tracing-config`, { body }) + return patch(`/apps/${appId}/trace-config`, { body }) } export const removeTracingConfig: Fetcher<CommonResponse, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { - return del(`/apps/${appId}/tracing-config`, { + return del(`/apps/${appId}/trace-config`, { body: { tracing_provider: provider, }, From a84826033bdf566c96ae3dbd54213fb0bd2ef2f0 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Tue, 18 Jun 2024 18:01:42 +0800 Subject: [PATCH 223/273] feat: update default reply when config is none --- api/controllers/console/app/app.py | 2 -- api/controllers/console/app/ops_trace.py | 2 +- ...31d46af369_remove_app_model_config_trace_config_.py | 10 +++++----- api/requirements.txt | 2 ++ api/services/ops_trace/ops_trace_service.py | 5 +++++ 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index bc9a3c04300311..a9e8b5fb4a2d33 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -291,8 +291,6 @@ def get(self, app_id): app_trace_config = OpsTraceService.get_app_tracing_config( app_id=app_id ) - if not app_trace_config: - raise BadRequest("Tracing config not found") return app_trace_config diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index 1316fb7cdbc630..382f1ed89b3c9b 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -26,7 +26,7 @@ def get(self, app_id): app_id=app_id, tracing_provider=args['tracing_provider'] ) if not trace_config: - raise TracingConfigNotExist() + return {"has_not_configured": True} return trace_config except Exception as e: raise e diff --git a/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py index 8d69fa86eb8487..64489b11f50136 100644 --- a/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py +++ b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py @@ -28,10 +28,10 @@ def upgrade(): sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), sa.Column('is_active', sa.Boolean(), server_default=sa.text('true'), nullable=False), - sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + sa.PrimaryKeyConstraint('id', name='trace_app_config_pkey') # 修改了主键约束名称以避免冲突 ) with op.batch_alter_table('trace_app_config', schema=None) as batch_op: - batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + batch_op.create_index('trace_app_config_app_id_idx', ['app_id'], unique=False) with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: batch_op.drop_index('tracing_app_config_app_id_idx') @@ -54,13 +54,13 @@ def downgrade(): sa.Column('tracing_config', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), - sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + sa.PrimaryKeyConstraint('id', name='trace_app_config_pkey') ) with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: - batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + batch_op.create_index('trace_app_config_app_id_idx', ['app_id'], unique=False) with op.batch_alter_table('trace_app_config', schema=None) as batch_op: - batch_op.drop_index('tracing_app_config_app_id_idx') + batch_op.drop_index('trace_app_config_app_id_idx') op.drop_table('trace_app_config') # ### end Alembic commands ### diff --git a/api/requirements.txt b/api/requirements.txt index 1930b8386bbee6..f86581fdb711e9 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -91,3 +91,5 @@ chromadb~=0.5.0 novita_client~=0.5.6 tenacity~=8.3.0 cos-python-sdk-v5==1.9.30 +langfuse==2.36.1 +langsmith==0.1.77 \ No newline at end of file diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index e5cf1b57d63db9..e4700dc685fd12 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -346,5 +346,10 @@ def get_app_tracing_config(cls, app_id: str): :return: """ app: App = db.session.query(App).filter(App.id == app_id).first() + if not app.tracing: + return { + "enabled": False, + "tracing_provider": None + } app_trace_config = json.loads(app.tracing) return app_trace_config From 273179b15552dfb3fbd49dd51a9d072a7a70b971 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 18:09:04 +0800 Subject: [PATCH 224/273] feat: can crud --- .../[appId]/overview/tracing/config-popup.tsx | 8 ++--- .../tracing/provider-config-modal.tsx | 2 +- web/service/apps.ts | 34 +------------------ 3 files changed, 5 insertions(+), 39 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 81372cfb569529..63178d6892f8cb 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -56,11 +56,9 @@ const ConfigPopup: FC<PopupProps> = ({ } }, [onChooseProvider]) - const handleConfigUpdated = useCallback(() => { - return (payload: LangSmithConfig | LangFuseConfig) => { - onConfigUpdated(currentProvider!, payload) - hideConfigModal() - } + const handleConfigUpdated = useCallback((payload: LangSmithConfig | LangFuseConfig) => { + onConfigUpdated(currentProvider!, payload) + hideConfigModal() }, [currentProvider, hideConfigModal, onConfigUpdated]) const handleConfigRemoved = useCallback(() => { diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 61855fbe8052b1..012ef5d22ac23b 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -139,7 +139,7 @@ const ProviderConfigModal: FC<Props> = ({ onChosen(type) } finally { - setIsSaving(true) + setIsSaving(false) } }, [appId, checkValid, config, isEdit, isSaving, onChosen, onSaved, t, type]) diff --git a/web/service/apps.ts b/web/service/apps.ts index 96904958ed45e3..cd71ceadae4556 100644 --- a/web/service/apps.ts +++ b/web/service/apps.ts @@ -126,10 +126,6 @@ export const fetchAppVoices: Fetcher<AppVoicesListResponse, { appId: string; lan // Tracing export const fetchTracingStatus: Fetcher<TracingStatus, { appId: string }> = ({ appId }) => { return get(`/apps/${appId}/trace`) - // return Promise.resolve({ - // enabled: false, - // tracing_provider: 'langfuse', - // }) } export const updateTracingStatus: Fetcher<CommonResponse, { appId: string; body: Record<string, any> }> = ({ appId, body }) => { @@ -142,30 +138,6 @@ export const fetchTracingConfig: Fetcher<TracingConfig & { has_not_configured: t tracing_provider: provider, }, }) - // if (provider === TracingProvider.langSmith) { - // // return Promise.resolve({ - // // has_not_configured: true, - // // }) - // return Promise.resolve({ - // tracing_provider: 'langsmith', - // tracing_config: { - // api_key: '123132*********************21', - // endpoint: 'https://api.langsmith.ai', - // project: 'test', - // }, - // }) - // } - // return Promise.resolve({ - // has_not_configured: true, - // }) - // return Promise.resolve({ - // tracing_provider: 'langfuse', - // tracing_config: { - // public_key: '123132*********************21', - // secret_key: '888877*********55', - // host: 'https://api.langfuse.ai', - // }, - // }) } export const addTracingConfig: Fetcher<CommonResponse, { appId: string; body: TracingConfig }> = ({ appId, body }) => { @@ -177,9 +149,5 @@ export const updateTracingConfig: Fetcher<CommonResponse, { appId: string; body: } export const removeTracingConfig: Fetcher<CommonResponse, { appId: string; provider: TracingProvider }> = ({ appId, provider }) => { - return del(`/apps/${appId}/trace-config`, { - body: { - tracing_provider: provider, - }, - }) + return del(`/apps/${appId}/trace-config?tracing_provider=${provider}`) } From ff17d331c0e38c5fbc54f0243cbf52c2d3df33af Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 18:27:39 +0800 Subject: [PATCH 225/273] chore: save and update --- .../[appId]/overview/tracing/panel.tsx | 20 +++++++++++++------ .../tracing/provider-config-modal.tsx | 7 ++++--- web/i18n/en-US/common.ts | 1 + web/i18n/zh-Hans/common.ts | 1 + 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 48e6d56daa5e0e..5b9fff9f32e08a 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -43,13 +43,15 @@ const Panel: FC = () => { const [tracingStatus, setTracingStatus] = useState<TracingStatus | null>(null) const enabled = tracingStatus?.enabled || false - const handleTracingStatusChange = async (tracingStatus: TracingStatus) => { + const handleTracingStatusChange = async (tracingStatus: TracingStatus, noToast?: boolean) => { await updateTracingStatus({ appId, body: tracingStatus }) setTracingStatus(tracingStatus) - Toast.notify({ - type: 'success', - message: t('common.api.success'), - }) + if (!noToast) { + Toast.notify({ + type: 'success', + message: t('common.api.success'), + }) + } } const handleTracingEnabledChange = (enabled: boolean) => { @@ -92,6 +94,12 @@ const Panel: FC = () => { setLangSmithConfig(null) else setLangFuseConfig(null) + if (provider === inUseTracingProvider) { + handleTracingStatusChange({ + enabled: false, + tracing_provider: null, + }, true) + } } useEffect(() => { @@ -155,7 +163,7 @@ const Panel: FC = () => { <div className='mb-3 flex justify-between items-center'> <Title /> <div className='flex items-center p-2 rounded-xl border-[0.5px] border-gray-200 shadow-xs hover:bg-gray-100'> - {!hasConfiguredTracing + {!inUseTracingProvider ? <> <TracingIcon size='md' className='mr-2' /> <div className='leading-5 text-sm font-semibold text-gray-700'>{t(`${I18N_PREFIX}.title`)}</div> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 012ef5d22ac23b..c5dd11ade275fd 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -53,6 +53,7 @@ const ProviderConfigModal: FC<Props> = ({ }) => { const { t } = useTranslation() const isEdit = !!payload + const isAdd = !isEdit const [isSaving, setIsSaving] = useState(false) const [config, setConfig] = useState<LangSmithConfig | LangFuseConfig>((() => { if (isEdit) @@ -135,13 +136,13 @@ const ProviderConfigModal: FC<Props> = ({ message: t('common.api.success'), }) onSaved(config) - if (!isEdit) + if (!isAdd) onChosen(type) } finally { setIsSaving(false) } - }, [appId, checkValid, config, isEdit, isSaving, onChosen, onSaved, t, type]) + }, [appId, checkValid, config, isAdd, isEdit, isSaving, onChosen, onSaved, t, type]) return ( <> @@ -246,7 +247,7 @@ const ProviderConfigModal: FC<Props> = ({ onClick={handleSave} loading={isSaving} > - {t('common.operation.save')} + {t(`common.operation.${isAdd ? 'saveAndEnable' : 'save'}`)} </Button> </div> diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index 4515ad7a213901..aa3b970889991a 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -12,6 +12,7 @@ const translation = { cancel: 'Cancel', clear: 'Clear', save: 'Save', + saveAndEnable: 'Save & Enable', edit: 'Edit', add: 'Add', added: 'Added', diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index 07b4367502071f..aa5b51fe04cd3a 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -12,6 +12,7 @@ const translation = { cancel: '取消', clear: '清空', save: '保存', + saveAndEnable: '保存并启用', edit: '编辑', add: '添加', added: '已添加', From 97dfe74f4fd55d9c6fb567110e51932672151939 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 18:33:40 +0800 Subject: [PATCH 226/273] chore: handle not auto save choose --- .../[appId]/overview/tracing/provider-config-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index c5dd11ade275fd..423d271c205821 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -136,7 +136,7 @@ const ProviderConfigModal: FC<Props> = ({ message: t('common.api.success'), }) onSaved(config) - if (!isAdd) + if (isAdd) onChosen(type) } finally { From 70921aa0246318d72a688b1393510bd3e92f4714 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Tue, 18 Jun 2024 18:37:08 +0800 Subject: [PATCH 227/273] fix: choose provide not enable --- .../app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 5b9fff9f32e08a..ae221e49e5a060 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -63,7 +63,7 @@ const Panel: FC = () => { const handleChooseProvider = (provider: TracingProvider) => { handleTracingStatusChange({ tracing_provider: provider, - enabled, + enabled: true, }) } const inUseTracingProvider: TracingProvider | null = tracingStatus?.tracing_provider || null From 757b8cfd78eeba336c6c43ca57841c77895a6f7d Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Wed, 19 Jun 2024 09:50:29 +0800 Subject: [PATCH 228/273] fix: advanced chat trace error --- api/core/app/apps/advanced_chat/app_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index a06ae902fed5db..f8a2cb0f64de3c 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -109,7 +109,6 @@ def generate( application_generate_entity=application_generate_entity, conversation=conversation, stream=stream, - tracing_instance=tracing_instance, ) def single_iteration_generate(self, app_model: App, From 14b5f63d9d63c77c54426598f7f2b993c40780fd Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Mon, 17 Jun 2024 00:26:43 +0800 Subject: [PATCH 229/273] feat: add llm ops tracing --- api/controllers/console/__init__.py | 1 + api/controllers/console/app/error.py | 12 + api/controllers/console/app/ops_trace.py | 79 ++ api/core/agent/cot_agent_runner.py | 24 +- api/core/agent/fc_agent_runner.py | 13 +- api/core/app/app_config/entities.py | 10 +- .../app/apps/advanced_chat/app_generator.py | 26 +- api/core/app/apps/advanced_chat/app_runner.py | 18 +- .../advanced_chat/generate_task_pipeline.py | 74 +- api/core/app/apps/agent_chat/app_generator.py | 21 +- api/core/app/apps/agent_chat/app_runner.py | 17 +- api/core/app/apps/base_app_runner.py | 17 +- api/core/app/apps/chat/app_generator.py | 24 +- api/core/app/apps/chat/app_runner.py | 4 +- api/core/app/apps/completion/app_generator.py | 19 +- api/core/app/apps/completion/app_runner.py | 4 +- .../app/apps/message_based_app_generator.py | 38 +- api/core/app/apps/workflow/app_generator.py | 46 +- .../apps/workflow/generate_task_pipeline.py | 20 +- .../easy_ui_based_generate_task_pipeline.py | 31 +- .../task_pipeline/workflow_cycle_manage.py | 82 +- .../agent_tool_callback_handler.py | 19 + api/core/llm_generator/llm_generator.py | 52 +- api/core/moderation/input_moderation.py | 40 +- api/core/rag/retrieval/dataset_retrieval.py | 141 ++-- api/core/tools/tool/workflow_tool.py | 10 +- api/core/tools/tool_engine.py | 33 +- api/core/workflow/nodes/tool/tool_node.py | 10 +- ...9b_update_appmodelconfig_and_add_table_.py | 49 ++ api/models/model.py | 36 +- api/pyproject.toml | 2 + api/services/app_generate_service.py | 3 +- api/services/conversation_service.py | 2 +- api/services/message_service.py | 27 +- api/services/ops_trace/base_trace_instance.py | 31 + api/services/ops_trace/langfuse_trace.py | 712 ++++++++++++++++++ api/services/ops_trace/langsmith_trace.py | 545 ++++++++++++++ api/services/ops_trace/ops_trace_service.py | 321 ++++++++ api/services/ops_trace/trace_queue_manager.py | 133 ++++ api/services/ops_trace/utils.py | 28 + 40 files changed, 2531 insertions(+), 243 deletions(-) create mode 100644 api/controllers/console/app/ops_trace.py create mode 100644 api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py create mode 100644 api/services/ops_trace/base_trace_instance.py create mode 100644 api/services/ops_trace/langfuse_trace.py create mode 100644 api/services/ops_trace/langsmith_trace.py create mode 100644 api/services/ops_trace/ops_trace_service.py create mode 100644 api/services/ops_trace/trace_queue_manager.py create mode 100644 api/services/ops_trace/utils.py diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 29eac070a08fcb..8c67fef95f5f4c 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -20,6 +20,7 @@ generator, message, model_config, + ops_trace, site, statistic, workflow, diff --git a/api/controllers/console/app/error.py b/api/controllers/console/app/error.py index fbe42fbd2a7135..cd7919b5e426bd 100644 --- a/api/controllers/console/app/error.py +++ b/api/controllers/console/app/error.py @@ -97,3 +97,15 @@ class DraftWorkflowNotSync(BaseHTTPException): error_code = 'draft_workflow_not_sync' description = "Workflow graph might have been modified, please refresh and resubmit." code = 400 + + +class TracingConfigNotExist(BaseHTTPException): + error_code = 'trace_config_not_exist' + description = "Trace config not exist." + code = 400 + + +class TracingConfigIsExist(BaseHTTPException): + error_code = 'trace_config_is_exist' + description = "Trace config is exist." + code = 400 diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py new file mode 100644 index 00000000000000..c4b3641b14ffec --- /dev/null +++ b/api/controllers/console/app/ops_trace.py @@ -0,0 +1,79 @@ +from flask_restful import Resource, reqparse + +from controllers.console import api +from controllers.console.app.error import TracingConfigIsExist, TracingConfigNotExist +from controllers.console.setup import setup_required +from controllers.console.wraps import account_initialization_required +from libs.login import login_required +from services.ops_trace.ops_trace_service import OpsTraceService + + +class TraceAppConfigApi(Resource): + """ + Manage trace app configurations + """ + + @setup_required + @login_required + @account_initialization_required + def get(self, app_id): + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='args') + args = parser.parse_args() + + try: + trace_config = OpsTraceService.get_tracing_app_config( + app_id=app_id, tracing_provider=args['tracing_provider'] + ) + if not trace_config: + raise TracingConfigNotExist() + return trace_config + except Exception as e: + raise e + + @setup_required + @login_required + @account_initialization_required + def post(self, app_id): + """Create a new trace app configuration""" + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='json') + parser.add_argument('tracing_config', type=dict, required=True, location='json') + args = parser.parse_args() + + try: + result = OpsTraceService.create_tracing_app_config( + app_id=app_id, + tracing_provider=args['tracing_provider'], + tracing_config=args['tracing_config'] + ) + if not result: + raise TracingConfigIsExist() + return {"result": "success"} + except Exception as e: + raise e + + @setup_required + @login_required + @account_initialization_required + def put(self, app_id): + """Update an existing trace app configuration""" + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='json') + parser.add_argument('tracing_config', type=dict, required=True, location='json') + args = parser.parse_args() + + try: + result = OpsTraceService.update_tracing_app_config( + app_id=app_id, + tracing_provider=args['tracing_provider'], + tracing_config=args['tracing_config'] + ) + if not result: + raise TracingConfigNotExist() + return {"result": "success"} + except Exception as e: + raise e + + +api.add_resource(TraceAppConfigApi, '/apps/<uuid:app_id>/trace-config') diff --git a/api/core/agent/cot_agent_runner.py b/api/core/agent/cot_agent_runner.py index 982477138b5261..31d1ec0cdb23df 100644 --- a/api/core/agent/cot_agent_runner.py +++ b/api/core/agent/cot_agent_runner.py @@ -1,7 +1,7 @@ import json from abc import ABC, abstractmethod from collections.abc import Generator -from typing import Union +from typing import Union, Optional from core.agent.base_agent_runner import BaseAgentRunner from core.agent.entities import AgentScratchpadUnit @@ -20,6 +20,7 @@ from core.tools.tool.tool import Tool from core.tools.tool_engine import ToolEngine from models.model import Message +from services.ops_trace.base_trace_instance import BaseTraceInstance class CotAgentRunner(BaseAgentRunner, ABC): @@ -32,9 +33,9 @@ class CotAgentRunner(BaseAgentRunner, ABC): _prompt_messages_tools: list[PromptMessage] = None def run(self, message: Message, - query: str, - inputs: dict[str, str], - ) -> Union[Generator, LLMResult]: + query: str, + inputs: dict[str, str], + ) -> Union[Generator, LLMResult]: """ Run Cot agent application """ @@ -183,7 +184,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): messages_ids=[], llm_usage=usage_dict['usage'] ) - + if not scratchpad.is_final(): self.queue_manager.publish(QueueAgentThoughtEvent( agent_thought_id=agent_thought.id @@ -209,7 +210,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): function_call_state = True # action is tool call, invoke tool tool_invoke_response, tool_invoke_meta = self._handle_invoke_action( - action=scratchpad.action, + action=scratchpad.action, tool_instances=tool_instances, message_file_ids=message_file_ids ) @@ -257,12 +258,12 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): # save agent thought self.save_agent_thought( - agent_thought=agent_thought, + agent_thought=agent_thought, tool_name='', tool_input={}, tool_invoke_meta={}, thought=final_answer, - observation={}, + observation={}, answer=final_answer, messages_ids=[] ) @@ -282,7 +283,9 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, tool_instances: dict[str, Tool], - message_file_ids: list[str]) -> tuple[str, ToolInvokeMeta]: + message_file_ids: list[str], + tracing_instance: Optional[BaseTraceInstance] = None + ) -> tuple[str, ToolInvokeMeta]: """ handle invoke action :param action: action @@ -312,7 +315,8 @@ def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, tenant_id=self.tenant_id, message=self.message, invoke_from=self.application_generate_entity.invoke_from, - agent_tool_callback=self.agent_callback + agent_tool_callback=self.agent_callback, + tracing_instance=tracing_instance, ) # publish files diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index d7b063eb92ec55..e64722d22ca58d 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -20,7 +20,9 @@ from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform from core.tools.entities.tool_entities import ToolInvokeMeta from core.tools.tool_engine import ToolEngine -from models.model import Message +from extensions.ext_database import db +from models.model import AppModelConfig, Message +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -50,6 +52,14 @@ def run(self, } final_answer = '' + # get tracing instance + app_id = app_config.app_id + app_model_config_id = app_config.app_model_config_id + app_model_config = db.session.query(AppModelConfig).filter_by(id=app_model_config_id).first() + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id, app_model_config=app_model_config + ) + def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): if not final_llm_usage_dict['usage']: final_llm_usage_dict['usage'] = usage @@ -243,6 +253,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): message=self.message, invoke_from=self.application_generate_entity.invoke_from, agent_tool_callback=self.agent_callback, + tracing_instance=tracing_instance ) # publish files for message_file, save_as in message_files: diff --git a/api/core/app/app_config/entities.py b/api/core/app/app_config/entities.py index d6b6d894166d7e..6b58df617d7825 100644 --- a/api/core/app/app_config/entities.py +++ b/api/core/app/app_config/entities.py @@ -183,6 +183,14 @@ class TextToSpeechEntity(BaseModel): language: Optional[str] = None +class TracingConfigEntity(BaseModel): + """ + Tracing Config Entity. + """ + enabled: bool + tracing_provider: str + + class FileExtraConfig(BaseModel): """ File Upload Entity. @@ -199,7 +207,7 @@ class AppAdditionalFeatures(BaseModel): more_like_this: bool = False speech_to_text: bool = False text_to_speech: Optional[TextToSpeechEntity] = None - + trace_config: Optional[TracingConfigEntity] = None class AppConfig(BaseModel): """ diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 3b1ee3578dea3c..a06ae902fed5db 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -3,7 +3,7 @@ import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Any, Optional, Union from flask import Flask, current_app from pydantic import ValidationError @@ -29,13 +29,15 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator): - def generate(self, app_model: App, - workflow: Workflow, - user: Union[Account, EndUser], - args: dict, - invoke_from: InvokeFrom, - stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: bool = True, + tracing_instance: Optional[Any] = None + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -45,6 +47,7 @@ def generate(self, app_model: App, :param args: request args :param invoke_from: invoke from source :param stream: is stream + :param tracing_instance: tracing instance """ if not args.get('query'): raise ValueError('query is required') @@ -105,7 +108,8 @@ def generate(self, app_model: App, invoke_from=invoke_from, application_generate_entity=application_generate_entity, conversation=conversation, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) def single_iteration_generate(self, app_model: App, @@ -227,7 +231,7 @@ def _generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, ) return AdvancedChatAppGenerateResponseConverter.convert( @@ -326,7 +330,7 @@ def _handle_advanced_chat_response(self, application_generate_entity: AdvancedCh ) try: - return generate_task_pipeline.process() + return generate_task_pipeline.process(workflow) except ValueError as e: if e.args[0] == "I/O operation on closed file.": # ignore this error raise GenerateTaskStoppedException() diff --git a/api/core/app/apps/advanced_chat/app_runner.py b/api/core/app/apps/advanced_chat/app_runner.py index de3632894de2ed..96e9319dda58d2 100644 --- a/api/core/app/apps/advanced_chat/app_runner.py +++ b/api/core/app/apps/advanced_chat/app_runner.py @@ -70,7 +70,8 @@ def run(self, application_generate_entity: AdvancedChatAppGenerateEntity, app_record=app_record, app_generate_entity=application_generate_entity, inputs=inputs, - query=query + query=query, + message_id=message.id ): return @@ -156,11 +157,14 @@ def get_workflow(self, app_model: App, workflow_id: str) -> Optional[Workflow]: # return workflow return workflow - def handle_input_moderation(self, queue_manager: AppQueueManager, - app_record: App, - app_generate_entity: AdvancedChatAppGenerateEntity, - inputs: dict, - query: str) -> bool: + def handle_input_moderation( + self, queue_manager: AppQueueManager, + app_record: App, + app_generate_entity: AdvancedChatAppGenerateEntity, + inputs: dict, + query: str, + message_id: str + ) -> bool: """ Handle input moderation :param queue_manager: application queue manager @@ -168,6 +172,7 @@ def handle_input_moderation(self, queue_manager: AppQueueManager, :param app_generate_entity: application generate entity :param inputs: inputs :param query: query + :param message_id: message id :return: """ try: @@ -178,6 +183,7 @@ def handle_input_moderation(self, queue_manager: AppQueueManager, app_generate_entity=app_generate_entity, inputs=inputs, query=query, + message_id=message_id, ) except ModerationException as e: self._stream_output( diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 7c70afc2ae393c..207e962b376efd 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -54,6 +54,7 @@ WorkflowNodeExecution, WorkflowRunStatus, ) +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -69,13 +70,15 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc _workflow_system_variables: dict[SystemVariable, Any] _iteration_nested_relations: dict[str, list[str]] - def __init__(self, application_generate_entity: AdvancedChatAppGenerateEntity, - workflow: Workflow, - queue_manager: AppQueueManager, - conversation: Conversation, - message: Message, - user: Union[Account, EndUser], - stream: bool) -> None: + def __init__( + self, application_generate_entity: AdvancedChatAppGenerateEntity, + workflow: Workflow, + queue_manager: AppQueueManager, + conversation: Conversation, + message: Message, + user: Union[Account, EndUser], + stream: bool + ) -> None: """ Initialize AdvancedChatAppGenerateTaskPipeline. :param application_generate_entity: application generate entity @@ -111,7 +114,10 @@ def __init__(self, application_generate_entity: AdvancedChatAppGenerateEntity, self._stream_generate_routes = self._get_stream_generate_routes() self._conversation_name_generate_thread = None - def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]: + def process( + self, + workflow: Optional[Workflow] = None + ) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]: """ Process generate task pipeline. :return: @@ -126,14 +132,14 @@ def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStrea self._application_generate_entity.query ) - generator = self._process_stream_response() + generator = self._process_stream_response(workflow) if self._stream: return self._to_stream_response(generator) else: return self._to_blocking_response(generator) def _to_blocking_response(self, generator: Generator[StreamResponse, None, None]) \ - -> ChatbotAppBlockingResponse: + -> ChatbotAppBlockingResponse: """ Process blocking response. :return: @@ -164,7 +170,7 @@ def _to_blocking_response(self, generator: Generator[StreamResponse, None, None] raise Exception('Queue listening stopped unexpectedly.') def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) \ - -> Generator[ChatbotAppStreamResponse, None, None]: + -> Generator[ChatbotAppStreamResponse, None, None]: """ To stream response. :return: @@ -177,11 +183,13 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) stream_response=stream_response ) - def _process_stream_response(self) -> Generator[StreamResponse, None, None]: + def _process_stream_response(self, workflow: Optional[Workflow] = None) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: """ + app_id = self._conversation.app_id + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) for message in self._queue_manager.listen(): event = message.event @@ -249,7 +257,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event) self._handle_iteration_operation(event) elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent): - workflow_run = self._handle_workflow_finished(event) + workflow_run = self._handle_workflow_finished(event, tracing_instance) if workflow_run: yield self._workflow_finish_to_stream_response( task_id=self._application_generate_entity.task_id, @@ -292,7 +300,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: continue if not self._is_stream_out_support( - event=event + event=event ): continue @@ -361,7 +369,7 @@ def _message_end_to_stream_response(self) -> MessageEndStreamResponse: id=self._message.id, **extras ) - + def _get_stream_generate_routes(self) -> dict[str, ChatflowStreamGenerateRoute]: """ Get stream generate routes. @@ -391,9 +399,9 @@ def _get_stream_generate_routes(self) -> dict[str, ChatflowStreamGenerateRoute]: ) return stream_generate_routes - + def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \ - -> list[str]: + -> list[str]: """ Get answer start at node id. :param graph: graph @@ -414,14 +422,14 @@ def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \ target_node = next((node for node in nodes if node.get('id') == target_node_id), None) if not target_node: return [] - + node_iteration_id = target_node.get('data', {}).get('iteration_id') # get iteration start node id for node in nodes: if node.get('id') == node_iteration_id: if node.get('data', {}).get('start_node_id') == target_node_id: return [target_node_id] - + return [] start_node_ids = [] @@ -457,7 +465,7 @@ def _get_answer_start_at_node_ids(self, graph: dict, target_node_id: str) \ start_node_ids.extend(sub_start_node_ids) return start_node_ids - + def _get_iteration_nested_relations(self, graph: dict) -> dict[str, list[str]]: """ Get iteration nested relations. @@ -466,18 +474,18 @@ def _get_iteration_nested_relations(self, graph: dict) -> dict[str, list[str]]: """ nodes = graph.get('nodes') - iteration_ids = [node.get('id') for node in nodes + iteration_ids = [node.get('id') for node in nodes if node.get('data', {}).get('type') in [ NodeType.ITERATION.value, NodeType.LOOP.value, - ]] + ]] return { iteration_id: [ node.get('id') for node in nodes if node.get('data', {}).get('iteration_id') == iteration_id ] for iteration_id in iteration_ids } - + def _generate_stream_outputs_when_node_started(self) -> Generator: """ Generate stream outputs. @@ -485,8 +493,8 @@ def _generate_stream_outputs_when_node_started(self) -> Generator: """ if self._task_state.current_stream_generate_state: route_chunks = self._task_state.current_stream_generate_state.generate_route[ - self._task_state.current_stream_generate_state.current_route_position: - ] + self._task_state.current_stream_generate_state.current_route_position: + ] for route_chunk in route_chunks: if route_chunk.type == 'text': @@ -506,7 +514,8 @@ def _generate_stream_outputs_when_node_started(self) -> Generator: # all route chunks are generated if self._task_state.current_stream_generate_state.current_route_position == len( - self._task_state.current_stream_generate_state.generate_route): + self._task_state.current_stream_generate_state.generate_route + ): self._task_state.current_stream_generate_state = None def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: @@ -519,7 +528,7 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: route_chunks = self._task_state.current_stream_generate_state.generate_route[ self._task_state.current_stream_generate_state.current_route_position:] - + for route_chunk in route_chunks: if route_chunk.type == 'text': route_chunk = cast(TextGenerateRouteChunk, route_chunk) @@ -551,7 +560,8 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: value = iteration_state.current_index elif value_selector[1] == 'item': value = iterator_selector[iteration_state.current_index] if iteration_state.current_index < len( - iterator_selector) else None + iterator_selector + ) else None else: # check chunk node id is before current node id or equal to current node id if route_chunk_node_id not in self._task_state.ran_node_execution_infos: @@ -562,14 +572,15 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: # get route chunk node execution info route_chunk_node_execution_info = self._task_state.ran_node_execution_infos[route_chunk_node_id] if (route_chunk_node_execution_info.node_type == NodeType.LLM - and latest_node_execution_info.node_type == NodeType.LLM): + and latest_node_execution_info.node_type == NodeType.LLM): # only LLM support chunk stream output self._task_state.current_stream_generate_state.current_route_position += 1 continue # get route chunk node execution route_chunk_node_execution = db.session.query(WorkflowNodeExecution).filter( - WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id).first() + WorkflowNodeExecution.id == route_chunk_node_execution_info.workflow_node_execution_id + ).first() outputs = route_chunk_node_execution.outputs_dict @@ -631,7 +642,8 @@ def _generate_stream_outputs_when_node_finished(self) -> Optional[Generator]: # all route chunks are generated if self._task_state.current_stream_generate_state.current_route_position == len( - self._task_state.current_stream_generate_state.generate_route): + self._task_state.current_stream_generate_state.generate_route + ): self._task_state.current_stream_generate_state = None def _is_stream_out_support(self, event: QueueTextChunkEvent) -> bool: diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index 407fb931ecb9bd..ca8fbe138efa37 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -3,7 +3,7 @@ import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Optional, Union from flask import Flask, current_app from pydantic import ValidationError @@ -22,6 +22,7 @@ from extensions.ext_database import db from models.account import Account from models.model import App, EndUser +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -70,6 +71,12 @@ def generate(self, app_model: App, conversation=conversation ) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # validate override model config override_model_config_dict = None if args.get('model_config'): @@ -142,6 +149,7 @@ def generate(self, app_model: App, 'queue_manager': queue_manager, 'conversation_id': conversation.id, 'message_id': message.id, + 'tracing_instance': tracing_instance, }) worker_thread.start() @@ -153,7 +161,8 @@ def generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return AgentChatAppGenerateResponseConverter.convert( @@ -165,7 +174,9 @@ def _generate_worker(self, flask_app: Flask, application_generate_entity: AgentChatAppGenerateEntity, queue_manager: AppQueueManager, conversation_id: str, - message_id: str) -> None: + message_id: str, + tracing_instance: Optional[Any] = None + ) -> None: """ Generate worker in a new thread. :param flask_app: Flask app @@ -173,6 +184,7 @@ def _generate_worker(self, flask_app: Flask, :param queue_manager: queue manager :param conversation_id: conversation ID :param message_id: message ID + :param tracing_instance: tracing instance :return: """ with flask_app.app_context(): @@ -187,7 +199,8 @@ def _generate_worker(self, flask_app: Flask, application_generate_entity=application_generate_entity, queue_manager=queue_manager, conversation=conversation, - message=message + message=message, + tracing_instance=tracing_instance ) except GenerateTaskStoppedException: pass diff --git a/api/core/app/apps/agent_chat/app_runner.py b/api/core/app/apps/agent_chat/app_runner.py index d6367300de26e3..a942522996ca00 100644 --- a/api/core/app/apps/agent_chat/app_runner.py +++ b/api/core/app/apps/agent_chat/app_runner.py @@ -1,5 +1,5 @@ import logging -from typing import cast +from typing import Any, Optional, cast from core.agent.cot_chat_agent_runner import CotChatAgentRunner from core.agent.cot_completion_agent_runner import CotCompletionAgentRunner @@ -28,16 +28,21 @@ class AgentChatAppRunner(AppRunner): """ Agent Application Runner """ - def run(self, application_generate_entity: AgentChatAppGenerateEntity, - queue_manager: AppQueueManager, - conversation: Conversation, - message: Message) -> None: + + def run( + self, application_generate_entity: AgentChatAppGenerateEntity, + queue_manager: AppQueueManager, + conversation: Conversation, + message: Message, + tracing_instance: Optional[Any] = None + ) -> None: """ Run assistant application :param application_generate_entity: application generate entity :param queue_manager: application queue manager :param conversation: conversation :param message: message + :param tracing_instance: tracing instance :return: """ app_config = application_generate_entity.app_config @@ -100,6 +105,7 @@ def run(self, application_generate_entity: AgentChatAppGenerateEntity, app_generate_entity=application_generate_entity, inputs=inputs, query=query, + message_id=message.id ) except ModerationException as e: self.direct_output( @@ -241,6 +247,7 @@ def run(self, application_generate_entity: AgentChatAppGenerateEntity, message=message, query=query, inputs=inputs, + tracing_instance=tracing_instance, ) # handle invoke result diff --git a/api/core/app/apps/base_app_runner.py b/api/core/app/apps/base_app_runner.py index 53f457cb116c02..1ccc9597cee3a9 100644 --- a/api/core/app/apps/base_app_runner.py +++ b/api/core/app/apps/base_app_runner.py @@ -338,11 +338,14 @@ def _handle_invoke_result_stream(self, invoke_result: Generator, ), PublishFrom.APPLICATION_MANAGER ) - def moderation_for_inputs(self, app_id: str, - tenant_id: str, - app_generate_entity: AppGenerateEntity, - inputs: dict, - query: str) -> tuple[bool, dict, str]: + def moderation_for_inputs( + self, app_id: str, + tenant_id: str, + app_generate_entity: AppGenerateEntity, + inputs: dict, + query: str, + message_id: str, + ) -> tuple[bool, dict, str]: """ Process sensitive_word_avoidance. :param app_id: app id @@ -350,6 +353,7 @@ def moderation_for_inputs(self, app_id: str, :param app_generate_entity: app generate entity :param inputs: inputs :param query: query + :param message_id: message id :return: """ moderation_feature = InputModeration() @@ -358,7 +362,8 @@ def moderation_for_inputs(self, app_id: str, tenant_id=tenant_id, app_config=app_generate_entity.app_config, inputs=inputs, - query=query if query else '' + query=query if query else '', + message_id=message_id, ) def check_hosting_moderation(self, application_generate_entity: EasyUIBasedAppGenerateEntity, diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index 505ada09db5c90..e333e3458cbbc8 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -22,17 +22,19 @@ from extensions.ext_database import db from models.account import Account from models.model import App, EndUser +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) class ChatAppGenerator(MessageBasedAppGenerator): - def generate(self, app_model: App, - user: Union[Account, EndUser], - args: Any, - invoke_from: InvokeFrom, - stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: Any, + invoke_from: InvokeFrom, + stream: bool = True, + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -41,6 +43,7 @@ def generate(self, app_model: App, :param args: request args :param invoke_from: invoke from source :param stream: is stream + :param tracing_instance: tracing instance """ if not args.get('query'): raise ValueError('query is required') @@ -121,6 +124,12 @@ def generate(self, app_model: App, message ) = self._init_generate_records(application_generate_entity, conversation) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # init queue manager queue_manager = MessageBasedAppQueueManager( task_id=application_generate_entity.task_id, @@ -149,7 +158,8 @@ def generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return ChatAppGenerateResponseConverter.convert( diff --git a/api/core/app/apps/chat/app_runner.py b/api/core/app/apps/chat/app_runner.py index 7d243d0726724e..0a029af86a1ce1 100644 --- a/api/core/app/apps/chat/app_runner.py +++ b/api/core/app/apps/chat/app_runner.py @@ -96,6 +96,7 @@ def run(self, application_generate_entity: ChatAppGenerateEntity, app_generate_entity=application_generate_entity, inputs=inputs, query=query, + message_id=message.id ) except ModerationException as e: self.direct_output( @@ -165,7 +166,8 @@ def run(self, application_generate_entity: ChatAppGenerateEntity, invoke_from=application_generate_entity.invoke_from, show_retrieve_source=app_config.additional_features.show_retrieve_source, hit_callback=hit_callback, - memory=memory + memory=memory, + message_id=message.id, ) # reorganize all inputs and template to prompt messages diff --git a/api/core/app/apps/completion/app_generator.py b/api/core/app/apps/completion/app_generator.py index 52d907b5353143..fcf00e685594dd 100644 --- a/api/core/app/apps/completion/app_generator.py +++ b/api/core/app/apps/completion/app_generator.py @@ -24,6 +24,7 @@ from models.model import App, EndUser, Message from services.errors.app import MoreLikeThisDisabledError from services.errors.message import MessageNotExistsError +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -114,6 +115,12 @@ def generate(self, app_model: App, message ) = self._init_generate_records(application_generate_entity) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # init queue manager queue_manager = MessageBasedAppQueueManager( task_id=application_generate_entity.task_id, @@ -141,7 +148,8 @@ def generate(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return CompletionAppGenerateResponseConverter.convert( @@ -273,6 +281,12 @@ def generate_more_like_this(self, app_model: App, message ) = self._init_generate_records(application_generate_entity) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + app_model_config=app_model_config, + ) + # init queue manager queue_manager = MessageBasedAppQueueManager( task_id=application_generate_entity.task_id, @@ -300,7 +314,8 @@ def generate_more_like_this(self, app_model: App, conversation=conversation, message=message, user=user, - stream=stream + stream=stream, + tracing_instance=tracing_instance, ) return CompletionAppGenerateResponseConverter.convert( diff --git a/api/core/app/apps/completion/app_runner.py b/api/core/app/apps/completion/app_runner.py index a3a9945bc0436b..2e701320148408 100644 --- a/api/core/app/apps/completion/app_runner.py +++ b/api/core/app/apps/completion/app_runner.py @@ -77,6 +77,7 @@ def run(self, application_generate_entity: CompletionAppGenerateEntity, app_generate_entity=application_generate_entity, inputs=inputs, query=query, + message_id=message.id ) except ModerationException as e: self.direct_output( @@ -124,7 +125,8 @@ def run(self, application_generate_entity: CompletionAppGenerateEntity, query=query, invoke_from=application_generate_entity.invoke_from, show_retrieve_source=app_config.additional_features.show_retrieve_source, - hit_callback=hit_callback + hit_callback=hit_callback, + message_id=message.id ) # reorganize all inputs and template to prompt messages diff --git a/api/core/app/apps/message_based_app_generator.py b/api/core/app/apps/message_based_app_generator.py index 6acf5da8df4d2a..11763d1c6cde53 100644 --- a/api/core/app/apps/message_based_app_generator.py +++ b/api/core/app/apps/message_based_app_generator.py @@ -1,7 +1,7 @@ import json import logging from collections.abc import Generator -from typing import Optional, Union +from typing import Any, Optional, Union from sqlalchemy import and_ @@ -35,22 +35,24 @@ class MessageBasedAppGenerator(BaseAppGenerator): - def _handle_response(self, application_generate_entity: Union[ - ChatAppGenerateEntity, - CompletionAppGenerateEntity, - AgentChatAppGenerateEntity, - AdvancedChatAppGenerateEntity - ], - queue_manager: AppQueueManager, - conversation: Conversation, - message: Message, - user: Union[Account, EndUser], - stream: bool = False) \ - -> Union[ - ChatbotAppBlockingResponse, - CompletionAppBlockingResponse, - Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None] - ]: + def _handle_response( + self, application_generate_entity: Union[ + ChatAppGenerateEntity, + CompletionAppGenerateEntity, + AgentChatAppGenerateEntity, + AdvancedChatAppGenerateEntity + ], + queue_manager: AppQueueManager, + conversation: Conversation, + message: Message, + user: Union[Account, EndUser], + stream: bool = False, + tracing_instance: Optional[Any] = None + ) -> Union[ + ChatbotAppBlockingResponse, + CompletionAppBlockingResponse, + Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None] + ]: """ Handle response. :param application_generate_entity: application generate entity @@ -72,7 +74,7 @@ def _handle_response(self, application_generate_entity: Union[ ) try: - return generate_task_pipeline.process() + return generate_task_pipeline.process(tracing_instance) except ValueError as e: if e.args[0] == "I/O operation on closed file.": # ignore this error raise GenerateTaskStoppedException() diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index c4324978d81fc3..829ccc8cab2b5a 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -3,7 +3,7 @@ import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Any, Optional, Union from flask import Flask, current_app from pydantic import ValidationError @@ -29,14 +29,16 @@ class WorkflowAppGenerator(BaseAppGenerator): - def generate(self, app_model: App, - workflow: Workflow, - user: Union[Account, EndUser], - args: dict, - invoke_from: InvokeFrom, - stream: bool = True, - call_depth: int = 0) \ - -> Union[dict, Generator[dict, None, None]]: + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: bool = True, + call_depth: int = 0, + tracing_instance: Optional[Any] = None + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -46,6 +48,8 @@ def generate(self, app_model: App, :param args: request args :param invoke_from: invoke from source :param stream: is stream + :param call_depth: call depth + :param tracing_instance: ops tracing instance """ inputs = args['inputs'] @@ -87,17 +91,18 @@ def generate(self, app_model: App, application_generate_entity=application_generate_entity, invoke_from=invoke_from, stream=stream, - call_depth=call_depth + call_depth=call_depth, ) - def _generate(self, app_model: App, - workflow: Workflow, - user: Union[Account, EndUser], - application_generate_entity: WorkflowAppGenerateEntity, - invoke_from: InvokeFrom, - stream: bool = True, - call_depth: int = 0) \ - -> Union[dict, Generator[dict, None, None]]: + def _generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + application_generate_entity: WorkflowAppGenerateEntity, + invoke_from: InvokeFrom, + stream: bool = True, + call_depth: int = 0 + ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -131,7 +136,7 @@ def _generate(self, app_model: App, workflow=workflow, queue_manager=queue_manager, user=user, - stream=stream + stream=stream, ) return WorkflowAppGenerateResponseConverter.convert( @@ -271,9 +276,10 @@ def _handle_response(self, application_generate_entity: WorkflowAppGenerateEntit user=user, stream=stream ) + app_id = application_generate_entity.app_config.app_id try: - return generate_task_pipeline.process() + return generate_task_pipeline.process(app_id, workflow) except ValueError as e: if e.args[0] == "I/O operation on closed file.": # ignore this error raise GenerateTaskStoppedException() diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 8d961e0993b96f..944c3736a140d1 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -1,6 +1,6 @@ import logging from collections.abc import Generator -from typing import Any, Union +from typing import Any, Optional, Union from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.entities.app_invoke_entities import ( @@ -48,6 +48,7 @@ WorkflowNodeExecution, WorkflowRun, ) +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -95,7 +96,11 @@ def __init__(self, application_generate_entity: WorkflowAppGenerateEntity, self._stream_generate_nodes = self._get_stream_generate_nodes() self._iteration_nested_relations = self._get_iteration_nested_relations(self._workflow.graph_dict) - def process(self) -> Union[WorkflowAppBlockingResponse, Generator[WorkflowAppStreamResponse, None, None]]: + def process( + self, + app_id: Optional[str] = None, + workflow: Optional[Workflow] = None, + ) -> Union[WorkflowAppBlockingResponse, Generator[WorkflowAppStreamResponse, None, None]]: """ Process generate task pipeline. :return: @@ -104,7 +109,7 @@ def process(self) -> Union[WorkflowAppBlockingResponse, Generator[WorkflowAppStr db.session.refresh(self._user) db.session.close() - generator = self._process_stream_response() + generator = self._process_stream_response(app_id, workflow) if self._stream: return self._to_stream_response(generator) else: @@ -158,11 +163,16 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) stream_response=stream_response ) - def _process_stream_response(self) -> Generator[StreamResponse, None, None]: + def _process_stream_response( + self, + app_id: Optional[str] = None, + workflow: Optional[Workflow] = None, + ) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: """ + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) for message in self._queue_manager.listen(): event = message.event @@ -215,7 +225,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event) self._handle_iteration_operation(event) elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent): - workflow_run = self._handle_workflow_finished(event) + workflow_run = self._handle_workflow_finished(event, tracing_instance) # save workflow app log self._save_workflow_app_log(workflow_run) diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py index ccb684d84b0c8f..da0731df3eff1f 100644 --- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py @@ -2,7 +2,7 @@ import logging import time from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom from core.app.entities.app_invoke_entities import ( @@ -50,6 +50,7 @@ from extensions.ext_database import db from models.account import Account from models.model import AppMode, Conversation, EndUser, Message, MessageAgentThought +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName logger = logging.getLogger(__name__) @@ -100,7 +101,10 @@ def __init__(self, application_generate_entity: Union[ self._conversation_name_generate_thread = None - def process(self) -> Union[ + def process( + self, + tracing_instance: Optional[Any] = None + ) -> Union[ ChatbotAppBlockingResponse, CompletionAppBlockingResponse, Generator[Union[ChatbotAppStreamResponse, CompletionAppStreamResponse], None, None] @@ -120,7 +124,7 @@ def process(self) -> Union[ self._application_generate_entity.query ) - generator = self._process_stream_response() + generator = self._process_stream_response(tracing_instance) if self._stream: return self._to_stream_response(generator) else: @@ -197,7 +201,9 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) stream_response=stream_response ) - def _process_stream_response(self) -> Generator[StreamResponse, None, None]: + def _process_stream_response( + self, tracing_instance: Optional[Any] = None + ) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: @@ -224,7 +230,7 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: yield self._message_replace_to_stream_response(answer=output_moderation_answer) # Save message - self._save_message() + self._save_message(tracing_instance) yield self._message_end_to_stream_response() elif isinstance(event, QueueRetrieverResourcesEvent): @@ -269,7 +275,9 @@ def _process_stream_response(self) -> Generator[StreamResponse, None, None]: if self._conversation_name_generate_thread: self._conversation_name_generate_thread.join() - def _save_message(self) -> None: + def _save_message( + self, tracing_instance: Optional[Any] = None, + ) -> None: """ Save message. :return: @@ -300,6 +308,17 @@ def _save_message(self) -> None: db.session.commit() + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.MESSAGE_TRACE, + conversation_id=self._conversation.id, + message_id=self._message.id + ) + ) + message_was_created.send( self._message, application_generate_entity=self._application_generate_entity, diff --git a/api/core/app/task_pipeline/workflow_cycle_manage.py b/api/core/app/task_pipeline/workflow_cycle_manage.py index 978a318279165f..5c0ffbe07bdd5b 100644 --- a/api/core/app/task_pipeline/workflow_cycle_manage.py +++ b/api/core/app/task_pipeline/workflow_cycle_manage.py @@ -1,7 +1,7 @@ import json import time from datetime import datetime, timezone -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.queue_entities import ( @@ -39,6 +39,8 @@ WorkflowRunStatus, WorkflowRunTriggeredFrom, ) +from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName class WorkflowCycleManage(WorkflowIterationCycleManage): @@ -94,11 +96,15 @@ def _init_workflow_run(self, workflow: Workflow, return workflow_run - def _workflow_run_success(self, workflow_run: WorkflowRun, - start_at: float, - total_tokens: int, - total_steps: int, - outputs: Optional[str] = None) -> WorkflowRun: + def _workflow_run_success( + self, workflow_run: WorkflowRun, + start_at: float, + total_tokens: int, + total_steps: int, + outputs: Optional[str] = None, + conversation_id: Optional[str] = None, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> WorkflowRun: """ Workflow run success :param workflow_run: workflow run @@ -106,6 +112,8 @@ def _workflow_run_success(self, workflow_run: WorkflowRun, :param total_tokens: total tokens :param total_steps: total steps :param outputs: outputs + :param conversation_id: conversation id + :param tracing_instance: tracing instance :return: """ workflow_run.status = WorkflowRunStatus.SUCCEEDED.value @@ -119,14 +127,29 @@ def _workflow_run_success(self, workflow_run: WorkflowRun, db.session.refresh(workflow_run) db.session.close() + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.WORKFLOW_TRACE, + workflow_run=workflow_run, + conversation_id=conversation_id, + ) + ) + return workflow_run - def _workflow_run_failed(self, workflow_run: WorkflowRun, - start_at: float, - total_tokens: int, - total_steps: int, - status: WorkflowRunStatus, - error: str) -> WorkflowRun: + def _workflow_run_failed( + self, workflow_run: WorkflowRun, + start_at: float, + total_tokens: int, + total_steps: int, + status: WorkflowRunStatus, + error: str, + conversation_id: Optional[str] = None, + tracing_instance: Optional[Any] = None + ) -> WorkflowRun: """ Workflow run failed :param workflow_run: workflow run @@ -148,6 +171,17 @@ def _workflow_run_failed(self, workflow_run: WorkflowRun, db.session.refresh(workflow_run) db.session.close() + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.WORKFLOW_TRACE, + workflow_run=workflow_run, + conversation_id=conversation_id, + ) + ) + return workflow_run def _init_node_execution_from_workflow_run(self, workflow_run: WorkflowRun, @@ -440,9 +474,9 @@ def _handle_node_finished(self, event: QueueNodeSucceededEvent | QueueNodeFailed current_node_execution = self._task_state.ran_node_execution_infos[event.node_id] workflow_node_execution = db.session.query(WorkflowNodeExecution).filter( WorkflowNodeExecution.id == current_node_execution.workflow_node_execution_id).first() - + execution_metadata = event.execution_metadata if isinstance(event, QueueNodeSucceededEvent) else None - + if self._iteration_state and self._iteration_state.current_iterations: if not execution_metadata: execution_metadata = {} @@ -470,7 +504,7 @@ def _handle_node_finished(self, event: QueueNodeSucceededEvent | QueueNodeFailed if execution_metadata and execution_metadata.get(NodeRunMetadataKey.TOTAL_TOKENS): self._task_state.total_tokens += ( int(execution_metadata.get(NodeRunMetadataKey.TOTAL_TOKENS))) - + if self._iteration_state: for iteration_node_id in self._iteration_state.current_iterations: data = self._iteration_state.current_iterations[iteration_node_id] @@ -496,13 +530,16 @@ def _handle_node_finished(self, event: QueueNodeSucceededEvent | QueueNodeFailed return workflow_node_execution - def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent) \ - -> Optional[WorkflowRun]: + def _handle_workflow_finished( + self, event: QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> Optional[WorkflowRun]: workflow_run = db.session.query(WorkflowRun).filter( WorkflowRun.id == self._task_state.workflow_run_id).first() if not workflow_run: return None + conversation_id = self._application_generate_entity.inputs.get('sys.conversation_id') if isinstance(event, QueueStopEvent): workflow_run = self._workflow_run_failed( workflow_run=workflow_run, @@ -510,7 +547,8 @@ def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceed total_tokens=self._task_state.total_tokens, total_steps=self._task_state.total_steps, status=WorkflowRunStatus.STOPPED, - error='Workflow stopped.' + error='Workflow stopped.', + conversation_id=conversation_id, ) latest_node_execution_info = self._task_state.latest_node_execution_info @@ -531,7 +569,9 @@ def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceed total_tokens=self._task_state.total_tokens, total_steps=self._task_state.total_steps, status=WorkflowRunStatus.FAILED, - error=event.error + error=event.error, + conversation_id=conversation_id, + tracing_instance=tracing_instance, ) else: if self._task_state.latest_node_execution_info: @@ -546,7 +586,9 @@ def _handle_workflow_finished(self, event: QueueStopEvent | QueueWorkflowSucceed start_at=self._task_state.start_at, total_tokens=self._task_state.total_tokens, total_steps=self._task_state.total_steps, - outputs=outputs + outputs=outputs, + conversation_id=conversation_id, + tracing_instance=tracing_instance, ) self._task_state.workflow_run_id = workflow_run.id diff --git a/api/core/callback_handler/agent_tool_callback_handler.py b/api/core/callback_handler/agent_tool_callback_handler.py index ac5076cd012d0d..d70161f64c1ebf 100644 --- a/api/core/callback_handler/agent_tool_callback_handler.py +++ b/api/core/callback_handler/agent_tool_callback_handler.py @@ -3,6 +3,8 @@ from pydantic import BaseModel +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName + _TEXT_COLOR_MAPPING = { "blue": "36;1", "yellow": "33;1", @@ -51,6 +53,9 @@ def on_tool_end( tool_name: str, tool_inputs: dict[str, Any], tool_outputs: str, + message_id: Optional[str] = None, + timer: Optional[Any] = None, + tracing_instance: Optional[Any] = None, ) -> None: """If not the final action, print out observation.""" print_text("\n[on_tool_end]\n", color=self.color) @@ -59,6 +64,20 @@ def on_tool_end( print_text("Outputs: " + str(tool_outputs)[:1000] + "\n", color=self.color) print_text("\n") + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.TOOL_TRACE, + message_id=message_id, + tool_name=tool_name, + tool_inputs=tool_inputs, + tool_outputs=tool_outputs, + timer=timer, + ) + ) + def on_tool_error( self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any ) -> None: diff --git a/api/core/llm_generator/llm_generator.py b/api/core/llm_generator/llm_generator.py index 14de8649c637e7..c705cc754286c7 100644 --- a/api/core/llm_generator/llm_generator.py +++ b/api/core/llm_generator/llm_generator.py @@ -1,5 +1,6 @@ import json import logging +from typing import Optional from core.llm_generator.output_parser.errors import OutputParserException from core.llm_generator.output_parser.rule_config_generator import RuleConfigGeneratorOutputParser @@ -10,11 +11,16 @@ from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError from core.prompt.utils.prompt_template_parser import PromptTemplateParser +from extensions.ext_database import db +from models.model import Conversation +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName +from services.ops_trace.utils import measure_time class LLMGenerator: @classmethod - def generate_conversation_name(cls, tenant_id: str, query): + def generate_conversation_name(cls, tenant_id: str, query, conversation_id: Optional[str] = None): prompt = CONVERSATION_TITLE_PROMPT if len(query) > 2000: @@ -29,18 +35,19 @@ def generate_conversation_name(cls, tenant_id: str, query): tenant_id=tenant_id, model_type=ModelType.LLM, ) - prompts = [UserPromptMessage(content=prompt)] - response = model_instance.invoke_llm( - prompt_messages=prompts, - model_parameters={ - "max_tokens": 100, - "temperature": 1 - }, - stream=False - ) - answer = response.message.content + with measure_time() as timer: + response = model_instance.invoke_llm( + prompt_messages=prompts, + model_parameters={ + "max_tokens": 100, + "temperature": 1 + }, + stream=False + ) + + answer = response.message.content result_dict = json.loads(answer) answer = result_dict['Your Output'] name = answer.strip() @@ -48,6 +55,29 @@ def generate_conversation_name(cls, tenant_id: str, query): if len(name) > 75: name = name[:75] + '...' + # get tracing instance + conversation_data: Conversation = db.query(Conversation).filter(Conversation.id == conversation_id).first() + app_id = conversation_data.app_id + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id=conversation_data.message_id) + + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id, app_model_config=app_model_config + ) + + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.CONVERSATION_TRACE, + conversation_id=conversation_id, + generate_conversation_name=name, + inputs=prompt, + timer=timer, + tenant_id=tenant_id, + ) + ) + return name @classmethod diff --git a/api/core/moderation/input_moderation.py b/api/core/moderation/input_moderation.py index 8fbc0c2d5003f6..0d915f74fed64d 100644 --- a/api/core/moderation/input_moderation.py +++ b/api/core/moderation/input_moderation.py @@ -3,16 +3,21 @@ from core.app.app_config.entities import AppConfig from core.moderation.base import ModerationAction, ModerationException from core.moderation.factory import ModerationFactory +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.utils import measure_time logger = logging.getLogger(__name__) class InputModeration: - def check(self, app_id: str, - tenant_id: str, - app_config: AppConfig, - inputs: dict, - query: str) -> tuple[bool, dict, str]: + def check( + self, app_id: str, + tenant_id: str, + app_config: AppConfig, + inputs: dict, + query: str, + message_id: str, + ) -> tuple[bool, dict, str]: """ Process sensitive_word_avoidance. :param app_id: app id @@ -20,6 +25,7 @@ def check(self, app_id: str, :param app_config: app config :param inputs: inputs :param query: query + :param message_id: message id :return: """ if not app_config.sensitive_word_avoidance: @@ -35,8 +41,30 @@ def check(self, app_id: str, config=sensitive_word_avoidance_config.config ) - moderation_result = moderation_factory.moderation_for_inputs(inputs, query) + with measure_time() as timer: + moderation_result = moderation_factory.moderation_for_inputs(inputs, query) + from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName + + # get tracing instance + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id, app_model_config=app_model_config + ) + + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.MODERATION_TRACE, + message_id=message_id, + moderation_result=moderation_result, + inputs=inputs, + timer=timer + ) + ) + if not moderation_result.flagged: return False, inputs, query diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index b42a441a3f12ad..878587e09cb7af 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -23,6 +23,9 @@ from extensions.ext_database import db from models.dataset import Dataset, DatasetQuery, DocumentSegment from models.dataset import Document as DatasetDocument +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName +from services.ops_trace.utils import measure_time default_retrieval_model = { 'search_method': 'semantic_search', @@ -37,14 +40,17 @@ class DatasetRetrieval: - def retrieve(self, app_id: str, user_id: str, tenant_id: str, - model_config: ModelConfigWithCredentialsEntity, - config: DatasetEntity, - query: str, - invoke_from: InvokeFrom, - show_retrieve_source: bool, - hit_callback: DatasetIndexToolCallbackHandler, - memory: Optional[TokenBufferMemory] = None) -> Optional[str]: + def retrieve( + self, app_id: str, user_id: str, tenant_id: str, + model_config: ModelConfigWithCredentialsEntity, + config: DatasetEntity, + query: str, + invoke_from: InvokeFrom, + show_retrieve_source: bool, + hit_callback: DatasetIndexToolCallbackHandler, + message_id: str, + memory: Optional[TokenBufferMemory] = None, + ) -> Optional[str]: """ Retrieve dataset. :param app_id: app_id @@ -56,6 +62,7 @@ def retrieve(self, app_id: str, user_id: str, tenant_id: str, :param invoke_from: invoke from :param show_retrieve_source: show retrieve source :param hit_callback: hit callback + :param message_id: message id :param memory: memory :return: """ @@ -112,15 +119,20 @@ def retrieve(self, app_id: str, user_id: str, tenant_id: str, all_documents = [] user_from = 'account' if invoke_from in [InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER] else 'end_user' if retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE: - all_documents = self.single_retrieve(app_id, tenant_id, user_id, user_from, available_datasets, query, - model_instance, - model_config, planning_strategy) + all_documents = self.single_retrieve( + app_id, tenant_id, user_id, user_from, available_datasets, query, + model_instance, + model_config, planning_strategy, message_id + ) elif retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE: - all_documents = self.multiple_retrieve(app_id, tenant_id, user_id, user_from, - available_datasets, query, retrieve_config.top_k, - retrieve_config.score_threshold, - retrieve_config.reranking_model.get('reranking_provider_name'), - retrieve_config.reranking_model.get('reranking_model_name')) + all_documents = self.multiple_retrieve( + app_id, tenant_id, user_id, user_from, + available_datasets, query, retrieve_config.top_k, + retrieve_config.score_threshold, + retrieve_config.reranking_model.get('reranking_provider_name'), + retrieve_config.reranking_model.get('reranking_model_name'), + message_id, + ) document_score_list = {} for item in all_documents: @@ -188,16 +200,18 @@ def retrieve(self, app_id: str, user_id: str, tenant_id: str, return str("\n".join(document_context_list)) return '' - def single_retrieve(self, app_id: str, - tenant_id: str, - user_id: str, - user_from: str, - available_datasets: list, - query: str, - model_instance: ModelInstance, - model_config: ModelConfigWithCredentialsEntity, - planning_strategy: PlanningStrategy, - ): + def single_retrieve( + self, app_id: str, + tenant_id: str, + user_id: str, + user_from: str, + available_datasets: list, + query: str, + model_instance: ModelInstance, + model_config: ModelConfigWithCredentialsEntity, + planning_strategy: PlanningStrategy, + message_id: Optional[str] = None, + ): tools = [] for dataset in available_datasets: description = dataset.description @@ -250,27 +264,35 @@ def single_retrieve(self, app_id: str, if score_threshold_enabled: score_threshold = retrieval_model_config.get("score_threshold") - results = RetrievalService.retrieve(retrival_method=retrival_method, dataset_id=dataset.id, - query=query, - top_k=top_k, score_threshold=score_threshold, - reranking_model=reranking_model) + with measure_time() as timer: + results = RetrievalService.retrieve( + retrival_method=retrival_method, dataset_id=dataset.id, + query=query, + top_k=top_k, score_threshold=score_threshold, + reranking_model=reranking_model + ) self._on_query(query, [dataset_id], app_id, user_from, user_id) + if results: - self._on_retrival_end(results) + self._on_retrival_end(results, message_id, timer) + return results return [] - def multiple_retrieve(self, - app_id: str, - tenant_id: str, - user_id: str, - user_from: str, - available_datasets: list, - query: str, - top_k: int, - score_threshold: float, - reranking_provider_name: str, - reranking_model_name: str): + def multiple_retrieve( + self, + app_id: str, + tenant_id: str, + user_id: str, + user_from: str, + available_datasets: list, + query: str, + top_k: int, + score_threshold: float, + reranking_provider_name: str, + reranking_model_name: str, + message_id: Optional[str] = None, + ): threads = [] all_documents = [] dataset_ids = [dataset.id for dataset in available_datasets] @@ -296,15 +318,23 @@ def multiple_retrieve(self, ) rerank_runner = RerankRunner(rerank_model_instance) - all_documents = rerank_runner.run(query, all_documents, - score_threshold, - top_k) + + with measure_time() as timer: + all_documents = rerank_runner.run( + query, all_documents, + score_threshold, + top_k + ) self._on_query(query, dataset_ids, app_id, user_from, user_id) + if all_documents: - self._on_retrival_end(all_documents) + self._on_retrival_end(all_documents, message_id, timer) + return all_documents - def _on_retrival_end(self, documents: list[Document]) -> None: + def _on_retrival_end( + self, documents: list[Document], message_id: Optional[str] = None, timer: Optional[dict] = None + ) -> None: """Handle retrival end.""" for document in documents: query = db.session.query(DocumentSegment).filter( @@ -323,6 +353,23 @@ def _on_retrival_end(self, documents: list[Document]) -> None: db.session.commit() + # get tracing instance + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model_config.app_id, app_model_config=app_model_config + ) + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.DATASET_RETRIEVAL_TRACE, + message_id=message_id, + documents=documents, + timer=timer + ) + ) + def _on_query(self, query: str, dataset_ids: list[str], app_id: str, user_from: str, user_id: str) -> None: """ Handle query. diff --git a/api/core/tools/tool/workflow_tool.py b/api/core/tools/tool/workflow_tool.py index 122b663f943be3..2c362d80c47831 100644 --- a/api/core/tools/tool/workflow_tool.py +++ b/api/core/tools/tool/workflow_tool.py @@ -1,7 +1,7 @@ import json import logging from copy import deepcopy -from typing import Any, Union +from typing import Any, Optional, Union from core.file.file_obj import FileTransferMethod, FileVar from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolProviderType @@ -31,9 +31,10 @@ def tool_provider_type(self) -> ToolProviderType: :return: the tool provider type """ return ToolProviderType.WORKFLOW - - def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \ - -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + def _invoke( + self, user_id: str, tool_parameters: dict[str, Any], tracing_instance: Optional[Any] = None + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: """ invoke the tool """ @@ -56,6 +57,7 @@ def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) \ invoke_from=self.runtime.invoke_from, stream=False, call_depth=self.workflow_call_depth + 1, + tracing_instance=tracing_instance, ) data = result.get('data', {}) diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py index 16fe9051e3b34a..bf96461cc1447e 100644 --- a/api/core/tools/tool_engine.py +++ b/api/core/tools/tool_engine.py @@ -1,7 +1,7 @@ from copy import deepcopy from datetime import datetime, timezone from mimetypes import guess_type -from typing import Union +from typing import Optional, Union from yarl import URL @@ -24,6 +24,7 @@ from core.tools.utils.message_transformer import ToolFileMessageTransformer from extensions.ext_database import db from models.model import Message, MessageFile +from services.ops_trace.base_trace_instance import BaseTraceInstance class ToolEngine: @@ -31,10 +32,12 @@ class ToolEngine: Tool runtime engine take care of the tool executions. """ @staticmethod - def agent_invoke(tool: Tool, tool_parameters: Union[str, dict], - user_id: str, tenant_id: str, message: Message, invoke_from: InvokeFrom, - agent_tool_callback: DifyAgentCallbackHandler) \ - -> tuple[str, list[tuple[MessageFile, bool]], ToolInvokeMeta]: + def agent_invoke( + tool: Tool, tool_parameters: Union[str, dict], + user_id: str, tenant_id: str, message: Message, invoke_from: InvokeFrom, + agent_tool_callback: DifyAgentCallbackHandler, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> tuple[str, list[tuple[MessageFile, bool]], ToolInvokeMeta]: """ Agent invokes the tool with the given arguments. """ @@ -82,9 +85,11 @@ def agent_invoke(tool: Tool, tool_parameters: Union[str, dict], # hit the callback handler agent_tool_callback.on_tool_end( - tool_name=tool.identity.name, - tool_inputs=tool_parameters, - tool_outputs=plain_text + tool_name=tool.identity.name, + tool_inputs=tool_parameters, + tool_outputs=plain_text, + message_id=message.id, + tracing_instance=tracing_instance, ) # transform tool invoke message to get LLM friendly message @@ -120,8 +125,9 @@ def agent_invoke(tool: Tool, tool_parameters: Union[str, dict], def workflow_invoke(tool: Tool, tool_parameters: dict, user_id: str, workflow_id: str, workflow_tool_callback: DifyWorkflowCallbackHandler, - workflow_call_depth: int) \ - -> list[ToolInvokeMessage]: + workflow_call_depth: int, + tracing_instance: Optional[BaseTraceInstance] = None + ) -> list[ToolInvokeMessage]: """ Workflow invokes the tool with the given arguments. """ @@ -139,9 +145,10 @@ def workflow_invoke(tool: Tool, tool_parameters: dict, # hit the callback handler workflow_tool_callback.on_tool_end( - tool_name=tool.identity.name, - tool_inputs=tool_parameters, - tool_outputs=response + tool_name=tool.identity.name, + tool_inputs=tool_parameters, + tool_outputs=response, + tracing_instance=tracing_instance, ) return response diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 2a472fc8d2cbf4..42d8648594ad5f 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -13,7 +13,9 @@ from core.workflow.nodes.base_node import BaseNode from core.workflow.nodes.tool.entities import ToolNodeData from core.workflow.utils.variable_template_parser import VariableTemplateParser -from models.workflow import WorkflowNodeExecutionStatus +from extensions.ext_database import db +from models.workflow import Workflow, WorkflowNodeExecutionStatus +from services.ops_trace.ops_trace_service import OpsTraceService class ToolNode(BaseNode): @@ -54,6 +56,11 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: # get parameters parameters = self._generate_parameters(variable_pool, node_data, tool_runtime) + # get tracing instance + workflow: Workflow = db.session.query(Workflow).filter(Workflow.id == self.workflow_id).first() + app_id = workflow.app_id + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + try: messages = ToolEngine.workflow_invoke( tool=tool_runtime, @@ -62,6 +69,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: workflow_id=self.workflow_id, workflow_tool_callback=DifyWorkflowCallbackHandler(), workflow_call_depth=self.workflow_call_depth, + tracing_instance=tracing_instance ) except Exception as e: return NodeRunResult( diff --git a/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py new file mode 100644 index 00000000000000..cffd36a0cd3ee8 --- /dev/null +++ b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py @@ -0,0 +1,49 @@ +"""update AppModelConfig and add table TracingAppConfig + +Revision ID: 04c602f5dc9b +Revises: 4e99a8df00ff +Create Date: 2024-06-12 07:49:07.666510 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '04c602f5dc9b' +down_revision = '4e99a8df00ff' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tracing_app_configs', + sa.Column('id', models.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('app_id', models.StringUUID(), nullable=False), + sa.Column('tracing_provider', sa.String(length=255), nullable=True), + sa.Column('tracing_config', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.add_column(sa.Column('trace_config', sa.Text(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.drop_column('trace_config') + + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + op.drop_table('tracing_app_configs') + # ### end Alembic commands ### diff --git a/api/models/model.py b/api/models/model.py index 657db5a5c2e274..d290776f2f6186 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -6,7 +6,7 @@ from flask import current_app, request from flask_login import UserMixin -from sqlalchemy import Float, text +from sqlalchemy import Float, func, text from core.file.tool_file_parser import ToolFileParser from core.file.upload_file_parser import UploadFileParser @@ -233,6 +233,7 @@ class AppModelConfig(db.Model): dataset_configs = db.Column(db.Text) external_data_tools = db.Column(db.Text) file_upload = db.Column(db.Text) + trace_config = db.Column(db.Text) @property def app(self): @@ -1327,3 +1328,36 @@ class TagBinding(db.Model): target_id = db.Column(StringUUID, nullable=True) created_by = db.Column(StringUUID, nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + + +class TracingAppConfig(db.Model): + __tablename__ = 'tracing_app_configs' + __table_args__ = ( + db.PrimaryKeyConstraint('id', name='tracing_app_config_pkey'), + db.Index('tracing_app_config_app_id_idx', 'app_id'), + ) + + id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) + app_id = db.Column(StringUUID, nullable=False) + tracing_provider = db.Column(db.String(255), nullable=True) + tracing_config = db.Column(db.JSON, nullable=True) + created_at = db.Column(db.DateTime, nullable=False, server_default=func.now()) + updated_at = db.Column(db.DateTime, nullable=False, server_default=func.now(), onupdate=func.now()) + + @property + def tracing_config_dict(self): + return self.tracing_config if self.tracing_config else {} + + @property + def tracing_config_str(self): + return json.dumps(self.tracing_config_dict) + + def to_dict(self): + return { + 'id': self.id, + 'app_id': self.app_id, + 'tracing_provider': self.tracing_provider, + 'tracing_config': self.tracing_config_dict, + "created_at": self.created_at.__str__() if self.created_at else None, + 'updated_at': self.updated_at.__str__() if self.updated_at else None, + } diff --git a/api/pyproject.toml b/api/pyproject.toml index a83d98b43842df..e91b3ecc859890 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -184,6 +184,8 @@ tcvectordb = "1.3.2" chromadb = "~0.5.1" tenacity = "~8.3.0" cos-python-sdk-v5 = "1.9.30" +langfuse = "^2.36.1" +langsmith = "^0.1.77" novita-client = "^0.5.6" [tool.poetry.group.dev] diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index f73a6dcbb686b1..f73a88fdd11451 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -18,7 +18,8 @@ def generate(cls, app_model: App, user: Union[Account, EndUser], args: Any, invoke_from: InvokeFrom, - streaming: bool = True) -> Union[dict, Generator[dict, None, None]]: + streaming: bool = True, + ) -> Union[dict, Generator[dict, None, None]]: """ App Content Generate :param app_model: app model diff --git a/api/services/conversation_service.py b/api/services/conversation_service.py index 5c2fb83b7249e5..44a264087cbd69 100644 --- a/api/services/conversation_service.py +++ b/api/services/conversation_service.py @@ -96,7 +96,7 @@ def auto_generate_name(cls, app_model: App, conversation: Conversation): # generate conversation name try: - name = LLMGenerator.generate_conversation_name(app_model.tenant_id, message.query) + name = LLMGenerator.generate_conversation_name(app_model.tenant_id, message.query, conversation.id) conversation.name = name except: pass diff --git a/api/services/message_service.py b/api/services/message_service.py index e826dcc6bf1455..49555e55588a77 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -19,6 +19,9 @@ MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError, ) +from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName +from services.ops_trace.utils import measure_time from services.workflow_service import WorkflowService @@ -262,9 +265,27 @@ def get_suggested_questions_after_answer(cls, app_model: App, user: Optional[Uni message_limit=3, ) - questions = LLMGenerator.generate_suggested_questions_after_answer( - tenant_id=app_model.tenant_id, - histories=histories + with measure_time() as timer: + questions = LLMGenerator.generate_suggested_questions_after_answer( + tenant_id=app_model.tenant_id, + histories=histories + ) + + # get tracing instance + app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model_config.app_id, app_model_config=app_model_config ) + if tracing_instance: + trace_manager = TraceQueueManager() + trace_manager.add_trace_task( + TraceTask( + tracing_instance, + TraceTaskName.SUGGESTED_QUESTION_TRACE, + message_id=message_id, + suggested_question=questions, + timer=timer + ) + ) return questions diff --git a/api/services/ops_trace/base_trace_instance.py b/api/services/ops_trace/base_trace_instance.py new file mode 100644 index 00000000000000..d785c09b75cd79 --- /dev/null +++ b/api/services/ops_trace/base_trace_instance.py @@ -0,0 +1,31 @@ +from abc import ABC, abstractmethod + + +class BaseTraceInstance(ABC): + @abstractmethod + def __init__(self): + ... + + @abstractmethod + def message_trace(self, **kwargs): + return kwargs + + @abstractmethod + def moderation_trace(self, **kwargs): + return kwargs + + @abstractmethod + def suggested_question_trace(self, **kwargs): + return kwargs + + @abstractmethod + def dataset_retrieval_trace(self, **kwargs): + return kwargs + + @abstractmethod + def tool_trace(self, **kwargs): + return kwargs + + @abstractmethod + def generate_name_trace(self, **kwargs): + return kwargs diff --git a/api/services/ops_trace/langfuse_trace.py b/api/services/ops_trace/langfuse_trace.py new file mode 100644 index 00000000000000..af6badbd96e626 --- /dev/null +++ b/api/services/ops_trace/langfuse_trace.py @@ -0,0 +1,712 @@ +import json +import os +from datetime import datetime, timedelta +from enum import Enum +from typing import Any, Optional, Union + +from langfuse import Langfuse +from pydantic import BaseModel, Field, field_validator +from pydantic_core.core_schema import ValidationInfo + +from core.moderation.base import ModerationInputsResult +from extensions.ext_database import db +from models.dataset import Document +from models.model import Message, MessageAgentThought, MessageFile +from models.workflow import WorkflowNodeExecution, WorkflowRun +from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.utils import filter_none_values + + +def validate_input_output(v, field_name): + """ + Validate input output + :param v: + :param field_name: + :return: + """ + if v == {} or v is None: + return v + if isinstance(v, str): + return {field_name: v} + elif isinstance(v, list): + if len(v) > 0 and isinstance(v[0], dict): + return {"message": v} + else: + return {field_name: v} + return v + + +class LevelEnum(str, Enum): + DEBUG = "DEBUG" + WARNING = "WARNING" + ERROR = "ERROR" + DEFAULT = "DEFAULT" + + +class LangfuseTrace(BaseModel): + """ + Langfuse trace model + """ + id: Optional[str] = Field( + default=None, + description="The id of the trace can be set, defaults to a random id. Used to link traces to external systems " + "or when creating a distributed trace. Traces are upserted on id.", + ) + name: Optional[str] = Field( + default=None, + description="Identifier of the trace. Useful for sorting/filtering in the UI.", + ) + input: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The input of the trace. Can be any JSON object." + ) + output: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The output of the trace. Can be any JSON object." + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="Additional metadata of the trace. Can be any JSON object. Metadata is merged when being updated " + "via the API.", + ) + user_id: Optional[str] = Field( + default=None, + description="The id of the user that triggered the execution. Used to provide user-level analytics.", + ) + session_id: Optional[str] = Field( + default=None, + description="Used to group multiple traces into a session in Langfuse. Use your own session/thread identifier.", + ) + version: Optional[str] = Field( + default=None, + description="The version of the trace type. Used to understand how changes to the trace type affect metrics. " + "Useful in debugging.", + ) + release: Optional[str] = Field( + default=None, + description="The release identifier of the current deployment. Used to understand how changes of different " + "deployments affect metrics. Useful in debugging.", + ) + tags: Optional[list[str]] = Field( + default=None, + description="Tags are used to categorize or label traces. Traces can be filtered by tags in the UI and GET " + "API. Tags can also be changed in the UI. Tags are merged and never deleted via the API.", + ) + public: Optional[bool] = Field( + default=None, + description="You can make a trace public to share it via a public link. This allows others to view the trace " + "without needing to log in or be members of your Langfuse project.", + ) + + @field_validator("input", "output") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + return validate_input_output(v, field_name) + + +class LangfuseSpan(BaseModel): + """ + Langfuse span model + """ + id: Optional[str] = Field( + default=None, + description="The id of the span can be set, otherwise a random id is generated. Spans are upserted on id.", + ) + session_id: Optional[str] = Field( + default=None, + description="Used to group multiple spans into a session in Langfuse. Use your own session/thread identifier.", + ) + trace_id: Optional[str] = Field( + default=None, + description="The id of the trace the span belongs to. Used to link spans to traces.", + ) + user_id: Optional[str] = Field( + default=None, + description="The id of the user that triggered the execution. Used to provide user-level analytics.", + ) + start_time: Optional[datetime | str] = Field( + default_factory=datetime.now, + description="The time at which the span started, defaults to the current time.", + ) + end_time: Optional[datetime | str] = Field( + default=None, + description="The time at which the span ended. Automatically set by span.end().", + ) + name: Optional[str] = Field( + default=None, + description="Identifier of the span. Useful for sorting/filtering in the UI.", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="Additional metadata of the span. Can be any JSON object. Metadata is merged when being updated " + "via the API.", + ) + level: Optional[str] = Field( + default=None, + description="The level of the span. Can be DEBUG, DEFAULT, WARNING or ERROR. Used for sorting/filtering of " + "traces with elevated error levels and for highlighting in the UI.", + ) + status_message: Optional[str] = Field( + default=None, + description="The status message of the span. Additional field for context of the event. E.g. the error " + "message of an error event.", + ) + input: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The input of the span. Can be any JSON object." + ) + output: Optional[Union[str, dict[str, Any], list, None]] = Field( + default=None, description="The output of the span. Can be any JSON object." + ) + version: Optional[str] = Field( + default=None, + description="The version of the span type. Used to understand how changes to the span type affect metrics. " + "Useful in debugging.", + ) + + @field_validator("input", "output") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + return validate_input_output(v, field_name) + + +class UnitEnum(str, Enum): + CHARACTERS = "CHARACTERS" + TOKENS = "TOKENS" + SECONDS = "SECONDS" + MILLISECONDS = "MILLISECONDS" + IMAGES = "IMAGES" + + +class GenerationUsage(BaseModel): + promptTokens: Optional[int] = None + completionTokens: Optional[int] = None + totalTokens: Optional[int] = None + input: Optional[int] = None + output: Optional[int] = None + total: Optional[int] = None + unit: Optional[UnitEnum] = None + inputCost: Optional[float] = None + outputCost: Optional[float] = None + totalCost: Optional[float] = None + + +class LangfuseGeneration(BaseModel): + id: Optional[str] = Field( + default=None, + description="The id of the generation can be set, defaults to random id.", + ) + trace_id: Optional[str] = Field( + default=None, + description="The id of the trace the generation belongs to. Used to link generations to traces.", + ) + parent_observation_id: Optional[str] = Field( + default=None, + description="The id of the observation the generation belongs to. Used to link generations to observations.", + ) + name: Optional[str] = Field( + default=None, + description="Identifier of the generation. Useful for sorting/filtering in the UI.", + ) + start_time: Optional[datetime | str] = Field( + default_factory=datetime.now, + description="The time at which the generation started, defaults to the current time.", + ) + completion_start_time: Optional[datetime | str] = Field( + default=None, + description="The time at which the completion started (streaming). Set it to get latency analytics broken " + "down into time until completion started and completion duration.", + ) + end_time: Optional[datetime | str] = Field( + default=None, + description="The time at which the generation ended. Automatically set by generation.end().", + ) + model: Optional[str] = Field( + default=None, description="The name of the model used for the generation." + ) + model_parameters: Optional[dict[str, Any]] = Field( + default=None, + description="The parameters of the model used for the generation; can be any key-value pairs.", + ) + input: Optional[Any] = Field( + default=None, + description="The prompt used for the generation. Can be any string or JSON object.", + ) + output: Optional[Any] = Field( + default=None, + description="The completion generated by the model. Can be any string or JSON object.", + ) + usage: Optional[GenerationUsage] = Field( + default=None, + description="The usage object supports the OpenAi structure with tokens and a more generic version with " + "detailed costs and units.", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="Additional metadata of the generation. Can be any JSON object. Metadata is merged when being " + "updated via the API.", + ) + level: Optional[LevelEnum] = Field( + default=None, + description="The level of the generation. Can be DEBUG, DEFAULT, WARNING or ERROR. Used for sorting/filtering " + "of traces with elevated error levels and for highlighting in the UI.", + ) + status_message: Optional[str] = Field( + default=None, + description="The status message of the generation. Additional field for context of the event. E.g. the error " + "message of an error event.", + ) + version: Optional[str] = Field( + default=None, + description="The version of the generation type. Used to understand how changes to the span type affect " + "metrics. Useful in debugging.", + ) + + @field_validator("input", "output") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + return validate_input_output(v, field_name) + + +class LangFuseDataTrace(BaseTraceInstance): + def __init__( + self, + langfuse_client_public_key: str = None, + langfuse_client_secret_key: str = None, + langfuse_client_host: str = "https://cloud.langfuse.com", + ): + super().__init__() + self.langfuse_client = Langfuse( + public_key=langfuse_client_public_key, + secret_key=langfuse_client_secret_key, + host=langfuse_client_host, + ) + self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001") + + def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): + conversion_id = kwargs.get("conversation_id") + workflow_id = workflow_run.workflow_id + tenant_id = workflow_run.tenant_id + workflow_run_id = workflow_run.id + workflow_run_elapsed_time = workflow_run.elapsed_time + workflow_run_status = workflow_run.status + workflow_run_inputs = ( + json.loads(workflow_run.inputs) if workflow_run.inputs else {} + ) + workflow_run_outputs = ( + json.loads(workflow_run.outputs) if workflow_run.outputs else {} + ) + workflow_run_version = workflow_run.version + error = workflow_run.error if workflow_run.error else "" + + total_tokens = workflow_run.total_tokens + + file_list = workflow_run_inputs.get("sys.file") if workflow_run_inputs.get("sys.file") else [] + query = workflow_run_inputs.get("query") or workflow_run_inputs.get("sys.query") or "" + + metadata = { + "workflow_id": workflow_id, + "conversation_id": conversion_id, + "workflow_run_id": workflow_run_id, + "tenant_id": tenant_id, + "elapsed_time": workflow_run_elapsed_time, + "status": workflow_run_status, + "version": workflow_run_version, + "total_tokens": total_tokens, + "file_list": file_list, + } + + trace_data = LangfuseTrace( + id=workflow_run_id, + name=f"workflow_{workflow_run_id}", + user_id=tenant_id, + input=query, + output=workflow_run_outputs, + metadata=metadata, + session_id=conversion_id, + tags=["workflow"], + ) + + self.add_trace(langfuse_trace_data=trace_data) + + # through workflow_run_id get all_nodes_execution + workflow_nodes_executions = ( + db.session.query(WorkflowNodeExecution) + .filter(WorkflowNodeExecution.workflow_run_id == workflow_run_id) + .order_by(WorkflowNodeExecution.created_at) + .all() + ) + + for node_execution in workflow_nodes_executions: + node_execution_id = node_execution.id + tenant_id = node_execution.tenant_id + app_id = node_execution.app_id + node_name = node_execution.title + node_type = node_execution.node_type + status = node_execution.status + inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} + outputs = ( + json.loads(node_execution.outputs) if node_execution.outputs else {} + ) + created_at = node_execution.created_at if node_execution.created_at else datetime.now() + finished_at = node_execution.finished_at if node_execution.finished_at else datetime.now() + metadata = json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} + metadata.update( + { + "workflow_run_id": workflow_run_id, + "node_execution_id": node_execution_id, + "tenant_id": tenant_id, + "app_id": app_id, + "node_name": node_name, + "node_type": node_type, + "status": status, + } + ) + + process_data = json.loads(node_execution.process_data) if node_execution.process_data else {} + if process_data and process_data.get("model_mode") == "chat": + # add generation + node_total_tokens = json.loads(node_execution.execution_metadata).get("total_tokens") + generation_usage = GenerationUsage( + totalTokens=node_total_tokens, + ) + + langfuse_generation_data = LangfuseGeneration( + name=f"{node_name}_{node_execution_id}", + trace_id=workflow_run_id, + start_time=created_at, + end_time=finished_at, + input=inputs, + output=outputs, + metadata=metadata, + level=LevelEnum.DEFAULT if status == 'succeeded' else LevelEnum.ERROR, + status_message=error if error else "", + usage=generation_usage, + ) + + self.add_generation(langfuse_generation_data) + + # add span + span_data = LangfuseSpan( + name=f"{node_name}_{node_execution_id}", + input=inputs, + output=outputs, + trace_id=workflow_run_id, + start_time=created_at, + end_time=finished_at, + metadata=metadata, + level=LevelEnum.DEFAULT if status == 'succeeded' else LevelEnum.ERROR, + status_message=error if error else "", + ) + + self.add_span(langfuse_span_data=span_data) + + def message_trace(self, message_id: str, conversation_id: str, **kwargs): + message_data = kwargs.get("message_data") + conversation_mode = kwargs.get("conversation_mode") + message_tokens = message_data.message_tokens + answer_tokens = message_data.answer_tokens + total_tokens = message_tokens + answer_tokens + error = message_data.error if message_data.error else "" + input = message_data.message + file_list = input[0].get("files", []) + provider_response_latency = message_data.provider_response_latency + created_at = message_data.created_at + end_time = created_at + timedelta(seconds=provider_response_latency) + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else "" + file_list.append(file_url) + + metadata = { + "conversation_id": conversation_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + + trace_data = LangfuseTrace( + id=message_id, + user_id=message_data.from_end_user_id if message_data.from_end_user_id else message_data.from_account_id, + name=f"message_{message_id}", + input={ + "message": input, + "files": file_list, + "message_tokens": message_tokens, + "answer_tokens": answer_tokens, + "total_tokens": total_tokens, + "error": error, + "provider_response_latency": provider_response_latency, + "created_at": created_at, + }, + output=message_data.answer, + metadata=metadata, + session_id=conversation_id, + tags=["message", str(conversation_mode)], + ) + self.add_trace(langfuse_trace_data=trace_data) + + # start add span + generation_usage = GenerationUsage( + totalTokens=total_tokens, + input=message_tokens, + output=answer_tokens, + total=total_tokens, + unit=UnitEnum.TOKENS, + ) + + langfuse_generation_data = LangfuseGeneration( + name=f"generation_{message_id}", + trace_id=message_id, + start_time=created_at, + end_time=end_time, + model=message_data.model_id, + input=input, + output=message_data.answer, + metadata=metadata, + level=LevelEnum.DEFAULT if message_data.status != 'error' else LevelEnum.ERROR, + status_message=message_data.error if message_data.error else "", + usage=generation_usage, + ) + + self.add_generation(langfuse_generation_data) + + def moderation_trace(self, message_id: str, moderation_result: ModerationInputsResult, **kwargs): + inputs = kwargs.get("inputs") + message_data = kwargs.get("message_data") + flagged = moderation_result.flagged + action = moderation_result.action + preset_response = moderation_result.preset_response + query = moderation_result.query + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "message_id": message_id, + "action": action, + "preset_response": preset_response, + "query": query, + } + + span_data = LangfuseSpan( + name="moderation", + input=inputs, + output={ + "action": action, + "flagged": flagged, + "preset_response": preset_response, + "inputs": inputs, + }, + trace_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.created_at, + metadata=metadata, + ) + + self.add_span(langfuse_span_data=span_data) + + def suggested_question_trace(self, message_id: str, suggested_question: str, **kwargs): + message_data = kwargs.get("message_data") + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + inputs = message_data.query + + metadata = { + "message_id": message_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + + span_data = LangfuseSpan( + name="suggested_question", + input=inputs, + output=suggested_question, + trace_id=message_id, + start_time=start_time, + end_time=end_time, + metadata=metadata, + level=LevelEnum.DEFAULT if message_data.status != 'error' else LevelEnum.ERROR, + status_message=message_data.error if message_data.error else "", + ) + + self.add_span(langfuse_span_data=span_data) + + def dataset_retrieval_trace(self, message_id: str, documents: list[Document], **kwargs): + message_data = kwargs.get("message_data") + inputs = message_data.query if message_data.query else message_data.inputs + metadata = { + "message_id": message_id, + "documents": documents + } + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + dataset_retrieval_span_data = LangfuseSpan( + name="dataset_retrieval", + input=inputs, + output={"documents": documents}, + trace_id=message_id, + start_time=start_time, + end_time=end_time, + metadata=metadata, + ) + + self.add_span(langfuse_span_data=dataset_retrieval_span_data) + + def tool_trace(self, message_id: str, tool_name: str, tool_inputs: dict[str, Any], tool_outputs: str, **kwargs): + message_data: Message = kwargs.get("message_data") + created_time = message_data.created_at + end_time = message_data.updated_at + tool_config = {} + time_cost = 0 + error = None + tool_parameters = {} + + agent_thoughts: list[MessageAgentThought] = message_data.agent_thoughts + for agent_thought in agent_thoughts: + if tool_name in agent_thought.tools: + created_time = agent_thought.created_at + tool_meta_data = agent_thought.tool_meta.get(tool_name, {}) + tool_config = tool_meta_data.get('tool_config', {}) + time_cost = tool_meta_data.get('time_cost', 0) + end_time = created_time + timedelta(seconds=time_cost) + error = tool_meta_data.get('error', "") + tool_parameters = tool_meta_data.get('tool_parameters', {}) + + metadata = { + "message_id": message_id, + "tool_name": tool_name, + "tool_inputs": tool_inputs, + "tool_outputs": tool_outputs, + "tool_config": tool_config, + "time_cost": time_cost, + "error": error, + "tool_parameters": tool_parameters, + } + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + if message_file_data: + message_file_id = message_file_data.id if message_file_data else None + type = message_file_data.type + created_by_role = message_file_data.created_by_role + created_user_id = message_file_data.created_by + + metadata.update( + { + "message_file_id": message_file_id, + "created_by_role": created_by_role, + "created_user_id": created_user_id, + "type": type, + } + ) + + tool_span_data = LangfuseSpan( + name=tool_name, + input=tool_inputs, + output=tool_outputs, + trace_id=message_id, + start_time=created_time, + end_time=end_time, + metadata=metadata, + level=LevelEnum.DEFAULT if error == "" else LevelEnum.ERROR, + status_message=error, + ) + + self.add_span(langfuse_span_data=tool_span_data) + + def generate_name_trace(self, conversation_id: str, inputs: str, generate_conversation_name: str, **kwargs): + timer = kwargs.get("timer") + tenant_id = kwargs.get("tenant_id") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "conversation_id": conversation_id, + } + + name_generation_trace_data = LangfuseTrace( + name="generate_name", + input=inputs, + output=generate_conversation_name, + user_id=tenant_id, + metadata=metadata, + session_id=conversation_id, + ) + + self.add_trace(langfuse_trace_data=name_generation_trace_data) + + name_generation_span_data = LangfuseSpan( + name="generate_name", + input=inputs, + output=generate_conversation_name, + trace_id=conversation_id, + start_time=start_time, + end_time=end_time, + metadata=metadata, + ) + self.add_span(langfuse_span_data=name_generation_span_data) + + def add_trace(self, langfuse_trace_data: Optional[LangfuseTrace] = None): + format_trace_data = ( + filter_none_values(langfuse_trace_data.model_dump()) if langfuse_trace_data else {} + ) + try: + self.langfuse_client.trace(**format_trace_data) + print("LangFuse Trace created successfully") + except Exception as e: + raise f"LangFuse Failed to create trace: {str(e)}" + + def add_span(self, langfuse_span_data: Optional[LangfuseSpan] = None): + format_span_data = ( + filter_none_values(langfuse_span_data.model_dump()) if langfuse_span_data else {} + ) + try: + self.langfuse_client.span(**format_span_data) + print("LangFuse Span created successfully") + except Exception as e: + raise f"LangFuse Failed to create span: {str(e)}" + + def update_span(self, span, langfuse_span_data: Optional[LangfuseSpan] = None): + format_span_data = ( + filter_none_values(langfuse_span_data.model_dump()) if langfuse_span_data else {} + ) + + span.end(**format_span_data) + + def add_generation( + self, langfuse_generation_data: Optional[LangfuseGeneration] = None + ): + format_generation_data = ( + filter_none_values(langfuse_generation_data.model_dump()) + if langfuse_generation_data + else {} + ) + try: + self.langfuse_client.generation(**format_generation_data) + print("LangFuse Generation created successfully") + except Exception as e: + raise f"LangFuse Failed to create generation: {str(e)}" + + def update_generation( + self, generation, langfuse_generation_data: Optional[LangfuseGeneration] = None + ): + format_generation_data = ( + filter_none_values(langfuse_generation_data.model_dump()) + if langfuse_generation_data + else {} + ) + + generation.end(**format_generation_data) diff --git a/api/services/ops_trace/langsmith_trace.py b/api/services/ops_trace/langsmith_trace.py new file mode 100644 index 00000000000000..ef0afdfeb91681 --- /dev/null +++ b/api/services/ops_trace/langsmith_trace.py @@ -0,0 +1,545 @@ +import json +import os +from datetime import datetime, timedelta +from enum import Enum +from typing import Any, Optional, Union + +from langsmith import Client +from pydantic import BaseModel, Field, field_validator +from pydantic_core.core_schema import ValidationInfo + +from core.moderation.base import ModerationInputsResult +from extensions.ext_database import db +from models.dataset import Document +from models.model import Message, MessageAgentThought, MessageFile +from models.workflow import WorkflowNodeExecution, WorkflowRun +from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.utils import filter_none_values + + +class LangSmithRunType(str, Enum): + tool = "tool" + chain = "chain" + llm = "llm" + retriever = "retriever" + embedding = "embedding" + prompt = "prompt" + parser = "parser" + + +class LangSmithTokenUsage(BaseModel): + input_tokens: Optional[int] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + +class LangSmithMultiModel(BaseModel): + file_list: Optional[list[str]] = Field(None, description="List of files") + + +class LangSmithRunModel(LangSmithTokenUsage, LangSmithMultiModel): + name: Optional[str] = Field(..., description="Name of the run") + inputs: Optional[Union[str, dict[str, Any], list, None]] = Field(None, description="Inputs of the run") + outputs: Optional[Union[str, dict[str, Any], list, None]] = Field(None, description="Outputs of the run") + run_type: LangSmithRunType = Field(..., description="Type of the run") + start_time: Optional[datetime | str] = Field(None, description="Start time of the run") + end_time: Optional[datetime | str] = Field(None, description="End time of the run") + extra: Optional[dict[str, Any]] = Field( + None, description="Extra information of the run" + ) + error: Optional[str] = Field(None, description="Error message of the run") + serialized: Optional[dict[str, Any]] = Field( + None, description="Serialized data of the run" + ) + parent_run_id: Optional[str] = Field(None, description="Parent run ID") + events: Optional[list[dict[str, Any]]] = Field( + None, description="Events associated with the run" + ) + tags: Optional[list[str]] = Field(None, description="Tags associated with the run") + trace_id: Optional[str] = Field( + None, description="Trace ID associated with the run" + ) + dotted_order: Optional[str] = Field(None, description="Dotted order of the run") + id: Optional[str] = Field(None, description="ID of the run") + session_id: Optional[str] = Field( + None, description="Session ID associated with the run" + ) + session_name: Optional[str] = Field( + None, description="Session name associated with the run" + ) + reference_example_id: Optional[str] = Field( + None, description="Reference example ID associated with the run" + ) + input_attachments: Optional[dict[str, Any]] = Field( + None, description="Input attachments of the run" + ) + output_attachments: Optional[dict[str, Any]] = Field( + None, description="Output attachments of the run" + ) + + @field_validator("inputs", "outputs") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + values = info.data + if v == {} or v is None: + return v + usage_metadata = { + "input_tokens": values.get('input_tokens', 0), + "output_tokens": values.get('output_tokens', 0), + "total_tokens": values.get('total_tokens', 0), + } + file_list = values.get("file_list", []) + if isinstance(v, str): + return { + field_name: v, + "file_list": file_list, + "usage_metadata": usage_metadata, + } + elif isinstance(v, list): + if len(v) > 0 and isinstance(v[0], dict): + data = { + "message": v, + "usage_metadata": usage_metadata, + "file_list": file_list, + } + return data + else: + return {field_name: v} + if isinstance(v, dict): + v["usage_metadata"] = usage_metadata + v["file_list"] = file_list + return v + return v + + @field_validator("start_time", "end_time") + def format_time(cls, v, info: ValidationInfo): + if not isinstance(v, datetime): + raise ValueError(f"{info.field_name} must be a datetime object") + else: + return v.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +class LangSmithRunUpdateModel(BaseModel): + run_id: str = Field(..., description="ID of the run") + trace_id: Optional[str] = Field( + None, description="Trace ID associated with the run" + ) + dotted_order: Optional[str] = Field(None, description="Dotted order of the run") + parent_run_id: Optional[str] = Field(None, description="Parent run ID") + end_time: Optional[datetime | str] = Field(None, description="End time of the run") + error: Optional[str] = Field(None, description="Error message of the run") + inputs: Optional[dict[str, Any]] = Field(None, description="Inputs of the run") + outputs: Optional[dict[str, Any]] = Field(None, description="Outputs of the run") + events: Optional[list[dict[str, Any]]] = Field( + None, description="Events associated with the run" + ) + tags: Optional[list[str]] = Field(None, description="Tags associated with the run") + extra: Optional[dict[str, Any]] = Field( + None, description="Extra information of the run" + ) + input_attachments: Optional[dict[str, Any]] = Field( + None, description="Input attachments of the run" + ) + output_attachments: Optional[dict[str, Any]] = Field( + None, description="Output attachments of the run" + ) + + +class LangSmithDataTrace(BaseTraceInstance): + def __init__( + self, + langsmith_key: str = None, + project_name: str = None, + endpoint: str = "https://api.smith.langchain.com" + ): + super().__init__() + self.langsmith_key = langsmith_key + self.project_name = project_name + self.project_id = None + self.langsmith_client = Client( + api_key=langsmith_key, api_url=endpoint + ) + self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001") + + def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): + conversion_id = kwargs.get("conversation_id") + workflow_id = workflow_run.workflow_id + tenant_id = workflow_run.tenant_id + workflow_run_id = workflow_run.id + workflow_run_created_at = workflow_run.created_at + workflow_run_finished_at = workflow_run.finished_at + workflow_run_elapsed_time = workflow_run.elapsed_time + workflow_run_status = workflow_run.status + workflow_run_inputs = ( + json.loads(workflow_run.inputs) if workflow_run.inputs else {} + ) + workflow_run_outputs = ( + json.loads(workflow_run.outputs) if workflow_run.outputs else {} + ) + workflow_run_version = workflow_run.version + error = workflow_run.error if workflow_run.error else "" + + total_tokens = workflow_run.total_tokens + + file_list = workflow_run_inputs.get("sys.file") if workflow_run_inputs.get("sys.file") else [] + query = workflow_run_inputs.get("query") or workflow_run_inputs.get("sys.query") or "" + + metadata = { + "workflow_id": workflow_id, + "conversation_id": conversion_id, + "workflow_run_id": workflow_run_id, + "tenant_id": tenant_id, + "elapsed_time": workflow_run_elapsed_time, + "status": workflow_run_status, + "version": workflow_run_version, + "total_tokens": total_tokens, + } + + langsmith_run = LangSmithRunModel( + file_list=file_list, + total_tokens=total_tokens, + id=workflow_run_id, + name=f"workflow_run_{workflow_run_id}", + inputs=query, + run_type=LangSmithRunType.tool, + start_time=workflow_run_created_at, + end_time=workflow_run_finished_at, + outputs=workflow_run_outputs, + extra={ + "metadata": metadata, + }, + error=error, + tags=["workflow"], + ) + + self.add_run(langsmith_run) + + # through workflow_run_id get all_nodes_execution + workflow_nodes_executions = ( + db.session.query(WorkflowNodeExecution) + .filter(WorkflowNodeExecution.workflow_run_id == workflow_run_id) + .order_by(WorkflowNodeExecution.created_at) + .all() + ) + + for node_execution in workflow_nodes_executions: + node_execution_id = node_execution.id + tenant_id = node_execution.tenant_id + app_id = node_execution.app_id + node_name = node_execution.title + node_type = node_execution.node_type + status = node_execution.status + inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} + outputs = ( + json.loads(node_execution.outputs) if node_execution.outputs else {} + ) + created_at = node_execution.created_at if node_execution.created_at else datetime.now() + finished_at = node_execution.finished_at if node_execution.finished_at else datetime.now() + execution_metadata = ( + json.loads(node_execution.execution_metadata) + if node_execution.execution_metadata + else {} + ) + node_total_tokens = execution_metadata.get("total_tokens", 0) + + metadata = json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} + metadata.update( + { + "node_execution_id": node_execution_id, + "tenant_id": tenant_id, + "app_id": app_id, + "app_name": node_name, + "node_type": node_type, + "status": status, + } + ) + + process_data = json.loads(node_execution.process_data) if node_execution.process_data else {} + if process_data and process_data.get("model_mode") == "chat": + run_type = LangSmithRunType.llm + elif node_type == "knowledge-retrieval": + run_type = LangSmithRunType.retriever + else: + run_type = LangSmithRunType.tool + + langsmith_run = LangSmithRunModel( + total_tokens=node_total_tokens, + name=f"{node_name}_{node_execution_id}", + inputs=inputs, + run_type=run_type, + start_time=created_at, + end_time=finished_at, + outputs=outputs, + file_list=file_list, + extra={ + "metadata": metadata, + }, + parent_run_id=workflow_run_id, + tags=["node_execution"], + ) + + self.add_run(langsmith_run) + + def message_trace(self, message_id: str, conversation_id: str, **kwargs): + message_data = kwargs.get("message_data") + conversation_mode = kwargs.get("conversation_mode") + message_tokens = message_data.message_tokens + answer_tokens = message_data.answer_tokens + total_tokens = message_tokens + answer_tokens + error = message_data.error if message_data.error else "" + inputs = message_data.message + file_list = inputs[0].get("files", []) + provider_response_latency = message_data.provider_response_latency + created_at = message_data.created_at + end_time = created_at + timedelta(seconds=provider_response_latency) + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else "" + file_list.append(file_url) + + metadata = { + "conversation_id": conversation_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + message_run = LangSmithRunModel( + input_tokens=message_tokens, + output_tokens=answer_tokens, + total_tokens=total_tokens, + id=message_id, + name=f"message_{message_id}", + inputs=inputs, + run_type=LangSmithRunType.llm, + start_time=created_at, + end_time=end_time, + outputs=message_data.answer, + extra={ + "metadata": metadata, + }, + tags=["message", str(conversation_mode)], + error=error, + file_list=file_list, + ) + self.add_run(message_run) + + def moderation_trace(self, message_id: str, moderation_result: ModerationInputsResult, **kwargs): + inputs = kwargs.get("inputs") + message_data = kwargs.get("message_data") + flagged = moderation_result.flagged + action = moderation_result.action + preset_response = moderation_result.preset_response + query = moderation_result.query + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "message_id": message_id, + "action": action, + "preset_response": preset_response, + "query": query, + } + + langsmith_run = LangSmithRunModel( + name="moderation", + inputs=inputs, + outputs={ + "action": action, + "flagged": flagged, + "preset_response": preset_response, + "inputs": inputs, + }, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["moderation"], + parent_run_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.updated_at, + ) + + self.add_run(langsmith_run) + + def suggested_question_trace(self, message_id: str, suggested_question: str, **kwargs): + message_data = kwargs.get("message_data") + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + inputs = message_data.query + + metadata = { + "message_id": message_id, + "ls_provider": message_data.model_provider, + "ls_model_name": message_data.model_id, + "status": message_data.status, + "from_end_user_id": message_data.from_account_id, + "from_account_id": message_data.from_account_id, + "agent_based": message_data.agent_based, + "workflow_run_id": message_data.workflow_run_id, + "from_source": message_data.from_source, + } + + suggested_question_run = LangSmithRunModel( + name="suggested_question", + inputs=inputs, + outputs=suggested_question, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["suggested_question"], + parent_run_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.updated_at, + ) + + self.add_run(suggested_question_run) + + def dataset_retrieval_trace(self, message_id: str, documents: list[Document], **kwargs): + message_data = kwargs.get("message_data") + inputs = message_data.query if message_data.query else message_data.inputs + metadata = { + "message_id": message_id, + "documents": documents + } + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + dataset_retrieval_run = LangSmithRunModel( + name="dataset_retrieval", + inputs=inputs, + outputs={"documents": documents}, + run_type=LangSmithRunType.retriever, + extra={ + "metadata": metadata, + }, + tags=["dataset_retrieval"], + parent_run_id=message_id, + start_time=start_time or message_data.created_at, + end_time=end_time or message_data.updated_at, + ) + + self.add_run(dataset_retrieval_run) + + def tool_trace(self, message_id: str, tool_name: str, tool_inputs: dict[str, Any], tool_outputs: str, **kwargs): + message_data: Message = kwargs.get("message_data") + created_time = message_data.created_at + end_time = message_data.updated_at + tool_config = {} + time_cost = 0 + error = "" + tool_parameters = {} + file_url = "" + + agent_thoughts: list[MessageAgentThought] = message_data.agent_thoughts + for agent_thought in agent_thoughts: + if tool_name in agent_thought.tools: + created_time = agent_thought.created_at + tool_meta_data = agent_thought.tool_meta.get(tool_name, {}) + tool_config = tool_meta_data.get('tool_config', {}) + time_cost = tool_meta_data.get('time_cost', 0) + end_time = created_time + timedelta(seconds=time_cost) + error = tool_meta_data.get('error', "") + tool_parameters = tool_meta_data.get('tool_parameters', {}) + + metadata = { + "message_id": message_id, + "tool_name": tool_name, + "tool_inputs": tool_inputs, + "tool_outputs": tool_outputs, + "tool_config": tool_config, + "time_cost": time_cost, + "error": error, + "tool_parameters": tool_parameters, + } + + # get message file data + message_file_data: MessageFile = kwargs.get("message_file_data") + if message_file_data: + message_file_id = message_file_data.id if message_file_data else None + type = message_file_data.type + created_by_role = message_file_data.created_by_role + created_user_id = message_file_data.created_by + file_url = f"{self.file_base_url}/{message_file_data.url}" + + metadata.update( + { + "message_file_id": message_file_id, + "created_by_role": created_by_role, + "created_user_id": created_user_id, + "type": type, + } + ) + + tool_run = LangSmithRunModel( + name=tool_name, + inputs=tool_inputs, + outputs=tool_outputs, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["tool", tool_name], + parent_run_id=message_id, + start_time=created_time, + end_time=end_time, + file_list=[file_url], + ) + + self.add_run(tool_run) + + def generate_name_trace(self, conversation_id: str, inputs: str, generate_conversation_name: str, **kwargs): + timer = kwargs.get("timer") + start_time = timer.get("start") + end_time = timer.get("end") + + metadata = { + "conversation_id": conversation_id, + } + + name_run = LangSmithRunModel( + name="generate_name", + inputs=inputs, + outputs=generate_conversation_name, + run_type=LangSmithRunType.tool, + extra={ + "metadata": metadata, + }, + tags=["generate_name"], + start_time=start_time or datetime.now(), + end_time=end_time or datetime.now(), + ) + + self.add_run(name_run) + + def add_run(self, run_data: LangSmithRunModel): + data = run_data.model_dump() + if self.project_id: + data["session_id"] = self.project_id + elif self.project_name: + data["session_name"] = self.project_name + + data = filter_none_values(data) + try: + self.langsmith_client.create_run(**data) + print("LangSmith Run created successfully.") + except Exception as e: + raise f"LangSmith Failed to create run: {str(e)}" + + def update_run(self, update_run_data: LangSmithRunUpdateModel): + data = update_run_data.model_dump() + data = filter_none_values(data) + try: + self.langsmith_client.update_run(**data) + print("LangSmith Run updated successfully.") + except Exception as e: + raise f"LangSmith Failed to update run: {str(e)}" diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py new file mode 100644 index 00000000000000..8072ca3948a830 --- /dev/null +++ b/api/services/ops_trace/ops_trace_service.py @@ -0,0 +1,321 @@ +import json +from enum import Enum +from typing import Optional + +from pydantic import BaseModel + +from core.app.app_config.entities import AppAdditionalFeatures +from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token +from extensions.ext_database import db +from models.model import App, AppModelConfig, Conversation, Message, TracingAppConfig +from models.workflow import Workflow +from services.ops_trace.langfuse_trace import LangFuseDataTrace +from services.ops_trace.langsmith_trace import LangSmithDataTrace + + +class TracingProviderEnum(Enum): + LANGFUSE = 'langfuse' + LANGSMITH = 'langSmith' + + +class LangfuseConfig(BaseModel): + """ + Model class for Langfuse tracing config. + """ + public_key: str + secret_key: str + host: str + + +class LangsmithConfig(BaseModel): + """ + Model class for Langsmith tracing config. + """ + api_key: str + project: str + endpoint: str + + +class OpsTraceService: + @classmethod + def get_tracing_app_config(cls, app_id: str, tracing_provider: str): + """ + Get tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :return: + """ + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config_data: + return None + + # decrypt_token and obfuscated_token + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + decrypt_tracing_config = cls.decrypt_tracing_config(tenant_id, tracing_provider, trace_config_data.tracing_config) + decrypt_tracing_config = cls.obfuscated_decrypt_token(tracing_provider, decrypt_tracing_config) + + trace_config_data.tracing_config = decrypt_tracing_config + + return trace_config_data.to_dict() + + @classmethod + def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_config: dict): + """ + Create tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + # check if trace config already exists + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if trace_config_data: + return None + + # get tenant id + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) + trace_config_data = TracingAppConfig( + app_id=app_id, + tracing_provider=tracing_provider, + tracing_config=tracing_config, + ) + db.session.add(trace_config_data) + db.session.commit() + + return trace_config_data.to_dict() + + @classmethod + def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_config: dict): + """ + Update tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + # check if trace config already exists + trace_config = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config: + return None + + # get tenant id + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) + + trace_config.tracing_config = tracing_config + db.session.commit() + + return trace_config.to_dict() + + @classmethod + def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): + """ + Encrypt tracing config + :param tenant_id: tenant id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + tracing_config = LangfuseConfig(**tracing_config) + encrypt_public_key = encrypt_token(tenant_id, tracing_config.public_key) + encrypt_secret_key = encrypt_token(tenant_id, tracing_config.secret_key) + tracing_config = LangfuseConfig( + public_key=encrypt_public_key, + secret_key=encrypt_secret_key, + host=tracing_config.host + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + tracing_config = LangsmithConfig(**tracing_config) + encrypt_api_key = encrypt_token(tenant_id, tracing_config.api_key) + tracing_config = LangsmithConfig( + api_key=encrypt_api_key, + project=tracing_config.project, + endpoint=tracing_config.endpoint + ) + + if isinstance(tracing_config, BaseModel): + return tracing_config.dict() + return tracing_config + + @classmethod + def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): + """ + Decrypt tracing config + :param tenant_id: tenant id + :param tracing_provider: tracing provider + :param tracing_config: tracing config + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + tracing_config = LangfuseConfig(**tracing_config) + decrypt_public_key = decrypt_token(tenant_id, tracing_config.public_key) + decrypt_secret_key = decrypt_token(tenant_id, tracing_config.secret_key) + tracing_config = LangfuseConfig( + public_key=decrypt_public_key, + secret_key=decrypt_secret_key, + host=tracing_config.host + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + tracing_config = LangsmithConfig(**tracing_config) + decrypt_api_key = decrypt_token(tenant_id, tracing_config.api_key) + tracing_config = LangsmithConfig( + api_key=decrypt_api_key, + project=tracing_config.project, + endpoint=tracing_config.endpoint + ) + + if isinstance(tracing_config, BaseModel): + return tracing_config.dict() + return tracing_config + + @classmethod + def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config:dict): + """ + Decrypt tracing config + :param tracing_provider: tracing provider + :param decrypt_tracing_config: tracing config + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + decrypt_tracing_config = LangfuseConfig(**decrypt_tracing_config) + decrypt_public_key = decrypt_tracing_config.public_key + decrypt_secret_key = decrypt_tracing_config.secret_key + obfuscated_public_key = obfuscated_token(decrypt_public_key) + obfuscated_secret_key = obfuscated_token(decrypt_secret_key) + decrypt_tracing_config = LangfuseConfig( + public_key=obfuscated_public_key, + secret_key=obfuscated_secret_key, + host=decrypt_tracing_config.host + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + decrypt_tracing_config = LangsmithConfig(**decrypt_tracing_config) + decrypt_api_key = decrypt_tracing_config.api_key + obfuscated_api_key = obfuscated_token(decrypt_api_key) + decrypt_tracing_config = LangsmithConfig( + api_key=obfuscated_api_key, + project=decrypt_tracing_config.project, + endpoint=decrypt_tracing_config.endpoint + ) + + return decrypt_tracing_config.dict() + + @classmethod + def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): + """ + Get decrypted tracing config + :param app_id: app id + :param tracing_provider: tracing provider + :return: + """ + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config_data: + return None + + # decrypt_token + tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id + decrypt_tracing_config = cls.decrypt_tracing_config( + tenant_id, tracing_provider, trace_config_data.tracing_config + ) + + return decrypt_tracing_config + + @classmethod + def get_ops_trace_instance( + cls, + app_id: str, + workflow: Optional[Workflow] = None, + app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None + ): + """ + Get ops trace through model config + :param app_id: app_id + :param workflow: workflow + :param app_model_config: app_model_config + :return: + """ + tracing_instance = None + app_ops_trace_config = None + + # get trace configuration from available sources + if app_model_config is not None: + if isinstance(app_model_config, AppAdditionalFeatures): + app_ops_trace_config = app_model_config.trace_config + elif isinstance(app_model_config, AppModelConfig): + app_ops_trace_config = json.loads( + app_model_config.trace_config + ) if app_model_config.trace_config else None + elif workflow: + features_data = json.loads(workflow.features) + app_ops_trace_config = features_data.get('trace_config') if features_data else None + else: + # As a last resort, fetch from the database + trace_config_data = db.session.query(AppModelConfig.trace_config).filter( + AppModelConfig.app_id == app_id + ).order_by(AppModelConfig.updated_at.desc()).first() + if trace_config_data: + app_ops_trace_config = json.loads(trace_config_data.trace_config) + else: + raise ValueError('Trace config not found') + + if app_ops_trace_config is not None: + tracing_provider = app_ops_trace_config.get('tracing_provider') + else: + return None + + # decrypt_token + decrypt_trace_config = cls.get_decrypted_tracing_config(app_id, tracing_provider) + if app_ops_trace_config.get('enabled'): + tracing_provider = tracing_provider + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + langfuse_client_public_key = decrypt_trace_config.get('public_key') + langfuse_client_secret_key = decrypt_trace_config.get('secret_key') + langfuse_host = decrypt_trace_config.get('host') + tracing_instance = LangFuseDataTrace( + langfuse_client_public_key, + langfuse_client_secret_key, + langfuse_host, + ) + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + langsmith_api_key = decrypt_trace_config.get('api_key') + langsmith_project = decrypt_trace_config.get('project') + langsmith_endpoint = decrypt_trace_config.get('endpoint') + print(langsmith_api_key, langsmith_project, langsmith_endpoint) + tracing_instance = LangSmithDataTrace( + langsmith_api_key, + langsmith_project, + langsmith_endpoint, + ) + + return tracing_instance + + return None + + @classmethod + def get_app_config_through_message_id(cls, message_id: str): + app_model_config = None + message_data = db.session.query(Message).filter(Message.id == message_id).first() + conversation_id = message_data.conversation_id + conversation_data = db.session.query(Conversation).filter(Conversation.id == conversation_id).first() + + if conversation_data.app_model_config_id: + app_model_config = db.session.query(AppModelConfig).filter( + AppModelConfig.id == conversation_data.app_model_config_id + ).first() + elif conversation_data.app_model_config_id is None and conversation_data.override_model_configs: + app_model_config = conversation_data.override_model_configs + + return app_model_config diff --git a/api/services/ops_trace/trace_queue_manager.py b/api/services/ops_trace/trace_queue_manager.py new file mode 100644 index 00000000000000..bb65d7e1a2cde0 --- /dev/null +++ b/api/services/ops_trace/trace_queue_manager.py @@ -0,0 +1,133 @@ +import queue +import threading +from enum import Enum + +from extensions.ext_database import db +from models.model import Conversation, MessageFile +from services.ops_trace.utils import get_message_data + + +class TraceTaskName(str, Enum): + CONVERSATION_TRACE = 'conversation_trace' + WORKFLOW_TRACE = 'workflow_trace' + MESSAGE_TRACE = 'message_trace' + MODERATION_TRACE = 'moderation_trace' + SUGGESTED_QUESTION_TRACE = 'suggested_question_trace' + DATASET_RETRIEVAL_TRACE = 'dataset_retrieval_trace' + TOOL_TRACE = 'tool_trace' + GENERATE_NAME_TRACE = 'generate_name_trace' + + +class TraceTask: + def __init__(self, trace_instance, trace_type, **kwargs): + self.trace_instance = trace_instance + self.trace_type = trace_type + self.kwargs = kwargs + + def execute(self): + method_name, processed_kwargs = self.preprocess() + method = getattr(self.trace_instance, method_name) + method(**processed_kwargs) + + def preprocess(self): + if self.trace_type == TraceTaskName.CONVERSATION_TRACE: + return TraceTaskName.CONVERSATION_TRACE, self.process_conversation_trace(**self.kwargs) + if self.trace_type == TraceTaskName.WORKFLOW_TRACE: + return TraceTaskName.WORKFLOW_TRACE, self.process_workflow_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.MESSAGE_TRACE: + return TraceTaskName.MESSAGE_TRACE, self.process_message_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.MODERATION_TRACE: + return TraceTaskName.MODERATION_TRACE, self.process_moderation_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.SUGGESTED_QUESTION_TRACE: + return TraceTaskName.SUGGESTED_QUESTION_TRACE, self.process_suggested_question_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.DATASET_RETRIEVAL_TRACE: + return TraceTaskName.DATASET_RETRIEVAL_TRACE, self.process_dataset_retrieval_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.TOOL_TRACE: + return TraceTaskName.TOOL_TRACE, self.process_tool_trace(**self.kwargs) + elif self.trace_type == TraceTaskName.GENERATE_NAME_TRACE: + return TraceTaskName.GENERATE_NAME_TRACE, self.process_generate_name_trace(**self.kwargs) + else: + return '', {} + + # process methods for different trace types + def process_conversation_trace(self, **kwargs): + return kwargs + + def process_workflow_trace(self, **kwargs): + return kwargs + + def process_message_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + message_file_data = db.session.query(MessageFile).filter_by(message_id=message_id).first() + conversation_mode = db.session.query(Conversation.mode).filter_by(id=message_data.conversation_id).first() + conversation_mode = conversation_mode[0] + kwargs['message_data'] = message_data + kwargs['message_file_data'] = message_file_data + kwargs['conversation_mode'] = conversation_mode + return kwargs + + def process_moderation_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + kwargs['message_data'] = message_data + return kwargs + + def process_suggested_question_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + kwargs['message_data'] = message_data + return kwargs + + def process_dataset_retrieval_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + kwargs['message_data'] = message_data + return kwargs + + def process_tool_trace(self, **kwargs): + message_id = kwargs.get('message_id') + message_data = get_message_data(message_id) + if not message_data: + return {} + message_file_data = db.session.query(MessageFile).filter_by(message_id=message_id).first() + kwargs['message_data'] = message_data + kwargs['message_file_data'] = message_file_data + return kwargs + + def process_generate_name_trace(self, **kwargs): + return kwargs + + +class TraceQueueManager: + def __init__(self): + from app import app + self.app = app + self.queue = queue.Queue() + self.is_running = True + self.thread = threading.Thread(target=self.process_queue) + self.thread.start() + + def stop(self): + self.is_running = False + + def process_queue(self): + with self.app.app_context(): + while self.is_running: + try: + task = self.queue.get(timeout=1) + task.execute() + self.queue.task_done() + except queue.Empty: + self.stop() + + def add_trace_task(self, trace_task): + self.queue.put(trace_task) diff --git a/api/services/ops_trace/utils.py b/api/services/ops_trace/utils.py new file mode 100644 index 00000000000000..c556947380d9e5 --- /dev/null +++ b/api/services/ops_trace/utils.py @@ -0,0 +1,28 @@ +from contextlib import contextmanager +from datetime import datetime + +from extensions.ext_database import db +from models.model import Message + + +def filter_none_values(data: dict): + for key, value in data.items(): + if value is None: + continue + if isinstance(value, datetime): + data[key] = value.isoformat() + return {key: value for key, value in data.items() if value is not None} + + +def get_message_data(message_id): + return db.session.query(Message).filter(Message.id == message_id).first() + + +@contextmanager +def measure_time(): + timing_info = {'start': datetime.now(), 'end': None} + try: + yield timing_info + finally: + timing_info['end'] = datetime.now() + print(f"Execution time: {timing_info['end'] - timing_info['start']}") \ No newline at end of file From e61d9d53f372a97a4a87c2d374b43c235ee44cdf Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Mon, 17 Jun 2024 10:33:57 +0800 Subject: [PATCH 230/273] feat: add remove tracing app --- api/controllers/console/app/ops_trace.py | 20 ++++++++++++++++++++ api/services/ops_trace/ops_trace_service.py | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index c4b3641b14ffec..d421a913ac09e5 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -75,5 +75,25 @@ def put(self, app_id): except Exception as e: raise e + @setup_required + @login_required + @account_initialization_required + def delete(self, app_id): + """Delete an existing trace app configuration""" + parser = reqparse.RequestParser() + parser.add_argument('tracing_provider', type=str, required=True, location='args') + args = parser.parse_args() + + try: + result = OpsTraceService.delete_tracing_app_config( + app_id=app_id, + tracing_provider=args['tracing_provider'] + ) + if not result: + raise TracingConfigNotExist() + return {"result": "success"} + except Exception as e: + raise e + api.add_resource(TraceAppConfigApi, '/apps/<uuid:app_id>/trace-config') diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 8072ca3948a830..0b604367d4c907 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -117,6 +117,26 @@ def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c return trace_config.to_dict() + @classmethod + def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): + """ + Delete tracing app config + :param app_id: app id + :param tracing_provider: tracing provider + :return: + """ + trace_config = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + ).first() + + if not trace_config: + return None + + db.session.delete(trace_config) + db.session.commit() + + return True + @classmethod def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): """ @@ -279,7 +299,6 @@ def get_ops_trace_instance( # decrypt_token decrypt_trace_config = cls.get_decrypted_tracing_config(app_id, tracing_provider) if app_ops_trace_config.get('enabled'): - tracing_provider = tracing_provider if tracing_provider == TracingProviderEnum.LANGFUSE.value: langfuse_client_public_key = decrypt_trace_config.get('public_key') langfuse_client_secret_key = decrypt_trace_config.get('secret_key') From 429fe5c25aa88cb98661e4c1f85c65ec9f56ef62 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Mon, 17 Jun 2024 18:09:45 +0800 Subject: [PATCH 231/273] feat: update trace table --- ...9b_update_appmodelconfig_and_add_table_.py | 2 +- .../versions/2a3aebbbf4bb_add_app_tracing.py | 39 +++++++++++ ...9_remove_app_model_config_trace_config_.py | 66 +++++++++++++++++++ api/models/model.py | 8 ++- api/services/ops_trace/ops_trace_service.py | 28 ++++---- 5 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py create mode 100644 api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py diff --git a/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py index cffd36a0cd3ee8..316a44edce0c55 100644 --- a/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py +++ b/api/migrations/versions/04c602f5dc9b_update_appmodelconfig_and_add_table_.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = '04c602f5dc9b' -down_revision = '4e99a8df00ff' +down_revision = '7b45942e39bb' branch_labels = None depends_on = None diff --git a/api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py b/api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py new file mode 100644 index 00000000000000..09ef5e186cd089 --- /dev/null +++ b/api/migrations/versions/2a3aebbbf4bb_add_app_tracing.py @@ -0,0 +1,39 @@ +"""add app tracing + +Revision ID: 2a3aebbbf4bb +Revises: c031d46af369 +Create Date: 2024-06-17 10:08:54.803701 + +""" +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '2a3aebbbf4bb' +down_revision = 'c031d46af369' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('apps', schema=None) as batch_op: + batch_op.add_column(sa.Column('tracing', sa.Text(), nullable=True)) + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + with op.batch_alter_table('apps', schema=None) as batch_op: + batch_op.drop_column('tracing') + + # ### end Alembic commands ### diff --git a/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py new file mode 100644 index 00000000000000..8d69fa86eb8487 --- /dev/null +++ b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py @@ -0,0 +1,66 @@ +"""remove app model config trace config and rename trace app config + +Revision ID: c031d46af369 +Revises: 04c602f5dc9b +Create Date: 2024-06-17 10:01:00.255189 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +import models as models + +# revision identifiers, used by Alembic. +revision = 'c031d46af369' +down_revision = '04c602f5dc9b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('trace_app_config', + sa.Column('id', models.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('app_id', models.StringUUID(), nullable=False), + sa.Column('tracing_provider', sa.String(length=255), nullable=True), + sa.Column('tracing_config', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), + sa.Column('is_active', sa.Boolean(), server_default=sa.text('true'), nullable=False), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.drop_column('trace_config') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('app_model_configs', schema=None) as batch_op: + batch_op.add_column(sa.Column('trace_config', sa.TEXT(), autoincrement=False, nullable=True)) + + op.create_table('tracing_app_configs', + sa.Column('id', sa.UUID(), server_default=sa.text('uuid_generate_v4()'), autoincrement=False, nullable=False), + sa.Column('app_id', sa.UUID(), autoincrement=False, nullable=False), + sa.Column('tracing_provider', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('tracing_config', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + op.drop_table('trace_app_config') + # ### end Alembic commands ### diff --git a/api/models/model.py b/api/models/model.py index d290776f2f6186..8a6b324343e68c 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -73,6 +73,7 @@ class App(db.Model): is_demo = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) is_public = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) is_universal = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) + tracing = db.Column(db.Text, nullable=True) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) @@ -233,7 +234,6 @@ class AppModelConfig(db.Model): dataset_configs = db.Column(db.Text) external_data_tools = db.Column(db.Text) file_upload = db.Column(db.Text) - trace_config = db.Column(db.Text) @property def app(self): @@ -1330,8 +1330,8 @@ class TagBinding(db.Model): created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) -class TracingAppConfig(db.Model): - __tablename__ = 'tracing_app_configs' +class TraceAppConfig(db.Model): + __tablename__ = 'trace_app_config' __table_args__ = ( db.PrimaryKeyConstraint('id', name='tracing_app_config_pkey'), db.Index('tracing_app_config_app_id_idx', 'app_id'), @@ -1343,6 +1343,7 @@ class TracingAppConfig(db.Model): tracing_config = db.Column(db.JSON, nullable=True) created_at = db.Column(db.DateTime, nullable=False, server_default=func.now()) updated_at = db.Column(db.DateTime, nullable=False, server_default=func.now(), onupdate=func.now()) + is_active = db.Column(db.Boolean, nullable=False, server_default=db.text('true')) @property def tracing_config_dict(self): @@ -1358,6 +1359,7 @@ def to_dict(self): 'app_id': self.app_id, 'tracing_provider': self.tracing_provider, 'tracing_config': self.tracing_config_dict, + "is_active": self.is_active, "created_at": self.created_at.__str__() if self.created_at else None, 'updated_at': self.updated_at.__str__() if self.updated_at else None, } diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 0b604367d4c907..9cc88a10e4639d 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -7,7 +7,7 @@ from core.app.app_config.entities import AppAdditionalFeatures from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from extensions.ext_database import db -from models.model import App, AppModelConfig, Conversation, Message, TracingAppConfig +from models.model import App, AppModelConfig, Conversation, Message, TraceAppConfig from models.workflow import Workflow from services.ops_trace.langfuse_trace import LangFuseDataTrace from services.ops_trace.langsmith_trace import LangSmithDataTrace @@ -15,7 +15,7 @@ class TracingProviderEnum(Enum): LANGFUSE = 'langfuse' - LANGSMITH = 'langSmith' + LANGSMITH = 'langsmith' class LangfuseConfig(BaseModel): @@ -45,8 +45,8 @@ def get_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: @@ -71,8 +71,8 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :return: """ # check if trace config already exists - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if trace_config_data: @@ -81,7 +81,7 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c # get tenant id tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) - trace_config_data = TracingAppConfig( + trace_config_data = TraceAppConfig( app_id=app_id, tracing_provider=tracing_provider, tracing_config=tracing_config, @@ -101,8 +101,8 @@ def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :return: """ # check if trace config already exists - trace_config = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -125,8 +125,8 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -136,7 +136,7 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): db.session.commit() return True - + @classmethod def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): """ @@ -238,8 +238,8 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: From 6f53b7579715af7190af1d6d478b1e2090e29092 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Tue, 18 Jun 2024 12:34:59 +0800 Subject: [PATCH 232/273] feat: change table struct --- api/controllers/console/app/app.py | 45 +++++++++-- api/core/agent/cot_agent_runner.py | 39 ++++++---- api/core/agent/fc_agent_runner.py | 7 +- .../advanced_chat/generate_task_pipeline.py | 2 +- api/core/app/apps/agent_chat/app_generator.py | 1 - api/core/app/apps/chat/app_generator.py | 1 - api/core/app/apps/completion/app_generator.py | 6 +- .../apps/workflow/generate_task_pipeline.py | 2 +- api/core/llm_generator/llm_generator.py | 3 +- api/core/moderation/input_moderation.py | 3 +- api/core/rag/retrieval/dataset_retrieval.py | 5 +- api/core/workflow/nodes/tool/tool_node.py | 2 +- api/fields/app_fields.py | 1 + api/services/message_service.py | 4 +- api/services/ops_trace/ops_trace_service.py | 74 +++++++++++-------- 15 files changed, 123 insertions(+), 72 deletions(-) diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 082838334ae040..bc9a3c04300311 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -1,4 +1,3 @@ -import json import uuid from flask_login import current_user @@ -9,17 +8,14 @@ from controllers.console.app.wraps import get_app_model from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check -from core.tools.tool_manager import ToolManager -from core.tools.utils.configuration import ToolParameterConfigurationManager from fields.app_fields import ( app_detail_fields, app_detail_fields_with_site, app_pagination_fields, ) from libs.login import login_required -from models.model import App, AppMode, AppModelConfig from services.app_service import AppService -from services.tag_service import TagService +from services.ops_trace.ops_trace_service import OpsTraceService ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion'] @@ -286,6 +282,44 @@ def post(self, app_model): return app_model +class AppTraceApi(Resource): + @setup_required + @login_required + @account_initialization_required + def get(self, app_id): + """Get app trace""" + app_trace_config = OpsTraceService.get_app_tracing_config( + app_id=app_id + ) + if not app_trace_config: + raise BadRequest("Tracing config not found") + + return app_trace_config + + @setup_required + @login_required + @account_initialization_required + def post(self, app_id): + # add app trace + if not current_user.is_admin_or_owner: + raise Forbidden() + parser = reqparse.RequestParser() + parser.add_argument('enabled', type=bool, required=True, location='json') + parser.add_argument('tracing_provider', type=str, required=True, location='json') + args = parser.parse_args() + + try: + OpsTraceService.update_app_tracing_config( + app_id=app_id, + enabled=args['enabled'], + tracing_provider=args['tracing_provider'], + ) + except Exception as e: + raise e + + return {"result": "success"} + + api.add_resource(AppListApi, '/apps') api.add_resource(AppImportApi, '/apps/import') api.add_resource(AppApi, '/apps/<uuid:app_id>') @@ -295,3 +329,4 @@ def post(self, app_model): api.add_resource(AppIconApi, '/apps/<uuid:app_id>/icon') api.add_resource(AppSiteStatus, '/apps/<uuid:app_id>/site-enable') api.add_resource(AppApiStatus, '/apps/<uuid:app_id>/api-enable') +api.add_resource(AppTraceApi, '/apps/<uuid:app_id>/trace') diff --git a/api/core/agent/cot_agent_runner.py b/api/core/agent/cot_agent_runner.py index 31d1ec0cdb23df..9f3c7ea3bba9bf 100644 --- a/api/core/agent/cot_agent_runner.py +++ b/api/core/agent/cot_agent_runner.py @@ -1,7 +1,7 @@ import json from abc import ABC, abstractmethod from collections.abc import Generator -from typing import Union, Optional +from typing import Optional, Union from core.agent.base_agent_runner import BaseAgentRunner from core.agent.entities import AgentScratchpadUnit @@ -21,6 +21,7 @@ from core.tools.tool_engine import ToolEngine from models.model import Message from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.ops_trace_service import OpsTraceService class CotAgentRunner(BaseAgentRunner, ABC): @@ -32,7 +33,8 @@ class CotAgentRunner(BaseAgentRunner, ABC): _query: str = None _prompt_messages_tools: list[PromptMessage] = None - def run(self, message: Message, + def run( + self, message: Message, query: str, inputs: dict[str, str], ) -> Union[Generator, LLMResult]: @@ -43,6 +45,12 @@ def run(self, message: Message, self._repack_app_generate_entity(app_generate_entity) self._init_react_state(query) + # get tracing instance + app_id = self.app_config.app_id + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_id + ) + # check model mode if 'Observation' not in app_generate_entity.model_conf.stop: if app_generate_entity.model_conf.provider not in self._ignore_observation_providers: @@ -212,7 +220,8 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): tool_invoke_response, tool_invoke_meta = self._handle_invoke_action( action=scratchpad.action, tool_instances=tool_instances, - message_file_ids=message_file_ids + message_file_ids=message_file_ids, + tracing_instance=tracing_instance, ) scratchpad.observation = tool_invoke_response scratchpad.agent_response = tool_invoke_response @@ -238,8 +247,7 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): # update prompt tool message for prompt_tool in self._prompt_messages_tools: - self.update_prompt_message_tool( - tool_instances[prompt_tool.name], prompt_tool) + self.update_prompt_message_tool(tool_instances[prompt_tool.name], prompt_tool) iteration_step += 1 @@ -276,12 +284,11 @@ def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): message=AssistantPromptMessage( content=final_answer ), - usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage( - ), + usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(), system_fingerprint='' )), PublishFrom.APPLICATION_MANAGER) - def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, + def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, tool_instances: dict[str, Tool], message_file_ids: list[str], tracing_instance: Optional[BaseTraceInstance] = None @@ -300,7 +307,7 @@ def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, if not tool_instance: answer = f"there is not a tool named {tool_call_name}" return answer, ToolInvokeMeta.error_instance(answer) - + if isinstance(tool_call_args, str): try: tool_call_args = json.loads(tool_call_args) @@ -322,8 +329,7 @@ def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, # publish files for message_file, save_as in message_files: if save_as: - self.variables_pool.set_file( - tool_name=tool_call_name, value=message_file.id, name=save_as) + self.variables_pool.set_file(tool_name=tool_call_name, value=message_file.id, name=save_as) # publish message file self.queue_manager.publish(QueueMessageFileEvent( @@ -354,7 +360,7 @@ def _fill_in_inputs_from_external_data_tools(self, instruction: str, inputs: dic continue return instruction - + def _init_react_state(self, query) -> None: """ init agent scratchpad @@ -362,7 +368,7 @@ def _init_react_state(self, query) -> None: self._query = query self._agent_scratchpad = [] self._historic_prompt_messages = self._organize_historic_prompt_messages() - + @abstractmethod def _organize_prompt_messages(self) -> list[PromptMessage]: """ @@ -394,6 +400,13 @@ def _organize_historic_prompt_messages(self, current_session_messages: list[Prom scratchpads: list[AgentScratchpadUnit] = [] current_scratchpad: AgentScratchpadUnit = None + self.history_prompt_messages = AgentHistoryPromptTransform( + model_config=self.model_config, + prompt_messages=current_session_messages or [], + history_messages=self.history_prompt_messages, + memory=self.memory + ).get_prompt() + for message in self.history_prompt_messages: if isinstance(message, AssistantPromptMessage): if not current_scratchpad: diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index e64722d22ca58d..3ddd43a771602e 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -20,8 +20,7 @@ from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform from core.tools.entities.tool_entities import ToolInvokeMeta from core.tools.tool_engine import ToolEngine -from extensions.ext_database import db -from models.model import AppModelConfig, Message +from models.model import Message from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -54,10 +53,8 @@ def run(self, # get tracing instance app_id = app_config.app_id - app_model_config_id = app_config.app_model_config_id - app_model_config = db.session.query(AppModelConfig).filter_by(id=app_model_config_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id, app_model_config=app_model_config + app_id=app_id ) def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 207e962b376efd..0d461528bdc737 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -189,7 +189,7 @@ def _process_stream_response(self, workflow: Optional[Workflow] = None) -> Gener :return: """ app_id = self._conversation.app_id - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) for message in self._queue_manager.listen(): event = message.event diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index ca8fbe138efa37..431a8b88a15d5f 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -74,7 +74,6 @@ def generate(self, app_model: App, # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( app_id=app_model.id, - app_model_config=app_model_config, ) # validate override model config diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index e333e3458cbbc8..2dde1331891cc7 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -127,7 +127,6 @@ def generate( # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( app_id=app_model.id, - app_model_config=app_model_config, ) # init queue manager diff --git a/api/core/app/apps/completion/app_generator.py b/api/core/app/apps/completion/app_generator.py index fcf00e685594dd..186a11557af565 100644 --- a/api/core/app/apps/completion/app_generator.py +++ b/api/core/app/apps/completion/app_generator.py @@ -117,8 +117,7 @@ def generate(self, app_model: App, # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model.id, - app_model_config=app_model_config, + app_id=app_model.id ) # init queue manager @@ -283,8 +282,7 @@ def generate_more_like_this(self, app_model: App, # get tracing instance tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model.id, - app_model_config=app_model_config, + app_id=app_model.id ) # init queue manager diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 944c3736a140d1..54bd3fbf0db264 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -172,7 +172,7 @@ def _process_stream_response( Process stream response. :return: """ - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) for message in self._queue_manager.listen(): event = message.event diff --git a/api/core/llm_generator/llm_generator.py b/api/core/llm_generator/llm_generator.py index c705cc754286c7..4059f637c58e2a 100644 --- a/api/core/llm_generator/llm_generator.py +++ b/api/core/llm_generator/llm_generator.py @@ -58,10 +58,9 @@ def generate_conversation_name(cls, tenant_id: str, query, conversation_id: Opti # get tracing instance conversation_data: Conversation = db.query(Conversation).filter(Conversation.id == conversation_id).first() app_id = conversation_data.app_id - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id=conversation_data.message_id) tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id, app_model_config=app_model_config + app_id=app_id ) if tracing_instance: diff --git a/api/core/moderation/input_moderation.py b/api/core/moderation/input_moderation.py index 0d915f74fed64d..c835c97eda0e34 100644 --- a/api/core/moderation/input_moderation.py +++ b/api/core/moderation/input_moderation.py @@ -47,9 +47,8 @@ def check( from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName # get tracing instance - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id, app_model_config=app_model_config + app_id=app_id ) if tracing_instance: diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index 878587e09cb7af..7d7645b8e68fab 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -23,6 +23,7 @@ from extensions.ext_database import db from models.dataset import Dataset, DatasetQuery, DocumentSegment from models.dataset import Document as DatasetDocument +from models.model import Message from services.ops_trace.ops_trace_service import OpsTraceService from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName from services.ops_trace.utils import measure_time @@ -354,9 +355,9 @@ def _on_retrival_end( db.session.commit() # get tracing instance - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + app_id = db.session.query(Message.app_id).filter(Message.id == message_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model_config.app_id, app_model_config=app_model_config + app_id=app_id ) if tracing_instance: trace_manager = TraceQueueManager() diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 42d8648594ad5f..45a2e812197f06 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -59,7 +59,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: # get tracing instance workflow: Workflow = db.session.query(Workflow).filter(Workflow.id == self.workflow_id).first() app_id = workflow.app_id - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id, workflow=workflow) + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) try: messages = ToolEngine.workflow_invoke( diff --git a/api/fields/app_fields.py b/api/fields/app_fields.py index 212c3e7f179862..1a0a1b815d6325 100644 --- a/api/fields/app_fields.py +++ b/api/fields/app_fields.py @@ -50,6 +50,7 @@ 'enable_site': fields.Boolean, 'enable_api': fields.Boolean, 'model_config': fields.Nested(model_config_fields, attribute='app_model_config', allow_null=True), + 'tracing': fields.Raw, 'created_at': TimestampField } diff --git a/api/services/message_service.py b/api/services/message_service.py index 49555e55588a77..e43d244d58bdbd 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -272,9 +272,9 @@ def get_suggested_questions_after_answer(cls, app_model: App, user: Optional[Uni ) # get tracing instance - app_model_config = OpsTraceService.get_app_config_through_message_id(message_id) + app_id = db.session.query(Message.app_id).filter(Message.id == message_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model_config.app_id, app_model_config=app_model_config + app_id=app_id, ) if tracing_instance: trace_manager = TraceQueueManager() diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 9cc88a10e4639d..e5cf1b57d63db9 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -27,7 +27,7 @@ class LangfuseConfig(BaseModel): host: str -class LangsmithConfig(BaseModel): +class LangSmithConfig(BaseModel): """ Model class for Langsmith tracing config. """ @@ -156,9 +156,9 @@ def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c host=tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangsmithConfig(**tracing_config) + tracing_config = LangSmithConfig(**tracing_config) encrypt_api_key = encrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangsmithConfig( + tracing_config = LangSmithConfig( api_key=encrypt_api_key, project=tracing_config.project, endpoint=tracing_config.endpoint @@ -187,9 +187,9 @@ def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c host=tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangsmithConfig(**tracing_config) + tracing_config = LangSmithConfig(**tracing_config) decrypt_api_key = decrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangsmithConfig( + tracing_config = LangSmithConfig( api_key=decrypt_api_key, project=tracing_config.project, endpoint=tracing_config.endpoint @@ -219,10 +219,10 @@ def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config: host=decrypt_tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - decrypt_tracing_config = LangsmithConfig(**decrypt_tracing_config) + decrypt_tracing_config = LangSmithConfig(**decrypt_tracing_config) decrypt_api_key = decrypt_tracing_config.api_key obfuscated_api_key = obfuscated_token(decrypt_api_key) - decrypt_tracing_config = LangsmithConfig( + decrypt_tracing_config = LangSmithConfig( api_key=obfuscated_api_key, project=decrypt_tracing_config.project, endpoint=decrypt_tracing_config.endpoint @@ -256,9 +256,9 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): @classmethod def get_ops_trace_instance( cls, - app_id: str, + app_id, workflow: Optional[Workflow] = None, - app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None + app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None, ): """ Get ops trace through model config @@ -268,29 +268,10 @@ def get_ops_trace_instance( :return: """ tracing_instance = None - app_ops_trace_config = None - - # get trace configuration from available sources - if app_model_config is not None: - if isinstance(app_model_config, AppAdditionalFeatures): - app_ops_trace_config = app_model_config.trace_config - elif isinstance(app_model_config, AppModelConfig): - app_ops_trace_config = json.loads( - app_model_config.trace_config - ) if app_model_config.trace_config else None - elif workflow: - features_data = json.loads(workflow.features) - app_ops_trace_config = features_data.get('trace_config') if features_data else None - else: - # As a last resort, fetch from the database - trace_config_data = db.session.query(AppModelConfig.trace_config).filter( - AppModelConfig.app_id == app_id - ).order_by(AppModelConfig.updated_at.desc()).first() - if trace_config_data: - app_ops_trace_config = json.loads(trace_config_data.trace_config) - else: - raise ValueError('Trace config not found') - + app: App = db.session.query(App).filter( + App.id == app_id + ).first() + app_ops_trace_config = json.loads(app.tracing) if app.tracing else None if app_ops_trace_config is not None: tracing_provider = app_ops_trace_config.get('tracing_provider') else: @@ -338,3 +319,32 @@ def get_app_config_through_message_id(cls, message_id: str): app_model_config = conversation_data.override_model_configs return app_model_config + + @classmethod + def update_app_tracing_config(cls, app_id: str, enabled: bool, tracing_provider: str): + """ + Update app tracing config + :param app_id: app id + :param enabled: enabled + :param tracing_provider: tracing provider + :return: + """ + app_config: App = db.session.query(App).filter(App.id == app_id).first() + app_config.tracing = json.dumps( + { + "enabled": enabled, + "tracing_provider": tracing_provider, + } + ) + db.session.commit() + + @classmethod + def get_app_tracing_config(cls, app_id: str): + """ + Get app tracing config + :param app_id: app id + :return: + """ + app: App = db.session.query(App).filter(App.id == app_id).first() + app_trace_config = json.loads(app.tracing) + return app_trace_config From 4ef224a8e60986a35057acff3a270ccb8b05efb9 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Tue, 18 Jun 2024 14:17:18 +0800 Subject: [PATCH 233/273] feat: change TraceAppConfigApi request type patch --- api/controllers/console/app/ops_trace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index d421a913ac09e5..1316fb7cdbc630 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -56,7 +56,7 @@ def post(self, app_id): @setup_required @login_required @account_initialization_required - def put(self, app_id): + def patch(self, app_id): """Update an existing trace app configuration""" parser = reqparse.RequestParser() parser.add_argument('tracing_provider', type=str, required=True, location='json') From c9a36e642c825b7cd8f82d4801728eeec06e7a99 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Tue, 18 Jun 2024 18:01:42 +0800 Subject: [PATCH 234/273] feat: update default reply when config is none --- api/controllers/console/app/app.py | 2 -- api/controllers/console/app/ops_trace.py | 2 +- ...31d46af369_remove_app_model_config_trace_config_.py | 10 +++++----- api/requirements.txt | 2 ++ api/services/ops_trace/ops_trace_service.py | 5 +++++ 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index bc9a3c04300311..a9e8b5fb4a2d33 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -291,8 +291,6 @@ def get(self, app_id): app_trace_config = OpsTraceService.get_app_tracing_config( app_id=app_id ) - if not app_trace_config: - raise BadRequest("Tracing config not found") return app_trace_config diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index 1316fb7cdbc630..382f1ed89b3c9b 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -26,7 +26,7 @@ def get(self, app_id): app_id=app_id, tracing_provider=args['tracing_provider'] ) if not trace_config: - raise TracingConfigNotExist() + return {"has_not_configured": True} return trace_config except Exception as e: raise e diff --git a/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py index 8d69fa86eb8487..64489b11f50136 100644 --- a/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py +++ b/api/migrations/versions/c031d46af369_remove_app_model_config_trace_config_.py @@ -28,10 +28,10 @@ def upgrade(): sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False), sa.Column('is_active', sa.Boolean(), server_default=sa.text('true'), nullable=False), - sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + sa.PrimaryKeyConstraint('id', name='trace_app_config_pkey') # 修改了主键约束名称以避免冲突 ) with op.batch_alter_table('trace_app_config', schema=None) as batch_op: - batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + batch_op.create_index('trace_app_config_app_id_idx', ['app_id'], unique=False) with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: batch_op.drop_index('tracing_app_config_app_id_idx') @@ -54,13 +54,13 @@ def downgrade(): sa.Column('tracing_config', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True), sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False), - sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + sa.PrimaryKeyConstraint('id', name='trace_app_config_pkey') ) with op.batch_alter_table('tracing_app_configs', schema=None) as batch_op: - batch_op.create_index('tracing_app_config_app_id_idx', ['app_id'], unique=False) + batch_op.create_index('trace_app_config_app_id_idx', ['app_id'], unique=False) with op.batch_alter_table('trace_app_config', schema=None) as batch_op: - batch_op.drop_index('tracing_app_config_app_id_idx') + batch_op.drop_index('trace_app_config_app_id_idx') op.drop_table('trace_app_config') # ### end Alembic commands ### diff --git a/api/requirements.txt b/api/requirements.txt index 7ab636e226e604..4e00646c0b102c 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -91,3 +91,5 @@ chromadb~=0.5.1 novita_client~=0.5.6 tenacity~=8.3.0 cos-python-sdk-v5==1.9.30 +langfuse==2.36.1 +langsmith==0.1.77 \ No newline at end of file diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index e5cf1b57d63db9..e4700dc685fd12 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -346,5 +346,10 @@ def get_app_tracing_config(cls, app_id: str): :return: """ app: App = db.session.query(App).filter(App.id == app_id).first() + if not app.tracing: + return { + "enabled": False, + "tracing_provider": None + } app_trace_config = json.loads(app.tracing) return app_trace_config From 4d41ae7ef968c1347cb99ae328513e8c5f0614a9 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Wed, 19 Jun 2024 09:50:29 +0800 Subject: [PATCH 235/273] fix: advanced chat trace error --- api/core/app/apps/advanced_chat/app_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index a06ae902fed5db..f8a2cb0f64de3c 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -109,7 +109,6 @@ def generate( application_generate_entity=application_generate_entity, conversation=conversation, stream=stream, - tracing_instance=tracing_instance, ) def single_iteration_generate(self, app_model: App, From 348f5813c1084ccf5581dae203d44a258e667c1b Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 13:35:22 +0800 Subject: [PATCH 236/273] chore: config disabled tooltip --- .../[appId]/overview/tracing/config-popup.tsx | 28 +++++++++++++++---- web/i18n/en-US/app.ts | 1 + web/i18n/zh-Hans/app.ts | 1 + 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 63178d6892f8cb..6e1e98b7717fb3 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -10,6 +10,7 @@ import { TracingProvider } from './type' import ProviderConfigModal from './provider-config-modal' import Indicator from '@/app/components/header/indicator' import Switch from '@/app/components/base/switch' +import TooltipPlus from '@/app/components/base/tooltip-plus' const I18N_PREFIX = 'app.tracing' @@ -68,6 +69,16 @@ const ConfigPopup: FC<PopupProps> = ({ const providerAllConfigured = langSmithConfig && langFuseConfig const providerAllNotConfigured = !langSmithConfig && !langFuseConfig + + const switchContent = ( + <Switch + className='ml-3' + defaultValue={enabled} + onChange={onStatusChange} + size='l' + disabled={providerAllNotConfigured} + /> + ) const langSmithPanel = ( <ProviderPanel type={TracingProvider.langSmith} @@ -100,12 +111,17 @@ const ConfigPopup: FC<PopupProps> = ({ <div className='ml-1.5 text-xs font-semibold text-gray-500 uppercase'> {t(`${I18N_PREFIX}.${enabled ? 'enabled' : 'disabled'}`)} </div> - <Switch - className='ml-3' - defaultValue={enabled} - onChange={onStatusChange} - size='l' - /> + {providerAllNotConfigured + ? ( + <TooltipPlus + popupContent={t(`${I18N_PREFIX}.disabledTip`)} + > + {switchContent} + + </TooltipPlus> + ) + : switchContent} + </div> </div> diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index a975f91ea27806..6153c1187377f0 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -93,6 +93,7 @@ const translation = { expand: 'Expand', tracing: 'Tracing', disabled: 'Disabled', + disabledTip: 'Please config provider first', enabled: 'In Service', tracingDescription: 'Capture the full context of app execution, including LLM calls, context, prompts, HTTP requests, and more, to a third-party tracing platform.', configProviderTitle: { diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 641f8774f395f1..4008a247c9e76a 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -92,6 +92,7 @@ const translation = { expand: '展开', tracing: '追踪', disabled: '已禁用', + disabledTip: '请先配置提供商', enabled: '已启用', tracingDescription: '捕获应用程序执行的完整上下文，包括 LLM 调用、上下文、提示、HTTP 请求等，发送到第三方跟踪平台。', configProviderTitle: { From 7004af54a793a1c3fab99c8daf5ed8a3bd5fd2a7 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 13:39:26 +0800 Subject: [PATCH 237/273] fix: remove order --- .../[appId]/overview/tracing/provider-config-modal.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 423d271c205821..94f7508c34d14f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -70,7 +70,6 @@ const ProviderConfigModal: FC<Props> = ({ }] = useBoolean(false) const handleRemove = useCallback(async () => { - hideRemoveConfirm() await removeTracingConfig({ appId, provider: type, @@ -80,6 +79,7 @@ const ProviderConfigModal: FC<Props> = ({ message: t('common.api.remove'), }) onRemoved() + hideRemoveConfirm() }, [hideRemoveConfirm, appId, type, t, onRemoved]) const handleConfigChange = useCallback((key: string) => { From b63663d17c2d4a098bc75c40fce29d32c558bc44 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 13:47:18 +0800 Subject: [PATCH 238/273] feat: observalibity name --- web/i18n/en-US/common.ts | 2 +- web/i18n/zh-Hans/common.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index aa3b970889991a..2f65e5ee0845ea 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -440,7 +440,7 @@ const translation = { latestAvailable: 'Dify {{version}} is the latest version available.', }, appMenus: { - overview: 'Overview', + overview: 'Observability', promptEng: 'Orchestrate', apiAccess: 'API Access', logAndAnn: 'Logs & Ann.', diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index aa5b51fe04cd3a..9981c749088bc5 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -436,7 +436,7 @@ const translation = { latestAvailable: 'Dify {{version}} 已是最新版本。', }, appMenus: { - overview: '概览', + overview: '可观察性', promptEng: '编排', apiAccess: '访问 API', logAndAnn: '日志与标注', From 2c2dc704183bbec8ac226c537ef4029cfe908f85 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 14:55:13 +0800 Subject: [PATCH 239/273] feat: add permission --- .../overview/tracing/config-button.tsx | 4 +++ .../[appId]/overview/tracing/config-popup.tsx | 26 ++++++++++++------- .../[appId]/overview/tracing/panel.tsx | 5 ++++ .../overview/tracing/provider-panel.tsx | 25 +++++++++++------- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index fff7ad33b70ea6..a6e2d59373ecb6 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -16,6 +16,7 @@ import { const I18N_PREFIX = 'app.tracing' type Props = { + readOnly: boolean className?: string hasConfigured: boolean } & PopupProps @@ -31,6 +32,9 @@ const ConfigBtn: FC<Props> = ({ setOpen(v => !v) }, [setOpen]) + if (popupProps.readOnly && !hasConfigured) + return null + const triggerContent = hasConfigured ? ( <div className={cn(className, 'p-1 rounded-md hover:bg-black/5 cursor-pointer')}> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 6e1e98b7717fb3..7aa1fca96d8a3f 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -16,6 +16,7 @@ const I18N_PREFIX = 'app.tracing' export type PopupProps = { appId: string + readOnly: boolean enabled: boolean onStatusChange: (enabled: boolean) => void chosenProvider: TracingProvider | null @@ -28,6 +29,7 @@ export type PopupProps = { const ConfigPopup: FC<PopupProps> = ({ appId, + readOnly, enabled, onStatusChange, chosenProvider, @@ -82,6 +84,7 @@ const ConfigPopup: FC<PopupProps> = ({ const langSmithPanel = ( <ProviderPanel type={TracingProvider.langSmith} + readOnly={readOnly} hasConfigured={!!langSmithConfig} onConfig={handleOnConfig(TracingProvider.langSmith)} isChosen={chosenProvider === TracingProvider.langSmith} @@ -92,6 +95,7 @@ const ConfigPopup: FC<PopupProps> = ({ const langfusePanel = ( <ProviderPanel type={TracingProvider.langfuse} + readOnly={readOnly} hasConfigured={!!langFuseConfig} onConfig={handleOnConfig(TracingProvider.langfuse)} isChosen={chosenProvider === TracingProvider.langfuse} @@ -111,16 +115,20 @@ const ConfigPopup: FC<PopupProps> = ({ <div className='ml-1.5 text-xs font-semibold text-gray-500 uppercase'> {t(`${I18N_PREFIX}.${enabled ? 'enabled' : 'disabled'}`)} </div> - {providerAllNotConfigured - ? ( - <TooltipPlus - popupContent={t(`${I18N_PREFIX}.disabledTip`)} - > - {switchContent} + {!readOnly && ( + <> + {providerAllNotConfigured + ? ( + <TooltipPlus + popupContent={t(`${I18N_PREFIX}.disabledTip`)} + > + {switchContent} - </TooltipPlus> - ) - : switchContent} + </TooltipPlus> + ) + : switchContent} + </> + )} </div> </div> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index ae221e49e5a060..05f8f87e6787f7 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -15,6 +15,7 @@ import Indicator from '@/app/components/header/indicator' import { fetchTracingConfig as doFetchTracingConfig, fetchTracingStatus, updateTracingStatus } from '@/service/apps' import type { TracingStatus } from '@/models/app' import Toast from '@/app/components/base/toast' +import { useAppContext } from '@/context/app-context' const I18N_PREFIX = 'app.tracing' @@ -36,6 +37,8 @@ const Panel: FC = () => { const pathname = usePathname() const matched = pathname.match(/\/app\/([^/]+)/) const appId = (matched?.length && matched[1]) ? matched[1] : '' + const { isCurrentWorkspaceEditor } = useAppContext() + const readOnly = !isCurrentWorkspaceEditor const [isLoaded, { setTrue: setLoaded, @@ -142,6 +145,7 @@ const Panel: FC = () => { <div className='flex items-center space-x-1'> <ConfigButton appId={appId} + readOnly={readOnly} hasConfigured={false} enabled={enabled} onStatusChange={handleTracingEnabledChange} @@ -185,6 +189,7 @@ const Panel: FC = () => { <ConfigButton appId={appId} + readOnly={readOnly} hasConfigured className='ml-2' enabled={enabled} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx index a4759ef0ab536d..54b211ab34b4cd 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx @@ -11,6 +11,7 @@ const I18N_PREFIX = 'app.tracing' type Props = { type: TracingProvider + readOnly: boolean isChosen: boolean onChoose: () => void hasConfigured: boolean @@ -26,6 +27,7 @@ const getIcon = (type: TracingProvider) => { const ProviderPanel: FC<Props> = ({ type, + readOnly, isChosen, onChoose, hasConfigured, @@ -41,13 +43,13 @@ const ProviderPanel: FC<Props> = ({ const handleChosen = useCallback((e: React.MouseEvent) => { e.stopPropagation() - if (isChosen || !hasConfigured) + if (isChosen || !hasConfigured || readOnly) return onChoose() - }, [hasConfigured, isChosen, onChoose]) + }, [hasConfigured, isChosen, onChoose, readOnly]) return ( <div - className={cn(isChosen ? 'border-primary-400' : 'border-transparent', !isChosen && hasConfigured && 'cursor-pointer', 'px-4 py-3 rounded-xl border-[1.5px] bg-gray-100')} + className={cn(isChosen ? 'border-primary-400' : 'border-transparent', !isChosen && hasConfigured && !readOnly && 'cursor-pointer', 'px-4 py-3 rounded-xl border-[1.5px] bg-gray-100')} onClick={handleChosen} > <div className={'flex justify-between items-center space-x-1'}> @@ -55,13 +57,16 @@ const ProviderPanel: FC<Props> = ({ <Icon className='h-6' /> {isChosen && <div className='ml-1 flex items-center h-4 px-1 rounded-[4px] border border-primary-500 leading-4 text-xs font-medium text-primary-500 uppercase '>{t(`${I18N_PREFIX}.inUse`)}</div>} </div> - <div - className='flex px-2 items-center h-6 bg-white rounded-md border-[0.5px] border-gray-200 shadow-xs cursor-pointer text-gray-700 space-x-1' - onClick={handleConfigBtnClick} - > - <Settings04 className='w-3 h-3' /> - <div className='text-xs font-medium'>{t(`${I18N_PREFIX}.config`)}</div> - </div> + {!readOnly && ( + <div + className='flex px-2 items-center h-6 bg-white rounded-md border-[0.5px] border-gray-200 shadow-xs cursor-pointer text-gray-700 space-x-1' + onClick={handleConfigBtnClick} + > + <Settings04 className='w-3 h-3' /> + <div className='text-xs font-medium'>{t(`${I18N_PREFIX}.config`)}</div> + </div> + )} + </div> <div className='mt-2 leading-4 text-xs font-normal text-gray-500'> {t(`${I18N_PREFIX}.${type}.description`)} From 54154f76a47bdee3d6d35c536edc0b36a7d30959 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 15:00:47 +0800 Subject: [PATCH 240/273] feat: use new btn --- .../[appId]/overview/tracing/config-button.tsx | 2 +- .../[appId]/overview/tracing/provider-config-modal.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index a6e2d59373ecb6..44478b73fe4cae 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -42,7 +42,7 @@ const ConfigBtn: FC<Props> = ({ </div> ) : ( - <Button type='primary' + <Button variant='primary' className={cn(className, '!h-8 !px-3 select-none')} > <Settings04 className='mr-1 w-4 h-4' /> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 94f7508c34d14f..0873401eb51820 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -243,7 +243,7 @@ const ProviderConfigModal: FC<Props> = ({ </Button> <Button className='h-9 text-sm font-medium' - type='primary' + variant='primary' onClick={handleSave} loading={isSaving} > From 91331150662b0b53005b78b8d741bbbd8cef1679 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Wed, 19 Jun 2024 15:27:35 +0800 Subject: [PATCH 241/273] fix: update api config.py --- api/config.py | 76 +-------------------------------------------------- 1 file changed, 1 insertion(+), 75 deletions(-) diff --git a/api/config.py b/api/config.py index d5bfad88ae7cd1..cc9d676a1ce1f4 100644 --- a/api/config.py +++ b/api/config.py @@ -219,80 +219,6 @@ def __init__(self): self.CHROMA_AUTH_PROVIDER = get_env('CHROMA_AUTH_PROVIDER') self.CHROMA_AUTH_CREDENTIALS = get_env('CHROMA_AUTH_CREDENTIALS') - # ------------------------ - # Mail Configurations. - # ------------------------ - self.MAIL_TYPE = get_env('MAIL_TYPE') - self.MAIL_DEFAULT_SEND_FROM = get_env('MAIL_DEFAULT_SEND_FROM') - self.RESEND_API_KEY = get_env('RESEND_API_KEY') - self.RESEND_API_URL = get_env('RESEND_API_URL') - # SMTP settings - self.SMTP_SERVER = get_env('SMTP_SERVER') - self.SMTP_PORT = get_env('SMTP_PORT') - self.SMTP_USERNAME = get_env('SMTP_USERNAME') - self.SMTP_PASSWORD = get_env('SMTP_PASSWORD') - self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS') - self.SMTP_OPPORTUNISTIC_TLS = get_bool_env('SMTP_OPPORTUNISTIC_TLS') - - # ------------------------ - # Workspace Configurations. - # ------------------------ - self.INVITE_EXPIRY_HOURS = int(get_env('INVITE_EXPIRY_HOURS')) - - # ------------------------ - # Sentry Configurations. - # ------------------------ - self.SENTRY_DSN = get_env('SENTRY_DSN') - self.SENTRY_TRACES_SAMPLE_RATE = float(get_env('SENTRY_TRACES_SAMPLE_RATE')) - self.SENTRY_PROFILES_SAMPLE_RATE = float(get_env('SENTRY_PROFILES_SAMPLE_RATE')) - - # ------------------------ - # Business Configurations. - # ------------------------ - - # multi model send image format, support base64, url, default is base64 - self.MULTIMODAL_SEND_IMAGE_FORMAT = get_env('MULTIMODAL_SEND_IMAGE_FORMAT') - - # Dataset Configurations. - self.CLEAN_DAY_SETTING = get_env('CLEAN_DAY_SETTING') - - # File upload Configurations. - self.UPLOAD_FILE_SIZE_LIMIT = int(get_env('UPLOAD_FILE_SIZE_LIMIT')) - self.UPLOAD_FILE_BATCH_LIMIT = int(get_env('UPLOAD_FILE_BATCH_LIMIT')) - self.UPLOAD_IMAGE_FILE_SIZE_LIMIT = int(get_env('UPLOAD_IMAGE_FILE_SIZE_LIMIT')) - self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT') - - # RAG ETL Configurations. - self.ETL_TYPE = get_env('ETL_TYPE') - self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL') - self.UNSTRUCTURED_API_KEY = get_env('UNSTRUCTURED_API_KEY') - self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE') - - # Indexing Configurations. - self.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH = get_env('INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH') - - # Tool Configurations. - self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE') - - self.WORKFLOW_MAX_EXECUTION_STEPS = int(get_env('WORKFLOW_MAX_EXECUTION_STEPS')) - self.WORKFLOW_MAX_EXECUTION_TIME = int(get_env('WORKFLOW_MAX_EXECUTION_TIME')) - self.WORKFLOW_CALL_MAX_DEPTH = int(get_env('WORKFLOW_CALL_MAX_DEPTH')) - self.APP_MAX_EXECUTION_TIME = int(get_env('APP_MAX_EXECUTION_TIME')) - - # Moderation in app Configurations. - self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE')) - - # Notion integration setting - self.NOTION_CLIENT_ID = get_env('NOTION_CLIENT_ID') - self.NOTION_CLIENT_SECRET = get_env('NOTION_CLIENT_SECRET') - self.NOTION_INTEGRATION_TYPE = get_env('NOTION_INTEGRATION_TYPE') - self.NOTION_INTERNAL_SECRET = get_env('NOTION_INTERNAL_SECRET') - self.NOTION_INTEGRATION_TOKEN = get_env('NOTION_INTEGRATION_TOKEN') - - # Firecrawl integration setting - self.FIRECRAWL_API_KEY = get_env('FIRECRAWL_API_KEY') - self.FIRECRAWL_BASE_URL = get_env('FIRECRAWL_BASE_URL') - # ------------------------ # Platform Configurations. # ------------------------ @@ -325,4 +251,4 @@ def __init__(self): # fetch app templates mode, remote, builtin, db(only for dify SaaS), default: remote self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE') - self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN') + self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN') \ No newline at end of file From a70ace824f2bc31a965d1d332aa212926130a78b Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Wed, 19 Jun 2024 15:48:51 +0800 Subject: [PATCH 242/273] feat: add trace ops api check --- api/controllers/console/app/ops_trace.py | 1 + api/services/ops_trace/langfuse_trace.py | 7 +++ api/services/ops_trace/langsmith_trace.py | 14 +++++- api/services/ops_trace/ops_trace_service.py | 52 ++++++++++++++++++--- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index 382f1ed89b3c9b..c3e9ebac1c2aa4 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -69,6 +69,7 @@ def patch(self, app_id): tracing_provider=args['tracing_provider'], tracing_config=args['tracing_config'] ) + print("==============", result) if not result: raise TracingConfigNotExist() return {"result": "success"} diff --git a/api/services/ops_trace/langfuse_trace.py b/api/services/ops_trace/langfuse_trace.py index af6badbd96e626..6a037b4ad7e20d 100644 --- a/api/services/ops_trace/langfuse_trace.py +++ b/api/services/ops_trace/langfuse_trace.py @@ -710,3 +710,10 @@ def update_generation( ) generation.end(**format_generation_data) + + def api_check(self): + try: + return self.langfuse_client.auth_check() + except Exception as e: + print(f"LangFuse API check failed: {str(e)}") + return False diff --git a/api/services/ops_trace/langsmith_trace.py b/api/services/ops_trace/langsmith_trace.py index ef0afdfeb91681..529ce1b07aaa78 100644 --- a/api/services/ops_trace/langsmith_trace.py +++ b/api/services/ops_trace/langsmith_trace.py @@ -14,7 +14,7 @@ from models.model import Message, MessageAgentThought, MessageFile from models.workflow import WorkflowNodeExecution, WorkflowRun from services.ops_trace.base_trace_instance import BaseTraceInstance -from services.ops_trace.utils import filter_none_values +from services.ops_trace.utils import filter_none_values, replace_text_with_content class LangSmithRunType(str, Enum): @@ -97,6 +97,8 @@ def ensure_dict(cls, v, info: ValidationInfo): } elif isinstance(v, list): if len(v) > 0 and isinstance(v[0], dict): + # rename text to content + replace_text_with_content(data=v) data = { "message": v, "usage_metadata": usage_metadata, @@ -543,3 +545,13 @@ def update_run(self, update_run_data: LangSmithRunUpdateModel): print("LangSmith Run updated successfully.") except Exception as e: raise f"LangSmith Failed to update run: {str(e)}" + + def api_check(self): + try: + random_project_name = f"test_project_{datetime.now().strftime('%Y%m%d%H%M%S')}" + self.langsmith_client.create_project(project_name=random_project_name) + self.langsmith_client.delete_project(project_name=random_project_name) + return True + except Exception as e: + print(f"LangSmith API check failed: {str(e)}") + return False diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index e4700dc685fd12..db5c5cc92bf131 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -1,14 +1,11 @@ import json from enum import Enum -from typing import Optional from pydantic import BaseModel -from core.app.app_config.entities import AppAdditionalFeatures from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from extensions.ext_database import db from models.model import App, AppModelConfig, Conversation, Message, TraceAppConfig -from models.workflow import Workflow from services.ops_trace.langfuse_trace import LangFuseDataTrace from services.ops_trace.langsmith_trace import LangSmithDataTrace @@ -70,6 +67,13 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :param tracing_config: tracing config :return: """ + if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: + raise {"error": f"Invalid tracing provider: {tracing_provider}"} + + # api check + if not cls.check_trace_config_is_effective(tracing_config, tracing_provider): + return {"error": "Tracing config is not effective"} + # check if trace config already exists trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider @@ -100,6 +104,13 @@ def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :param tracing_config: tracing config :return: """ + if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: + raise ValueError(f"Invalid tracing provider: {tracing_provider}") + + # api check + if not cls.check_trace_config_is_effective(tracing_config, tracing_provider): + raise ValueError("Invalid Credentials") + # check if trace config already exists trace_config = db.session.query(TraceAppConfig).filter( TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider @@ -257,14 +268,10 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): def get_ops_trace_instance( cls, app_id, - workflow: Optional[Workflow] = None, - app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None, ): """ Get ops trace through model config :param app_id: app_id - :param workflow: workflow - :param app_model_config: app_model_config :return: """ tracing_instance = None @@ -329,7 +336,11 @@ def update_app_tracing_config(cls, app_id: str, enabled: bool, tracing_provider: :param tracing_provider: tracing provider :return: """ + # auth check + if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: + raise ValueError(f"Invalid tracing provider: {tracing_provider}") app_config: App = db.session.query(App).filter(App.id == app_id).first() + app_config.tracing = json.dumps( { "enabled": enabled, @@ -353,3 +364,30 @@ def get_app_tracing_config(cls, app_id: str): } app_trace_config = json.loads(app.tracing) return app_trace_config + + @staticmethod + def check_trace_config_is_effective(tracing_config: dict, tracing_provider: str): + """ + Check trace config is effective + :param tracing_config: tracing config + :param tracing_provider: tracing provider + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + tracing_config = LangfuseConfig(**tracing_config) + langfuse_trace_instance = LangFuseDataTrace( + tracing_config.public_key, + tracing_config.secret_key, + tracing_config.host, + ) + return langfuse_trace_instance.api_check() + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + tracing_config = LangSmithConfig(**tracing_config) + langsmith_trace_instance = LangSmithDataTrace( + tracing_config.api_key, + tracing_config.project, + tracing_config.endpoint, + ) + return langsmith_trace_instance.api_check() + else: + raise ValueError(f"Unsupported tracing provider: {tracing_provider}") From c3ddc2ba54c9f6813baf5f2789cfb42d1ce279d4 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 17:13:06 +0800 Subject: [PATCH 243/273] chore: click triger show popup --- .../overview/tracing/config-button.tsx | 22 +++++++-- .../[appId]/overview/tracing/panel.tsx | 45 ++++++++++--------- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx index 44478b73fe4cae..6b65af08243deb 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-button.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React, { useCallback, useState } from 'react' +import React, { useCallback, useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' import type { PopupProps } from './config-popup' @@ -19,19 +19,34 @@ type Props = { readOnly: boolean className?: string hasConfigured: boolean + controlShowPopup?: number } & PopupProps const ConfigBtn: FC<Props> = ({ className, hasConfigured, + controlShowPopup, ...popupProps }) => { const { t } = useTranslation() - const [open, setOpen] = useState(false) + const [open, doSetOpen] = useState(false) + const openRef = useRef(open) + const setOpen = useCallback((v: boolean) => { + doSetOpen(v) + openRef.current = v + }, [doSetOpen]) + const handleTrigger = useCallback(() => { - setOpen(v => !v) + setOpen(!openRef.current) }, [setOpen]) + useEffect(() => { + if (controlShowPopup) + // setOpen(!openRef.current) + setOpen(true) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [controlShowPopup]) + if (popupProps.readOnly && !hasConfigured) return null @@ -57,6 +72,7 @@ const ConfigBtn: FC<Props> = ({ placement='bottom-end' offset={{ mainAxis: 12, + crossAxis: hasConfigured ? 8 : 0, }} > <PortalToFollowElemTrigger onClick={handleTrigger}> diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 05f8f87e6787f7..98817c33ac31cf 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React, { useEffect, useState } from 'react' +import React, { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from 'classnames' import { usePathname } from 'next/navigation' @@ -116,7 +116,10 @@ const Panel: FC = () => { }, []) const [isFold, setFold] = useState(false) - + const [controlShowPopup, setControlShowPopup] = useState<number>(0) + const showPopup = useCallback(() => { + setControlShowPopup(Date.now()) + }, [setControlShowPopup]) if (!isLoaded) { return <div className='mb-3'> <Title /> @@ -164,7 +167,7 @@ const Panel: FC = () => { } return ( - <div className='mb-3 flex justify-between items-center'> + <div className='mb-3 flex justify-between items-center cursor-pointer' onClick={showPopup}> <Title /> <div className='flex items-center p-2 rounded-xl border-[0.5px] border-gray-200 shadow-xs hover:bg-gray-100'> {!inUseTracingProvider @@ -186,26 +189,28 @@ const Panel: FC = () => { {hasConfiguredTracing && ( <div className='ml-2 w-px h-3.5 bg-gray-200'></div> )} - - <ConfigButton - appId={appId} - readOnly={readOnly} - hasConfigured - className='ml-2' - enabled={enabled} - onStatusChange={handleTracingEnabledChange} - chosenProvider={inUseTracingProvider} - onChooseProvider={handleChooseProvider} - langSmithConfig={langSmithConfig} - langFuseConfig={langFuseConfig} - onConfigUpdated={handleTracingConfigUpdated} - onConfigRemoved={handleTracingConfigRemoved} - /> + <div className='flex items-center' onClick={e => e.stopPropagation()}> + <ConfigButton + appId={appId} + readOnly={readOnly} + hasConfigured + className='ml-2' + enabled={enabled} + onStatusChange={handleTracingEnabledChange} + chosenProvider={inUseTracingProvider} + onChooseProvider={handleChooseProvider} + langSmithConfig={langSmithConfig} + langFuseConfig={langFuseConfig} + onConfigUpdated={handleTracingConfigUpdated} + onConfigRemoved={handleTracingConfigRemoved} + controlShowPopup={controlShowPopup} + /> + </div> {!hasConfiguredTracing && ( - <> + <div className='flex items-center' onClick={e => e.stopPropagation()}> <div className='mx-2 w-px h-3.5 bg-gray-200'></div> <ToggleExpandBtn isFold={isFold} onFoldChange={setFold} /> - </> + </div> )} </div> </div> From af8b40b87a304dd08cfe79d709c186912199d82d Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 17:15:18 +0800 Subject: [PATCH 244/273] chore: keep the same pos as languse --- .../tracing/provider-config-modal.tsx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index 0873401eb51820..e4760f5d3de0e8 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -103,10 +103,10 @@ const ProviderConfigModal: FC<Props> = ({ if (type === TracingProvider.langfuse) { const postData = config as LangFuseConfig - if (!postData.public_key) - errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.publicKey`) }) if (!errorMessage && !postData.secret_key) errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.secretKey`) }) + if (!postData.public_key) + errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.publicKey`) }) } return errorMessage @@ -187,14 +187,6 @@ const ProviderConfigModal: FC<Props> = ({ )} {type === TracingProvider.langfuse && ( <> - <Field - label={t(`${I18N_PREFIX}.publicKey`)!} - labelClassName='!text-sm' - isRequired - value={(config as LangFuseConfig).public_key} - onChange={handleConfigChange('public_key')} - placeholder={t(`${I18N_PREFIX}.placeholder`, { key: t(`${I18N_PREFIX}.publicKey`) })!} - /> <Field label={t(`${I18N_PREFIX}.secretKey`)!} labelClassName='!text-sm' @@ -203,6 +195,14 @@ const ProviderConfigModal: FC<Props> = ({ onChange={handleConfigChange('secret_key')} placeholder={t(`${I18N_PREFIX}.placeholder`, { key: t(`${I18N_PREFIX}.secretKey`) })!} /> + <Field + label={t(`${I18N_PREFIX}.publicKey`)!} + labelClassName='!text-sm' + isRequired + value={(config as LangFuseConfig).public_key} + onChange={handleConfigChange('public_key')} + placeholder={t(`${I18N_PREFIX}.placeholder`, { key: t(`${I18N_PREFIX}.publicKey`) })!} + /> <Field label='Host' labelClassName='!text-sm' From 661f694c0034a4caa1e603e22ce1d24ba8fbeda8 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Wed, 19 Jun 2024 17:21:09 +0800 Subject: [PATCH 245/273] chore: always hide key string value --- .../(appDetailLayout)/[appId]/overview/tracing/panel.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index 98817c33ac31cf..e3053d2e8d4dd7 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -85,11 +85,13 @@ const Panel: FC = () => { setLangFuseConfig(langFuseConfig as LangFuseConfig) } - const handleTracingConfigUpdated = (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig) => { + const handleTracingConfigUpdated = async (provider: TracingProvider) => { + // call api to hide secret key value + const { tracing_config } = await doFetchTracingConfig({ appId, provider }) if (provider === TracingProvider.langSmith) - setLangSmithConfig(payload as LangSmithConfig) + setLangSmithConfig(tracing_config as LangSmithConfig) else - setLangFuseConfig(payload as LangFuseConfig) + setLangFuseConfig(tracing_config as LangFuseConfig) } const handleTracingConfigRemoved = (provider: TracingProvider) => { From c7bd901298130de2b131e1117faa0987600c4a34 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Wed, 19 Jun 2024 17:41:46 +0800 Subject: [PATCH 246/273] fix: uuid error --- api/core/agent/cot_agent_runner.py | 3 +-- api/core/agent/fc_agent_runner.py | 3 +-- .../apps/advanced_chat/generate_task_pipeline.py | 3 +-- api/core/rag/retrieval/dataset_retrieval.py | 4 +--- api/services/message_service.py | 3 +-- api/services/ops_trace/ops_trace_service.py | 14 +++++++++++++- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/api/core/agent/cot_agent_runner.py b/api/core/agent/cot_agent_runner.py index 9f3c7ea3bba9bf..c08adc5c7d5838 100644 --- a/api/core/agent/cot_agent_runner.py +++ b/api/core/agent/cot_agent_runner.py @@ -46,9 +46,8 @@ def run( self._init_react_state(query) # get tracing instance - app_id = self.app_config.app_id tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id + app_id=self.app_config.app_id ) # check model mode diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index 3ddd43a771602e..721a8b7f03fc0e 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -52,9 +52,8 @@ def run(self, final_answer = '' # get tracing instance - app_id = app_config.app_id tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id + app_id=app_config.app_id ) def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index 0d461528bdc737..ec93eb2d5dfada 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -188,8 +188,7 @@ def _process_stream_response(self, workflow: Optional[Workflow] = None) -> Gener Process stream response. :return: """ - app_id = self._conversation.app_id - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) + tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=self._conversation.app_id) for message in self._queue_manager.listen(): event = message.event diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index 7d7645b8e68fab..4aa441e4bce467 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -23,7 +23,6 @@ from extensions.ext_database import db from models.dataset import Dataset, DatasetQuery, DocumentSegment from models.dataset import Document as DatasetDocument -from models.model import Message from services.ops_trace.ops_trace_service import OpsTraceService from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName from services.ops_trace.utils import measure_time @@ -355,9 +354,8 @@ def _on_retrival_end( db.session.commit() # get tracing instance - app_id = db.session.query(Message.app_id).filter(Message.id == message_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id + message_id=message_id ) if tracing_instance: trace_manager = TraceQueueManager() diff --git a/api/services/message_service.py b/api/services/message_service.py index e43d244d58bdbd..b5a32c1b770bff 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -272,9 +272,8 @@ def get_suggested_questions_after_answer(cls, app_model: App, user: Optional[Uni ) # get tracing instance - app_id = db.session.query(Message.app_id).filter(Message.id == message_id).first() tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id, + message_id=message_id ) if tracing_instance: trace_manager = TraceQueueManager() diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index db5c5cc92bf131..9e9b76006b1ea0 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -1,5 +1,7 @@ import json from enum import Enum +from typing import Union +from uuid import UUID from pydantic import BaseModel @@ -267,18 +269,28 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): @classmethod def get_ops_trace_instance( cls, - app_id, + app_id: Union[UUID, str] = None, + message_id: str = None ): """ Get ops trace through model config :param app_id: app_id + :param message_id: message_id :return: """ + if message_id: + record: Message = db.session.query(Message).filter(Message.id == message_id).first() + app_id = record.app_id + + if isinstance(app_id, UUID): + app_id = str(app_id) + tracing_instance = None app: App = db.session.query(App).filter( App.id == app_id ).first() app_ops_trace_config = json.loads(app.tracing) if app.tracing else None + if app_ops_trace_config is not None: tracing_provider = app_ops_trace_config.get('tracing_provider') else: From 7221f10b750d8021bb5ac514df696efd2f0b3594 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Wed, 19 Jun 2024 18:19:19 +0800 Subject: [PATCH 247/273] feat: update workflow trace conversation_id --- api/core/app/task_pipeline/workflow_cycle_manage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/core/app/task_pipeline/workflow_cycle_manage.py b/api/core/app/task_pipeline/workflow_cycle_manage.py index 5c0ffbe07bdd5b..c76c1c5e418e87 100644 --- a/api/core/app/task_pipeline/workflow_cycle_manage.py +++ b/api/core/app/task_pipeline/workflow_cycle_manage.py @@ -532,14 +532,16 @@ def _handle_node_finished(self, event: QueueNodeSucceededEvent | QueueNodeFailed def _handle_workflow_finished( self, event: QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent, - tracing_instance: Optional[BaseTraceInstance] = None + tracing_instance: Optional[BaseTraceInstance] = None, + conversation_id: Optional[str] = None ) -> Optional[WorkflowRun]: workflow_run = db.session.query(WorkflowRun).filter( WorkflowRun.id == self._task_state.workflow_run_id).first() if not workflow_run: return None - conversation_id = self._application_generate_entity.inputs.get('sys.conversation_id') + if conversation_id is None: + conversation_id = self._application_generate_entity.inputs.get('sys.conversation_id') if isinstance(event, QueueStopEvent): workflow_run = self._workflow_run_failed( workflow_run=workflow_run, From 0ed2ed83cbf0f3b5b1732fa427d6120a201f4180 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Thu, 20 Jun 2024 10:54:37 +0800 Subject: [PATCH 248/273] feat: add loading effect --- .../[appId]/overview/tracing/panel.tsx | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index e3053d2e8d4dd7..ba127f62abb11d 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -16,6 +16,7 @@ import { fetchTracingConfig as doFetchTracingConfig, fetchTracingStatus, updateT import type { TracingStatus } from '@/models/app' import Toast from '@/app/components/base/toast' import { useAppContext } from '@/context/app-context' +import Loading from '@/app/components/base/loading' const I18N_PREFIX = 'app.tracing' @@ -27,7 +28,7 @@ const Title = ({ const { t } = useTranslation() return ( - <div className={cn(className, 'text-lg font-semibold text-gray-900')}> + <div className={cn(className, 'flex items-center text-lg font-semibold text-gray-900')}> {t('appOverview.overview.title')} </div> ) @@ -123,14 +124,19 @@ const Panel: FC = () => { setControlShowPopup(Date.now()) }, [setControlShowPopup]) if (!isLoaded) { - return <div className='mb-3'> - <Title /> - </div> + return ( + <div className='flex items-center justify-between mb-3'> + <Title className='h-[41px]' /> + <div className='w-[200px]'> + <Loading /> + </div> + </div> + ) } if (!isFold && !hasConfiguredTracing) { return ( - <div className='mb-3'> + <div className={cn('mb-3')}> <Title /> <div className='mt-2 flex justify-between p-3 pr-4 items-center bg-white border-[0.5px] border-black/8 rounded-xl shadow-md'> <div className='flex space-x-2'> @@ -169,8 +175,8 @@ const Panel: FC = () => { } return ( - <div className='mb-3 flex justify-between items-center cursor-pointer' onClick={showPopup}> - <Title /> + <div className={cn('mb-3 flex justify-between items-center cursor-pointer')} onClick={showPopup}> + <Title className='h-[41px]' /> <div className='flex items-center p-2 rounded-xl border-[0.5px] border-gray-200 shadow-xs hover:bg-gray-100'> {!inUseTracingProvider ? <> From 423bb4c1bbabd5d6aadf116dd2190c92b4f91734 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Thu, 20 Jun 2024 11:54:51 +0800 Subject: [PATCH 249/273] feat: format input and out --- .../advanced_chat/generate_task_pipeline.py | 4 +- .../apps/workflow/generate_task_pipeline.py | 2 +- api/core/llm_generator/llm_generator.py | 10 +-- api/services/ops_trace/langfuse_trace.py | 44 ++++++++--- api/services/ops_trace/langsmith_trace.py | 76 +++++++++++++++---- api/services/ops_trace/ops_trace_service.py | 10 ++- api/services/ops_trace/utils.py | 17 ++++- 7 files changed, 129 insertions(+), 34 deletions(-) diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index ec93eb2d5dfada..e30c58f0de55a8 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -256,7 +256,9 @@ def _process_stream_response(self, workflow: Optional[Workflow] = None) -> Gener yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event) self._handle_iteration_operation(event) elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent): - workflow_run = self._handle_workflow_finished(event, tracing_instance) + workflow_run = self._handle_workflow_finished( + event, tracing_instance=tracing_instance, conversation_id=self._conversation.id + ) if workflow_run: yield self._workflow_finish_to_stream_response( task_id=self._application_generate_entity.task_id, diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 54bd3fbf0db264..982fab01a0bdc3 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -225,7 +225,7 @@ def _process_stream_response( yield self._handle_iteration_to_stream_response(self._application_generate_entity.task_id, event) self._handle_iteration_operation(event) elif isinstance(event, QueueStopEvent | QueueWorkflowSucceededEvent | QueueWorkflowFailedEvent): - workflow_run = self._handle_workflow_finished(event, tracing_instance) + workflow_run = self._handle_workflow_finished(event, tracing_instance=tracing_instance) # save workflow app log self._save_workflow_app_log(workflow_run) diff --git a/api/core/llm_generator/llm_generator.py b/api/core/llm_generator/llm_generator.py index 4059f637c58e2a..808f2a941f143c 100644 --- a/api/core/llm_generator/llm_generator.py +++ b/api/core/llm_generator/llm_generator.py @@ -11,8 +11,6 @@ from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError from core.prompt.utils.prompt_template_parser import PromptTemplateParser -from extensions.ext_database import db -from models.model import Conversation from services.ops_trace.ops_trace_service import OpsTraceService from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName from services.ops_trace.utils import measure_time @@ -56,19 +54,15 @@ def generate_conversation_name(cls, tenant_id: str, query, conversation_id: Opti name = name[:75] + '...' # get tracing instance - conversation_data: Conversation = db.query(Conversation).filter(Conversation.id == conversation_id).first() - app_id = conversation_data.app_id - tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id + conversation_id=conversation_id ) - if tracing_instance: trace_manager = TraceQueueManager() trace_manager.add_trace_task( TraceTask( tracing_instance, - TraceTaskName.CONVERSATION_TRACE, + TraceTaskName.SUGGESTED_QUESTION_TRACE, conversation_id=conversation_id, generate_conversation_name=name, inputs=prompt, diff --git a/api/services/ops_trace/langfuse_trace.py b/api/services/ops_trace/langfuse_trace.py index 6a037b4ad7e20d..b9e5642cd09dee 100644 --- a/api/services/ops_trace/langfuse_trace.py +++ b/api/services/ops_trace/langfuse_trace.py @@ -14,7 +14,7 @@ from models.model import Message, MessageAgentThought, MessageFile from models.workflow import WorkflowNodeExecution, WorkflowRun from services.ops_trace.base_trace_instance import BaseTraceInstance -from services.ops_trace.utils import filter_none_values +from services.ops_trace.utils import filter_none_values, replace_text_with_content def validate_input_output(v, field_name): @@ -27,12 +27,24 @@ def validate_input_output(v, field_name): if v == {} or v is None: return v if isinstance(v, str): - return {field_name: v} + return [ + { + "role": "assistant" if field_name == "output" else "user", + "content": v, + } + ] elif isinstance(v, list): if len(v) > 0 and isinstance(v[0], dict): - return {"message": v} + v = replace_text_with_content(data=v) + return v else: - return {field_name: v} + return [ + { + "role": "assistant" if field_name == "output" else "user", + "content": str(v), + } + ] + return v @@ -187,6 +199,11 @@ class GenerationUsage(BaseModel): outputCost: Optional[float] = None totalCost: Optional[float] = None + @field_validator("input", "output") + def ensure_dict(cls, v, info: ValidationInfo): + field_name = info.field_name + return validate_input_output(v, field_name) + class LangfuseGeneration(BaseModel): id: Optional[str] = Field( @@ -514,7 +531,7 @@ def suggested_question_trace(self, message_id: str, suggested_question: str, **k timer = kwargs.get("timer") start_time = timer.get("start") end_time = timer.get("end") - inputs = message_data.query + input = message_data.query metadata = { "message_id": message_id, @@ -528,19 +545,28 @@ def suggested_question_trace(self, message_id: str, suggested_question: str, **k "from_source": message_data.from_source, } - span_data = LangfuseSpan( + generation_usage = GenerationUsage( + totalTokens=len(suggested_question), + input=len(input), + output=len(suggested_question), + total=len(suggested_question), + unit=UnitEnum.CHARACTERS, + ) + + generation_data = LangfuseGeneration( name="suggested_question", - input=inputs, - output=suggested_question, + input=input, + output=str(suggested_question), trace_id=message_id, start_time=start_time, end_time=end_time, metadata=metadata, level=LevelEnum.DEFAULT if message_data.status != 'error' else LevelEnum.ERROR, status_message=message_data.error if message_data.error else "", + usage=generation_usage, ) - self.add_span(langfuse_span_data=span_data) + self.add_generation(langfuse_generation_data=generation_data) def dataset_retrieval_trace(self, message_id: str, documents: list[Document], **kwargs): message_data = kwargs.get("message_data") diff --git a/api/services/ops_trace/langsmith_trace.py b/api/services/ops_trace/langsmith_trace.py index 529ce1b07aaa78..e9453138ae00a3 100644 --- a/api/services/ops_trace/langsmith_trace.py +++ b/api/services/ops_trace/langsmith_trace.py @@ -90,23 +90,52 @@ def ensure_dict(cls, v, info: ValidationInfo): } file_list = values.get("file_list", []) if isinstance(v, str): - return { - field_name: v, - "file_list": file_list, - "usage_metadata": usage_metadata, - } + if field_name == "inputs": + return { + "messages": { + "role": "user", + "content": v, + "usage_metadata": usage_metadata, + "file_list": file_list, + }, + } + elif field_name == "outputs": + return { + "choices": { + "role": "ai", + "content": v, + "usage_metadata": usage_metadata, + "file_list": file_list, + }, + } elif isinstance(v, list): + data = {} if len(v) > 0 and isinstance(v[0], dict): # rename text to content - replace_text_with_content(data=v) - data = { - "message": v, - "usage_metadata": usage_metadata, - "file_list": file_list, - } + v = replace_text_with_content(data=v) + if field_name == "inputs": + data = { + "messages": v, + } + elif field_name == "outputs": + data = { + "choices": { + "role": "ai", + "content": v, + "usage_metadata": usage_metadata, + "file_list": file_list, + }, + } return data else: - return {field_name: v} + return { + "choices": { + "role": "ai" if field_name == "outputs" else "user", + "content": str(v), + "usage_metadata": usage_metadata, + "file_list": file_list, + }, + } if isinstance(v, dict): v["usage_metadata"] = usage_metadata v["file_list"] = file_list @@ -318,7 +347,7 @@ def message_trace(self, message_id: str, conversation_id: str, **kwargs): id=message_id, name=f"message_{message_id}", inputs=inputs, - run_type=LangSmithRunType.llm, + run_type=LangSmithRunType.chain, start_time=created_at, end_time=end_time, outputs=message_data.answer, @@ -331,6 +360,27 @@ def message_trace(self, message_id: str, conversation_id: str, **kwargs): ) self.add_run(message_run) + # create llm run parented to message run + llm_run = LangSmithRunModel( + input_tokens=message_tokens, + output_tokens=answer_tokens, + total_tokens=total_tokens, + name=f"llm_{message_id}", + inputs=inputs, + run_type=LangSmithRunType.llm, + start_time=created_at, + end_time=end_time, + outputs=message_data.answer, + extra={ + "metadata": metadata, + }, + parent_run_id=message_id, + tags=["llm", str(conversation_mode)], + error=error, + file_list=file_list, + ) + self.add_run(llm_run) + def moderation_trace(self, message_id: str, moderation_result: ModerationInputsResult, **kwargs): inputs = kwargs.get("inputs") message_data = kwargs.get("message_data") diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 9e9b76006b1ea0..b402f0cc65c04e 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -270,14 +270,22 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): def get_ops_trace_instance( cls, app_id: Union[UUID, str] = None, - message_id: str = None + message_id: str = None, + conversation_id: str = None ): """ Get ops trace through model config :param app_id: app_id :param message_id: message_id + :param conversation_id: conversation_id :return: """ + if conversation_id: + conversation_data: Conversation = db.session.query(Conversation).filter( + Conversation.id == conversation_id + ).first() + app_id = conversation_data.app_id + if message_id: record: Message = db.session.query(Message).filter(Message.id == message_id).first() app_id = record.app_id diff --git a/api/services/ops_trace/utils.py b/api/services/ops_trace/utils.py index c556947380d9e5..2b12db0f482c6d 100644 --- a/api/services/ops_trace/utils.py +++ b/api/services/ops_trace/utils.py @@ -25,4 +25,19 @@ def measure_time(): yield timing_info finally: timing_info['end'] = datetime.now() - print(f"Execution time: {timing_info['end'] - timing_info['start']}") \ No newline at end of file + print(f"Execution time: {timing_info['end'] - timing_info['start']}") + + +def replace_text_with_content(data): + if isinstance(data, dict): + new_data = {} + for key, value in data.items(): + if key == 'text': + new_data['content'] = value + else: + new_data[key] = replace_text_with_content(value) + return new_data + elif isinstance(data, list): + return [replace_text_with_content(item) for item in data] + else: + return data From 4e13f6e5e86de208dce316f5d1800a31d0d6eb15 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Thu, 20 Jun 2024 14:31:37 +0800 Subject: [PATCH 250/273] feat: add ops trace encrypt_config decrypt_config obfuscate_config --- api/.env.example | 4 +- api/controllers/console/app/app.py | 13 ++- api/controllers/console/app/ops_trace.py | 1 - api/services/ops_trace/__init__.py | 0 api/services/ops_trace/langfuse_trace.py | 30 ++++++- api/services/ops_trace/langsmith_trace.py | 25 +++++- api/services/ops_trace/model.py | 26 ++++++ api/services/ops_trace/ops_trace_service.py | 89 +++------------------ 8 files changed, 91 insertions(+), 97 deletions(-) create mode 100644 api/services/ops_trace/__init__.py create mode 100644 api/services/ops_trace/model.py diff --git a/api/.env.example b/api/.env.example index 9a15a650bbd017..8c59a8104b70b3 100644 --- a/api/.env.example +++ b/api/.env.example @@ -223,7 +223,5 @@ WORKFLOW_CALL_MAX_DEPTH=5 # App configuration APP_MAX_EXECUTION_TIME=1200 -# Firecrawl Web Extractor -FIRECRAWL_API_KEY= -FIRECRAWL_BASE_URL= + diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index a9e8b5fb4a2d33..2f49222450490f 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -306,14 +306,11 @@ def post(self, app_id): parser.add_argument('tracing_provider', type=str, required=True, location='json') args = parser.parse_args() - try: - OpsTraceService.update_app_tracing_config( - app_id=app_id, - enabled=args['enabled'], - tracing_provider=args['tracing_provider'], - ) - except Exception as e: - raise e + OpsTraceService.update_app_tracing_config( + app_id=app_id, + enabled=args['enabled'], + tracing_provider=args['tracing_provider'], + ) return {"result": "success"} diff --git a/api/controllers/console/app/ops_trace.py b/api/controllers/console/app/ops_trace.py index c3e9ebac1c2aa4..382f1ed89b3c9b 100644 --- a/api/controllers/console/app/ops_trace.py +++ b/api/controllers/console/app/ops_trace.py @@ -69,7 +69,6 @@ def patch(self, app_id): tracing_provider=args['tracing_provider'], tracing_config=args['tracing_config'] ) - print("==============", result) if not result: raise TracingConfigNotExist() return {"result": "success"} diff --git a/api/services/ops_trace/__init__.py b/api/services/ops_trace/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/services/ops_trace/langfuse_trace.py b/api/services/ops_trace/langfuse_trace.py index b9e5642cd09dee..ec859a698029b8 100644 --- a/api/services/ops_trace/langfuse_trace.py +++ b/api/services/ops_trace/langfuse_trace.py @@ -1,4 +1,5 @@ import json +import logging import os from datetime import datetime, timedelta from enum import Enum @@ -8,14 +9,17 @@ from pydantic import BaseModel, Field, field_validator from pydantic_core.core_schema import ValidationInfo +from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from core.moderation.base import ModerationInputsResult from extensions.ext_database import db from models.dataset import Document from models.model import Message, MessageAgentThought, MessageFile from models.workflow import WorkflowNodeExecution, WorkflowRun from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.model import LangfuseConfig from services.ops_trace.utils import filter_none_values, replace_text_with_content +logger = logging.getLogger(__name__) def validate_input_output(v, field_name): """ @@ -691,7 +695,7 @@ def add_trace(self, langfuse_trace_data: Optional[LangfuseTrace] = None): ) try: self.langfuse_client.trace(**format_trace_data) - print("LangFuse Trace created successfully") + logger.debug("LangFuse Trace created successfully") except Exception as e: raise f"LangFuse Failed to create trace: {str(e)}" @@ -701,7 +705,7 @@ def add_span(self, langfuse_span_data: Optional[LangfuseSpan] = None): ) try: self.langfuse_client.span(**format_span_data) - print("LangFuse Span created successfully") + logger.debug("LangFuse Span created successfully") except Exception as e: raise f"LangFuse Failed to create span: {str(e)}" @@ -722,7 +726,7 @@ def add_generation( ) try: self.langfuse_client.generation(**format_generation_data) - print("LangFuse Generation created successfully") + logger.debug("LangFuse Generation created successfully") except Exception as e: raise f"LangFuse Failed to create generation: {str(e)}" @@ -741,5 +745,23 @@ def api_check(self): try: return self.langfuse_client.auth_check() except Exception as e: - print(f"LangFuse API check failed: {str(e)}") + logger.debug(f"LangFuse API check failed: {str(e)}") return False + + @classmethod + def obfuscate_config(cls, config: LangfuseConfig): + public_key = obfuscated_token(config.public_key) + secret_key = obfuscated_token(config.secret_key) + return LangfuseConfig(public_key=public_key, secret_key=secret_key, host=config.host) + + @classmethod + def encrypt_config(cls, tenant_id, config: LangfuseConfig): + decrypt_public_key = encrypt_token(tenant_id, config.public_key) + decrypt_secret_key = encrypt_token(tenant_id, config.secret_key) + return LangfuseConfig(public_key=decrypt_public_key, secret_key=decrypt_secret_key, host=config.host) + + @classmethod + def decrypt_config(cls, tenant_id, config: LangfuseConfig): + decrypt_public_key = decrypt_token(tenant_id, config.public_key) + decrypt_secret_key = decrypt_token(tenant_id, config.secret_key) + return LangfuseConfig(public_key=decrypt_public_key, secret_key=decrypt_secret_key, host=config.host) diff --git a/api/services/ops_trace/langsmith_trace.py b/api/services/ops_trace/langsmith_trace.py index e9453138ae00a3..bc9c86717d0a4b 100644 --- a/api/services/ops_trace/langsmith_trace.py +++ b/api/services/ops_trace/langsmith_trace.py @@ -1,4 +1,5 @@ import json +import logging import os from datetime import datetime, timedelta from enum import Enum @@ -8,14 +9,17 @@ from pydantic import BaseModel, Field, field_validator from pydantic_core.core_schema import ValidationInfo +from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from core.moderation.base import ModerationInputsResult from extensions.ext_database import db from models.dataset import Document from models.model import Message, MessageAgentThought, MessageFile from models.workflow import WorkflowNodeExecution, WorkflowRun from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.model import LangSmithConfig from services.ops_trace.utils import filter_none_values, replace_text_with_content +logger = logging.getLogger(__name__) class LangSmithRunType(str, Enum): tool = "tool" @@ -583,7 +587,7 @@ def add_run(self, run_data: LangSmithRunModel): data = filter_none_values(data) try: self.langsmith_client.create_run(**data) - print("LangSmith Run created successfully.") + logger.debug("LangSmith Run created successfully.") except Exception as e: raise f"LangSmith Failed to create run: {str(e)}" @@ -592,7 +596,7 @@ def update_run(self, update_run_data: LangSmithRunUpdateModel): data = filter_none_values(data) try: self.langsmith_client.update_run(**data) - print("LangSmith Run updated successfully.") + logger.debug("LangSmith Run updated successfully.") except Exception as e: raise f"LangSmith Failed to update run: {str(e)}" @@ -603,5 +607,20 @@ def api_check(self): self.langsmith_client.delete_project(project_name=random_project_name) return True except Exception as e: - print(f"LangSmith API check failed: {str(e)}") + logger.debug(f"LangSmith API check failed: {str(e)}") return False + + @classmethod + def obfuscate_config(cls, config: LangSmithConfig): + api_key = obfuscated_token(config.api_key) + return LangSmithConfig(api_key=api_key, project=config.project, endpoint=config.endpoint) + + @classmethod + def encrypt_config(cls, tenant_id, config: LangSmithConfig): + api_key = encrypt_token(tenant_id, config.api_key) + return LangSmithConfig(api_key=api_key, project=config.project, endpoint=config.endpoint) + + @classmethod + def decrypt_config(cls, tenant_id, config: LangSmithConfig): + api_key = decrypt_token(tenant_id, config.api_key) + return LangSmithConfig(api_key=api_key, project=config.project, endpoint=config.endpoint) diff --git a/api/services/ops_trace/model.py b/api/services/ops_trace/model.py new file mode 100644 index 00000000000000..51dca08137e773 --- /dev/null +++ b/api/services/ops_trace/model.py @@ -0,0 +1,26 @@ +from enum import Enum + +from pydantic import BaseModel + + +class TracingProviderEnum(Enum): + LANGFUSE = 'langfuse' + LANGSMITH = 'langsmith' + + +class LangfuseConfig(BaseModel): + """ + Model class for Langfuse tracing config. + """ + public_key: str + secret_key: str + host: str + + +class LangSmithConfig(BaseModel): + """ + Model class for Langsmith tracing config. + """ + api_key: str + project: str + endpoint: str diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index b402f0cc65c04e..4abb676efa7553 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -1,38 +1,12 @@ import json -from enum import Enum from typing import Union from uuid import UUID -from pydantic import BaseModel - -from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from extensions.ext_database import db from models.model import App, AppModelConfig, Conversation, Message, TraceAppConfig from services.ops_trace.langfuse_trace import LangFuseDataTrace from services.ops_trace.langsmith_trace import LangSmithDataTrace - - -class TracingProviderEnum(Enum): - LANGFUSE = 'langfuse' - LANGSMITH = 'langsmith' - - -class LangfuseConfig(BaseModel): - """ - Model class for Langfuse tracing config. - """ - public_key: str - secret_key: str - host: str - - -class LangSmithConfig(BaseModel): - """ - Model class for Langsmith tracing config. - """ - api_key: str - project: str - endpoint: str +from services.ops_trace.model import LangfuseConfig, LangSmithConfig, TracingProviderEnum class OpsTraceService: @@ -161,25 +135,12 @@ def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c """ if tracing_provider == TracingProviderEnum.LANGFUSE.value: tracing_config = LangfuseConfig(**tracing_config) - encrypt_public_key = encrypt_token(tenant_id, tracing_config.public_key) - encrypt_secret_key = encrypt_token(tenant_id, tracing_config.secret_key) - tracing_config = LangfuseConfig( - public_key=encrypt_public_key, - secret_key=encrypt_secret_key, - host=tracing_config.host - ) + tracing_config = LangFuseDataTrace.encrypt_config(tenant_id, tracing_config) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: tracing_config = LangSmithConfig(**tracing_config) - encrypt_api_key = encrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangSmithConfig( - api_key=encrypt_api_key, - project=tracing_config.project, - endpoint=tracing_config.endpoint - ) + tracing_config = LangSmithDataTrace.encrypt_config(tenant_id, tracing_config) - if isinstance(tracing_config, BaseModel): - return tracing_config.dict() - return tracing_config + return tracing_config.model_dump() @classmethod def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): @@ -192,25 +153,12 @@ def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c """ if tracing_provider == TracingProviderEnum.LANGFUSE.value: tracing_config = LangfuseConfig(**tracing_config) - decrypt_public_key = decrypt_token(tenant_id, tracing_config.public_key) - decrypt_secret_key = decrypt_token(tenant_id, tracing_config.secret_key) - tracing_config = LangfuseConfig( - public_key=decrypt_public_key, - secret_key=decrypt_secret_key, - host=tracing_config.host - ) + tracing_config = LangFuseDataTrace.decrypt_config(tenant_id, tracing_config) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: tracing_config = LangSmithConfig(**tracing_config) - decrypt_api_key = decrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangSmithConfig( - api_key=decrypt_api_key, - project=tracing_config.project, - endpoint=tracing_config.endpoint - ) + tracing_config = LangSmithDataTrace.decrypt_config(tenant_id, tracing_config) - if isinstance(tracing_config, BaseModel): - return tracing_config.dict() - return tracing_config + return tracing_config.model_dump() @classmethod def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config:dict): @@ -220,28 +168,14 @@ def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config: :param decrypt_tracing_config: tracing config :return: """ + obfuscate_config = None if tracing_provider == TracingProviderEnum.LANGFUSE.value: decrypt_tracing_config = LangfuseConfig(**decrypt_tracing_config) - decrypt_public_key = decrypt_tracing_config.public_key - decrypt_secret_key = decrypt_tracing_config.secret_key - obfuscated_public_key = obfuscated_token(decrypt_public_key) - obfuscated_secret_key = obfuscated_token(decrypt_secret_key) - decrypt_tracing_config = LangfuseConfig( - public_key=obfuscated_public_key, - secret_key=obfuscated_secret_key, - host=decrypt_tracing_config.host - ) + obfuscate_config = LangFuseDataTrace.obfuscate_config(decrypt_tracing_config) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: decrypt_tracing_config = LangSmithConfig(**decrypt_tracing_config) - decrypt_api_key = decrypt_tracing_config.api_key - obfuscated_api_key = obfuscated_token(decrypt_api_key) - decrypt_tracing_config = LangSmithConfig( - api_key=obfuscated_api_key, - project=decrypt_tracing_config.project, - endpoint=decrypt_tracing_config.endpoint - ) - - return decrypt_tracing_config.dict() + obfuscate_config = LangSmithDataTrace.obfuscate_config(decrypt_tracing_config) + return obfuscate_config.model_dump() @classmethod def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): @@ -320,7 +254,6 @@ def get_ops_trace_instance( langsmith_api_key = decrypt_trace_config.get('api_key') langsmith_project = decrypt_trace_config.get('project') langsmith_endpoint = decrypt_trace_config.get('endpoint') - print(langsmith_api_key, langsmith_project, langsmith_endpoint) tracing_instance = LangSmithDataTrace( langsmith_api_key, langsmith_project, From e694ec9a49ecb6bd725980b8d0e10465f355ca6a Mon Sep 17 00:00:00 2001 From: -LAN- <laipz8200@outlook.com> Date: Fri, 21 Jun 2024 11:11:49 +0800 Subject: [PATCH 251/273] refactor(service): Improve module export. --- api/services/__init__.py | 4 +++- api/services/errors/__init__.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/api/services/__init__.py b/api/services/__init__.py index 20e68ab6d94cf2..6891436314b299 100644 --- a/api/services/__init__.py +++ b/api/services/__init__.py @@ -1 +1,3 @@ -import services.errors +from . import errors + +__all__ = ['errors'] diff --git a/api/services/errors/__init__.py b/api/services/errors/__init__.py index 493919d373bb17..bb5711145c0a44 100644 --- a/api/services/errors/__init__.py +++ b/api/services/errors/__init__.py @@ -1,6 +1,29 @@ +from . import ( + account, + app, + app_model_config, + audio, + base, + completion, + conversation, + dataset, + document, + file, + index, + message, +) + __all__ = [ - 'base', 'conversation', 'message', 'index', 'app_model_config', 'account', 'document', 'dataset', - 'app', 'completion', 'audio', 'file' + "base", + "conversation", + "message", + "index", + "app_model_config", + "account", + "document", + "dataset", + "app", + "completion", + "audio", + "file", ] - -from . import * From 89aa72224a9fdb1a5cf167dd617e6faa1d3480ec Mon Sep 17 00:00:00 2001 From: -LAN- <laipz8200@outlook.com> Date: Fri, 21 Jun 2024 11:47:15 +0800 Subject: [PATCH 252/273] feat(api/auth): switch-to-stateful-authentication --- api/app.py | 41 +++++++++++++-------------- api/controllers/console/auth/login.py | 13 +++++---- api/controllers/console/auth/oauth.py | 5 ++-- api/controllers/console/setup.py | 4 +-- api/libs/helper.py | 2 +- api/services/account_service.py | 30 ++++++++++++++++---- 6 files changed, 58 insertions(+), 37 deletions(-) diff --git a/api/app.py b/api/app.py index 52461aac9304e9..82ec64e6b7c61b 100644 --- a/api/app.py +++ b/api/app.py @@ -2,7 +2,7 @@ from configs.app_configs import DifyConfigs -if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true': +if not os.environ.get("DEBUG") or os.environ.get("DEBUG", "false").lower() != 'true': from gevent import monkey monkey.patch_all() @@ -152,27 +152,26 @@ def initialize_extensions(app): @login_manager.request_loader def load_user_from_request(request_from_flask_login): """Load user based on the request.""" - if request.blueprint in ['console', 'inner_api']: - # Check if the user_id contains a dot, indicating the old format - auth_header = request.headers.get('Authorization', '') - if not auth_header: - auth_token = request.args.get('_token') - if not auth_token: - raise Unauthorized('Invalid Authorization token.') - else: - if ' ' not in auth_header: - raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.') - auth_scheme, auth_token = auth_header.split(None, 1) - auth_scheme = auth_scheme.lower() - if auth_scheme != 'bearer': - raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.') - - decoded = PassportService().verify(auth_token) - user_id = decoded.get('user_id') - - return AccountService.load_user(user_id) - else: + if request.blueprint not in ['console', 'inner_api']: return None + # Check if the user_id contains a dot, indicating the old format + auth_header = request.headers.get('Authorization', '') + if not auth_header: + auth_token = request.args.get('_token') + if not auth_token: + raise Unauthorized('Invalid Authorization token.') + else: + if ' ' not in auth_header: + raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.') + auth_scheme, auth_token = auth_header.split(None, 1) + auth_scheme = auth_scheme.lower() + if auth_scheme != 'bearer': + raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.') + + decoded = PassportService().verify(auth_token) + user_id = decoded.get('user_id') + + return AccountService.load_logged_in_account(account_id=user_id, token=auth_token) @login_manager.unauthorized_handler diff --git a/api/controllers/console/auth/login.py b/api/controllers/console/auth/login.py index 8a24e584138ba8..67d6dc8e95f718 100644 --- a/api/controllers/console/auth/login.py +++ b/api/controllers/console/auth/login.py @@ -1,3 +1,5 @@ +from typing import cast + import flask_login from flask import current_app, request from flask_restful import Resource, reqparse @@ -5,8 +7,9 @@ import services from controllers.console import api from controllers.console.setup import setup_required -from libs.helper import email +from libs.helper import email, get_remote_ip from libs.password import valid_password +from models.account import Account from services.account_service import AccountService, TenantService @@ -34,10 +37,7 @@ def post(self): if len(tenants) == 0: return {'result': 'fail', 'data': 'workspace not found, please contact system admin to invite you to join in a workspace'} - AccountService.update_last_login(account, request) - - # todo: return the user info - token = AccountService.get_account_jwt_token(account) + token = AccountService.login(account, ip_address=get_remote_ip(request)) return {'result': 'success', 'data': token} @@ -46,6 +46,9 @@ class LogoutApi(Resource): @setup_required def get(self): + account = cast(Account, flask_login.current_user) + token = request.headers.get('Authorization', '').split(' ')[1] + AccountService.logout(account=account, token=token) flask_login.logout_user() return {'result': 'success'} diff --git a/api/controllers/console/auth/oauth.py b/api/controllers/console/auth/oauth.py index e5b80e9a57be33..2e4a627e066033 100644 --- a/api/controllers/console/auth/oauth.py +++ b/api/controllers/console/auth/oauth.py @@ -8,6 +8,7 @@ from constants.languages import languages from extensions.ext_database import db +from libs.helper import get_remote_ip from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo from models.account import Account, AccountStatus from services.account_service import AccountService, RegisterService, TenantService @@ -78,9 +79,7 @@ def get(self, provider: str): TenantService.create_owner_tenant_if_not_exist(account) - AccountService.update_last_login(account, request) - - token = AccountService.get_account_jwt_token(account) + token = AccountService.login(account, ip_address=get_remote_ip(request)) return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?console_token={token}') diff --git a/api/controllers/console/setup.py b/api/controllers/console/setup.py index 1911559cff527d..a8fdde2791c426 100644 --- a/api/controllers/console/setup.py +++ b/api/controllers/console/setup.py @@ -4,7 +4,7 @@ from flask_restful import Resource, reqparse from extensions.ext_database import db -from libs.helper import email, str_len +from libs.helper import email, get_remote_ip, str_len from libs.password import valid_password from models.model import DifySetup from services.account_service import AccountService, RegisterService, TenantService @@ -61,7 +61,7 @@ def post(self): TenantService.create_owner_tenant_if_not_exist(account) setup() - AccountService.update_last_login(account, request) + AccountService.update_last_login(account, ip_address=get_remote_ip(request)) return {'result': 'success'}, 201 diff --git a/api/libs/helper.py b/api/libs/helper.py index fa326c5a532d05..ebabb2ea474b6a 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -140,7 +140,7 @@ def generate_string(n): return result -def get_remote_ip(request): +def get_remote_ip(request) -> str: if request.headers.get('CF-Connecting-IP'): return request.headers.get('Cf-Connecting-Ip') elif request.headers.getlist("X-Forwarded-For"): diff --git a/api/services/account_service.py b/api/services/account_service.py index 7551c9cb4bb767..2c401aad911791 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -13,7 +13,6 @@ from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created from extensions.ext_redis import redis_client -from libs.helper import get_remote_ip from libs.passport import PassportService from libs.password import compare_password, hash_password, valid_password from libs.rsa import generate_key_pair @@ -67,10 +66,10 @@ def load_user(user_id: str) -> Account: @staticmethod - def get_account_jwt_token(account): + def get_account_jwt_token(account, *, exp: timedelta = timedelta(days=30)): payload = { "user_id": account.id, - "exp": datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30), + "exp": datetime.now(timezone.utc).replace(tzinfo=None) + exp, "iss": current_app.config['EDITION'], "sub": 'Console API Passport', } @@ -195,14 +194,35 @@ def update_account(account, **kwargs): return account @staticmethod - def update_last_login(account: Account, request) -> None: + def update_last_login(account: Account, *, ip_address: str) -> None: """Update last login time and ip""" account.last_login_at = datetime.now(timezone.utc).replace(tzinfo=None) - account.last_login_ip = get_remote_ip(request) + account.last_login_ip = ip_address db.session.add(account) db.session.commit() logging.info(f'Account {account.id} logged in successfully.') + @staticmethod + def login(account: Account, *, ip_address: Optional[str] = None): + if ip_address: + AccountService.update_last_login(account, ip_address=ip_address) + exp = timedelta(days=30) + token = AccountService.get_account_jwt_token(account, exp=exp) + redis_client.set(_get_login_cache_key(account_id=account.id, token=token), '1', ex=int(exp.total_seconds())) + return token + + @staticmethod + def logout(*, account: Account, token: str): + redis_client.delete(_get_login_cache_key(account_id=account.id, token=token)) + + @staticmethod + def load_logged_in_account(*, account_id: str, token: str): + if not redis_client.get(_get_login_cache_key(account_id=account_id, token=token)): + return None + return AccountService.load_user(account_id) + +def _get_login_cache_key(*, account_id: str, token: str): + return f"account_login:{account_id}:{token}" class TenantService: From c38fd13a989fde63c36da8be5ad13dae031bb466 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Fri, 21 Jun 2024 13:53:22 +0800 Subject: [PATCH 253/273] feat: generate tracing_instance in app_generator --- api/core/agent/cot_agent_runner.py | 5 +--- api/core/agent/fc_agent_runner.py | 5 +--- .../app/apps/advanced_chat/app_generator.py | 12 +++++++--- .../advanced_chat/generate_task_pipeline.py | 6 ++--- api/core/app/apps/agent_chat/app_generator.py | 10 +++++--- api/core/app/apps/agent_chat/app_runner.py | 6 ++--- api/core/app/apps/base_app_runner.py | 1 + api/core/app/apps/chat/app_generator.py | 14 +++++------ api/core/app/apps/chat/app_runner.py | 2 +- api/core/app/apps/completion/app_generator.py | 8 ++++++- api/core/app/apps/completion/app_runner.py | 2 +- .../app/apps/message_based_app_generator.py | 4 ++-- api/core/app/apps/workflow/app_generator.py | 9 ++++++- .../apps/workflow/generate_task_pipeline.py | 7 ++---- api/core/app/entities/app_invoke_entities.py | 3 +++ api/core/moderation/input_moderation.py | 24 ++++++++----------- api/core/rag/retrieval/dataset_retrieval.py | 9 +++---- api/core/tools/tool_engine.py | 2 -- api/core/workflow/nodes/tool/tool_node.py | 10 +------- api/services/message_service.py | 1 + api/services/ops_trace/langfuse_trace.py | 8 ++++--- api/services/ops_trace/langsmith_trace.py | 6 +++-- api/services/ops_trace/ops_trace_service.py | 3 ++- 23 files changed, 82 insertions(+), 75 deletions(-) diff --git a/api/core/agent/cot_agent_runner.py b/api/core/agent/cot_agent_runner.py index c08adc5c7d5838..1f2db3fb7b4168 100644 --- a/api/core/agent/cot_agent_runner.py +++ b/api/core/agent/cot_agent_runner.py @@ -21,7 +21,6 @@ from core.tools.tool_engine import ToolEngine from models.model import Message from services.ops_trace.base_trace_instance import BaseTraceInstance -from services.ops_trace.ops_trace_service import OpsTraceService class CotAgentRunner(BaseAgentRunner, ABC): @@ -46,9 +45,7 @@ def run( self._init_react_state(query) # get tracing instance - tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=self.app_config.app_id - ) + tracing_instance = app_generate_entity.tracing_instance # check model mode if 'Observation' not in app_generate_entity.model_conf.stop: diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index 721a8b7f03fc0e..6b2997655f1c1b 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -21,7 +21,6 @@ from core.tools.entities.tool_entities import ToolInvokeMeta from core.tools.tool_engine import ToolEngine from models.model import Message -from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -52,9 +51,7 @@ def run(self, final_answer = '' # get tracing instance - tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_config.app_id - ) + tracing_instance = app_generate_entity.tracing_instance def increase_usage(final_llm_usage_dict: dict[str, LLMUsage], usage: LLMUsage): if not final_llm_usage_dict['usage']: diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index f8a2cb0f64de3c..8060bf036f3943 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -3,7 +3,7 @@ import threading import uuid from collections.abc import Generator -from typing import Any, Optional, Union +from typing import Union from flask import Flask, current_app from pydantic import ValidationError @@ -24,6 +24,7 @@ from models.account import Account from models.model import App, Conversation, EndUser, Message from models.workflow import Workflow +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -36,7 +37,6 @@ def generate( args: dict, invoke_from: InvokeFrom, stream: bool = True, - tracing_instance: Optional[Any] = None ) -> Union[dict, Generator[dict, None, None]]: """ Generate App response. @@ -87,6 +87,11 @@ def generate( workflow=workflow ) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id + ) + # init application generate entity application_generate_entity = AdvancedChatAppGenerateEntity( task_id=str(uuid.uuid4()), @@ -98,7 +103,8 @@ def generate( user_id=user.id, stream=stream, invoke_from=invoke_from, - extras=extras + extras=extras, + tracing_instance=tracing_instance ) return self._generate( diff --git a/api/core/app/apps/advanced_chat/generate_task_pipeline.py b/api/core/app/apps/advanced_chat/generate_task_pipeline.py index e30c58f0de55a8..f4c97033d47c1d 100644 --- a/api/core/app/apps/advanced_chat/generate_task_pipeline.py +++ b/api/core/app/apps/advanced_chat/generate_task_pipeline.py @@ -54,7 +54,6 @@ WorkflowNodeExecution, WorkflowRunStatus, ) -from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -132,7 +131,7 @@ def process( self._application_generate_entity.query ) - generator = self._process_stream_response(workflow) + generator = self._process_stream_response(self._application_generate_entity.tracing_instance) if self._stream: return self._to_stream_response(generator) else: @@ -183,12 +182,11 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) stream_response=stream_response ) - def _process_stream_response(self, workflow: Optional[Workflow] = None) -> Generator[StreamResponse, None, None]: + def _process_stream_response(self, tracing_instance) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: """ - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=self._conversation.app_id) for message in self._queue_manager.listen(): event = message.event diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index 431a8b88a15d5f..3e93fad59cf893 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -109,6 +109,11 @@ def generate(self, app_model: App, override_config_dict=override_model_config_dict ) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + ) + # init application generate entity application_generate_entity = AgentChatAppGenerateEntity( task_id=str(uuid.uuid4()), @@ -122,7 +127,8 @@ def generate(self, app_model: App, stream=stream, invoke_from=invoke_from, extras=extras, - call_depth=0 + call_depth=0, + tracing_instance=tracing_instance ) # init generate records @@ -161,7 +167,6 @@ def generate(self, app_model: App, message=message, user=user, stream=stream, - tracing_instance=tracing_instance, ) return AgentChatAppGenerateResponseConverter.convert( @@ -199,7 +204,6 @@ def _generate_worker(self, flask_app: Flask, queue_manager=queue_manager, conversation=conversation, message=message, - tracing_instance=tracing_instance ) except GenerateTaskStoppedException: pass diff --git a/api/core/app/apps/agent_chat/app_runner.py b/api/core/app/apps/agent_chat/app_runner.py index a942522996ca00..1444a71257e720 100644 --- a/api/core/app/apps/agent_chat/app_runner.py +++ b/api/core/app/apps/agent_chat/app_runner.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Optional, cast +from typing import cast from core.agent.cot_chat_agent_runner import CotChatAgentRunner from core.agent.cot_completion_agent_runner import CotCompletionAgentRunner @@ -34,7 +34,6 @@ def run( queue_manager: AppQueueManager, conversation: Conversation, message: Message, - tracing_instance: Optional[Any] = None ) -> None: """ Run assistant application @@ -225,7 +224,7 @@ def run( runner_cls = FunctionCallAgentRunner else: raise ValueError(f"Invalid agent strategy: {agent_entity.strategy}") - + runner = runner_cls( tenant_id=app_config.tenant_id, application_generate_entity=application_generate_entity, @@ -247,7 +246,6 @@ def run( message=message, query=query, inputs=inputs, - tracing_instance=tracing_instance, ) # handle invoke result diff --git a/api/core/app/apps/base_app_runner.py b/api/core/app/apps/base_app_runner.py index 1ccc9597cee3a9..b0e55bbf605a8d 100644 --- a/api/core/app/apps/base_app_runner.py +++ b/api/core/app/apps/base_app_runner.py @@ -364,6 +364,7 @@ def moderation_for_inputs( inputs=inputs, query=query if query else '', message_id=message_id, + tracing_instance=app_generate_entity.tracing_instance ) def check_hosting_moderation(self, application_generate_entity: EasyUIBasedAppGenerateEntity, diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index 2dde1331891cc7..093e54f3701623 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -103,6 +103,11 @@ def generate( override_config_dict=override_model_config_dict ) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + ) + # init application generate entity application_generate_entity = ChatAppGenerateEntity( task_id=str(uuid.uuid4()), @@ -115,7 +120,8 @@ def generate( user_id=user.id, stream=stream, invoke_from=invoke_from, - extras=extras + extras=extras, + tracing_instance=tracing_instance, ) # init generate records @@ -124,11 +130,6 @@ def generate( message ) = self._init_generate_records(application_generate_entity, conversation) - # get tracing instance - tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_model.id, - ) - # init queue manager queue_manager = MessageBasedAppQueueManager( task_id=application_generate_entity.task_id, @@ -158,7 +159,6 @@ def generate( message=message, user=user, stream=stream, - tracing_instance=tracing_instance, ) return ChatAppGenerateResponseConverter.convert( diff --git a/api/core/app/apps/chat/app_runner.py b/api/core/app/apps/chat/app_runner.py index 0a029af86a1ce1..89a498eb3607f9 100644 --- a/api/core/app/apps/chat/app_runner.py +++ b/api/core/app/apps/chat/app_runner.py @@ -155,7 +155,7 @@ def run(self, application_generate_entity: ChatAppGenerateEntity, application_generate_entity.invoke_from ) - dataset_retrieval = DatasetRetrieval() + dataset_retrieval = DatasetRetrieval(application_generate_entity) context = dataset_retrieval.retrieve( app_id=app_record.id, user_id=application_generate_entity.user_id, diff --git a/api/core/app/apps/completion/app_generator.py b/api/core/app/apps/completion/app_generator.py index 186a11557af565..2bb6e5d04f9b65 100644 --- a/api/core/app/apps/completion/app_generator.py +++ b/api/core/app/apps/completion/app_generator.py @@ -95,6 +95,11 @@ def generate(self, app_model: App, override_config_dict=override_model_config_dict ) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + ) + # init application generate entity application_generate_entity = CompletionAppGenerateEntity( task_id=str(uuid.uuid4()), @@ -106,7 +111,8 @@ def generate(self, app_model: App, user_id=user.id, stream=stream, invoke_from=invoke_from, - extras=extras + extras=extras, + tracing_instance=tracing_instance ) # init generate records diff --git a/api/core/app/apps/completion/app_runner.py b/api/core/app/apps/completion/app_runner.py index 2e701320148408..f0e5f9ae173c39 100644 --- a/api/core/app/apps/completion/app_runner.py +++ b/api/core/app/apps/completion/app_runner.py @@ -115,7 +115,7 @@ def run(self, application_generate_entity: CompletionAppGenerateEntity, if dataset_config and dataset_config.retrieve_config.query_variable: query = inputs.get(dataset_config.retrieve_config.query_variable, "") - dataset_retrieval = DatasetRetrieval() + dataset_retrieval = DatasetRetrieval(application_generate_entity) context = dataset_retrieval.retrieve( app_id=app_record.id, user_id=application_generate_entity.user_id, diff --git a/api/core/app/apps/message_based_app_generator.py b/api/core/app/apps/message_based_app_generator.py index 11763d1c6cde53..fbfd2353011bca 100644 --- a/api/core/app/apps/message_based_app_generator.py +++ b/api/core/app/apps/message_based_app_generator.py @@ -1,7 +1,7 @@ import json import logging from collections.abc import Generator -from typing import Any, Optional, Union +from typing import Optional, Union from sqlalchemy import and_ @@ -47,7 +47,6 @@ def _handle_response( message: Message, user: Union[Account, EndUser], stream: bool = False, - tracing_instance: Optional[Any] = None ) -> Union[ ChatbotAppBlockingResponse, CompletionAppBlockingResponse, @@ -74,6 +73,7 @@ def _handle_response( ) try: + tracing_instance = application_generate_entity.tracing_instance return generate_task_pipeline.process(tracing_instance) except ValueError as e: if e.args[0] == "I/O operation on closed file.": # ignore this error diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index 829ccc8cab2b5a..d2d4947e783e7d 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -24,6 +24,7 @@ from models.account import Account from models.model import App, EndUser from models.workflow import Workflow +from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -72,6 +73,11 @@ def generate( workflow=workflow ) + # get tracing instance + tracing_instance = OpsTraceService.get_ops_trace_instance( + app_id=app_model.id, + ) + # init application generate entity application_generate_entity = WorkflowAppGenerateEntity( task_id=str(uuid.uuid4()), @@ -81,7 +87,8 @@ def generate( user_id=user.id, stream=stream, invoke_from=invoke_from, - call_depth=call_depth + call_depth=call_depth, + tracing_instance=tracing_instance ) return self._generate( diff --git a/api/core/app/apps/workflow/generate_task_pipeline.py b/api/core/app/apps/workflow/generate_task_pipeline.py index 982fab01a0bdc3..49a3cbbbad6f6d 100644 --- a/api/core/app/apps/workflow/generate_task_pipeline.py +++ b/api/core/app/apps/workflow/generate_task_pipeline.py @@ -48,7 +48,6 @@ WorkflowNodeExecution, WorkflowRun, ) -from services.ops_trace.ops_trace_service import OpsTraceService logger = logging.getLogger(__name__) @@ -109,7 +108,7 @@ def process( db.session.refresh(self._user) db.session.close() - generator = self._process_stream_response(app_id, workflow) + generator = self._process_stream_response(self._application_generate_entity.tracing_instance) if self._stream: return self._to_stream_response(generator) else: @@ -165,14 +164,12 @@ def _to_stream_response(self, generator: Generator[StreamResponse, None, None]) def _process_stream_response( self, - app_id: Optional[str] = None, - workflow: Optional[Workflow] = None, + tracing_instance: Optional[Any] = None ) -> Generator[StreamResponse, None, None]: """ Process stream response. :return: """ - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) for message in self._queue_manager.listen(): event = message.event diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.py index f27a110870b111..77905e296c652e 100644 --- a/api/core/app/entities/app_invoke_entities.py +++ b/api/core/app/entities/app_invoke_entities.py @@ -89,6 +89,9 @@ class AppGenerateEntity(BaseModel): # extra parameters, like: auto_generate_conversation_name extras: dict[str, Any] = {} + # tracing instance + tracing_instance: Optional[Any] = None + class EasyUIBasedAppGenerateEntity(AppGenerateEntity): """ diff --git a/api/core/moderation/input_moderation.py b/api/core/moderation/input_moderation.py index c835c97eda0e34..a5965992071ae9 100644 --- a/api/core/moderation/input_moderation.py +++ b/api/core/moderation/input_moderation.py @@ -1,9 +1,10 @@ import logging +from typing import Any, Optional from core.app.app_config.entities import AppConfig from core.moderation.base import ModerationAction, ModerationException from core.moderation.factory import ModerationFactory -from services.ops_trace.ops_trace_service import OpsTraceService +from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName from services.ops_trace.utils import measure_time logger = logging.getLogger(__name__) @@ -11,12 +12,13 @@ class InputModeration: def check( - self, app_id: str, - tenant_id: str, - app_config: AppConfig, - inputs: dict, - query: str, - message_id: str, + self, app_id: str, + tenant_id: str, + app_config: AppConfig, + inputs: dict, + query: str, + message_id: str, + tracing_instance: Optional[Any] = None ) -> tuple[bool, dict, str]: """ Process sensitive_word_avoidance. @@ -26,6 +28,7 @@ def check( :param inputs: inputs :param query: query :param message_id: message id + :param tracing_instance: tracing instance :return: """ if not app_config.sensitive_word_avoidance: @@ -44,13 +47,6 @@ def check( with measure_time() as timer: moderation_result = moderation_factory.moderation_for_inputs(inputs, query) - from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName - - # get tracing instance - tracing_instance = OpsTraceService.get_ops_trace_instance( - app_id=app_id - ) - if tracing_instance: trace_manager = TraceQueueManager() trace_manager.add_trace_task( diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index c1fb14913e8cf1..ae1644e205019d 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -24,7 +24,6 @@ from extensions.ext_database import db from models.dataset import Dataset, DatasetQuery, DocumentSegment from models.dataset import Document as DatasetDocument -from services.ops_trace.ops_trace_service import OpsTraceService from services.ops_trace.trace_queue_manager import TraceQueueManager, TraceTask, TraceTaskName from services.ops_trace.utils import measure_time @@ -41,6 +40,9 @@ class DatasetRetrieval: + def __init__(self, application_generate_entity=None): + self.application_generate_entity = application_generate_entity + def retrieve( self, app_id: str, user_id: str, tenant_id: str, model_config: ModelConfigWithCredentialsEntity, @@ -355,9 +357,8 @@ def _on_retrival_end( db.session.commit() # get tracing instance - tracing_instance = OpsTraceService.get_ops_trace_instance( - message_id=message_id - ) + tracing_instance = self.application_generate_entity.tracing_instance if self.application_generate_entity else None + if tracing_instance: trace_manager = TraceQueueManager() trace_manager.add_trace_task( diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py index bf96461cc1447e..34dea1e82b510f 100644 --- a/api/core/tools/tool_engine.py +++ b/api/core/tools/tool_engine.py @@ -126,7 +126,6 @@ def workflow_invoke(tool: Tool, tool_parameters: dict, user_id: str, workflow_id: str, workflow_tool_callback: DifyWorkflowCallbackHandler, workflow_call_depth: int, - tracing_instance: Optional[BaseTraceInstance] = None ) -> list[ToolInvokeMessage]: """ Workflow invokes the tool with the given arguments. @@ -148,7 +147,6 @@ def workflow_invoke(tool: Tool, tool_parameters: dict, tool_name=tool.identity.name, tool_inputs=tool_parameters, tool_outputs=response, - tracing_instance=tracing_instance, ) return response diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 45a2e812197f06..2a472fc8d2cbf4 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -13,9 +13,7 @@ from core.workflow.nodes.base_node import BaseNode from core.workflow.nodes.tool.entities import ToolNodeData from core.workflow.utils.variable_template_parser import VariableTemplateParser -from extensions.ext_database import db -from models.workflow import Workflow, WorkflowNodeExecutionStatus -from services.ops_trace.ops_trace_service import OpsTraceService +from models.workflow import WorkflowNodeExecutionStatus class ToolNode(BaseNode): @@ -56,11 +54,6 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: # get parameters parameters = self._generate_parameters(variable_pool, node_data, tool_runtime) - # get tracing instance - workflow: Workflow = db.session.query(Workflow).filter(Workflow.id == self.workflow_id).first() - app_id = workflow.app_id - tracing_instance = OpsTraceService.get_ops_trace_instance(app_id=app_id) - try: messages = ToolEngine.workflow_invoke( tool=tool_runtime, @@ -69,7 +62,6 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: workflow_id=self.workflow_id, workflow_tool_callback=DifyWorkflowCallbackHandler(), workflow_call_depth=self.workflow_call_depth, - tracing_instance=tracing_instance ) except Exception as e: return NodeRunResult( diff --git a/api/services/message_service.py b/api/services/message_service.py index b5a32c1b770bff..4922b775319cd9 100644 --- a/api/services/message_service.py +++ b/api/services/message_service.py @@ -275,6 +275,7 @@ def get_suggested_questions_after_answer(cls, app_model: App, user: Optional[Uni tracing_instance = OpsTraceService.get_ops_trace_instance( message_id=message_id ) + if tracing_instance: trace_manager = TraceQueueManager() trace_manager.add_trace_task( diff --git a/api/services/ops_trace/langfuse_trace.py b/api/services/ops_trace/langfuse_trace.py index ec859a698029b8..3fbda83973f72d 100644 --- a/api/services/ops_trace/langfuse_trace.py +++ b/api/services/ops_trace/langfuse_trace.py @@ -341,7 +341,7 @@ def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): input=query, output=workflow_run_outputs, metadata=metadata, - session_id=conversion_id, + session_id=conversion_id if conversion_id else workflow_run_id, tags=["workflow"], ) @@ -351,7 +351,7 @@ def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): workflow_nodes_executions = ( db.session.query(WorkflowNodeExecution) .filter(WorkflowNodeExecution.workflow_run_id == workflow_run_id) - .order_by(WorkflowNodeExecution.created_at) + .order_by(WorkflowNodeExecution.index.desc()) .all() ) @@ -367,7 +367,9 @@ def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): json.loads(node_execution.outputs) if node_execution.outputs else {} ) created_at = node_execution.created_at if node_execution.created_at else datetime.now() - finished_at = node_execution.finished_at if node_execution.finished_at else datetime.now() + elapsed_time = node_execution.elapsed_time + finished_at = created_at + timedelta(seconds=elapsed_time) + metadata = json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} metadata.update( { diff --git a/api/services/ops_trace/langsmith_trace.py b/api/services/ops_trace/langsmith_trace.py index bc9c86717d0a4b..1e4a9710c98a73 100644 --- a/api/services/ops_trace/langsmith_trace.py +++ b/api/services/ops_trace/langsmith_trace.py @@ -253,7 +253,7 @@ def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): workflow_nodes_executions = ( db.session.query(WorkflowNodeExecution) .filter(WorkflowNodeExecution.workflow_run_id == workflow_run_id) - .order_by(WorkflowNodeExecution.created_at) + .order_by(WorkflowNodeExecution.index.desc()) .all() ) @@ -269,7 +269,9 @@ def workflow_trace(self, workflow_run: WorkflowRun, **kwargs): json.loads(node_execution.outputs) if node_execution.outputs else {} ) created_at = node_execution.created_at if node_execution.created_at else datetime.now() - finished_at = node_execution.finished_at if node_execution.finished_at else datetime.now() + elapsed_time = node_execution.elapsed_time + finished_at = created_at + timedelta(seconds=elapsed_time) + execution_metadata = ( json.loads(node_execution.execution_metadata) if node_execution.execution_metadata diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 4abb676efa7553..f765ef70afa724 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -43,7 +43,8 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :param tracing_config: tracing config :return: """ - if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: + if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, + TracingProviderEnum.LANGSMITH.value] and tracing_provider != "": raise {"error": f"Invalid tracing provider: {tracing_provider}"} # api check From 95c882934e79b463ac221cc54f63d0817802277c Mon Sep 17 00:00:00 2001 From: "Pan, Wen-Ming" <smalljelly2018@gmail.com> Date: Fri, 21 Jun 2024 16:45:56 +0800 Subject: [PATCH 254/273] feat: add support for Vertex AI claude-3-5-sonnet@20240620 (#5475) Co-authored-by: Wenming Pan <pwm@google.com> --- .../llm/anthropic.claude-3.5-sonnet.yaml | 55 +++++++++++++++++++ .../model_providers/vertex_ai/llm/llm.py | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 api/core/model_runtime/model_providers/vertex_ai/llm/anthropic.claude-3.5-sonnet.yaml diff --git a/api/core/model_runtime/model_providers/vertex_ai/llm/anthropic.claude-3.5-sonnet.yaml b/api/core/model_runtime/model_providers/vertex_ai/llm/anthropic.claude-3.5-sonnet.yaml new file mode 100644 index 00000000000000..c64384e6a2f153 --- /dev/null +++ b/api/core/model_runtime/model_providers/vertex_ai/llm/anthropic.claude-3.5-sonnet.yaml @@ -0,0 +1,55 @@ +model: claude-3-5-sonnet@20240620 +label: + en_US: Claude 3.5 Sonnet +model_type: llm +features: + - agent-thought + - vision +model_properties: + mode: chat + context_size: 200000 +parameter_rules: + - name: max_tokens + use_template: max_tokens + required: true + type: int + default: 4096 + min: 1 + max: 4096 + help: + zh_Hans: 停止前生成的最大令牌数。请注意，Anthropic Claude 模型可能会在达到 max_tokens 的值之前停止生成令牌。不同的 Anthropic Claude 模型对此参数具有不同的最大值。 + en_US: The maximum number of tokens to generate before stopping. Note that Anthropic Claude models might stop generating tokens before reaching the value of max_tokens. Different Anthropic Claude models have different maximum values for this parameter. + - name: temperature + use_template: temperature + required: false + type: float + default: 1 + min: 0.0 + max: 1.0 + help: + zh_Hans: 生成内容的随机性。 + en_US: The amount of randomness injected into the response. + - name: top_p + required: false + type: float + default: 0.999 + min: 0.000 + max: 1.000 + help: + zh_Hans: 在核采样中，Anthropic Claude 按概率递减顺序计算每个后续标记的所有选项的累积分布，并在达到 top_p 指定的特定概率时将其切断。您应该更改温度或top_p，但不能同时更改两者。 + en_US: In nucleus sampling, Anthropic Claude computes the cumulative distribution over all the options for each subsequent token in decreasing probability order and cuts it off once it reaches a particular probability specified by top_p. You should alter either temperature or top_p, but not both. + - name: top_k + required: false + type: int + default: 0 + min: 0 + # tip docs from aws has error, max value is 500 + max: 500 + help: + zh_Hans: 对于每个后续标记，仅从前 K 个选项中进行采样。使用 top_k 删除长尾低概率响应。 + en_US: Only sample from the top K options for each subsequent token. Use top_k to remove long tail low probability responses. +pricing: + input: '0.003' + output: '0.015' + unit: '0.001' + currency: USD diff --git a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py index 6b2258f873789d..804c3535fbdb7e 100644 --- a/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py +++ b/api/core/model_runtime/model_providers/vertex_ai/llm/llm.py @@ -100,7 +100,7 @@ def _generate_anthropic(self, model: str, credentials: dict, prompt_messages: li token = credentials.token # Vertex AI Anthropic Claude3 Opus model available in us-east5 region, Sonnet and Haiku available in us-central1 region - if 'opus' in model: + if 'opus' or 'claude-3-5-sonnet' in model: location = 'us-east5' else: location = 'us-central1' From 91d38a535f9a4254cf3abba3fd00a9c7641aeb2e Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:49:33 +0800 Subject: [PATCH 255/273] fix: max_tokens of qwen-plus & qwen-plus-chat (#5480) --- .../model_providers/tongyi/llm/qwen-plus-chat.yaml | 4 ++-- .../model_runtime/model_providers/tongyi/llm/qwen-plus.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus-chat.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus-chat.yaml index ae3ec0fc040a2f..5681f5c7b06665 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus-chat.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus-chat.yaml @@ -22,9 +22,9 @@ parameter_rules: - name: max_tokens use_template: max_tokens type: int - default: 1500 + default: 2000 min: 1 - max: 1500 + max: 2000 help: zh_Hans: 用于指定模型在生成内容时token的最大数量，它定义了生成的上限，但不保证每次都会生成到这个数量。 en_US: It is used to specify the maximum number of tokens when the model generates content. It defines the upper limit of generation, but does not guarantee that this number will be generated every time. diff --git a/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml b/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml index bfa04792a0c642..71dabb55f07fd8 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml +++ b/api/core/model_runtime/model_providers/tongyi/llm/qwen-plus.yaml @@ -20,9 +20,9 @@ parameter_rules: - name: max_tokens use_template: max_tokens type: int - default: 1500 + default: 2000 min: 1 - max: 1500 + max: 2000 help: zh_Hans: 用于指定模型在生成内容时token的最大数量，它定义了生成的上限，但不保证每次都会生成到这个数量。 en_US: It is used to specify the maximum number of tokens when the model generates content. It defines the upper limit of generation, but does not guarantee that this number will be generated every time. From 455d75eb744109a4d17bd4054a3f9ce0c4ae6307 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Mon, 17 Jun 2024 10:33:57 +0800 Subject: [PATCH 256/273] feat: add remove tracing app --- api/services/ops_trace/ops_trace_service.py | 176 ++++++-------------- 1 file changed, 51 insertions(+), 125 deletions(-) diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 1f98a3186fa8d7..0b604367d4c907 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -1,20 +1,21 @@ import json from enum import Enum -from typing import Union -from uuid import UUID +from typing import Optional from pydantic import BaseModel +from core.app.app_config.entities import AppAdditionalFeatures from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from extensions.ext_database import db -from models.model import App, AppModelConfig, Conversation, Message, TraceAppConfig +from models.model import App, AppModelConfig, Conversation, Message, TracingAppConfig +from models.workflow import Workflow from services.ops_trace.langfuse_trace import LangFuseDataTrace from services.ops_trace.langsmith_trace import LangSmithDataTrace class TracingProviderEnum(Enum): LANGFUSE = 'langfuse' - LANGSMITH = 'langsmith' + LANGSMITH = 'langSmith' class LangfuseConfig(BaseModel): @@ -26,7 +27,7 @@ class LangfuseConfig(BaseModel): host: str -class LangSmithConfig(BaseModel): +class LangsmithConfig(BaseModel): """ Model class for Langsmith tracing config. """ @@ -44,8 +45,8 @@ def get_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( - TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: @@ -69,17 +70,9 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :param tracing_config: tracing config :return: """ - if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, - TracingProviderEnum.LANGSMITH.value] and tracing_provider != "": - raise {"error": f"Invalid tracing provider: {tracing_provider}"} - - # api check - if not cls.check_trace_config_is_effective(tracing_config, tracing_provider): - return {"error": "Tracing config is not effective"} - # check if trace config already exists - trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( - TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider ).first() if trace_config_data: @@ -88,7 +81,7 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c # get tenant id tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) - trace_config_data = TraceAppConfig( + trace_config_data = TracingAppConfig( app_id=app_id, tracing_provider=tracing_provider, tracing_config=tracing_config, @@ -107,16 +100,9 @@ def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :param tracing_config: tracing config :return: """ - if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: - raise ValueError(f"Invalid tracing provider: {tracing_provider}") - - # api check - if not cls.check_trace_config_is_effective(tracing_config, tracing_provider): - raise ValueError("Invalid Credentials") - # check if trace config already exists - trace_config = db.session.query(TraceAppConfig).filter( - TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -139,8 +125,8 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config = db.session.query(TraceAppConfig).filter( - TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -150,7 +136,7 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): db.session.commit() return True - + @classmethod def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): """ @@ -170,9 +156,9 @@ def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c host=tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangSmithConfig(**tracing_config) + tracing_config = LangsmithConfig(**tracing_config) encrypt_api_key = encrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangSmithConfig( + tracing_config = LangsmithConfig( api_key=encrypt_api_key, project=tracing_config.project, endpoint=tracing_config.endpoint @@ -201,9 +187,9 @@ def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c host=tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangSmithConfig(**tracing_config) + tracing_config = LangsmithConfig(**tracing_config) decrypt_api_key = decrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangSmithConfig( + tracing_config = LangsmithConfig( api_key=decrypt_api_key, project=tracing_config.project, endpoint=tracing_config.endpoint @@ -233,10 +219,10 @@ def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config: host=decrypt_tracing_config.host ) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - decrypt_tracing_config = LangSmithConfig(**decrypt_tracing_config) + decrypt_tracing_config = LangsmithConfig(**decrypt_tracing_config) decrypt_api_key = decrypt_tracing_config.api_key obfuscated_api_key = obfuscated_token(decrypt_api_key) - decrypt_tracing_config = LangSmithConfig( + decrypt_tracing_config = LangsmithConfig( api_key=obfuscated_api_key, project=decrypt_tracing_config.project, endpoint=decrypt_tracing_config.endpoint @@ -252,8 +238,8 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( - TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider + trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( + TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: @@ -270,35 +256,40 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): @classmethod def get_ops_trace_instance( cls, - app_id: Union[UUID, str] = None, - message_id: str = None, - conversation_id: str = None + app_id: str, + workflow: Optional[Workflow] = None, + app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None ): """ Get ops trace through model config :param app_id: app_id - :param message_id: message_id - :param conversation_id: conversation_id + :param workflow: workflow + :param app_model_config: app_model_config :return: """ - if conversation_id: - conversation_data: Conversation = db.session.query(Conversation).filter( - Conversation.id == conversation_id - ).first() - app_id = conversation_data.app_id - - if message_id: - record: Message = db.session.query(Message).filter(Message.id == message_id).first() - app_id = record.app_id - - if isinstance(app_id, UUID): - app_id = str(app_id) - tracing_instance = None - app: App = db.session.query(App).filter( - App.id == app_id - ).first() - app_ops_trace_config = json.loads(app.tracing) if app.tracing else None + app_ops_trace_config = None + + # get trace configuration from available sources + if app_model_config is not None: + if isinstance(app_model_config, AppAdditionalFeatures): + app_ops_trace_config = app_model_config.trace_config + elif isinstance(app_model_config, AppModelConfig): + app_ops_trace_config = json.loads( + app_model_config.trace_config + ) if app_model_config.trace_config else None + elif workflow: + features_data = json.loads(workflow.features) + app_ops_trace_config = features_data.get('trace_config') if features_data else None + else: + # As a last resort, fetch from the database + trace_config_data = db.session.query(AppModelConfig.trace_config).filter( + AppModelConfig.app_id == app_id + ).order_by(AppModelConfig.updated_at.desc()).first() + if trace_config_data: + app_ops_trace_config = json.loads(trace_config_data.trace_config) + else: + raise ValueError('Trace config not found') if app_ops_trace_config is not None: tracing_provider = app_ops_trace_config.get('tracing_provider') @@ -347,68 +338,3 @@ def get_app_config_through_message_id(cls, message_id: str): app_model_config = conversation_data.override_model_configs return app_model_config - - @classmethod - def update_app_tracing_config(cls, app_id: str, enabled: bool, tracing_provider: str): - """ - Update app tracing config - :param app_id: app id - :param enabled: enabled - :param tracing_provider: tracing provider - :return: - """ - # auth check - if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: - raise ValueError(f"Invalid tracing provider: {tracing_provider}") - app_config: App = db.session.query(App).filter(App.id == app_id).first() - - app_config.tracing = json.dumps( - { - "enabled": enabled, - "tracing_provider": tracing_provider, - } - ) - db.session.commit() - - @classmethod - def get_app_tracing_config(cls, app_id: str): - """ - Get app tracing config - :param app_id: app id - :return: - """ - app: App = db.session.query(App).filter(App.id == app_id).first() - if not app.tracing: - return { - "enabled": False, - "tracing_provider": None - } - app_trace_config = json.loads(app.tracing) - return app_trace_config - - @staticmethod - def check_trace_config_is_effective(tracing_config: dict, tracing_provider: str): - """ - Check trace config is effective - :param tracing_config: tracing config - :param tracing_provider: tracing provider - :return: - """ - if tracing_provider == TracingProviderEnum.LANGFUSE.value: - tracing_config = LangfuseConfig(**tracing_config) - langfuse_trace_instance = LangFuseDataTrace( - tracing_config.public_key, - tracing_config.secret_key, - tracing_config.host, - ) - return langfuse_trace_instance.api_check() - elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangSmithConfig(**tracing_config) - langsmith_trace_instance = LangSmithDataTrace( - tracing_config.api_key, - tracing_config.project, - tracing_config.endpoint, - ) - return langsmith_trace_instance.api_check() - else: - raise ValueError(f"Unsupported tracing provider: {tracing_provider}") From a589edc967b4f16328922ee775a89c93ae3a4500 Mon Sep 17 00:00:00 2001 From: Joe <1264204425@qq.com> Date: Thu, 20 Jun 2024 14:31:37 +0800 Subject: [PATCH 257/273] feat: add ops trace encrypt_config decrypt_config obfuscate_config --- api/controllers/console/app/app.py | 13 +- api/services/ops_trace/langfuse_trace.py | 30 ++- api/services/ops_trace/langsmith_trace.py | 25 +- api/services/ops_trace/ops_trace_service.py | 254 ++++++++++---------- 4 files changed, 183 insertions(+), 139 deletions(-) diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index a9e8b5fb4a2d33..2f49222450490f 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -306,14 +306,11 @@ def post(self, app_id): parser.add_argument('tracing_provider', type=str, required=True, location='json') args = parser.parse_args() - try: - OpsTraceService.update_app_tracing_config( - app_id=app_id, - enabled=args['enabled'], - tracing_provider=args['tracing_provider'], - ) - except Exception as e: - raise e + OpsTraceService.update_app_tracing_config( + app_id=app_id, + enabled=args['enabled'], + tracing_provider=args['tracing_provider'], + ) return {"result": "success"} diff --git a/api/services/ops_trace/langfuse_trace.py b/api/services/ops_trace/langfuse_trace.py index 740e3829b52a11..3fbda83973f72d 100644 --- a/api/services/ops_trace/langfuse_trace.py +++ b/api/services/ops_trace/langfuse_trace.py @@ -1,4 +1,5 @@ import json +import logging import os from datetime import datetime, timedelta from enum import Enum @@ -8,14 +9,17 @@ from pydantic import BaseModel, Field, field_validator from pydantic_core.core_schema import ValidationInfo +from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from core.moderation.base import ModerationInputsResult from extensions.ext_database import db from models.dataset import Document from models.model import Message, MessageAgentThought, MessageFile from models.workflow import WorkflowNodeExecution, WorkflowRun from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.model import LangfuseConfig from services.ops_trace.utils import filter_none_values, replace_text_with_content +logger = logging.getLogger(__name__) def validate_input_output(v, field_name): """ @@ -693,7 +697,7 @@ def add_trace(self, langfuse_trace_data: Optional[LangfuseTrace] = None): ) try: self.langfuse_client.trace(**format_trace_data) - print("LangFuse Trace created successfully") + logger.debug("LangFuse Trace created successfully") except Exception as e: raise f"LangFuse Failed to create trace: {str(e)}" @@ -703,7 +707,7 @@ def add_span(self, langfuse_span_data: Optional[LangfuseSpan] = None): ) try: self.langfuse_client.span(**format_span_data) - print("LangFuse Span created successfully") + logger.debug("LangFuse Span created successfully") except Exception as e: raise f"LangFuse Failed to create span: {str(e)}" @@ -724,7 +728,7 @@ def add_generation( ) try: self.langfuse_client.generation(**format_generation_data) - print("LangFuse Generation created successfully") + logger.debug("LangFuse Generation created successfully") except Exception as e: raise f"LangFuse Failed to create generation: {str(e)}" @@ -743,5 +747,23 @@ def api_check(self): try: return self.langfuse_client.auth_check() except Exception as e: - print(f"LangFuse API check failed: {str(e)}") + logger.debug(f"LangFuse API check failed: {str(e)}") return False + + @classmethod + def obfuscate_config(cls, config: LangfuseConfig): + public_key = obfuscated_token(config.public_key) + secret_key = obfuscated_token(config.secret_key) + return LangfuseConfig(public_key=public_key, secret_key=secret_key, host=config.host) + + @classmethod + def encrypt_config(cls, tenant_id, config: LangfuseConfig): + decrypt_public_key = encrypt_token(tenant_id, config.public_key) + decrypt_secret_key = encrypt_token(tenant_id, config.secret_key) + return LangfuseConfig(public_key=decrypt_public_key, secret_key=decrypt_secret_key, host=config.host) + + @classmethod + def decrypt_config(cls, tenant_id, config: LangfuseConfig): + decrypt_public_key = decrypt_token(tenant_id, config.public_key) + decrypt_secret_key = decrypt_token(tenant_id, config.secret_key) + return LangfuseConfig(public_key=decrypt_public_key, secret_key=decrypt_secret_key, host=config.host) diff --git a/api/services/ops_trace/langsmith_trace.py b/api/services/ops_trace/langsmith_trace.py index 58da34666de78f..1e4a9710c98a73 100644 --- a/api/services/ops_trace/langsmith_trace.py +++ b/api/services/ops_trace/langsmith_trace.py @@ -1,4 +1,5 @@ import json +import logging import os from datetime import datetime, timedelta from enum import Enum @@ -8,14 +9,17 @@ from pydantic import BaseModel, Field, field_validator from pydantic_core.core_schema import ValidationInfo +from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from core.moderation.base import ModerationInputsResult from extensions.ext_database import db from models.dataset import Document from models.model import Message, MessageAgentThought, MessageFile from models.workflow import WorkflowNodeExecution, WorkflowRun from services.ops_trace.base_trace_instance import BaseTraceInstance +from services.ops_trace.model import LangSmithConfig from services.ops_trace.utils import filter_none_values, replace_text_with_content +logger = logging.getLogger(__name__) class LangSmithRunType(str, Enum): tool = "tool" @@ -585,7 +589,7 @@ def add_run(self, run_data: LangSmithRunModel): data = filter_none_values(data) try: self.langsmith_client.create_run(**data) - print("LangSmith Run created successfully.") + logger.debug("LangSmith Run created successfully.") except Exception as e: raise f"LangSmith Failed to create run: {str(e)}" @@ -594,7 +598,7 @@ def update_run(self, update_run_data: LangSmithRunUpdateModel): data = filter_none_values(data) try: self.langsmith_client.update_run(**data) - print("LangSmith Run updated successfully.") + logger.debug("LangSmith Run updated successfully.") except Exception as e: raise f"LangSmith Failed to update run: {str(e)}" @@ -605,5 +609,20 @@ def api_check(self): self.langsmith_client.delete_project(project_name=random_project_name) return True except Exception as e: - print(f"LangSmith API check failed: {str(e)}") + logger.debug(f"LangSmith API check failed: {str(e)}") return False + + @classmethod + def obfuscate_config(cls, config: LangSmithConfig): + api_key = obfuscated_token(config.api_key) + return LangSmithConfig(api_key=api_key, project=config.project, endpoint=config.endpoint) + + @classmethod + def encrypt_config(cls, tenant_id, config: LangSmithConfig): + api_key = encrypt_token(tenant_id, config.api_key) + return LangSmithConfig(api_key=api_key, project=config.project, endpoint=config.endpoint) + + @classmethod + def decrypt_config(cls, tenant_id, config: LangSmithConfig): + api_key = decrypt_token(tenant_id, config.api_key) + return LangSmithConfig(api_key=api_key, project=config.project, endpoint=config.endpoint) diff --git a/api/services/ops_trace/ops_trace_service.py b/api/services/ops_trace/ops_trace_service.py index 0b604367d4c907..4abb676efa7553 100644 --- a/api/services/ops_trace/ops_trace_service.py +++ b/api/services/ops_trace/ops_trace_service.py @@ -1,39 +1,12 @@ import json -from enum import Enum -from typing import Optional +from typing import Union +from uuid import UUID -from pydantic import BaseModel - -from core.app.app_config.entities import AppAdditionalFeatures -from core.helper.encrypter import decrypt_token, encrypt_token, obfuscated_token from extensions.ext_database import db -from models.model import App, AppModelConfig, Conversation, Message, TracingAppConfig -from models.workflow import Workflow +from models.model import App, AppModelConfig, Conversation, Message, TraceAppConfig from services.ops_trace.langfuse_trace import LangFuseDataTrace from services.ops_trace.langsmith_trace import LangSmithDataTrace - - -class TracingProviderEnum(Enum): - LANGFUSE = 'langfuse' - LANGSMITH = 'langSmith' - - -class LangfuseConfig(BaseModel): - """ - Model class for Langfuse tracing config. - """ - public_key: str - secret_key: str - host: str - - -class LangsmithConfig(BaseModel): - """ - Model class for Langsmith tracing config. - """ - api_key: str - project: str - endpoint: str +from services.ops_trace.model import LangfuseConfig, LangSmithConfig, TracingProviderEnum class OpsTraceService: @@ -45,8 +18,8 @@ def get_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: @@ -70,9 +43,16 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :param tracing_config: tracing config :return: """ + if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: + raise {"error": f"Invalid tracing provider: {tracing_provider}"} + + # api check + if not cls.check_trace_config_is_effective(tracing_config, tracing_provider): + return {"error": "Tracing config is not effective"} + # check if trace config already exists - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if trace_config_data: @@ -81,7 +61,7 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c # get tenant id tenant_id = db.session.query(App).filter(App.id == app_id).first().tenant_id tracing_config = cls.encrypt_tracing_config(tenant_id, tracing_provider, tracing_config) - trace_config_data = TracingAppConfig( + trace_config_data = TraceAppConfig( app_id=app_id, tracing_provider=tracing_provider, tracing_config=tracing_config, @@ -100,9 +80,16 @@ def update_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c :param tracing_config: tracing config :return: """ + if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: + raise ValueError(f"Invalid tracing provider: {tracing_provider}") + + # api check + if not cls.check_trace_config_is_effective(tracing_config, tracing_provider): + raise ValueError("Invalid Credentials") + # check if trace config already exists - trace_config = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -125,8 +112,8 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config: @@ -136,7 +123,7 @@ def delete_tracing_app_config(cls, app_id: str, tracing_provider: str): db.session.commit() return True - + @classmethod def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): """ @@ -148,25 +135,12 @@ def encrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c """ if tracing_provider == TracingProviderEnum.LANGFUSE.value: tracing_config = LangfuseConfig(**tracing_config) - encrypt_public_key = encrypt_token(tenant_id, tracing_config.public_key) - encrypt_secret_key = encrypt_token(tenant_id, tracing_config.secret_key) - tracing_config = LangfuseConfig( - public_key=encrypt_public_key, - secret_key=encrypt_secret_key, - host=tracing_config.host - ) + tracing_config = LangFuseDataTrace.encrypt_config(tenant_id, tracing_config) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangsmithConfig(**tracing_config) - encrypt_api_key = encrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangsmithConfig( - api_key=encrypt_api_key, - project=tracing_config.project, - endpoint=tracing_config.endpoint - ) + tracing_config = LangSmithConfig(**tracing_config) + tracing_config = LangSmithDataTrace.encrypt_config(tenant_id, tracing_config) - if isinstance(tracing_config, BaseModel): - return tracing_config.dict() - return tracing_config + return tracing_config.model_dump() @classmethod def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_config: dict): @@ -179,25 +153,12 @@ def decrypt_tracing_config(cls, tenant_id: str, tracing_provider: str, tracing_c """ if tracing_provider == TracingProviderEnum.LANGFUSE.value: tracing_config = LangfuseConfig(**tracing_config) - decrypt_public_key = decrypt_token(tenant_id, tracing_config.public_key) - decrypt_secret_key = decrypt_token(tenant_id, tracing_config.secret_key) - tracing_config = LangfuseConfig( - public_key=decrypt_public_key, - secret_key=decrypt_secret_key, - host=tracing_config.host - ) + tracing_config = LangFuseDataTrace.decrypt_config(tenant_id, tracing_config) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - tracing_config = LangsmithConfig(**tracing_config) - decrypt_api_key = decrypt_token(tenant_id, tracing_config.api_key) - tracing_config = LangsmithConfig( - api_key=decrypt_api_key, - project=tracing_config.project, - endpoint=tracing_config.endpoint - ) + tracing_config = LangSmithConfig(**tracing_config) + tracing_config = LangSmithDataTrace.decrypt_config(tenant_id, tracing_config) - if isinstance(tracing_config, BaseModel): - return tracing_config.dict() - return tracing_config + return tracing_config.model_dump() @classmethod def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config:dict): @@ -207,28 +168,14 @@ def obfuscated_decrypt_token(cls, tracing_provider: str, decrypt_tracing_config: :param decrypt_tracing_config: tracing config :return: """ + obfuscate_config = None if tracing_provider == TracingProviderEnum.LANGFUSE.value: decrypt_tracing_config = LangfuseConfig(**decrypt_tracing_config) - decrypt_public_key = decrypt_tracing_config.public_key - decrypt_secret_key = decrypt_tracing_config.secret_key - obfuscated_public_key = obfuscated_token(decrypt_public_key) - obfuscated_secret_key = obfuscated_token(decrypt_secret_key) - decrypt_tracing_config = LangfuseConfig( - public_key=obfuscated_public_key, - secret_key=obfuscated_secret_key, - host=decrypt_tracing_config.host - ) + obfuscate_config = LangFuseDataTrace.obfuscate_config(decrypt_tracing_config) elif tracing_provider == TracingProviderEnum.LANGSMITH.value: - decrypt_tracing_config = LangsmithConfig(**decrypt_tracing_config) - decrypt_api_key = decrypt_tracing_config.api_key - obfuscated_api_key = obfuscated_token(decrypt_api_key) - decrypt_tracing_config = LangsmithConfig( - api_key=obfuscated_api_key, - project=decrypt_tracing_config.project, - endpoint=decrypt_tracing_config.endpoint - ) - - return decrypt_tracing_config.dict() + decrypt_tracing_config = LangSmithConfig(**decrypt_tracing_config) + obfuscate_config = LangSmithDataTrace.obfuscate_config(decrypt_tracing_config) + return obfuscate_config.model_dump() @classmethod def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): @@ -238,8 +185,8 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): :param tracing_provider: tracing provider :return: """ - trace_config_data: TracingAppConfig = db.session.query(TracingAppConfig).filter( - TracingAppConfig.app_id == app_id, TracingAppConfig.tracing_provider == tracing_provider + trace_config_data: TraceAppConfig = db.session.query(TraceAppConfig).filter( + TraceAppConfig.app_id == app_id, TraceAppConfig.tracing_provider == tracing_provider ).first() if not trace_config_data: @@ -256,40 +203,35 @@ def get_decrypted_tracing_config(cls, app_id: str, tracing_provider: str): @classmethod def get_ops_trace_instance( cls, - app_id: str, - workflow: Optional[Workflow] = None, - app_model_config: Optional[AppModelConfig | AppAdditionalFeatures] = None + app_id: Union[UUID, str] = None, + message_id: str = None, + conversation_id: str = None ): """ Get ops trace through model config :param app_id: app_id - :param workflow: workflow - :param app_model_config: app_model_config + :param message_id: message_id + :param conversation_id: conversation_id :return: """ + if conversation_id: + conversation_data: Conversation = db.session.query(Conversation).filter( + Conversation.id == conversation_id + ).first() + app_id = conversation_data.app_id + + if message_id: + record: Message = db.session.query(Message).filter(Message.id == message_id).first() + app_id = record.app_id + + if isinstance(app_id, UUID): + app_id = str(app_id) + tracing_instance = None - app_ops_trace_config = None - - # get trace configuration from available sources - if app_model_config is not None: - if isinstance(app_model_config, AppAdditionalFeatures): - app_ops_trace_config = app_model_config.trace_config - elif isinstance(app_model_config, AppModelConfig): - app_ops_trace_config = json.loads( - app_model_config.trace_config - ) if app_model_config.trace_config else None - elif workflow: - features_data = json.loads(workflow.features) - app_ops_trace_config = features_data.get('trace_config') if features_data else None - else: - # As a last resort, fetch from the database - trace_config_data = db.session.query(AppModelConfig.trace_config).filter( - AppModelConfig.app_id == app_id - ).order_by(AppModelConfig.updated_at.desc()).first() - if trace_config_data: - app_ops_trace_config = json.loads(trace_config_data.trace_config) - else: - raise ValueError('Trace config not found') + app: App = db.session.query(App).filter( + App.id == app_id + ).first() + app_ops_trace_config = json.loads(app.tracing) if app.tracing else None if app_ops_trace_config is not None: tracing_provider = app_ops_trace_config.get('tracing_provider') @@ -312,7 +254,6 @@ def get_ops_trace_instance( langsmith_api_key = decrypt_trace_config.get('api_key') langsmith_project = decrypt_trace_config.get('project') langsmith_endpoint = decrypt_trace_config.get('endpoint') - print(langsmith_api_key, langsmith_project, langsmith_endpoint) tracing_instance = LangSmithDataTrace( langsmith_api_key, langsmith_project, @@ -338,3 +279,68 @@ def get_app_config_through_message_id(cls, message_id: str): app_model_config = conversation_data.override_model_configs return app_model_config + + @classmethod + def update_app_tracing_config(cls, app_id: str, enabled: bool, tracing_provider: str): + """ + Update app tracing config + :param app_id: app id + :param enabled: enabled + :param tracing_provider: tracing provider + :return: + """ + # auth check + if tracing_provider not in [TracingProviderEnum.LANGFUSE.value, TracingProviderEnum.LANGSMITH.value]: + raise ValueError(f"Invalid tracing provider: {tracing_provider}") + app_config: App = db.session.query(App).filter(App.id == app_id).first() + + app_config.tracing = json.dumps( + { + "enabled": enabled, + "tracing_provider": tracing_provider, + } + ) + db.session.commit() + + @classmethod + def get_app_tracing_config(cls, app_id: str): + """ + Get app tracing config + :param app_id: app id + :return: + """ + app: App = db.session.query(App).filter(App.id == app_id).first() + if not app.tracing: + return { + "enabled": False, + "tracing_provider": None + } + app_trace_config = json.loads(app.tracing) + return app_trace_config + + @staticmethod + def check_trace_config_is_effective(tracing_config: dict, tracing_provider: str): + """ + Check trace config is effective + :param tracing_config: tracing config + :param tracing_provider: tracing provider + :return: + """ + if tracing_provider == TracingProviderEnum.LANGFUSE.value: + tracing_config = LangfuseConfig(**tracing_config) + langfuse_trace_instance = LangFuseDataTrace( + tracing_config.public_key, + tracing_config.secret_key, + tracing_config.host, + ) + return langfuse_trace_instance.api_check() + elif tracing_provider == TracingProviderEnum.LANGSMITH.value: + tracing_config = LangSmithConfig(**tracing_config) + langsmith_trace_instance = LangSmithDataTrace( + tracing_config.api_key, + tracing_config.project, + tracing_config.endpoint, + ) + return langsmith_trace_instance.api_check() + else: + raise ValueError(f"Unsupported tracing provider: {tracing_provider}") From 27f0ae84167b91b7ea818fe6dc518b997b203626 Mon Sep 17 00:00:00 2001 From: Bowen Liang <liangbowen@gf.com.cn> Date: Sat, 22 Jun 2024 01:34:08 +0800 Subject: [PATCH 258/273] build: support Poetry for depencencies tool in api's Dockerfile (#5105) Co-authored-by: takatost <takatost@gmail.com> --- .github/workflows/api-tests.yml | 2 +- .github/workflows/build-push.yml | 113 ++++++++++++++++++++++++++----- .github/workflows/style.yml | 5 +- api/.dockerignore | 5 +- api/Dockerfile | 33 ++++++--- 5 files changed, 128 insertions(+), 30 deletions(-) diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 4af4daadeba3a8..612d698e3bc6a8 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -108,7 +108,7 @@ jobs: - name: Poetry check run: | - poetry check -C api + poetry check -C api --lock poetry show -C api - name: Install dependencies diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 89d301c4fd8a6e..2678f23a770acd 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -8,6 +8,10 @@ on: release: types: [published] +concurrency: + group: build-push-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + env: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -15,19 +19,35 @@ env: DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }} jobs: - build-and-push: - runs-on: ubuntu-latest + build: + runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }} if: github.repository == 'langgenius/dify' strategy: matrix: include: - - service_name: "web" - image_name_env: "DIFY_WEB_IMAGE_NAME" - context: "web" - - service_name: "api" + - service_name: "build-api-amd64" + image_name_env: "DIFY_API_IMAGE_NAME" + context: "api" + platform: linux/amd64 + - service_name: "build-api-arm64" image_name_env: "DIFY_API_IMAGE_NAME" context: "api" + platform: linux/arm64 + - service_name: "build-web-amd64" + image_name_env: "DIFY_WEB_IMAGE_NAME" + context: "web" + platform: linux/amd64 + - service_name: "build-web-arm64" + image_name_env: "DIFY_WEB_IMAGE_NAME" + context: "web" + platform: linux/arm64 + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -40,7 +60,66 @@ jobs: username: ${{ env.DOCKERHUB_USER }} password: ${{ env.DOCKERHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env[matrix.image_name_env] }} + + - name: Build Docker image + id: build + uses: docker/build-push-action@v6 + with: + context: "{{defaultContext}}:${{ matrix.context }}" + platforms: ${{ matrix.platform }} + build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=${{ matrix.service_name }} + cache-to: type=gha,mode=max,scope=${{ matrix.service_name }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + create-manifest: + needs: build + runs-on: ubuntu-latest + if: github.repository == 'langgenius/dify' + strategy: + matrix: + include: + - service_name: "merge-api-images" + image_name_env: "DIFY_API_IMAGE_NAME" + context: "api" + - service_name: "merge-web-images" + image_name_env: "DIFY_WEB_IMAGE_NAME" + context: "web" + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-${{ matrix.context }}-* + merge-multiple: true + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ env.DOCKERHUB_USER }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Extract metadata for Docker id: meta uses: docker/metadata-action@v5 with: @@ -51,14 +130,12 @@ jobs: type=sha,enable=true,priority=100,prefix=,suffix=,format=long type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }} - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: "{{defaultContext}}:${{ matrix.context }}" - platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env[matrix.image_name_env] }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env[matrix.image_name_env] }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index cc4302117f80f9..f6092c86337d85 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -99,7 +99,7 @@ jobs: **.sh **.yaml **.yml - Dockerfile + **Dockerfile dev/** - name: Super-linter @@ -113,7 +113,8 @@ jobs: IGNORE_GITIGNORED_FILES: true VALIDATE_BASH: true VALIDATE_BASH_EXEC: true - VALIDATE_GITHUB_ACTIONS: true + # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck + # VALIDATE_GITHUB_ACTIONS: true VALIDATE_DOCKERFILE_HADOLINT: true VALIDATE_XML: true VALIDATE_YAML: true diff --git a/api/.dockerignore b/api/.dockerignore index 0ee003b912310a..91a5254ea7e151 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -8,4 +8,7 @@ logs *.log* # jetbrains -.idea \ No newline at end of file +.idea + +# venv +.venv \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index 96b230e173b42a..15fd9d88e080f8 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,18 +1,28 @@ # base image -FROM python:3.10-slim-bookworm AS base +FROM python:3.10-slim-bookworm as base -LABEL maintainer="takatost@gmail.com" +WORKDIR /app/api + +# Install Poetry +ENV POETRY_VERSION=1.8.3 +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir --upgrade poetry==${POETRY_VERSION} + +# Configure Poetry +ENV POETRY_CACHE_DIR=/tmp/poetry_cache +ENV POETRY_NO_INTERACTION=1 +ENV POETRY_VIRTUALENVS_IN_PROJECT=true +ENV POETRY_VIRTUALENVS_CREATE=true -# install packages FROM base as packages RUN apt-get update \ && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev -COPY requirements.txt /requirements.txt +# Install Python dependencies +COPY pyproject.toml poetry.lock ./ +RUN poetry install --sync --no-cache --no-root -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --prefix=/pkg -r requirements.txt # production stage FROM base AS production @@ -37,13 +47,20 @@ RUN apt-get update \ && apt-get autoremove \ && rm -rf /var/lib/apt/lists/* -COPY --from=packages /pkg /usr/local +# Copy Python environment and packages +ENV VIRTUAL_ENV=/app/api/.venv +COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Copy source code COPY . /app/api/ +# Copy entrypoint COPY docker/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh + ARG COMMIT_SHA ENV COMMIT_SHA ${COMMIT_SHA} -ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] From 6a09409ec9d498d2a4f5706ee87263a61c0c137b Mon Sep 17 00:00:00 2001 From: tmuife <43266626@qq.com> Date: Sat, 22 Jun 2024 01:48:07 +0800 Subject: [PATCH 259/273] Add Oracle23ai as a vector datasource (#5342) Co-authored-by: walter from vm <walter.jin@oracle.com> --- .github/workflows/api-tests.yml | 4 + api/configs/middleware/__init__.py | 2 + api/configs/middleware/vdb/oracle_configs.py | 34 + api/controllers/console/datasets/datasets.py | 4 +- .../rag/datasource/vdb/oracle/__init__.py | 0 .../rag/datasource/vdb/oracle/oraclevector.py | 239 +++++++ api/core/rag/datasource/vdb/vector_factory.py | 3 + api/core/rag/datasource/vdb/vector_type.py | 1 + api/poetry.lock | 650 ++++++++++-------- api/pyproject.toml | 1 + api/requirements.txt | 3 +- .../integration_tests/vdb/oracle/__init__.py | 0 .../vdb/oracle/test_oraclevector.py | 30 + docker/docker-compose.oracle.yaml | 18 + docker/docker-compose.yaml | 31 + docker/startupscripts/create_user.sql | 5 + 16 files changed, 718 insertions(+), 307 deletions(-) create mode 100644 api/configs/middleware/vdb/oracle_configs.py create mode 100644 api/core/rag/datasource/vdb/oracle/__init__.py create mode 100644 api/core/rag/datasource/vdb/oracle/oraclevector.py create mode 100644 api/tests/integration_tests/vdb/oracle/__init__.py create mode 100644 api/tests/integration_tests/vdb/oracle/test_oraclevector.py create mode 100644 docker/docker-compose.oracle.yaml create mode 100755 docker/startupscripts/create_user.sql diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 612d698e3bc6a8..76ea795de5828b 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -68,6 +68,7 @@ jobs: docker/docker-compose.pgvecto-rs.yaml docker/docker-compose.pgvector.yaml docker/docker-compose.chroma.yaml + docker/docker-compose.oracle.yaml services: | weaviate qdrant @@ -77,6 +78,7 @@ jobs: pgvecto-rs pgvector chroma + oracle - name: Test Vector Stores run: dev/pytest/pytest_vdb.sh @@ -145,6 +147,7 @@ jobs: docker/docker-compose.pgvecto-rs.yaml docker/docker-compose.pgvector.yaml docker/docker-compose.chroma.yaml + docker/docker-compose.oracle.yaml services: | weaviate qdrant @@ -154,6 +157,7 @@ jobs: pgvecto-rs pgvector chroma + oracle - name: Test Vector Stores run: poetry run -C api bash dev/pytest/pytest_vdb.sh diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index d715dbed1fa754..7aa6e9ccd52074 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -6,6 +6,7 @@ from configs.middleware.vdb.chroma_configs import ChromaConfigs from configs.middleware.vdb.milvus_configs import MilvusConfigs from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs +from configs.middleware.vdb.oracle_configs import OracleConfigs from configs.middleware.vdb.pgvector_configs import PGVectorConfigs from configs.middleware.vdb.pgvectors_configs import PGVectoRSConfigs from configs.middleware.vdb.qdrant_configs import QdrantConfigs @@ -61,5 +62,6 @@ class MiddlewareConfigs( TencentVectorDBConfigs, TiDBVectorConfigs, WeaviateConfigs, + OracleConfigs, ): pass diff --git a/api/configs/middleware/vdb/oracle_configs.py b/api/configs/middleware/vdb/oracle_configs.py new file mode 100644 index 00000000000000..a70fd3efc2d3f9 --- /dev/null +++ b/api/configs/middleware/vdb/oracle_configs.py @@ -0,0 +1,34 @@ +from typing import Optional + +from pydantic import BaseModel, Field, PositiveInt + + +class OracleConfigs(BaseModel): + """ + ORACLE configs + """ + + ORACLE_HOST: Optional[str] = Field( + description='ORACLE host', + default=None, + ) + + ORACLE_PORT: Optional[PositiveInt] = Field( + description='ORACLE port', + default=None, + ) + + ORACLE_USER: Optional[str] = Field( + description='ORACLE user', + default=None, + ) + + ORACLE_PASSWORD: Optional[str] = Field( + description='ORACLE password', + default=None, + ) + + ORACLE_DATABASE: Optional[str] = Field( + description='ORACLE database', + default=None, + ) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 4d6aadec8163ac..619ab4f7e2d5af 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -498,7 +498,7 @@ class DatasetRetrievalSettingApi(Resource): def get(self): vector_type = current_app.config['VECTOR_STORE'] match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: return { 'retrieval_method': [ RetrievalMethod.SEMANTIC_SEARCH @@ -522,7 +522,7 @@ class DatasetRetrievalSettingMockApi(Resource): @account_initialization_required def get(self, vector_type): match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCEN: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: return { 'retrieval_method': [ RetrievalMethod.SEMANTIC_SEARCH diff --git a/api/core/rag/datasource/vdb/oracle/__init__.py b/api/core/rag/datasource/vdb/oracle/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py new file mode 100644 index 00000000000000..c087ed0cd82290 --- /dev/null +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -0,0 +1,239 @@ +import array +import json +import uuid +from contextlib import contextmanager +from typing import Any + +import numpy +import oracledb +from flask import current_app +from pydantic import BaseModel, model_validator + +from core.rag.datasource.entity.embedding import Embeddings +from core.rag.datasource.vdb.vector_base import BaseVector +from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory +from core.rag.datasource.vdb.vector_type import VectorType +from core.rag.models.document import Document +from extensions.ext_redis import redis_client +from models.dataset import Dataset + +oracledb.defaults.fetch_lobs = False + + +class OracleVectorConfig(BaseModel): + host: str + port: int + user: str + password: str + database: str + + @model_validator(mode='before') + def validate_config(cls, values: dict) -> dict: + if not values["host"]: + raise ValueError("config ORACLE_HOST is required") + if not values["port"]: + raise ValueError("config ORACLE_PORT is required") + if not values["user"]: + raise ValueError("config ORACLE_USER is required") + if not values["password"]: + raise ValueError("config ORACLE_PASSWORD is required") + if not values["database"]: + raise ValueError("config ORACLE_DB is required") + return values + + +SQL_CREATE_TABLE = """ +CREATE TABLE IF NOT EXISTS {table_name} ( + id varchar2(100) + ,text CLOB NOT NULL + ,meta JSON + ,embedding vector NOT NULL +) +""" + + +class OracleVector(BaseVector): + def __init__(self, collection_name: str, config: OracleVectorConfig): + super().__init__(collection_name) + self.pool = self._create_connection_pool(config) + self.table_name = f"embedding_{collection_name}" + + def get_type(self) -> str: + return VectorType.ORACLE + + def numpy_converter_in(self, value): + if value.dtype == numpy.float64: + dtype = "d" + elif value.dtype == numpy.float32: + dtype = "f" + else: + dtype = "b" + return array.array(dtype, value) + + def input_type_handler(self, cursor, value, arraysize): + if isinstance(value, numpy.ndarray): + return cursor.var( + oracledb.DB_TYPE_VECTOR, + arraysize=arraysize, + inconverter=self.numpy_converter_in, + ) + + def numpy_converter_out(self, value): + if value.typecode == "b": + dtype = numpy.int8 + elif value.typecode == "f": + dtype = numpy.float32 + else: + dtype = numpy.float64 + return numpy.array(value, copy=False, dtype=dtype) + + def output_type_handler(self, cursor, metadata): + if metadata.type_code is oracledb.DB_TYPE_VECTOR: + return cursor.var( + metadata.type_code, + arraysize=cursor.arraysize, + outconverter=self.numpy_converter_out, + ) + def _create_connection_pool(self, config: OracleVectorConfig): + return oracledb.create_pool(user=config.user, password=config.password, dsn="{}:{}/{}".format(config.host, config.port, config.database), min=1, max=50, increment=1) + + + @contextmanager + def _get_cursor(self): + conn = self.pool.acquire() + conn.inputtypehandler = self.input_type_handler + conn.outputtypehandler = self.output_type_handler + cur = conn.cursor() + try: + yield cur + finally: + cur.close() + conn.commit() + conn.close() + + def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): + dimension = len(embeddings[0]) + self._create_collection(dimension) + return self.add_texts(texts, embeddings) + + def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs): + values = [] + pks = [] + for i, doc in enumerate(documents): + doc_id = doc.metadata.get("doc_id", str(uuid.uuid4())) + pks.append(doc_id) + values.append( + ( + doc_id, + doc.page_content, + json.dumps(doc.metadata), + #array.array("f", embeddings[i]), + numpy.array(embeddings[i]), + ) + ) + #print(f"INSERT INTO {self.table_name} (id, text, meta, embedding) VALUES (:1, :2, :3, :4)") + with self._get_cursor() as cur: + cur.executemany(f"INSERT INTO {self.table_name} (id, text, meta, embedding) VALUES (:1, :2, :3, :4)", values) + return pks + + def text_exists(self, id: str) -> bool: + with self._get_cursor() as cur: + cur.execute(f"SELECT id FROM {self.table_name} WHERE id = '%s'" % (id,)) + return cur.fetchone() is not None + + def get_by_ids(self, ids: list[str]) -> list[Document]: + with self._get_cursor() as cur: + cur.execute(f"SELECT meta, text FROM {self.table_name} WHERE id IN %s", (tuple(ids),)) + docs = [] + for record in cur: + docs.append(Document(page_content=record[1], metadata=record[0])) + return docs + #def get_ids_by_metadata_field(self, key: str, value: str): + # with self._get_cursor() as cur: + # cur.execute(f"SELECT id FROM {self.table_name} d WHERE d.meta.{key}='{value}'" ) + # idss = [] + # for record in cur: + # idss.append(record[0]) + # return idss + + #def delete_by_document_id(self, document_id: str): + # ids = self.get_ids_by_metadata_field('doc_id', document_id) + # if len(ids)>0: + # with self._get_cursor() as cur: + # cur.execute(f"delete FROM {self.table_name} d WHERE d.meta.doc_id in '%s'" % ("','".join(ids),)) + + + def delete_by_ids(self, ids: list[str]) -> None: + with self._get_cursor() as cur: + cur.execute(f"DELETE FROM {self.table_name} WHERE id IN %s" % (tuple(ids),)) + + def delete_by_metadata_field(self, key: str, value: str) -> None: + with self._get_cursor() as cur: + cur.execute(f"DELETE FROM {self.table_name} WHERE meta->>%s = %s", (key, value)) + + def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: + """ + Search the nearest neighbors to a vector. + + :param query_vector: The input vector to search for similar items. + :param top_k: The number of nearest neighbors to return, default is 5. + :return: List of Documents that are nearest to the query vector. + """ + top_k = kwargs.get("top_k", 5) + with self._get_cursor() as cur: + cur.execute( + f"SELECT meta, text, vector_distance(embedding,:1) AS distance FROM {self.table_name} ORDER BY distance fetch first {top_k} rows only" ,[numpy.array(query_vector)] + ) + docs = [] + score_threshold = kwargs.get("score_threshold") if kwargs.get("score_threshold") else 0.0 + for record in cur: + metadata, text, distance = record + score = 1 - distance + metadata["score"] = score + if score > score_threshold: + docs.append(Document(page_content=text, metadata=metadata)) + return docs + + def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: + # do not support bm25 search + return [] + + def delete(self) -> None: + with self._get_cursor() as cur: + cur.execute(f"DROP TABLE IF EXISTS {self.table_name}") + + def _create_collection(self, dimension: int): + cache_key = f"vector_indexing_{self._collection_name}" + lock_name = f"{cache_key}_lock" + with redis_client.lock(lock_name, timeout=20): + collection_exist_cache_key = f"vector_indexing_{self._collection_name}" + if redis_client.get(collection_exist_cache_key): + return + + with self._get_cursor() as cur: + cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name)) + redis_client.set(collection_exist_cache_key, 1, ex=3600) + + +class OracleVectorFactory(AbstractVectorFactory): + def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> OracleVector: + if dataset.index_struct_dict: + class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"] + collection_name = class_prefix + else: + dataset_id = dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id) + dataset.index_struct = json.dumps( + self.gen_index_struct_dict(VectorType.ORACLE, collection_name)) + + config = current_app.config + return OracleVector( + collection_name=collection_name, + config=OracleVectorConfig( + host=config.get("ORACLE_HOST"), + port=config.get("ORACLE_PORT"), + user=config.get("ORACLE_USER"), + password=config.get("ORACLE_PASSWORD"), + database=config.get("ORACLE_DATABASE"), + ), + ) diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 8882cb2170b3a8..719e2b9a23cbb4 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -78,6 +78,9 @@ def get_vector_factory(vector_type: str) -> type[AbstractVectorFactory]: case VectorType.TENCENT: from core.rag.datasource.vdb.tencent.tencent_vector import TencentVectorFactory return TencentVectorFactory + case VectorType.ORACLE: + from core.rag.datasource.vdb.oracle.oraclevector import OracleVectorFactory + return OracleVectorFactory case VectorType.OPENSEARCH: from core.rag.datasource.vdb.opensearch.opensearch_vector import OpenSearchVectorFactory return OpenSearchVectorFactory diff --git a/api/core/rag/datasource/vdb/vector_type.py b/api/core/rag/datasource/vdb/vector_type.py index 4a27e52706d369..dbd5afcb3ea751 100644 --- a/api/core/rag/datasource/vdb/vector_type.py +++ b/api/core/rag/datasource/vdb/vector_type.py @@ -12,3 +12,4 @@ class VectorType(str, Enum): WEAVIATE = 'weaviate' OPENSEARCH = 'opensearch' TENCENT = 'tencent' + ORACLE = 'oracle' diff --git a/api/poetry.lock b/api/poetry.lock index cadac30d9ba862..89140ce75ec80a 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -572,54 +572,53 @@ crt = ["awscrt (==0.19.12)"] [[package]] name = "bottleneck" -version = "1.3.8" +version = "1.4.0" description = "Fast NumPy array functions written in C" optional = false python-versions = "*" files = [ - {file = "Bottleneck-1.3.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:865c8ed5b798c0198b0b80553e09cc0d890c4f5feb3d81d31661517ca7819fa3"}, - {file = "Bottleneck-1.3.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d073a31e259d40b25e29dbba80f73abf38afe98fd730c79dad7edd9a0ad6cff5"}, - {file = "Bottleneck-1.3.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b806b277ab47495032822f55f43b8d336e4b7e73f8506ed34d3ea3da6d644abc"}, - {file = "Bottleneck-1.3.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:770b517609916adeb39d3b1a386a29bc316da03dd61e7ee6e8a38325b80cc327"}, - {file = "Bottleneck-1.3.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2948502b0394ee419945b55b092585222a505c61d41a874c741be49f2cac056f"}, - {file = "Bottleneck-1.3.8-cp310-cp310-win32.whl", hash = "sha256:271b6333522beb8aee32e640ba49a2064491d2c10317baa58a5996be3dd443e4"}, - {file = "Bottleneck-1.3.8-cp310-cp310-win_amd64.whl", hash = "sha256:d41000ea7ca196b5fd39d6fccd34bf0704c8831731cedd2da2dcae3c6ac49c42"}, - {file = "Bottleneck-1.3.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0a7f454394cd3642498b6e077e70f4a6b9fd46a8eb908c83ac737fdc9f9a98c"}, - {file = "Bottleneck-1.3.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c4ea8b9024dcb4e83b5c118a3c8faa863ace2ad572849da548a74a8ee4e8f2a"}, - {file = "Bottleneck-1.3.8-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f40724b6e965ff5b88b333d4a10097b1629e60c0db21bb3d08c24d7b1a904a16"}, - {file = "Bottleneck-1.3.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4bd7183b8dcca89d0e65abe4507c19667dd31dacfbcc8ed705bad642f26a46e1"}, - {file = "Bottleneck-1.3.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:20aa31a7d9d747c499ace1610a6e1f7aba6e3d4a9923e0312f6b4b6d68a59af3"}, - {file = "Bottleneck-1.3.8-cp311-cp311-win32.whl", hash = "sha256:350520105d9449e6565b3f0c4ce1f80a0b3e4d63695ebbf29db41f62e13f6461"}, - {file = "Bottleneck-1.3.8-cp311-cp311-win_amd64.whl", hash = "sha256:167a278902775defde7dfded6e98e3707dfe54971ffd9aec25c43bc74e4e381a"}, - {file = "Bottleneck-1.3.8-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c6e93ed45c6c83392f73d0333b310b38772df7eb78c120c1447245691bdedaf4"}, - {file = "Bottleneck-1.3.8-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3400f47dda0196b5af50b0b0678e33cc8c42e52e55ae0a63cdfed60725659bc"}, - {file = "Bottleneck-1.3.8-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fba5fd1805c71b2eeea50bea93d59be449c4af23ebd8da5f75fd74fd0331e314"}, - {file = "Bottleneck-1.3.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:60139c5c3d2a9c1454a04af5ee981a9f56548d27fa36f264069b149a6e9b01ed"}, - {file = "Bottleneck-1.3.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:99fab17fa26c811ccad63e208314726e718ae6605314329eca09641954550523"}, - {file = "Bottleneck-1.3.8-cp312-cp312-win32.whl", hash = "sha256:d3ae2bb5d4168912e438e377cc1301fa01df949ba59cd86317b3e00404fd4a97"}, - {file = "Bottleneck-1.3.8-cp312-cp312-win_amd64.whl", hash = "sha256:bcba1d5d5328c50f94852ab521fcb26f35d9e0ccd928d120d56455d1a5bb743f"}, - {file = "Bottleneck-1.3.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8d01fd5389d3160d54619119987ac24b020fa6810b7b398fff4945892237b3da"}, - {file = "Bottleneck-1.3.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca25f0003ef65264942f6306d793e0f270ece8b406c5a293dfc7d878146e9f8"}, - {file = "Bottleneck-1.3.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf7763cf1516fa388c3587d12182fc1bc1c8089eab1a0a1bf09761f4c41af73c"}, - {file = "Bottleneck-1.3.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:38837c022350e2a656453f0e448416b7108cf67baccf11d04a0b3b70a48074dd"}, - {file = "Bottleneck-1.3.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:84ca5e741fae1c1796744dbdd0d2c1789cb74dd79c12ea8ec5834f83430f8520"}, - {file = "Bottleneck-1.3.8-cp37-cp37m-win32.whl", hash = "sha256:f4dfc22a3450227e692ef2ff4657639c33eec88ad04ee3ce29d1a23a4942da24"}, - {file = "Bottleneck-1.3.8-cp37-cp37m-win_amd64.whl", hash = "sha256:90b87eed152bbd760c4eb11473c2cf036abdb26e2f84caeb00787da74fb08c40"}, - {file = "Bottleneck-1.3.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54a1b5d9d63b2d9f2955f8542eea26c418f97873e0abf86ca52beea0208c9306"}, - {file = "Bottleneck-1.3.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:019dd142d1e870388fb0b649213a0d8e569cce784326e183deba8f17826edd9f"}, - {file = "Bottleneck-1.3.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b5ed34a540eb7df59f45da659af9f792306637de1c69c95f020294f3b9fc4a8"}, - {file = "Bottleneck-1.3.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b69fcd4d818bcf9d53497d8accd0d5f852a447728baaa33b9b7168f8c4221d06"}, - {file = "Bottleneck-1.3.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:02616a830bd477f5ba51103396092da4b9d83cea2e88f5b8069e3f4f7b796704"}, - {file = "Bottleneck-1.3.8-cp38-cp38-win32.whl", hash = "sha256:93d359fb83eb3bdd6635ef6e64835c38ffdc211441fc190549f286e6af98b5f6"}, - {file = "Bottleneck-1.3.8-cp38-cp38-win_amd64.whl", hash = "sha256:51c8bb3dffeb72c14f0382b80de76eabac6726d316babbd48f7e4056267d7910"}, - {file = "Bottleneck-1.3.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:84453548b0f722c3be912ce3c6b685917fea842bf1252eeb63714a2c1fd1ffc9"}, - {file = "Bottleneck-1.3.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92700867504a213cafa9b8d9be529bd6e18dc83366b2ba00e86e80769b93f678"}, - {file = "Bottleneck-1.3.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fadfd2f3931fdff42f4b9867eb02ed7c662d01e6099ff6b347b6ced791450651"}, - {file = "Bottleneck-1.3.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:cfbc4a3a934b677bfbc37ac8757c4e1264a76262b774259bd3fa8a265dbd668b"}, - {file = "Bottleneck-1.3.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3c74c18f86a1ffac22280b005df8bb8a58505ac6663c4d6807f39873c17dc347"}, - {file = "Bottleneck-1.3.8-cp39-cp39-win32.whl", hash = "sha256:211f881159e8adb3a57df2263028ae6dc89ec4328bfd43f3421e507406c28654"}, - {file = "Bottleneck-1.3.8-cp39-cp39-win_amd64.whl", hash = "sha256:8615eeb75009ba7c0a112a5a6a5154ed3d61fd6b0879631778b3e42e2d9a6d65"}, - {file = "Bottleneck-1.3.8.tar.gz", hash = "sha256:6780d896969ba7f53c8995ba90c87c548beb3db435dc90c60b9a10ed1ab4d868"}, + {file = "Bottleneck-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2110af22aa8c2779faba8aa021d6b559df04449bdf21d510eacd7910934189fe"}, + {file = "Bottleneck-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:381cbd1e52338fcdf9ff01c962e6aa187b2d8b3b369d42e779b6d33ac61f8d35"}, + {file = "Bottleneck-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a91e40bbb8452e77772614d882be2c34b3b514d9f15460f703293525a6e173d"}, + {file = "Bottleneck-1.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:59604949aea476f5075b965129eaa3c2d90891fd43b0dfaf2ad7621bb5db14a5"}, + {file = "Bottleneck-1.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c2c92545e1bc8e859d8d137aefa3b24843bd374b17c9814dafa3bbcea9fc4ec0"}, + {file = "Bottleneck-1.4.0-cp310-cp310-win32.whl", hash = "sha256:f63e79bfa2f82a7432c8b147ed321d01ca7769bc17cc04644286a4ce58d30549"}, + {file = "Bottleneck-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:d69907d8d679cb5091a3f479c46bf1076f149f6311ff3298bac5089b86a2fab1"}, + {file = "Bottleneck-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:67347b0f01f32a232a6269c37afc1c079e08f6455fa12e91f4a1cd12eb0d11a5"}, + {file = "Bottleneck-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1490348b3bbc0225523dc2c00c6bb3e66168c537d62797bd29783c0826c09838"}, + {file = "Bottleneck-1.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a704165552496cbcc8bcc5921bb679fd6fa66bb1e758888de091b1223231c9f0"}, + {file = "Bottleneck-1.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ffb4e4edf7997069719b9269926cc00a2a12c6e015422d1ebc2f621c4541396a"}, + {file = "Bottleneck-1.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5d6bf45ed58d5e7414c0011ef2da75474fe597a51970df83596b0bcb79c14c5e"}, + {file = "Bottleneck-1.4.0-cp311-cp311-win32.whl", hash = "sha256:ed209f8f3cb9954773764b0fa2510a7a9247ad245593187ac90bd0747771bc5c"}, + {file = "Bottleneck-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:d53f1a72b12cfd76b56934c33bc0cb7c1a295f23a2d3ffba8c764514c9b5e0ff"}, + {file = "Bottleneck-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e720ff24370324c84a82b1a18195274715c23181748b2b9e3dacad24198ca06f"}, + {file = "Bottleneck-1.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44305c70c2a1539b0ae968e033f301ad868a6146b47e3cccd73fdfe3fc07c4ee"}, + {file = "Bottleneck-1.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b4dac5d2a871b7bd296c2b92426daa27d5b07aa84ef2557db097d29135da4eb"}, + {file = "Bottleneck-1.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:fbcdd01db9e27741fb16a02b720cf02389d4b0b99cefe3c834c7df88c2d7412d"}, + {file = "Bottleneck-1.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:14b3334a39308fbb05dacd35ac100842aa9e9bc70afbdcebe43e46179d183fd0"}, + {file = "Bottleneck-1.4.0-cp312-cp312-win32.whl", hash = "sha256:520d7a83cd48b3f58e5df1a258acb547f8a5386a8c21ca9e1058d83a0d622fdf"}, + {file = "Bottleneck-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b1339b9ad3ee217253f246cde5c3789eb527cf9dd31ff0a1f5a8bf7fc89eadad"}, + {file = "Bottleneck-1.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2749602200aaa0e12a0f3f936dd6d4035384ad10d3acf7ac4f418c501683397"}, + {file = "Bottleneck-1.4.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb79a2ac135567694f13339f0bebcee96aec09c596b324b61cd7fd5e306f49d"}, + {file = "Bottleneck-1.4.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c6097bf39723e76ff5bba160daab92ae599df212c859db8d46648548584d04a8"}, + {file = "Bottleneck-1.4.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b5f72b66ccc0272de46b67346cf8490737ba2adc6a302664f5326e7741b6d5ab"}, + {file = "Bottleneck-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:9903f017b9d6f2f69ce241b424ddad7265624f64dc6eafbe257d45661febf8bd"}, + {file = "Bottleneck-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:834816c316ad184cae7ecb615b69876a42cd2cafb07ee66c57a9c1ccacb63339"}, + {file = "Bottleneck-1.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:03c43150f180d86a5633a6da788660d335983f6798fca306ba7f47ff27a1b7e7"}, + {file = "Bottleneck-1.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea333dbcadb780356c54f5c4fa7754f143573b57508fff43d5daf63298eb26a"}, + {file = "Bottleneck-1.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6179791c0119aec3708ef74ddadab8d183e3742adb93a9028718e8696bdf572b"}, + {file = "Bottleneck-1.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:220b72405f77aebb0137b733b464c2526ded471e4289ac1e840bab8852759a55"}, + {file = "Bottleneck-1.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8746f0f727997ce4c7457dc1fec4e4e3c0fdd8803514baa3d1c4ea6515ab04b2"}, + {file = "Bottleneck-1.4.0-cp38-cp38-win32.whl", hash = "sha256:6a36280ee33d9db799163f04e88b950261e590cc71d089f5e179b21680b5d491"}, + {file = "Bottleneck-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:de17e012694e6a987bb4eb050dd7f0cf939195a8e00cb23aa93ebee5fd5e64a8"}, + {file = "Bottleneck-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28260197ab8a4a6b7adf810523147b1a3e85607f4e26a0f685eb9d155cfc75af"}, + {file = "Bottleneck-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:90d5d188a0cca0b9655ff2904ee61e7f183079e97550be98c2541a2eec358a72"}, + {file = "Bottleneck-1.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2861ff645d236f1a6f5c6d1ddb3db37d19af1d91057bdc4fd7b76299a15b3079"}, + {file = "Bottleneck-1.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6136ce7dcf825c432a20b80ab1c460264a437d8430fff32536176147e0b6b832"}, + {file = "Bottleneck-1.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:889e6855b77345622b4ba927335d3118745d590492941f5f78554f157d259e92"}, + {file = "Bottleneck-1.4.0-cp39-cp39-win32.whl", hash = "sha256:817aa43a671ede696ea023d8f35839a391244662340cc95a0f46965dda8b35cf"}, + {file = "Bottleneck-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:23834d82177d6997f21fa63156550668cd07a9a6e5a1b66ea80f1a14ac6ffd07"}, + {file = "bottleneck-1.4.0.tar.gz", hash = "sha256:beb36df519b8709e7d357c0c9639b03b885ca6355bbf5e53752c685de51605b8"}, ] [package.dependencies] @@ -1088,13 +1087,13 @@ numpy = "*" [[package]] name = "chromadb" -version = "0.5.1" +version = "0.5.3" description = "Chroma." optional = false python-versions = ">=3.8" files = [ - {file = "chromadb-0.5.1-py3-none-any.whl", hash = "sha256:61f1f75a672b6edce7f1c8875c67e2aaaaf130dc1c1684431fbc42ad7240d01d"}, - {file = "chromadb-0.5.1.tar.gz", hash = "sha256:e2b2b6a34c2a949bedcaa42fa7775f40c7f6667848fc8094dcbf97fc0d30bee7"}, + {file = "chromadb-0.5.3-py3-none-any.whl", hash = "sha256:b3874f08356e291c68c6d2e177db472cd51f22f3af7b9746215b748fd1e29982"}, + {file = "chromadb-0.5.3.tar.gz", hash = "sha256:05d887f56a46b2e0fc6ac5ab979503a27b9ee50d5ca9e455f83b2fb9840cd026"}, ] [package.dependencies] @@ -1841,19 +1840,19 @@ files = [ [[package]] name = "duckduckgo-search" -version = "6.1.6" +version = "6.1.7" description = "Search for words, documents, images, news, maps and text translation using the DuckDuckGo.com search engine." optional = false python-versions = ">=3.8" files = [ - {file = "duckduckgo_search-6.1.6-py3-none-any.whl", hash = "sha256:6139ab17579e96ca7c5ed9398365245a36ecca8e7432545e3115ef90a9304eb7"}, - {file = "duckduckgo_search-6.1.6.tar.gz", hash = "sha256:42c83d58f4f1d717a580b89cc86861cbae59e46e75288243776c53349d006bf1"}, + {file = "duckduckgo_search-6.1.7-py3-none-any.whl", hash = "sha256:ec7d5becb8c392c0293ff9464938c1014896e1e14725c05adc306290a636fab2"}, + {file = "duckduckgo_search-6.1.7.tar.gz", hash = "sha256:c6fd8ba17fe9cd0a4f32e5b96984e959c3da865f9c2864bfcf82bf7ff9b7e8f0"}, ] [package.dependencies] click = ">=8.1.7" -orjson = ">=3.10.4" -pyreqwest-impersonate = ">=0.4.7" +orjson = ">=3.10.5" +pyreqwest-impersonate = ">=0.4.8" [package.extras] dev = ["mypy (>=1.10.0)", "pytest (>=8.2.2)", "pytest-asyncio (>=0.23.7)", "ruff (>=0.4.8)"] @@ -1861,13 +1860,13 @@ lxml = ["lxml (>=5.2.2)"] [[package]] name = "email-validator" -version = "2.1.1" +version = "2.1.2" description = "A robust email address syntax and deliverability validation library." optional = false python-versions = ">=3.8" files = [ - {file = "email_validator-2.1.1-py3-none-any.whl", hash = "sha256:97d882d174e2a65732fb43bfce81a3a834cbc1bde8bf419e30ef5ea976370a05"}, - {file = "email_validator-2.1.1.tar.gz", hash = "sha256:200a70680ba08904be6d1eef729205cc0d687634399a5924d842533efb824b84"}, + {file = "email_validator-2.1.2-py3-none-any.whl", hash = "sha256:d89f6324e13b1e39889eab7f9ca2f91dc9aebb6fa50a6d8bd4329ab50f251115"}, + {file = "email_validator-2.1.2.tar.gz", hash = "sha256:14c0f3d343c4beda37400421b39fa411bbe33a75df20825df73ad53e06a9f04c"}, ] [package.dependencies] @@ -2058,18 +2057,18 @@ sgmllib3k = "*" [[package]] name = "filelock" -version = "3.14.0" +version = "3.15.3" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.14.0-py3-none-any.whl", hash = "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f"}, - {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, + {file = "filelock-3.15.3-py3-none-any.whl", hash = "sha256:0151273e5b5d6cf753a61ec83b3a9b7d8821c39ae9af9d7ecf2f9e2f17404103"}, + {file = "filelock-3.15.3.tar.gz", hash = "sha256:e1199bf5194a2277273dacd50269f0d87d0682088a3c561c15674ea9005d8635"}, ] [package.extras] docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] typing = ["typing-extensions (>=4.8)"] [[package]] @@ -3904,32 +3903,32 @@ six = "*" [[package]] name = "llvmlite" -version = "0.42.0" +version = "0.43.0" description = "lightweight wrapper around basic LLVM functionality" optional = false python-versions = ">=3.9" files = [ - {file = "llvmlite-0.42.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3366938e1bf63d26c34fbfb4c8e8d2ded57d11e0567d5bb243d89aab1eb56098"}, - {file = "llvmlite-0.42.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c35da49666a21185d21b551fc3caf46a935d54d66969d32d72af109b5e7d2b6f"}, - {file = "llvmlite-0.42.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70f44ccc3c6220bd23e0ba698a63ec2a7d3205da0d848804807f37fc243e3f77"}, - {file = "llvmlite-0.42.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:763f8d8717a9073b9e0246998de89929071d15b47f254c10eef2310b9aac033d"}, - {file = "llvmlite-0.42.0-cp310-cp310-win_amd64.whl", hash = "sha256:8d90edf400b4ceb3a0e776b6c6e4656d05c7187c439587e06f86afceb66d2be5"}, - {file = "llvmlite-0.42.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ae511caed28beaf1252dbaf5f40e663f533b79ceb408c874c01754cafabb9cbf"}, - {file = "llvmlite-0.42.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81e674c2fe85576e6c4474e8c7e7aba7901ac0196e864fe7985492b737dbab65"}, - {file = "llvmlite-0.42.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb3975787f13eb97629052edb5017f6c170eebc1c14a0433e8089e5db43bcce6"}, - {file = "llvmlite-0.42.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5bece0cdf77f22379f19b1959ccd7aee518afa4afbd3656c6365865f84903f9"}, - {file = "llvmlite-0.42.0-cp311-cp311-win_amd64.whl", hash = "sha256:7e0c4c11c8c2aa9b0701f91b799cb9134a6a6de51444eff5a9087fc7c1384275"}, - {file = "llvmlite-0.42.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:08fa9ab02b0d0179c688a4216b8939138266519aaa0aa94f1195a8542faedb56"}, - {file = "llvmlite-0.42.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b2fce7d355068494d1e42202c7aff25d50c462584233013eb4470c33b995e3ee"}, - {file = "llvmlite-0.42.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebe66a86dc44634b59a3bc860c7b20d26d9aaffcd30364ebe8ba79161a9121f4"}, - {file = "llvmlite-0.42.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d47494552559e00d81bfb836cf1c4d5a5062e54102cc5767d5aa1e77ccd2505c"}, - {file = "llvmlite-0.42.0-cp312-cp312-win_amd64.whl", hash = "sha256:05cb7e9b6ce69165ce4d1b994fbdedca0c62492e537b0cc86141b6e2c78d5888"}, - {file = "llvmlite-0.42.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bdd3888544538a94d7ec99e7c62a0cdd8833609c85f0c23fcb6c5c591aec60ad"}, - {file = "llvmlite-0.42.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0936c2067a67fb8816c908d5457d63eba3e2b17e515c5fe00e5ee2bace06040"}, - {file = "llvmlite-0.42.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a78ab89f1924fc11482209f6799a7a3fc74ddc80425a7a3e0e8174af0e9e2301"}, - {file = "llvmlite-0.42.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7599b65c7af7abbc978dbf345712c60fd596aa5670496561cc10e8a71cebfb2"}, - {file = "llvmlite-0.42.0-cp39-cp39-win_amd64.whl", hash = "sha256:43d65cc4e206c2e902c1004dd5418417c4efa6c1d04df05c6c5675a27e8ca90e"}, - {file = "llvmlite-0.42.0.tar.gz", hash = "sha256:f92b09243c0cc3f457da8b983f67bd8e1295d0f5b3746c7a1861d7a99403854a"}, + {file = "llvmlite-0.43.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a289af9a1687c6cf463478f0fa8e8aa3b6fb813317b0d70bf1ed0759eab6f761"}, + {file = "llvmlite-0.43.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d4fd101f571a31acb1559ae1af30f30b1dc4b3186669f92ad780e17c81e91bc"}, + {file = "llvmlite-0.43.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d434ec7e2ce3cc8f452d1cd9a28591745de022f931d67be688a737320dfcead"}, + {file = "llvmlite-0.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6912a87782acdff6eb8bf01675ed01d60ca1f2551f8176a300a886f09e836a6a"}, + {file = "llvmlite-0.43.0-cp310-cp310-win_amd64.whl", hash = "sha256:14f0e4bf2fd2d9a75a3534111e8ebeb08eda2f33e9bdd6dfa13282afacdde0ed"}, + {file = "llvmlite-0.43.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3e8d0618cb9bfe40ac38a9633f2493d4d4e9fcc2f438d39a4e854f39cc0f5f98"}, + {file = "llvmlite-0.43.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0a9a1a39d4bf3517f2af9d23d479b4175ead205c592ceeb8b89af48a327ea57"}, + {file = "llvmlite-0.43.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1da416ab53e4f7f3bc8d4eeba36d801cc1894b9fbfbf2022b29b6bad34a7df2"}, + {file = "llvmlite-0.43.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977525a1e5f4059316b183fb4fd34fa858c9eade31f165427a3977c95e3ee749"}, + {file = "llvmlite-0.43.0-cp311-cp311-win_amd64.whl", hash = "sha256:d5bd550001d26450bd90777736c69d68c487d17bf371438f975229b2b8241a91"}, + {file = "llvmlite-0.43.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f99b600aa7f65235a5a05d0b9a9f31150c390f31261f2a0ba678e26823ec38f7"}, + {file = "llvmlite-0.43.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:35d80d61d0cda2d767f72de99450766250560399edc309da16937b93d3b676e7"}, + {file = "llvmlite-0.43.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eccce86bba940bae0d8d48ed925f21dbb813519169246e2ab292b5092aba121f"}, + {file = "llvmlite-0.43.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df6509e1507ca0760787a199d19439cc887bfd82226f5af746d6977bd9f66844"}, + {file = "llvmlite-0.43.0-cp312-cp312-win_amd64.whl", hash = "sha256:7a2872ee80dcf6b5dbdc838763d26554c2a18aa833d31a2635bff16aafefb9c9"}, + {file = "llvmlite-0.43.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9cd2a7376f7b3367019b664c21f0c61766219faa3b03731113ead75107f3b66c"}, + {file = "llvmlite-0.43.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18e9953c748b105668487b7c81a3e97b046d8abf95c4ddc0cd3c94f4e4651ae8"}, + {file = "llvmlite-0.43.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74937acd22dc11b33946b67dca7680e6d103d6e90eeaaaf932603bec6fe7b03a"}, + {file = "llvmlite-0.43.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9efc739cc6ed760f795806f67889923f7274276f0eb45092a1473e40d9b867"}, + {file = "llvmlite-0.43.0-cp39-cp39-win_amd64.whl", hash = "sha256:47e147cdda9037f94b399bf03bfd8a6b6b1f2f90be94a454e3386f006455a9b4"}, + {file = "llvmlite-0.43.0.tar.gz", hash = "sha256:ae2b5b5c3ef67354824fb75517c8db5fbe93bc02cd9671f3c62271626bc041d5"}, ] [[package]] @@ -4651,37 +4650,37 @@ requests = ">=2.27.1" [[package]] name = "numba" -version = "0.59.1" +version = "0.60.0" description = "compiling Python code using LLVM" optional = false python-versions = ">=3.9" files = [ - {file = "numba-0.59.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:97385a7f12212c4f4bc28f648720a92514bee79d7063e40ef66c2d30600fd18e"}, - {file = "numba-0.59.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b77aecf52040de2a1eb1d7e314497b9e56fba17466c80b457b971a25bb1576d"}, - {file = "numba-0.59.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3476a4f641bfd58f35ead42f4dcaf5f132569c4647c6f1360ccf18ee4cda3990"}, - {file = "numba-0.59.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:525ef3f820931bdae95ee5379c670d5c97289c6520726bc6937a4a7d4230ba24"}, - {file = "numba-0.59.1-cp310-cp310-win_amd64.whl", hash = "sha256:990e395e44d192a12105eca3083b61307db7da10e093972ca285c85bef0963d6"}, - {file = "numba-0.59.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:43727e7ad20b3ec23ee4fc642f5b61845c71f75dd2825b3c234390c6d8d64051"}, - {file = "numba-0.59.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:411df625372c77959570050e861981e9d196cc1da9aa62c3d6a836b5cc338966"}, - {file = "numba-0.59.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2801003caa263d1e8497fb84829a7ecfb61738a95f62bc05693fcf1733e978e4"}, - {file = "numba-0.59.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dd2842fac03be4e5324ebbbd4d2d0c8c0fc6e0df75c09477dd45b288a0777389"}, - {file = "numba-0.59.1-cp311-cp311-win_amd64.whl", hash = "sha256:0594b3dfb369fada1f8bb2e3045cd6c61a564c62e50cf1f86b4666bc721b3450"}, - {file = "numba-0.59.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1cce206a3b92836cdf26ef39d3a3242fec25e07f020cc4feec4c4a865e340569"}, - {file = "numba-0.59.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c8b4477763cb1fbd86a3be7050500229417bf60867c93e131fd2626edb02238"}, - {file = "numba-0.59.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d80bce4ef7e65bf895c29e3889ca75a29ee01da80266a01d34815918e365835"}, - {file = "numba-0.59.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7ad1d217773e89a9845886401eaaab0a156a90aa2f179fdc125261fd1105096"}, - {file = "numba-0.59.1-cp312-cp312-win_amd64.whl", hash = "sha256:5bf68f4d69dd3a9f26a9b23548fa23e3bcb9042e2935257b471d2a8d3c424b7f"}, - {file = "numba-0.59.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4e0318ae729de6e5dbe64c75ead1a95eb01fabfe0e2ebed81ebf0344d32db0ae"}, - {file = "numba-0.59.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0f68589740a8c38bb7dc1b938b55d1145244c8353078eea23895d4f82c8b9ec1"}, - {file = "numba-0.59.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:649913a3758891c77c32e2d2a3bcbedf4a69f5fea276d11f9119677c45a422e8"}, - {file = "numba-0.59.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9712808e4545270291d76b9a264839ac878c5eb7d8b6e02c970dc0ac29bc8187"}, - {file = "numba-0.59.1-cp39-cp39-win_amd64.whl", hash = "sha256:8d51ccd7008a83105ad6a0082b6a2b70f1142dc7cfd76deb8c5a862367eb8c86"}, - {file = "numba-0.59.1.tar.gz", hash = "sha256:76f69132b96028d2774ed20415e8c528a34e3299a40581bae178f0994a2f370b"}, -] - -[package.dependencies] -llvmlite = "==0.42.*" -numpy = ">=1.22,<1.27" + {file = "numba-0.60.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d761de835cd38fb400d2c26bb103a2726f548dc30368853121d66201672e651"}, + {file = "numba-0.60.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:159e618ef213fba758837f9837fb402bbe65326e60ba0633dbe6c7f274d42c1b"}, + {file = "numba-0.60.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1527dc578b95c7c4ff248792ec33d097ba6bef9eda466c948b68dfc995c25781"}, + {file = "numba-0.60.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe0b28abb8d70f8160798f4de9d486143200f34458d34c4a214114e445d7124e"}, + {file = "numba-0.60.0-cp310-cp310-win_amd64.whl", hash = "sha256:19407ced081d7e2e4b8d8c36aa57b7452e0283871c296e12d798852bc7d7f198"}, + {file = "numba-0.60.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a17b70fc9e380ee29c42717e8cc0bfaa5556c416d94f9aa96ba13acb41bdece8"}, + {file = "numba-0.60.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3fb02b344a2a80efa6f677aa5c40cd5dd452e1b35f8d1c2af0dfd9ada9978e4b"}, + {file = "numba-0.60.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5f4fde652ea604ea3c86508a3fb31556a6157b2c76c8b51b1d45eb40c8598703"}, + {file = "numba-0.60.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4142d7ac0210cc86432b818338a2bc368dc773a2f5cf1e32ff7c5b378bd63ee8"}, + {file = "numba-0.60.0-cp311-cp311-win_amd64.whl", hash = "sha256:cac02c041e9b5bc8cf8f2034ff6f0dbafccd1ae9590dc146b3a02a45e53af4e2"}, + {file = "numba-0.60.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d7da4098db31182fc5ffe4bc42c6f24cd7d1cb8a14b59fd755bfee32e34b8404"}, + {file = "numba-0.60.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38d6ea4c1f56417076ecf8fc327c831ae793282e0ff51080c5094cb726507b1c"}, + {file = "numba-0.60.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:62908d29fb6a3229c242e981ca27e32a6e606cc253fc9e8faeb0e48760de241e"}, + {file = "numba-0.60.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ebaa91538e996f708f1ab30ef4d3ddc344b64b5227b67a57aa74f401bb68b9d"}, + {file = "numba-0.60.0-cp312-cp312-win_amd64.whl", hash = "sha256:f75262e8fe7fa96db1dca93d53a194a38c46da28b112b8a4aca168f0df860347"}, + {file = "numba-0.60.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:01ef4cd7d83abe087d644eaa3d95831b777aa21d441a23703d649e06b8e06b74"}, + {file = "numba-0.60.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:819a3dfd4630d95fd574036f99e47212a1af41cbcb019bf8afac63ff56834449"}, + {file = "numba-0.60.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b983bd6ad82fe868493012487f34eae8bf7dd94654951404114f23c3466d34b"}, + {file = "numba-0.60.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c151748cd269ddeab66334bd754817ffc0cabd9433acb0f551697e5151917d25"}, + {file = "numba-0.60.0-cp39-cp39-win_amd64.whl", hash = "sha256:3031547a015710140e8c87226b4cfe927cac199835e5bf7d4fe5cb64e814e3ab"}, + {file = "numba-0.60.0.tar.gz", hash = "sha256:5df6158e5584eece5fc83294b949fd30b9f1125df7708862205217e068aabf16"}, +] + +[package.dependencies] +llvmlite = "==0.43.*" +numpy = ">=1.22,<2.1" [[package]] name = "numexpr" @@ -4879,13 +4878,13 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] [[package]] name = "openpyxl" -version = "3.1.3" +version = "3.1.4" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "openpyxl-3.1.3-py2.py3-none-any.whl", hash = "sha256:25071b558db709de9e8782c3d3e058af3b23ffb2fc6f40c8f0c45a154eced2c3"}, - {file = "openpyxl-3.1.3.tar.gz", hash = "sha256:8dd482e5350125b2388070bb2477927be2e8ebc27df61178709bc8c8751da2f9"}, + {file = "openpyxl-3.1.4-py2.py3-none-any.whl", hash = "sha256:ec17f6483f2b8f7c88c57e5e5d3b0de0e3fb9ac70edc084d28e864f5b33bbefd"}, + {file = "openpyxl-3.1.4.tar.gz", hash = "sha256:8d2c8adf5d20d6ce8f9bca381df86b534835e974ed0156dacefa76f68c1d69fb"}, ] [package.dependencies] @@ -5077,59 +5076,102 @@ files = [ {file = "opentelemetry_util_http-0.46b0.tar.gz", hash = "sha256:03b6e222642f9c7eae58d9132343e045b50aca9761fcb53709bd2b663571fdf6"}, ] +[[package]] +name = "oracledb" +version = "2.2.1" +description = "Python interface to Oracle Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "oracledb-2.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3dacef7c4dd3fca94728f05336076e063450bb57ea569e8dd67fae960aaf537e"}, + {file = "oracledb-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd8fdc93a65ae2e1c934a0e3e64cb01997ba004c48a986a37583f670dd344802"}, + {file = "oracledb-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531600569febef29806f058d0f0900127356caccba47785d7ec0fca4714af132"}, + {file = "oracledb-2.2.1-cp310-cp310-win32.whl", hash = "sha256:9bbd2c33a97a91d92178d6c4ffa8676b0da80b9fd1329a5e6a09e01b8b2472b5"}, + {file = "oracledb-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:708edcaddfefa1f58a75f72df2ea0d39980ae126db85ea59a4c83eab40b5f61e"}, + {file = "oracledb-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fb6d9a4d7400398b22edb9431334f9add884dec9877fd9c4ae531e1ccc6ee1fd"}, + {file = "oracledb-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07757c240afbb4f28112a6affc2c5e4e34b8a92e5bb9af81a40fba398da2b028"}, + {file = "oracledb-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63daec72f853c47179e98493e9b732909d96d495bdceb521c5973a3940d28142"}, + {file = "oracledb-2.2.1-cp311-cp311-win32.whl", hash = "sha256:fec5318d1e0ada7e4674574cb6c8d1665398e8b9c02982279107212f05df1660"}, + {file = "oracledb-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:5134dccb5a11bc755abf02fd49be6dc8141dfcae4b650b55d40509323d00b5c2"}, + {file = "oracledb-2.2.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ac5716bc9a48247fdf563f5f4ec097f5c9f074a60fd130cdfe16699208ca29b5"}, + {file = "oracledb-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c150bddb882b7c73fb462aa2d698744da76c363e404570ed11d05b65811d96c3"}, + {file = "oracledb-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193e1888411bc21187ade4b16b76820bd1e8f216e25602f6cd0a97d45723c1dc"}, + {file = "oracledb-2.2.1-cp312-cp312-win32.whl", hash = "sha256:44a960f8bbb0711af222e0a9690e037b6a2a382e0559ae8eeb9cfafe26c7a3bc"}, + {file = "oracledb-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:470136add32f0d0084225c793f12a52b61b52c3dc00c9cd388ec6a3db3a7643e"}, + {file = "oracledb-2.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:506f0027a2c4b6e33b8aabaebd00e4e31cc85134aa82fd855f4817917cfc9d5e"}, + {file = "oracledb-2.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b8b46e6579eaca3b1436fa57bd666ad041d7f4dd3f9237f21d132cc8b52c04"}, + {file = "oracledb-2.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a47019561c5cd76d1f19b3a528a98285dca9d915dd8559555f3074424ee9438"}, + {file = "oracledb-2.2.1-cp37-cp37m-win32.whl", hash = "sha256:4b433ea6465de03315bf7c121ad9272b4eef0ecaf235d1743b06557ee587bf6e"}, + {file = "oracledb-2.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6af95303446966c808f3a6c1c33cb0343e9bf8ec57841cc804de0eb1bfa337b5"}, + {file = "oracledb-2.2.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:7df0bebc28488655fbf64b9222d9a14e5ecd13254b426ef75da7adc80cbc18d9"}, + {file = "oracledb-2.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37564661ba93f5714969400fc8a57552e5ca4244d8ecc7044d29b4af4cf9a660"}, + {file = "oracledb-2.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9077cbbe7a2bad13e20af4276a1ef782029fc5601e9470b4b60f4bbb4144655b"}, + {file = "oracledb-2.2.1-cp38-cp38-win32.whl", hash = "sha256:406c1bacf8a12e993ffe148797a0eb98e62deac073195d5cfa076e78eea85c64"}, + {file = "oracledb-2.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:c1894be5800049c64cdba63f19b94bcb94c42e70f8a53d1dd2dfaa2882fa2096"}, + {file = "oracledb-2.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:78e64fa607b28f4de6ff4c6177ef10b8beae0b7fd43a76e78b2215defc1b73c6"}, + {file = "oracledb-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7d4999820f23bb5b28097885c8d18b6d6dce47a53aa59be66bf1c865c872b17"}, + {file = "oracledb-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0048148630b60fe42e598591be152bd863ef339dff1c3785b121313b94856223"}, + {file = "oracledb-2.2.1-cp39-cp39-win32.whl", hash = "sha256:49a16ccc64c52a83c9db40095d01b0f2ee7f8a20cb105c82ffc2f57151553cfd"}, + {file = "oracledb-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:9e76d46d8260e33442cac259278885adf90080f7d2117eaeb4b230504827860b"}, + {file = "oracledb-2.2.1.tar.gz", hash = "sha256:8464c6f0295f3318daf6c2c72c83c2dcbc37e13f8fd44e3e39ff8665f442d6b6"}, +] + +[package.dependencies] +cryptography = ">=3.2.1" + [[package]] name = "orjson" -version = "3.10.4" +version = "3.10.5" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false python-versions = ">=3.8" files = [ - {file = "orjson-3.10.4-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:afca963f19ca60c7aedadea9979f769139127288dd58ccf3f7c5e8e6dc62cabf"}, - {file = "orjson-3.10.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42b112eff36ba7ccc7a9d6b87e17b9d6bde4312d05e3ddf66bf5662481dee846"}, - {file = "orjson-3.10.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02b192eaba048b1039eca9a0cef67863bd5623042f5c441889a9957121d97e14"}, - {file = "orjson-3.10.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:827c3d0e4fc44242c82bfdb1a773235b8c0575afee99a9fa9a8ce920c14e440f"}, - {file = "orjson-3.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca8ec09724f10ec209244caeb1f9f428b6bb03f2eda9ed5e2c4dd7f2b7fabd44"}, - {file = "orjson-3.10.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8eaa5d531a8fde11993cbcb27e9acf7d9c457ba301adccb7fa3a021bfecab46c"}, - {file = "orjson-3.10.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e112aa7fc4ea67367ec5e86c39a6bb6c5719eddc8f999087b1759e765ddaf2d4"}, - {file = "orjson-3.10.4-cp310-none-win32.whl", hash = "sha256:1538844fb88446c42da3889f8c4ecce95a630b5a5ba18ecdfe5aea596f4dff21"}, - {file = "orjson-3.10.4-cp310-none-win_amd64.whl", hash = "sha256:de02811903a2e434127fba5389c3cc90f689542339a6e52e691ab7f693407b5a"}, - {file = "orjson-3.10.4-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:358afaec75de7237dfea08e6b1b25d226e33a1e3b6dc154fc99eb697f24a1ffa"}, - {file = "orjson-3.10.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb4e292c3198ab3d93e5f877301d2746be4ca0ba2d9c513da5e10eb90e19ff52"}, - {file = "orjson-3.10.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c39e57cf6323a39238490092985d5d198a7da4a3be013cc891a33fef13a536e"}, - {file = "orjson-3.10.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f86df433fc01361ff9270ad27455ce1ad43cd05e46de7152ca6adb405a16b2f6"}, - {file = "orjson-3.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c9966276a2c97e93e6cbe8286537f88b2a071827514f0d9d47a0aefa77db458"}, - {file = "orjson-3.10.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c499a14155a1f5a1e16e0cd31f6cf6f93965ac60a0822bc8340e7e2d3dac1108"}, - {file = "orjson-3.10.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3087023ce904a327c29487eb7e1f2c060070e8dbb9a3991b8e7952a9c6e62f38"}, - {file = "orjson-3.10.4-cp311-none-win32.whl", hash = "sha256:f965893244fe348b59e5ce560693e6dd03368d577ce26849b5d261ce31c70101"}, - {file = "orjson-3.10.4-cp311-none-win_amd64.whl", hash = "sha256:c212f06fad6aa6ce85d5665e91a83b866579f29441a47d3865c57329c0857357"}, - {file = "orjson-3.10.4-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:d0965a8b0131959833ca8a65af60285995d57ced0de2fd8f16fc03235975d238"}, - {file = "orjson-3.10.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27b64695d9f2aef3ae15a0522e370ec95c946aaea7f2c97a1582a62b3bdd9169"}, - {file = "orjson-3.10.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:867d882ddee6a20be4c8b03ae3d2b0333894d53ad632d32bd9b8123649577171"}, - {file = "orjson-3.10.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0667458f8a8ceb6dee5c08fec0b46195f92c474cbbec71dca2a6b7fd5b67b8d"}, - {file = "orjson-3.10.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3eac9befc4eaec1d1ff3bba6210576be4945332dde194525601c5ddb5c060d3"}, - {file = "orjson-3.10.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4343245443552eae240a33047a6d1bcac7a754ad4b1c57318173c54d7efb9aea"}, - {file = "orjson-3.10.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:30153e269eea43e98918d4d462a36a7065031d9246407dfff2579a4e457515c1"}, - {file = "orjson-3.10.4-cp312-none-win32.whl", hash = "sha256:1a7d092ee043abf3db19c2183115e80676495c9911843fdb3ebd48ca7b73079e"}, - {file = "orjson-3.10.4-cp312-none-win_amd64.whl", hash = "sha256:07a2adbeb8b9efe6d68fc557685954a1f19d9e33f5cc018ae1a89e96647c1b65"}, - {file = "orjson-3.10.4-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:f5a746f3d908bce1a1e347b9ca89864047533bdfab5a450066a0315f6566527b"}, - {file = "orjson-3.10.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:465b4a8a3e459f8d304c19071b4badaa9b267c59207a005a7dd9dfe13d3a423f"}, - {file = "orjson-3.10.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:35858d260728c434a3d91b60685ab32418318567e8902039837e1c2af2719e0b"}, - {file = "orjson-3.10.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8a5ba090d40c4460312dd69c232b38c2ff67a823185cfe667e841c9dd5c06841"}, - {file = "orjson-3.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5dde86755d064664e62e3612a166c28298aa8dfd35a991553faa58855ae739cc"}, - {file = "orjson-3.10.4-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:020a9e9001cfec85c156ef3b185ff758b62ef986cefdb8384c4579facd5ce126"}, - {file = "orjson-3.10.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3bf8e6e3388a2e83a86466c912387e0f0a765494c65caa7e865f99969b76ba0d"}, - {file = "orjson-3.10.4-cp38-none-win32.whl", hash = "sha256:c5a1cca6a4a3129db3da68a25dc0a459a62ae58e284e363b35ab304202d9ba9e"}, - {file = "orjson-3.10.4-cp38-none-win_amd64.whl", hash = "sha256:ecd97d98d7bee3e3d51d0b51c92c457f05db4993329eea7c69764f9820e27eb3"}, - {file = "orjson-3.10.4-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:71362daa330a2fc85553a1469185ac448547392a8f83d34e67779f8df3a52743"}, - {file = "orjson-3.10.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d24b59d1fecb0fd080c177306118a143f7322335309640c55ed9580d2044e363"}, - {file = "orjson-3.10.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e906670aea5a605b083ebb58d575c35e88cf880fa372f7cedaac3d51e98ff164"}, - {file = "orjson-3.10.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7ce32ed4bc4d632268e4978e595fe5ea07e026b751482b4a0feec48f66a90abc"}, - {file = "orjson-3.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1dcd34286246e0c5edd0e230d1da2daab2c1b465fcb6bac85b8d44057229d40a"}, - {file = "orjson-3.10.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c45d4b8c403e50beedb1d006a8916d9910ed56bceaf2035dc253618b44d0a161"}, - {file = "orjson-3.10.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:aaed3253041b5002a4f5bfdf6f7b5cce657d974472b0699a469d439beba40381"}, - {file = "orjson-3.10.4-cp39-none-win32.whl", hash = "sha256:9a4f41b7dbf7896f8dbf559b9b43dcd99e31e0d49ac1b59d74f52ce51ab10eb9"}, - {file = "orjson-3.10.4-cp39-none-win_amd64.whl", hash = "sha256:6c4eb7d867ed91cb61e6514cb4f457aa01d7b0fd663089df60a69f3d38b69d4c"}, - {file = "orjson-3.10.4.tar.gz", hash = "sha256:c912ed25b787c73fe994a5decd81c3f3b256599b8a87d410d799d5d52013af2a"}, + {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"}, + {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"}, + {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"}, + {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"}, + {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"}, + {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"}, + {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"}, + {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"}, + {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"}, + {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"}, + {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"}, + {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"}, + {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"}, + {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"}, + {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"}, + {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"}, ] [[package]] @@ -5498,20 +5540,20 @@ wcwidth = "*" [[package]] name = "proto-plus" -version = "1.23.0" +version = "1.24.0" description = "Beautiful, Pythonic protocol buffers." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, - {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, + {file = "proto-plus-1.24.0.tar.gz", hash = "sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445"}, + {file = "proto_plus-1.24.0-py3-none-any.whl", hash = "sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12"}, ] [package.dependencies] -protobuf = ">=3.19.0,<5.0.0dev" +protobuf = ">=3.19.0,<6.0.0dev" [package.extras] -testing = ["google-api-core[grpc] (>=1.31.5)"] +testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" @@ -5862,13 +5904,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pydantic-extra-types" -version = "2.8.1" +version = "2.8.2" description = "Extra Pydantic types." optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_extra_types-2.8.1-py3-none-any.whl", hash = "sha256:ca3fce71ee46bc1043bdf3d0e3c149a09ab162cb305c4ed8c501a5034a592dd6"}, - {file = "pydantic_extra_types-2.8.1.tar.gz", hash = "sha256:c7cabe403234658207dcefed3489f2e8bfc8f4a8e305e7ab25ee29eceed65b39"}, + {file = "pydantic_extra_types-2.8.2-py3-none-any.whl", hash = "sha256:f2400b3c3553fb7fa09a131967b4edf2d53f01ad9fa89d158784653f2e5c13d1"}, + {file = "pydantic_extra_types-2.8.2.tar.gz", hash = "sha256:4d2b3c52c1e2e4dfa31bf1d5a37b841b09e3c5a08ec2bffca0e07fc2ad7d5c4a"}, ] [package.dependencies] @@ -6071,59 +6113,59 @@ files = [ [[package]] name = "pyreqwest-impersonate" -version = "0.4.7" +version = "0.4.8" description = "HTTP client that can impersonate web browsers, mimicking their headers and `TLS/JA3/JA4/HTTP2` fingerprints" optional = false python-versions = ">=3.8" files = [ - {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c175dfc429c4231a6ce03841630b236f50995ca613ff1eea26fa4c75c730b562"}, - {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b3f83c50cef2d5ed0a9246318fd3ef3bfeabe286d4eabf92df4835c05a0be7dc"}, - {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f34930113aa42f47e0542418f6a67bdb2c23fe0e2fa1866f60b29280a036b829"}, - {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88d2792df548b845edd409a3e4284f76cb4fc2510fe4a69fde9e39d54910b935"}, - {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b27622d5183185dc63bcab9a7dd1de566688c63b844812b1d9366da7c459a494"}, - {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b7bf13d49ef127e659ed134129336e94f7107023ed0138c81a46321b9a580428"}, - {file = "pyreqwest_impersonate-0.4.7-cp310-none-win_amd64.whl", hash = "sha256:0cba006b076b85a875814a4b5dd8cb27f483ebeeb0de83984a3786060fe18e0d"}, - {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:370a8cb7a92b15749cbbe3ce7a9f09d35aac7d2a74505eb447f45419ea8ef2ff"}, - {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:33244ea10ccee08bac7a7ccdc3a8e6bef6e28f2466ed61de551fa24b76ee4b6a"}, - {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dba24fb6db822cbd9cbac32539893cc19cc06dd1820e03536e685b9fd2a2ffdd"}, - {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e001ed09fc364cc00578fd31c0ae44d543cf75daf06b2657c7a82dcd99336ce"}, - {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:608525535f078e85114fcd4eeba0f0771ffc7093c29208e9c0a55147502723bf"}, - {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38daedba0fc997e29cbc25c684a42a04aed38bfbcf85d8f1ffe8f87314d5f72f"}, - {file = "pyreqwest_impersonate-0.4.7-cp311-none-win_amd64.whl", hash = "sha256:d21f3e93ee0aecdc43d2914800bdf23501bde858d70ac7c0b06168f85f95bf22"}, - {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:5caeee29370a06a322ea6951730d21ec3c641ce46417fd2b5805b283564f2fef"}, - {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1c7aa4b428ed58370975d828a95eaf10561712e79a4e2eafca1746a4654a34a8"}, - {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:887249adcab35487a44a5428ccab2a6363642785b36649a732d5e649df568b8e"}, - {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60f932de8033c15323ba79a7470406ca8228e07aa60078dee5a18e89f0a9fc88"}, - {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a2e6332fd6d78623a22f4e747688fe9e6005b61b6f208936d5428d2a65d34b39"}, - {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:349b005eef323195685ba5cb2b6f302da0db481e59f03696ef57099f232f0c1f"}, - {file = "pyreqwest_impersonate-0.4.7-cp312-none-win_amd64.whl", hash = "sha256:5620025ac138a10c46a9b14c91b6f58114d50063ff865a2d02ad632751b67b29"}, - {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:ebf954e09b3dc800a7576c7bde9827b00064531364c7817356c7cc58eb4b46b2"}, - {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:112d9561f136548bd67d31cadb6b78d4c31751e526e62e09c6e581c2f1711455"}, - {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05213f5f014ecc6732d859a0f51b3dff0424748cc6e2d0d9a42aa1f7108b4eaa"}, - {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10fa70529a60fc043650ce03481fab7714e7519c3b06f5e81c95206b8b60aec6"}, - {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5b1288881eada1891db7e862c69b673fb159834a41f823b9b00fc52d0f096ccc"}, - {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:57ca562229c40615074f36e7f1ae5e57b8164f604eddb042132467c3a00fc2c5"}, - {file = "pyreqwest_impersonate-0.4.7-cp38-none-win_amd64.whl", hash = "sha256:c098ef1333511ea9a43be9a818fcc0866bd2caa63cdc9cf4ab48450ace675646"}, - {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:39d961330190bf2d59983ad16dafb4b42d5adcdfe7531ad099c8f3ab53f8d906"}, - {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0d793591784b89953422b1efaa17460f57f6116de25b3e3065d9fa6cf220ef18"}, - {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:945116bb9ffb7e45a87e313f47de28c4da889b14bda620aebc5ba9c3600425cf"}, - {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b96a0955c49f346786ee997c755561fecf33b7886cecef861fe4db15c7b23ad3"}, - {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ed997197f907ccce9b86a75163b5e78743bc469d2ddcf8a22d4d90c2595573cb"}, - {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1f54788f6fb0ee8b31c1eaadba81fb003efb406a768844e2a1a50b855f4806bf"}, - {file = "pyreqwest_impersonate-0.4.7-cp39-none-win_amd64.whl", hash = "sha256:0a679e81b0175dcc670a5ed47a5c184d7031ce16b5c58bf6b2c650ab9f2496c8"}, - {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bddb07e04e4006a2184608c44154983fdfa0ce2e230b0a7cec81cd4ba88dd07"}, - {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:780c53bfd2fbda151081165733fba5d5b1e17dd61999360110820942e351d011"}, - {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4bfa8ea763e6935e7660f8e885f1b00713b0d22f79a526c6ae6932b1856d1343"}, - {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:96b23b0688a63cbd6c39237461baa95162a69a15e9533789163aabcaf3f572fb"}, - {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b0eb56a8ad9d48952c613903d3ef6d8762d48dcec9807a509fee2a43e94ccac"}, - {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9330176494e260521ea0eaae349ca06128dc527400248c57b378597c470d335c"}, - {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:6343bc3392781ff470e5dc47fea9f77bb61d8831b07e901900d31c46decec5d1"}, - {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ecd598e16020a165029647ca80078311bf079e8317bf61c1b2fa824b8967e0db"}, - {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a38f3014ac31b08f5fb1ef4e1eb6c6e810f51f6cb815d0066ab3f34ec0f82d98"}, - {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db76a97068e5145f5b348037e09a91b2bed9c8eab92e79a3297b1306429fa839"}, - {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1596a8ef8f20bbfe606a90ad524946747846611c8633cbdfbad0a4298b538218"}, - {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:dcee18bc350b3d3a0455422c446f1f03f00eb762b3e470066e2bc4664fd7110d"}, - {file = "pyreqwest_impersonate-0.4.7.tar.gz", hash = "sha256:74ba7e6e4f4f753da4f71a7e5dc12625b296bd7d6ddd64093a1fbff14d8d5df7"}, + {file = "pyreqwest_impersonate-0.4.8-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:45cad57afe4e6f56078ed9a7a90d0dc839d19d3e7a70175c80af21017f383bfb"}, + {file = "pyreqwest_impersonate-0.4.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1986600253baf38f25fd07b8bdc1903359c26e5d34beb7d7d084845554b5664d"}, + {file = "pyreqwest_impersonate-0.4.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cca4e6e59b9ad0cd20bad6caed3ac96992cd9c1d3126ecdfcab2c0ac2b75376"}, + {file = "pyreqwest_impersonate-0.4.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ab6b32544491ee655264dab86fc8a58e47c4f87d196b28022d4007faf971a50"}, + {file = "pyreqwest_impersonate-0.4.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:64bd6299e7fc888bb7f7292cf3e29504c406e5d5d04afd37ca994ab8142d8ee4"}, + {file = "pyreqwest_impersonate-0.4.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e914b650dd953b8d9b24ef56aa4ecbfc16e399227b68accd818f8bf159e0c558"}, + {file = "pyreqwest_impersonate-0.4.8-cp310-none-win_amd64.whl", hash = "sha256:cb56a2149b0c4548a8e0158b071a943f33dae9b717f92b5c9ac34ccd1f5a958c"}, + {file = "pyreqwest_impersonate-0.4.8-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f62620e023490902feca0109f306e122e427feff7d59e03ecd22c69a89452367"}, + {file = "pyreqwest_impersonate-0.4.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:08d4c01d76da88cfe3d7d03b311b375ce3fb5a59130f93f0637bb755d6e56ff1"}, + {file = "pyreqwest_impersonate-0.4.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524e276bc460176c79d7ba4b9131d9db73c534586660371ebdf067749252a33"}, + {file = "pyreqwest_impersonate-0.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a22863bc0aaf02ca2f5d76c8130929ae680b7d82dfc1c28c1ed5f306ff626928"}, + {file = "pyreqwest_impersonate-0.4.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8cc82d57f6a91037e64a7aa9122f909576ef2a141a42ce599958ef9f8c4bc033"}, + {file = "pyreqwest_impersonate-0.4.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:da8a053308210e44fd8349f07f45442a0691ac932f2881e98b05cf9ac404b091"}, + {file = "pyreqwest_impersonate-0.4.8-cp311-none-win_amd64.whl", hash = "sha256:4baf3916c14364a815a64ead7f728afb61b37541933b2771f18dbb245029bb55"}, + {file = "pyreqwest_impersonate-0.4.8-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:78db05deed0b32c9c75f2b3168a3a9b7d5e36487b218cb839bfe7e2a143450cb"}, + {file = "pyreqwest_impersonate-0.4.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9af9446d605903c2b4e94621a9093f8d8a403729bc9cbfbcb62929f8238c838f"}, + {file = "pyreqwest_impersonate-0.4.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c55890181d8d81e66cac25a95e215dc9680645d01e9091b64449d5407ad9bc6"}, + {file = "pyreqwest_impersonate-0.4.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e69344e7ae9964502a8693da7ad77ebc3e1418ee197e2e394bc23c5d4970772a"}, + {file = "pyreqwest_impersonate-0.4.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b5db5c957a10d8cc2815085ba0b8fe09245b2f94c2225d9653a854a03b4217e1"}, + {file = "pyreqwest_impersonate-0.4.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03c19c21f63f9c91c590c4bbcc32cc2d8066b508c683a1d163b8c7d9816a01d5"}, + {file = "pyreqwest_impersonate-0.4.8-cp312-none-win_amd64.whl", hash = "sha256:0230610779129f74ff802c744643ce7589b1d07cba21d046fe3b574281c29581"}, + {file = "pyreqwest_impersonate-0.4.8-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b8cb9471ab4b2fa7e80d3ac4e580249ff988d782f2938ad1f0428433652b170d"}, + {file = "pyreqwest_impersonate-0.4.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8081a5ace2658be91519902bde9ddc5f94e1f850a39be196007a25e3da5bbfdc"}, + {file = "pyreqwest_impersonate-0.4.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69eababfa3200459276acd780a0f3eaf41d1fe7c02bd169e714cba422055b5b9"}, + {file = "pyreqwest_impersonate-0.4.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:632957fa671ebb841166e40913015de457225cb73600ef250c436c280e68bf45"}, + {file = "pyreqwest_impersonate-0.4.8-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2ce7ddef334b4e5c68f5ea1da1d65f686b8d84f4443059d128e0f069d3fa499a"}, + {file = "pyreqwest_impersonate-0.4.8-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6ce333d450b158d582e36317089a006440b4e66739a8e8849d170e4cb15e8c8d"}, + {file = "pyreqwest_impersonate-0.4.8-cp38-none-win_amd64.whl", hash = "sha256:9d9c85ce19db92362854f534807e470f03e905f283a7de6826dc79b790a8788e"}, + {file = "pyreqwest_impersonate-0.4.8-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:2503277f2a95a30e28e498570e2ed03ef4302f873054e8e21d6c0e607cbbc1d1"}, + {file = "pyreqwest_impersonate-0.4.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8260395ef4ddae325e8b30cef0391adde7bd35e1a1decf8c729e26391f09b52d"}, + {file = "pyreqwest_impersonate-0.4.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d8066b46d82bbaff5402d767e2f13d3449b8191c37bf8283e91d301a7159869"}, + {file = "pyreqwest_impersonate-0.4.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9c42f6343cfbd6663fb53edc9eb9feb4ebf6186b284e22368adc1eeb6a33854"}, + {file = "pyreqwest_impersonate-0.4.8-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ff534f491a059e74fb7f994876df86078b4b125dbecc53c098a298ecd55fa9c6"}, + {file = "pyreqwest_impersonate-0.4.8-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5b8fbf73b3ac513ddadafd338d61f79cd2370f0691d9175b2b92a45920920d6b"}, + {file = "pyreqwest_impersonate-0.4.8-cp39-none-win_amd64.whl", hash = "sha256:a26447c82665d0e361207c1a15e56b0ca54974aa6c1fdfa18c68f908dec78cbe"}, + {file = "pyreqwest_impersonate-0.4.8-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24a16b8d55309f0af0db9d04ff442b0c91afccf078a94809e7c3a71747a5c214"}, + {file = "pyreqwest_impersonate-0.4.8-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c8fada56465fc19179404cc9d5d5e1064f5dfe27405cb052f57a5b4fe06aed1"}, + {file = "pyreqwest_impersonate-0.4.8-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:a3d48d5abc146fd804395713427d944757a99254350e6a651e7d776818074aee"}, + {file = "pyreqwest_impersonate-0.4.8-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:475829fe9994c66258157a8d4adb1c038f44f79f901208ba656d547842337227"}, + {file = "pyreqwest_impersonate-0.4.8-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ef1ec0e97623bc0e18469418cc4dd2c59a2d5fddcae944de61e13c0b46f910e"}, + {file = "pyreqwest_impersonate-0.4.8-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91857b196de89e9b36d3f8629aa8772c0bbe7efef8334fe266956b1c192ec31c"}, + {file = "pyreqwest_impersonate-0.4.8-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:63831e407487b8a21bb51f97cd86a616c291d5138f8caec16ab6019cf6423935"}, + {file = "pyreqwest_impersonate-0.4.8-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:c30e61de93bcd0a9d3ca226b1ae5475002afde61e9d85018a6a4a040eeb86567"}, + {file = "pyreqwest_impersonate-0.4.8-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6c72c37b03bce9900f5dbb4f476af17253ec60c13bf7a7259f71a8dc1b036cb"}, + {file = "pyreqwest_impersonate-0.4.8-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b1f1096165741b5c2178ab15b0eb09b5de16dd39b1cc135767d72471f0a69ce"}, + {file = "pyreqwest_impersonate-0.4.8-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:70c940c0e4ef335e22a6c705b01f286ee44780b5909065d212d94d82ea2580cb"}, + {file = "pyreqwest_impersonate-0.4.8-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:81c06f21757602d85f16dbc1cbaee1121cd65455f65aed4c048b7dcda7be85c4"}, + {file = "pyreqwest_impersonate-0.4.8.tar.gz", hash = "sha256:1eba11d47bd17244c64fec1502cc26ee66cc5c8a3be131e408101ae2b455e5bc"}, ] [package.extras] @@ -6683,13 +6725,13 @@ test = ["coveralls", "pycodestyle", "pyflakes", "pylint", "pytest", "pytest-benc [[package]] name = "redis" -version = "5.0.5" +version = "5.0.6" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.7" files = [ - {file = "redis-5.0.5-py3-none-any.whl", hash = "sha256:30b47d4ebb6b7a0b9b40c1275a19b87bb6f46b3bed82a89012cf56dea4024ada"}, - {file = "redis-5.0.5.tar.gz", hash = "sha256:3417688621acf6ee368dec4a04dd95881be24efd34c79f00d31f62bb528800ae"}, + {file = "redis-5.0.6-py3-none-any.whl", hash = "sha256:c0d6d990850c627bbf7be01c5c4cbaadf67b48593e913bb71c9819c30df37eee"}, + {file = "redis-5.0.6.tar.gz", hash = "sha256:38473cd7c6389ad3e44a91f4c3eaf6bcb8a9f746007f29bf4fb20824ff0b2197"}, ] [package.dependencies] @@ -6909,28 +6951,28 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.4.8" +version = "0.4.9" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.4.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066"}, - {file = "ruff-0.4.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913"}, - {file = "ruff-0.4.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3"}, - {file = "ruff-0.4.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb"}, - {file = "ruff-0.4.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764"}, - {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883"}, - {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199"}, - {file = "ruff-0.4.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b"}, - {file = "ruff-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45"}, - {file = "ruff-0.4.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a"}, - {file = "ruff-0.4.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc"}, - {file = "ruff-0.4.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed"}, - {file = "ruff-0.4.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa"}, - {file = "ruff-0.4.8-py3-none-win32.whl", hash = "sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9"}, - {file = "ruff-0.4.8-py3-none-win_amd64.whl", hash = "sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d"}, - {file = "ruff-0.4.8-py3-none-win_arm64.whl", hash = "sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780"}, - {file = "ruff-0.4.8.tar.gz", hash = "sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268"}, + {file = "ruff-0.4.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b262ed08d036ebe162123170b35703aaf9daffecb698cd367a8d585157732991"}, + {file = "ruff-0.4.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:98ec2775fd2d856dc405635e5ee4ff177920f2141b8e2d9eb5bd6efd50e80317"}, + {file = "ruff-0.4.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4555056049d46d8a381f746680db1c46e67ac3b00d714606304077682832998e"}, + {file = "ruff-0.4.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e91175fbe48f8a2174c9aad70438fe9cb0a5732c4159b2a10a3565fea2d94cde"}, + {file = "ruff-0.4.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e8e7b95673f22e0efd3571fb5b0cf71a5eaaa3cc8a776584f3b2cc878e46bff"}, + {file = "ruff-0.4.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2d45ddc6d82e1190ea737341326ecbc9a61447ba331b0a8962869fcada758505"}, + {file = "ruff-0.4.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78de3fdb95c4af084087628132336772b1c5044f6e710739d440fc0bccf4d321"}, + {file = "ruff-0.4.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:06b60f91bfa5514bb689b500a25ba48e897d18fea14dce14b48a0c40d1635893"}, + {file = "ruff-0.4.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88bffe9c6a454bf8529f9ab9091c99490578a593cc9f9822b7fc065ee0712a06"}, + {file = "ruff-0.4.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:673bddb893f21ab47a8334c8e0ea7fd6598ecc8e698da75bcd12a7b9d0a3206e"}, + {file = "ruff-0.4.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8c1aff58c31948cc66d0b22951aa19edb5af0a3af40c936340cd32a8b1ab7438"}, + {file = "ruff-0.4.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:784d3ec9bd6493c3b720a0b76f741e6c2d7d44f6b2be87f5eef1ae8cc1d54c84"}, + {file = "ruff-0.4.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:732dd550bfa5d85af8c3c6cbc47ba5b67c6aed8a89e2f011b908fc88f87649db"}, + {file = "ruff-0.4.9-py3-none-win32.whl", hash = "sha256:8064590fd1a50dcf4909c268b0e7c2498253273309ad3d97e4a752bb9df4f521"}, + {file = "ruff-0.4.9-py3-none-win_amd64.whl", hash = "sha256:e0a22c4157e53d006530c902107c7f550b9233e9706313ab57b892d7197d8e52"}, + {file = "ruff-0.4.9-py3-none-win_arm64.whl", hash = "sha256:5d5460f789ccf4efd43f265a58538a2c24dbce15dbf560676e430375f20a8198"}, + {file = "ruff-0.4.9.tar.gz", hash = "sha256:f1cb0828ac9533ba0135d148d214e284711ede33640465e706772645483427e3"}, ] [[package]] @@ -7206,18 +7248,18 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "70.0.0" +version = "70.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, - {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, + {file = "setuptools-70.1.0-py3-none-any.whl", hash = "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267"}, + {file = "setuptools-70.1.0.tar.gz", hash = "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "sgmllib3k" @@ -7360,64 +7402,64 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.30" +version = "2.0.31" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.30-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b48154678e76445c7ded1896715ce05319f74b1e73cf82d4f8b59b46e9c0ddc"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2753743c2afd061bb95a61a51bbb6a1a11ac1c44292fad898f10c9839a7f75b2"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7bfc726d167f425d4c16269a9a10fe8630ff6d14b683d588044dcef2d0f6be7"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4f61ada6979223013d9ab83a3ed003ded6959eae37d0d685db2c147e9143797"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a365eda439b7a00732638f11072907c1bc8e351c7665e7e5da91b169af794af"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bba002a9447b291548e8d66fd8c96a6a7ed4f2def0bb155f4f0a1309fd2735d5"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-win32.whl", hash = "sha256:0138c5c16be3600923fa2169532205d18891b28afa817cb49b50e08f62198bb8"}, - {file = "SQLAlchemy-2.0.30-cp310-cp310-win_amd64.whl", hash = "sha256:99650e9f4cf3ad0d409fed3eec4f071fadd032e9a5edc7270cd646a26446feeb"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:955991a09f0992c68a499791a753523f50f71a6885531568404fa0f231832aa0"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f69e4c756ee2686767eb80f94c0125c8b0a0b87ede03eacc5c8ae3b54b99dc46"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69c9db1ce00e59e8dd09d7bae852a9add716efdc070a3e2068377e6ff0d6fdaa"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1429a4b0f709f19ff3b0cf13675b2b9bfa8a7e79990003207a011c0db880a13"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:efedba7e13aa9a6c8407c48facfdfa108a5a4128e35f4c68f20c3407e4376aa9"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:16863e2b132b761891d6c49f0a0f70030e0bcac4fd208117f6b7e053e68668d0"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-win32.whl", hash = "sha256:2ecabd9ccaa6e914e3dbb2aa46b76dede7eadc8cbf1b8083c94d936bcd5ffb49"}, - {file = "SQLAlchemy-2.0.30-cp311-cp311-win_amd64.whl", hash = "sha256:0b3f4c438e37d22b83e640f825ef0f37b95db9aa2d68203f2c9549375d0b2260"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5a79d65395ac5e6b0c2890935bad892eabb911c4aa8e8015067ddb37eea3d56c"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a5baf9267b752390252889f0c802ea13b52dfee5e369527da229189b8bd592e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cb5a646930c5123f8461f6468901573f334c2c63c795b9af350063a736d0134"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296230899df0b77dec4eb799bcea6fbe39a43707ce7bb166519c97b583cfcab3"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c62d401223f468eb4da32627bffc0c78ed516b03bb8a34a58be54d618b74d472"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3b69e934f0f2b677ec111b4d83f92dc1a3210a779f69bf905273192cf4ed433e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win32.whl", hash = "sha256:77d2edb1f54aff37e3318f611637171e8ec71472f1fdc7348b41dcb226f93d90"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win_amd64.whl", hash = "sha256:b6c7ec2b1f4969fc19b65b7059ed00497e25f54069407a8701091beb69e591a5"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5a8e3b0a7e09e94be7510d1661339d6b52daf202ed2f5b1f9f48ea34ee6f2d57"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b60203c63e8f984df92035610c5fb76d941254cf5d19751faab7d33b21e5ddc0"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1dc3eabd8c0232ee8387fbe03e0a62220a6f089e278b1f0aaf5e2d6210741ad"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:40ad017c672c00b9b663fcfcd5f0864a0a97828e2ee7ab0c140dc84058d194cf"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e42203d8d20dc704604862977b1470a122e4892791fe3ed165f041e4bf447a1b"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-win32.whl", hash = "sha256:2a4f4da89c74435f2bc61878cd08f3646b699e7d2eba97144030d1be44e27584"}, - {file = "SQLAlchemy-2.0.30-cp37-cp37m-win_amd64.whl", hash = "sha256:b6bf767d14b77f6a18b6982cbbf29d71bede087edae495d11ab358280f304d8e"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc0c53579650a891f9b83fa3cecd4e00218e071d0ba00c4890f5be0c34887ed3"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:311710f9a2ee235f1403537b10c7687214bb1f2b9ebb52702c5aa4a77f0b3af7"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:408f8b0e2c04677e9c93f40eef3ab22f550fecb3011b187f66a096395ff3d9fd"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a4b4fb0dd4d2669070fb05b8b8824afd0af57587393015baee1cf9890242d9"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a943d297126c9230719c27fcbbeab57ecd5d15b0bd6bfd26e91bfcfe64220621"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0a089e218654e740a41388893e090d2e2c22c29028c9d1353feb38638820bbeb"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-win32.whl", hash = "sha256:fa561138a64f949f3e889eb9ab8c58e1504ab351d6cf55259dc4c248eaa19da6"}, - {file = "SQLAlchemy-2.0.30-cp38-cp38-win_amd64.whl", hash = "sha256:7d74336c65705b986d12a7e337ba27ab2b9d819993851b140efdf029248e818e"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ae8c62fe2480dd61c532ccafdbce9b29dacc126fe8be0d9a927ca3e699b9491a"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2383146973a15435e4717f94c7509982770e3e54974c71f76500a0136f22810b"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8409de825f2c3b62ab15788635ccaec0c881c3f12a8af2b12ae4910a0a9aeef6"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0094c5dc698a5f78d3d1539853e8ecec02516b62b8223c970c86d44e7a80f6c7"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:edc16a50f5e1b7a06a2dcc1f2205b0b961074c123ed17ebda726f376a5ab0953"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f7703c2010355dd28f53deb644a05fc30f796bd8598b43f0ba678878780b6e4c"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-win32.whl", hash = "sha256:1f9a727312ff6ad5248a4367358e2cf7e625e98b1028b1d7ab7b806b7d757513"}, - {file = "SQLAlchemy-2.0.30-cp39-cp39-win_amd64.whl", hash = "sha256:a0ef36b28534f2a5771191be6edb44cc2673c7b2edf6deac6562400288664221"}, - {file = "SQLAlchemy-2.0.30-py3-none-any.whl", hash = "sha256:7108d569d3990c71e26a42f60474b4c02c8586c4681af5fd67e51a044fdea86a"}, - {file = "SQLAlchemy-2.0.30.tar.gz", hash = "sha256:2b1708916730f4830bc69d6f49d37f7698b5bd7530aca7f04f785f8849e95255"}, -] - -[package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, + {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, + {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} typing-extensions = ">=4.6.0" [package.extras] @@ -7537,13 +7579,13 @@ test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] name = "tencentcloud-sdk-python-common" -version = "3.0.1166" +version = "3.0.1172" description = "Tencent Cloud Common SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-common-3.0.1166.tar.gz", hash = "sha256:7e20a98f94cd82302f4f9a6c28cd1d1d90e1043767a9ff98eebe10def84ec7b9"}, - {file = "tencentcloud_sdk_python_common-3.0.1166-py2.py3-none-any.whl", hash = "sha256:e230159b275427c0ff95bd708df2ad625ab4a45ff495d9a89d4199d535ce68e9"}, + {file = "tencentcloud-sdk-python-common-3.0.1172.tar.gz", hash = "sha256:37b3b9f4a53caa070379afb6910ac989823eacd35169701405ddafb12ea14e9e"}, + {file = "tencentcloud_sdk_python_common-3.0.1172-py2.py3-none-any.whl", hash = "sha256:8915ddc713bcd7512e9d528ec36ad3e527990ab06f5e89f63941f2e5c23f4675"}, ] [package.dependencies] @@ -7551,17 +7593,17 @@ requests = ">=2.16.0" [[package]] name = "tencentcloud-sdk-python-hunyuan" -version = "3.0.1166" +version = "3.0.1172" description = "Tencent Cloud Hunyuan SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-hunyuan-3.0.1166.tar.gz", hash = "sha256:9be5f6ca91facdc40da91a0b9c300a0c54a83cf3792305d0e83c4216ca2a2e18"}, - {file = "tencentcloud_sdk_python_hunyuan-3.0.1166-py2.py3-none-any.whl", hash = "sha256:572d41d034a68a898ac74dd4d92f6b764cdb2b993cf71e6fbc52a40e65b0b4b4"}, + {file = "tencentcloud-sdk-python-hunyuan-3.0.1172.tar.gz", hash = "sha256:ae83b39c9da7302b10c4bffb7672ae95be72945b43e06a0b1ae9ac23bac2d43b"}, + {file = "tencentcloud_sdk_python_hunyuan-3.0.1172-py2.py3-none-any.whl", hash = "sha256:443908059ef1a00a798b7387f85e210d89c65b4f9db73629e53b3ec609b8528b"}, ] [package.dependencies] -tencentcloud-sdk-python-common = "3.0.1166" +tencentcloud-sdk-python-common = "3.0.1172" [[package]] name = "threadpoolctl" @@ -8998,4 +9040,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "5e63749820d62d42f8f0d38104ea135f68361bde660131a93fe7ad08141b51b1" +content-hash = "cac196b2ddb59d7873fb3380d87b622d002613d6dc1d271a5c15e46817a38c55" diff --git a/api/pyproject.toml b/api/pyproject.toml index f69ef26beee9f1..249113ddb9c1d7 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -187,6 +187,7 @@ tenacity = "~8.3.0" cos-python-sdk-v5 = "1.9.30" novita-client = "^0.5.6" opensearch-py = "2.4.0" +oracledb = "~2.2.1" [tool.poetry.group.dev] optional = true diff --git a/api/requirements.txt b/api/requirements.txt index 5c41ee9725bcda..831c23953abac2 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -92,4 +92,5 @@ chromadb~=0.5.1 novita_client~=0.5.6 tenacity~=8.3.0 opensearch-py==2.4.0 -cos-python-sdk-v5==1.9.30 \ No newline at end of file +cos-python-sdk-v5==1.9.30 +oracledb~=2.2.1 diff --git a/api/tests/integration_tests/vdb/oracle/__init__.py b/api/tests/integration_tests/vdb/oracle/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/integration_tests/vdb/oracle/test_oraclevector.py b/api/tests/integration_tests/vdb/oracle/test_oraclevector.py new file mode 100644 index 00000000000000..3252b0427609c6 --- /dev/null +++ b/api/tests/integration_tests/vdb/oracle/test_oraclevector.py @@ -0,0 +1,30 @@ +from core.rag.datasource.vdb.oracle.oraclevector import OracleVector, OracleVectorConfig +from core.rag.models.document import Document +from tests.integration_tests.vdb.test_vector_store import ( + AbstractVectorTest, + get_example_text, + setup_mock_redis, +) + + +class OracleVectorTest(AbstractVectorTest): + def __init__(self): + super().__init__() + self.vector = OracleVector( + collection_name=self.collection_name, + config=OracleVectorConfig( + host="localhost", + port=1521, + user="dify", + password="dify", + database="FREEPDB1", + ), + ) + + def search_by_full_text(self): + hits_by_full_text: list[Document] = self.vector.search_by_full_text(query=get_example_text()) + assert len(hits_by_full_text) == 0 + + +def test_oraclevector(setup_mock_redis): + OracleVectorTest().run_all_tests() diff --git a/docker/docker-compose.oracle.yaml b/docker/docker-compose.oracle.yaml new file mode 100644 index 00000000000000..527bd7f577161f --- /dev/null +++ b/docker/docker-compose.oracle.yaml @@ -0,0 +1,18 @@ +version: '3' +services: + # oracle 23 ai vector store. + oracle: + image: container-registry.oracle.com/database/free:latest + restart: always + ports: + - 1521:1521 + volumes: + - type: volume + source: oradata_vector + target: /opt/oracle/oradata + - ./startupscripts:/opt/oracle/scripts/startup + environment: + - ORACLE_PWD=Dify123456 + - ORACLE_CHARACTERSET=AL32UTF8 +volumes: + oradata_vector: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 1d3af7e4133077..0e0f997c97776e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -154,6 +154,12 @@ services: TIDB_VECTOR_USER: xxx.root TIDB_VECTOR_PASSWORD: xxxxxx TIDB_VECTOR_DATABASE: dify + # oracle configurations + ORACLE_HOST: oracle + ORACLE_PORT: 1521 + ORACLE_USER: dify + ORACLE_PASSWORD: dify + ORACLE_DATABASE: FREEPDB1 # Chroma configuration CHROMA_HOST: 127.0.0.1 CHROMA_PORT: 8000 @@ -350,6 +356,12 @@ services: TIDB_VECTOR_USER: xxx.root TIDB_VECTOR_PASSWORD: xxxxxx TIDB_VECTOR_DATABASE: dify + # oracle configurations + ORACLE_HOST: oracle + ORACLE_PORT: 1521 + ORACLE_USER: dify + ORACLE_PASSWORD: dify + ORACLE_DATABASE: FREEPDB1 # Chroma configuration CHROMA_HOST: 127.0.0.1 CHROMA_PORT: 8000 @@ -530,6 +542,22 @@ services: # timeout: 3s # retries: 30 + # The oracle vector database. + # Uncomment to use oracle23ai as vector store. Also need to Uncomment volumes block + # oracle: + # image: container-registry.oracle.com/database/free:latest + # restart: always + # ports: + # - 1521:1521 + # volumes: + # - type: volume + # source: oradata + # target: /opt/oracle/oradata + # - ./startupscripts:/opt/oracle/scripts/startup + # environment: + # - ORACLE_PWD=Dify123456 + # - ORACLE_CHARACTERSET=AL32UTF8 + # The nginx reverse proxy. # used for reverse proxying the API service and Web service. @@ -555,3 +583,6 @@ networks: ssrf_proxy_network: driver: bridge internal: true + +#volumes: +# oradata: diff --git a/docker/startupscripts/create_user.sql b/docker/startupscripts/create_user.sql new file mode 100755 index 00000000000000..b80e19c3b05a06 --- /dev/null +++ b/docker/startupscripts/create_user.sql @@ -0,0 +1,5 @@ +show pdbs; +ALTER SYSTEM SET PROCESSES=500 SCOPE=SPFILE; +alter session set container= freepdb1; +create user dify identified by dify DEFAULT TABLESPACE users quota unlimited on users; +grant DB_DEVELOPER_ROLE to dify; From 9a5c423d593f1fb214ddeeffd876ab17226a61ab Mon Sep 17 00:00:00 2001 From: takatost <takatost@users.noreply.github.com> Date: Sat, 22 Jun 2024 02:05:50 +0800 Subject: [PATCH 260/273] chore: remove pip support for api service (#5453) Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: Bowen Liang <liangbowen@gf.com.cn> --- .devcontainer/post_start_command.sh | 2 +- .github/workflows/api-tests.yml | 70 --------------------- api/README.md | 65 +------------------ api/requirements-dev.txt | 5 -- api/requirements.txt | 96 ----------------------------- 5 files changed, 4 insertions(+), 234 deletions(-) delete mode 100644 api/requirements-dev.txt diff --git a/.devcontainer/post_start_command.sh b/.devcontainer/post_start_command.sh index 633da9649cc9ed..e3d5a6d59d746e 100755 --- a/.devcontainer/post_start_command.sh +++ b/.devcontainer/post_start_command.sh @@ -1,3 +1,3 @@ #!/bin/bash -cd api && pip install -r requirements.txt \ No newline at end of file +poetry install -C api \ No newline at end of file diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 76ea795de5828b..4b75f886fdb292 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -14,76 +14,6 @@ concurrency: jobs: test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: - - "3.10" - - "3.11" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: | - ./api/requirements.txt - ./api/requirements-dev.txt - - - name: Install dependencies - run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt - - - name: Run Unit tests - run: dev/pytest/pytest_unit_tests.sh - - - name: Run ModelRuntime - run: dev/pytest/pytest_model_runtime.sh - - - name: Run Tool - run: dev/pytest/pytest_tools.sh - - - name: Set up Sandbox - uses: hoverkraft-tech/compose-action@v2.0.0 - with: - compose-file: | - docker/docker-compose.middleware.yaml - services: | - sandbox - ssrf_proxy - - - name: Run Workflow - run: dev/pytest/pytest_workflow.sh - - - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma) - uses: hoverkraft-tech/compose-action@v2.0.0 - with: - compose-file: | - docker/docker-compose.middleware.yaml - docker/docker-compose.qdrant.yaml - docker/docker-compose.milvus.yaml - docker/docker-compose.pgvecto-rs.yaml - docker/docker-compose.pgvector.yaml - docker/docker-compose.chroma.yaml - docker/docker-compose.oracle.yaml - services: | - weaviate - qdrant - etcd - minio - milvus-standalone - pgvecto-rs - pgvector - chroma - oracle - - - name: Test Vector Stores - run: dev/pytest/pytest_vdb.sh - - test-in-poetry: name: API Tests runs-on: ubuntu-latest strategy: diff --git a/api/README.md b/api/README.md index f49f2cf6d8332d..5f71dbe5f07f7b 100644 --- a/api/README.md +++ b/api/README.md @@ -2,6 +2,9 @@ ## Usage +> [!IMPORTANT] +> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`. + 1. Start the docker-compose stack The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`. @@ -29,8 +32,6 @@ Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment. - > Using pip can be found [below](#usage-with-pip). - 5. Install dependencies ```bash @@ -84,63 +85,3 @@ cd ../ poetry run -C api bash dev/pytest/pytest_all_tests.sh ``` - -## Usage with pip - -> [!NOTE] -> In the next version, we will deprecate pip as the primary package management tool for dify api service, currently Poetry and pip coexist. - -1. Start the docker-compose stack - - The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`. - - ```bash - cd ../docker - docker-compose -f docker-compose.middleware.yaml -p dify up -d - cd ../api - ``` - -2. Copy `.env.example` to `.env` -3. Generate a `SECRET_KEY` in the `.env` file. - - ```bash - sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env - ``` - -4. Create environment. - - If you use Anaconda, create a new environment and activate it - - ```bash - conda create --name dify python=3.10 - conda activate dify - ``` - -5. Install dependencies - - ```bash - pip install -r requirements.txt - ``` - -6. Run migrate - - Before the first launch, migrate the database to the latest version. - - ```bash - flask db upgrade - ``` - -7. Start backend: - - ```bash - flask run --host 0.0.0.0 --port=5001 --debug - ``` - -8. Setup your application by visiting <http://localhost:5001/console/api/setup> or other apis... -9. If you need to debug local async processing, please start the worker service. - - ```bash - celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail - ``` - - The started celery app handles the async tasks, e.g. dataset importing and documents indexing. diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt deleted file mode 100644 index 0391ac5969bcab..00000000000000 --- a/api/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -coverage~=7.2.4 -pytest~=8.1.1 -pytest-benchmark~=4.0.0 -pytest-env~=1.1.3 -pytest-mock~=3.14.0 diff --git a/api/requirements.txt b/api/requirements.txt index 831c23953abac2..e69de29bb2d1d6 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,96 +0,0 @@ -beautifulsoup4==4.12.2 -flask~=3.0.1 -Flask-SQLAlchemy~=3.0.5 -SQLAlchemy~=2.0.29 -Flask-Compress~=1.14 -flask-login~=0.6.3 -flask-migrate~=4.0.5 -flask-restful~=0.3.10 -flask-cors~=4.0.0 -gunicorn~=22.0.0 -gevent~=23.9.1 -openai~=1.29.0 -tiktoken~=0.7.0 -psycopg2-binary~=2.9.6 -pycryptodome==3.19.1 -python-dotenv==1.0.0 -Authlib==1.3.1 -boto3==1.34.123 -cachetools~=5.3.0 -weaviate-client~=3.21.0 -mailchimp-transactional~=1.0.50 -scikit-learn==1.2.2 -sentry-sdk[flask]~=1.39.2 -sympy==1.12 -jieba==0.42.1 -celery~=5.3.6 -redis[hiredis]~=5.0.3 -chardet~=5.1.0 -python-docx~=1.1.0 -pypdfium2~=4.17.0 -resend~=0.7.0 -pyjwt~=2.8.0 -anthropic~=0.23.1 -newspaper3k==0.2.8 -wikipedia==1.4.0 -readabilipy==0.2.0 -google-ai-generativelanguage==0.6.1 -google-api-core==2.18.0 -google-api-python-client==2.90.0 -google-auth==2.29.0 -google-auth-httplib2==0.2.0 -google-generativeai==0.5.0 -googleapis-common-protos==1.63.0 -google-cloud-storage==2.16.0 -replicate~=0.22.0 -websocket-client~=1.7.0 -dashscope[tokenizer]~=1.17.0 -huggingface_hub~=0.16.4 -transformers~=4.35.0 -tokenizers~=0.15.0 -pandas[performance,excel]~=2.2.2 -xinference-client==0.9.4 -safetensors~=0.4.3 -zhipuai==1.0.7 -werkzeug~=3.0.1 -pymilvus==2.3.1 -qdrant-client==1.7.3 -cohere~=5.2.4 -pyyaml~=6.0.1 -numpy~=1.26.4 -unstructured[docx,pptx,msg,md,ppt,epub]~=0.10.27 -bs4~=0.0.1 -markdown~=3.5.1 -httpx[socks]~=0.27.0 -matplotlib~=3.8.2 -yfinance~=0.2.40 -pydub~=0.25.1 -gmpy2~=2.1.5 -numexpr~=2.9.0 -duckduckgo-search~=6.1.5 -arxiv==2.1.0 -yarl~=1.9.4 -twilio~=9.0.4 -qrcode~=7.4.2 -azure-storage-blob==12.13.0 -azure-identity==1.16.1 -lxml==5.1.0 -pydantic~=2.7.4 -pydantic_extra_types~=2.8.1 -pydantic-settings~=2.3.3 -pgvecto-rs==0.1.4 -tcvectordb==1.3.2 -firecrawl-py==0.0.5 -oss2==2.18.5 -pgvector==0.2.5 -pymysql==1.1.1 -tidb-vector==0.0.9 -google-cloud-aiplatform==1.49.0 -vanna[postgres,mysql,clickhouse,duckdb]==0.5.5 -tencentcloud-sdk-python-hunyuan~=3.0.1158 -chromadb~=0.5.1 -novita_client~=0.5.6 -tenacity~=8.3.0 -opensearch-py==2.4.0 -cos-python-sdk-v5==1.9.30 -oracledb~=2.2.1 From 8890978ad3adfdf75c15d65d29727469761dfcb0 Mon Sep 17 00:00:00 2001 From: Bowen Liang <liangbowen@gf.com.cn> Date: Sat, 22 Jun 2024 09:54:25 +0800 Subject: [PATCH 261/273] chore: use singular style in config class name (#5489) --- api/app.py | 4 +- api/configs/{app_configs.py => app_config.py} | 22 ++--- api/configs/deploy/__init__.py | 2 +- api/configs/enterprise/__init__.py | 2 +- api/configs/extra/__init__.py | 10 +-- .../{notion_configs.py => notion_config.py} | 2 +- .../{sentry_configs.py => sentry_config.py} | 2 +- api/configs/feature/__init__.py | 90 +++++++++---------- api/configs/middleware/__init__.py | 6 +- .../{redis_configs.py => redis_config.py} | 2 +- .../test_dify_config.py} | 22 ++--- 11 files changed, 82 insertions(+), 82 deletions(-) rename api/configs/{app_configs.py => app_config.py} (63%) rename api/configs/extra/{notion_configs.py => notion_config.py} (96%) rename api/configs/extra/{sentry_configs.py => sentry_config.py} (93%) rename api/configs/middleware/{redis_configs.py => redis_config.py} (96%) rename api/tests/unit_tests/{settings/test_dify_settings.py => configs/test_dify_config.py} (69%) diff --git a/api/app.py b/api/app.py index 82ec64e6b7c61b..40a90fdfa7f546 100644 --- a/api/app.py +++ b/api/app.py @@ -1,6 +1,6 @@ import os -from configs.app_configs import DifyConfigs +from configs.app_config import DifyConfig if not os.environ.get("DEBUG") or os.environ.get("DEBUG", "false").lower() != 'true': from gevent import monkey @@ -83,7 +83,7 @@ def create_flask_app_with_configs() -> Flask: """ dify_app = DifyApp(__name__) dify_app.config.from_object(Config()) - dify_app.config.from_mapping(DifyConfigs().model_dump()) + dify_app.config.from_mapping(DifyConfig().model_dump()) return dify_app diff --git a/api/configs/app_configs.py b/api/configs/app_config.py similarity index 63% rename from api/configs/app_configs.py rename to api/configs/app_config.py index 96352e43d0e2ab..4467b84c8666e6 100644 --- a/api/configs/app_configs.py +++ b/api/configs/app_config.py @@ -1,14 +1,14 @@ from pydantic_settings import BaseSettings, SettingsConfigDict -from configs.deploy import DeploymentConfigs -from configs.enterprise import EnterpriseFeatureConfigs -from configs.extra import ExtraServiceConfigs -from configs.feature import FeatureConfigs -from configs.middleware import MiddlewareConfigs +from configs.deploy import DeploymentConfig +from configs.enterprise import EnterpriseFeatureConfig +from configs.extra import ExtraServiceConfig +from configs.feature import FeatureConfig +from configs.middleware import MiddlewareConfig from configs.packaging import PackagingInfo -class DifyConfigs( +class DifyConfig( # based on pydantic-settings BaseSettings, @@ -16,20 +16,20 @@ class DifyConfigs( PackagingInfo, # Deployment configs - DeploymentConfigs, + DeploymentConfig, # Feature configs - FeatureConfigs, + FeatureConfig, # Middleware configs - MiddlewareConfigs, + MiddlewareConfig, # Extra service configs - ExtraServiceConfigs, + ExtraServiceConfig, # Enterprise feature configs # **Before using, please contact business@dify.ai by email to inquire about licensing matters.** - EnterpriseFeatureConfigs, + EnterpriseFeatureConfig, ): model_config = SettingsConfigDict( diff --git a/api/configs/deploy/__init__.py b/api/configs/deploy/__init__.py index 9a1c1b84d7cfb7..f7b118201fe24f 100644 --- a/api/configs/deploy/__init__.py +++ b/api/configs/deploy/__init__.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, Field -class DeploymentConfigs(BaseModel): +class DeploymentConfig(BaseModel): """ Deployment configs """ diff --git a/api/configs/enterprise/__init__.py b/api/configs/enterprise/__init__.py index 37164ba2e3177f..39983036eb794f 100644 --- a/api/configs/enterprise/__init__.py +++ b/api/configs/enterprise/__init__.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, Field -class EnterpriseFeatureConfigs(BaseModel): +class EnterpriseFeatureConfig(BaseModel): """ Enterprise feature configs. **Before using, please contact business@dify.ai by email to inquire about licensing matters.** diff --git a/api/configs/extra/__init__.py b/api/configs/extra/__init__.py index f248fcb47a4961..358c12d63a3f86 100644 --- a/api/configs/extra/__init__.py +++ b/api/configs/extra/__init__.py @@ -1,12 +1,12 @@ from pydantic import BaseModel -from configs.extra.notion_configs import NotionConfigs -from configs.extra.sentry_configs import SentryConfigs +from configs.extra.notion_config import NotionConfig +from configs.extra.sentry_config import SentryConfig -class ExtraServiceConfigs( +class ExtraServiceConfig( # place the configs in alphabet order - NotionConfigs, - SentryConfigs, + NotionConfig, + SentryConfig, ): pass diff --git a/api/configs/extra/notion_configs.py b/api/configs/extra/notion_config.py similarity index 96% rename from api/configs/extra/notion_configs.py rename to api/configs/extra/notion_config.py index da96b3c8c57296..f8df28cefdeb1c 100644 --- a/api/configs/extra/notion_configs.py +++ b/api/configs/extra/notion_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field -class NotionConfigs(BaseModel): +class NotionConfig(BaseModel): """ Notion integration configs """ diff --git a/api/configs/extra/sentry_configs.py b/api/configs/extra/sentry_config.py similarity index 93% rename from api/configs/extra/sentry_configs.py rename to api/configs/extra/sentry_config.py index 33924199a308ec..8cdb8cf45a35fa 100644 --- a/api/configs/extra/sentry_configs.py +++ b/api/configs/extra/sentry_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, NonNegativeFloat -class SentryConfigs(BaseModel): +class SentryConfig(BaseModel): """ Sentry configs """ diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index e820761b06d8c4..4d7d0dcd1eb350 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -3,7 +3,7 @@ from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt -class SecurityConfigs(BaseModel): +class SecurityConfig(BaseModel): """ Secret Key configs """ @@ -16,7 +16,7 @@ class SecurityConfigs(BaseModel): ) -class AppExecutionConfigs(BaseModel): +class AppExecutionConfig(BaseModel): """ App Execution configs """ @@ -26,7 +26,7 @@ class AppExecutionConfigs(BaseModel): ) -class CodeExecutionSandboxConfigs(BaseModel): +class CodeExecutionSandboxConfig(BaseModel): """ Code Execution Sandbox configs """ @@ -41,7 +41,7 @@ class CodeExecutionSandboxConfigs(BaseModel): ) -class EndpointConfigs(BaseModel): +class EndpointConfig(BaseModel): """ Module URL configs """ @@ -70,7 +70,7 @@ class EndpointConfigs(BaseModel): ) -class FileAccessConfigs(BaseModel): +class FileAccessConfig(BaseModel): """ File Access configs """ @@ -89,7 +89,7 @@ class FileAccessConfigs(BaseModel): ) -class FileUploadConfigs(BaseModel): +class FileUploadConfig(BaseModel): """ File Uploading configs """ @@ -114,7 +114,7 @@ class FileUploadConfigs(BaseModel): ) -class HttpConfigs(BaseModel): +class HttpConfig(BaseModel): """ HTTP configs """ @@ -124,7 +124,7 @@ class HttpConfigs(BaseModel): ) -class InnerAPIConfigs(BaseModel): +class InnerAPIConfig(BaseModel): """ Inner API configs """ @@ -139,7 +139,7 @@ class InnerAPIConfigs(BaseModel): ) -class LoggingConfigs(BaseModel): +class LoggingConfig(BaseModel): """ Logging configs """ @@ -171,7 +171,7 @@ class LoggingConfigs(BaseModel): ) -class ModelLoadBalanceConfigs(BaseModel): +class ModelLoadBalanceConfig(BaseModel): """ Model load balance configs """ @@ -181,7 +181,7 @@ class ModelLoadBalanceConfigs(BaseModel): ) -class BillingConfigs(BaseModel): +class BillingConfig(BaseModel): """ Platform Billing Configurations """ @@ -191,7 +191,7 @@ class BillingConfigs(BaseModel): ) -class UpdateConfigs(BaseModel): +class UpdateConfig(BaseModel): """ Update configs """ @@ -201,7 +201,7 @@ class UpdateConfigs(BaseModel): ) -class WorkflowConfigs(BaseModel): +class WorkflowConfig(BaseModel): """ Workflow feature configs """ @@ -222,7 +222,7 @@ class WorkflowConfigs(BaseModel): ) -class OAuthConfigs(BaseModel): +class OAuthConfig(BaseModel): """ oauth configs """ @@ -252,7 +252,7 @@ class OAuthConfigs(BaseModel): ) -class ModerationConfigs(BaseModel): +class ModerationConfig(BaseModel): """ Moderation in app configs. """ @@ -264,7 +264,7 @@ class ModerationConfigs(BaseModel): ) -class ToolConfigs(BaseModel): +class ToolConfig(BaseModel): """ Tool configs """ @@ -275,7 +275,7 @@ class ToolConfigs(BaseModel): ) -class MailConfigs(BaseModel): +class MailConfig(BaseModel): """ Mail Configurations """ @@ -331,7 +331,7 @@ class MailConfigs(BaseModel): ) -class RagEtlConfigs(BaseModel): +class RagEtlConfig(BaseModel): """ RAG ETL Configurations. """ @@ -357,7 +357,7 @@ class RagEtlConfigs(BaseModel): ) -class DataSetConfigs(BaseModel): +class DataSetConfig(BaseModel): """ Dataset configs """ @@ -368,7 +368,7 @@ class DataSetConfigs(BaseModel): ) -class WorkspaceConfigs(BaseModel): +class WorkspaceConfig(BaseModel): """ Workspace configs """ @@ -379,7 +379,7 @@ class WorkspaceConfigs(BaseModel): ) -class IndexingConfigs(BaseModel): +class IndexingConfig(BaseModel): """ Indexing configs. """ @@ -390,36 +390,36 @@ class IndexingConfigs(BaseModel): ) -class ImageFormatConfigs(BaseModel): +class ImageFormatConfig(BaseModel): MULTIMODAL_SEND_IMAGE_FORMAT: str = Field( description='multi model send image format, support base64, url, default is base64', default='base64', ) -class FeatureConfigs( +class FeatureConfig( # place the configs in alphabet order - AppExecutionConfigs, - BillingConfigs, - CodeExecutionSandboxConfigs, - DataSetConfigs, - EndpointConfigs, - FileAccessConfigs, - FileUploadConfigs, - HttpConfigs, - ImageFormatConfigs, - InnerAPIConfigs, - IndexingConfigs, - LoggingConfigs, - MailConfigs, - ModelLoadBalanceConfigs, - ModerationConfigs, - OAuthConfigs, - RagEtlConfigs, - SecurityConfigs, - ToolConfigs, - UpdateConfigs, - WorkflowConfigs, - WorkspaceConfigs, + AppExecutionConfig, + BillingConfig, + CodeExecutionSandboxConfig, + DataSetConfig, + EndpointConfig, + FileAccessConfig, + FileUploadConfig, + HttpConfig, + ImageFormatConfig, + InnerAPIConfig, + IndexingConfig, + LoggingConfig, + MailConfig, + ModelLoadBalanceConfig, + ModerationConfig, + OAuthConfig, + RagEtlConfig, + SecurityConfig, + ToolConfig, + UpdateConfig, + WorkflowConfig, + WorkspaceConfig, ): pass diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index 7aa6e9ccd52074..f2093b918de95e 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -2,7 +2,7 @@ from pydantic import BaseModel, Field -from configs.middleware.redis_configs import RedisConfigs +from configs.middleware.redis_config import RedisConfig from configs.middleware.vdb.chroma_configs import ChromaConfigs from configs.middleware.vdb.milvus_configs import MilvusConfigs from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs @@ -44,10 +44,10 @@ class KeywordStoreConfigs(BaseModel): ) -class MiddlewareConfigs( +class MiddlewareConfig( # place the configs in alphabet order KeywordStoreConfigs, - RedisConfigs, + RedisConfig, StorageConfigs, # configs of vdb and vdb providers diff --git a/api/configs/middleware/redis_configs.py b/api/configs/middleware/redis_config.py similarity index 96% rename from api/configs/middleware/redis_configs.py rename to api/configs/middleware/redis_config.py index 9f98e7cabfefa7..4cc40bbe6d46c6 100644 --- a/api/configs/middleware/redis_configs.py +++ b/api/configs/middleware/redis_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt -class RedisConfigs(BaseModel): +class RedisConfig(BaseModel): """ Redis configs """ diff --git a/api/tests/unit_tests/settings/test_dify_settings.py b/api/tests/unit_tests/configs/test_dify_config.py similarity index 69% rename from api/tests/unit_tests/settings/test_dify_settings.py rename to api/tests/unit_tests/configs/test_dify_config.py index b5400d5c2c049d..b2e958b4d3fe21 100644 --- a/api/tests/unit_tests/settings/test_dify_settings.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -3,7 +3,7 @@ import pytest from flask import Flask -from configs.app_configs import DifyConfigs +from configs.app_config import DifyConfig EXAMPLE_ENV_FILENAME = '.env' @@ -19,32 +19,32 @@ def example_env_file(tmp_path, monkeypatch) -> str: return str(file_path) -def test_dify_configs_undefined_entry(example_env_file): +def test_dify_config_undefined_entry(example_env_file): # load dotenv file with pydantic-settings - settings = DifyConfigs(_env_file=example_env_file) + config = DifyConfig(_env_file=example_env_file) # entries not defined in app settings with pytest.raises(TypeError): # TypeError: 'AppSettings' object is not subscriptable - assert settings['LOG_LEVEL'] == 'INFO' + assert config['LOG_LEVEL'] == 'INFO' -def test_dify_configs(example_env_file): +def test_dify_config(example_env_file): # load dotenv file with pydantic-settings - settings = DifyConfigs(_env_file=example_env_file) + config = DifyConfig(_env_file=example_env_file) # constant values - assert settings.COMMIT_SHA == '' + assert config.COMMIT_SHA == '' # default values - assert settings.EDITION == 'SELF_HOSTED' - assert settings.API_COMPRESSION_ENABLED is False - assert settings.SENTRY_TRACES_SAMPLE_RATE == 1.0 + assert config.EDITION == 'SELF_HOSTED' + assert config.API_COMPRESSION_ENABLED is False + assert config.SENTRY_TRACES_SAMPLE_RATE == 1.0 def test_flask_configs(example_env_file): flask_app = Flask('app') - flask_app.config.from_mapping(DifyConfigs(_env_file=example_env_file).model_dump()) + flask_app.config.from_mapping(DifyConfig(_env_file=example_env_file).model_dump()) config = flask_app.config # configs read from dotenv directly From b05cc3a1e4f87a3b8137ce5c1e6087130e620ce8 Mon Sep 17 00:00:00 2001 From: Bowen Liang <liangbowen@gf.com.cn> Date: Sat, 22 Jun 2024 10:07:03 +0800 Subject: [PATCH 262/273] refactor: extract storage provider configs into dify configs (#5443) --- api/config.py | 37 ---------------- api/configs/middleware/__init__.py | 14 +++++- .../storage/aliyun_oss_storage_config.py | 39 ++++++++++++++++ .../storage/amazon_s3_storage_config.py | 44 +++++++++++++++++++ .../storage/azure_blob_storage_config.py | 29 ++++++++++++ .../storage/google_cloud_storage_config.py | 19 ++++++++ .../storage/tencent_cos_storage_config.py | 34 ++++++++++++++ 7 files changed, 178 insertions(+), 38 deletions(-) create mode 100644 api/configs/middleware/storage/aliyun_oss_storage_config.py create mode 100644 api/configs/middleware/storage/amazon_s3_storage_config.py create mode 100644 api/configs/middleware/storage/azure_blob_storage_config.py create mode 100644 api/configs/middleware/storage/google_cloud_storage_config.py create mode 100644 api/configs/middleware/storage/tencent_cos_storage_config.py diff --git a/api/config.py b/api/config.py index 4a3abc30fa632e..3c4721206e0874 100644 --- a/api/config.py +++ b/api/config.py @@ -9,8 +9,6 @@ 'DB_PORT': '5432', 'DB_DATABASE': 'dify', 'DB_CHARSET': '', - 'S3_USE_AWS_MANAGED_IAM': 'False', - 'S3_ADDRESS_STYLE': 'auto', 'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql', 'SQLALCHEMY_POOL_SIZE': 30, 'SQLALCHEMY_MAX_OVERFLOW': 10, @@ -100,41 +98,6 @@ def __init__(self): if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False - - # S3 Storage settings - self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM') - self.S3_ENDPOINT = get_env('S3_ENDPOINT') - self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME') - self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY') - self.S3_SECRET_KEY = get_env('S3_SECRET_KEY') - self.S3_REGION = get_env('S3_REGION') - self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE') - - # Azure Blob Storage settings - self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME') - self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY') - self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME') - self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL') - - # Aliyun Storage settings - self.ALIYUN_OSS_BUCKET_NAME = get_env('ALIYUN_OSS_BUCKET_NAME') - self.ALIYUN_OSS_ACCESS_KEY = get_env('ALIYUN_OSS_ACCESS_KEY') - self.ALIYUN_OSS_SECRET_KEY = get_env('ALIYUN_OSS_SECRET_KEY') - self.ALIYUN_OSS_ENDPOINT = get_env('ALIYUN_OSS_ENDPOINT') - self.ALIYUN_OSS_REGION = get_env('ALIYUN_OSS_REGION') - self.ALIYUN_OSS_AUTH_VERSION = get_env('ALIYUN_OSS_AUTH_VERSION') - - # Google Cloud Storage settings - self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME') - self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64') - - # Tencent Cos Storage settings - self.TENCENT_COS_BUCKET_NAME = get_env('TENCENT_COS_BUCKET_NAME') - self.TENCENT_COS_REGION = get_env('TENCENT_COS_REGION') - self.TENCENT_COS_SECRET_ID = get_env('TENCENT_COS_SECRET_ID') - self.TENCENT_COS_SECRET_KEY = get_env('TENCENT_COS_SECRET_KEY') - self.TENCENT_COS_SCHEME = get_env('TENCENT_COS_SCHEME') - # ------------------------ # Platform Configurations. # ------------------------ diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index f2093b918de95e..912e26d2a121bf 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -3,6 +3,11 @@ from pydantic import BaseModel, Field from configs.middleware.redis_config import RedisConfig +from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig +from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig +from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig +from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig +from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig from configs.middleware.vdb.chroma_configs import ChromaConfigs from configs.middleware.vdb.milvus_configs import MilvusConfigs from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs @@ -48,13 +53,21 @@ class MiddlewareConfig( # place the configs in alphabet order KeywordStoreConfigs, RedisConfig, + + # configs of storage and storage providers StorageConfigs, + AliyunOSSStorageConfig, + AzureBlobStorageConfig, + GoogleCloudStorageConfig, + TencentCloudCOSStorageConfig, + S3StorageConfig, # configs of vdb and vdb providers VectorStoreConfigs, ChromaConfigs, MilvusConfigs, OpenSearchConfigs, + OracleConfigs, PGVectorConfigs, PGVectoRSConfigs, QdrantConfigs, @@ -62,6 +75,5 @@ class MiddlewareConfig( TencentVectorDBConfigs, TiDBVectorConfigs, WeaviateConfigs, - OracleConfigs, ): pass diff --git a/api/configs/middleware/storage/aliyun_oss_storage_config.py b/api/configs/middleware/storage/aliyun_oss_storage_config.py new file mode 100644 index 00000000000000..67921149d6934b --- /dev/null +++ b/api/configs/middleware/storage/aliyun_oss_storage_config.py @@ -0,0 +1,39 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class AliyunOSSStorageConfig(BaseModel): + """ + Aliyun storage configs + """ + + ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field( + description='Aliyun storage ', + default=None, + ) + + ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field( + description='Aliyun storage access key', + default=None, + ) + + ALIYUN_OSS_SECRET_KEY: Optional[str] = Field( + description='Aliyun storage secret key', + default=None, + ) + + ALIYUN_OSS_ENDPOINT: Optional[str] = Field( + description='Aliyun storage endpoint URL', + default=None, + ) + + ALIYUN_OSS_REGION: Optional[str] = Field( + description='Aliyun storage region', + default=None, + ) + + ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field( + description='Aliyun storage authentication version', + default=None, + ) diff --git a/api/configs/middleware/storage/amazon_s3_storage_config.py b/api/configs/middleware/storage/amazon_s3_storage_config.py new file mode 100644 index 00000000000000..21fe425fa8d7fc --- /dev/null +++ b/api/configs/middleware/storage/amazon_s3_storage_config.py @@ -0,0 +1,44 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class S3StorageConfig(BaseModel): + """ + S3 storage configs + """ + + S3_ENDPOINT: Optional[str] = Field( + description='S3 storage endpoint', + default=None, + ) + + S3_REGION: Optional[str] = Field( + description='S3 storage region', + default=None, + ) + + S3_BUCKET_NAME: Optional[str] = Field( + description='S3 storage bucket name', + default=None, + ) + + S3_ACCESS_KEY: Optional[str] = Field( + description='S3 storage access key', + default=None, + ) + + S3_SECRET_KEY: Optional[str] = Field( + description='S3 storage secret key', + default=None, + ) + + S3_ADDRESS_STYLE: str = Field( + description='S3 storage address style', + default='auto', + ) + + S3_USE_AWS_MANAGED_IAM: bool = Field( + description='whether to use aws managed IAM for S3', + default=False, + ) diff --git a/api/configs/middleware/storage/azure_blob_storage_config.py b/api/configs/middleware/storage/azure_blob_storage_config.py new file mode 100644 index 00000000000000..a37aa496f16f07 --- /dev/null +++ b/api/configs/middleware/storage/azure_blob_storage_config.py @@ -0,0 +1,29 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class AzureBlobStorageConfig(BaseModel): + """ + Azure Blob storage configs + """ + + AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field( + description='Azure Blob account name', + default=None, + ) + + AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field( + description='Azure Blob account key', + default=None, + ) + + AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field( + description='Azure Blob container name', + default=None, + ) + + AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field( + description='Azure Blob account url', + default=None, + ) diff --git a/api/configs/middleware/storage/google_cloud_storage_config.py b/api/configs/middleware/storage/google_cloud_storage_config.py new file mode 100644 index 00000000000000..1f4d9f9883bb44 --- /dev/null +++ b/api/configs/middleware/storage/google_cloud_storage_config.py @@ -0,0 +1,19 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class GoogleCloudStorageConfig(BaseModel): + """ + Google Cloud storage configs + """ + + GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field( + description='Google Cloud storage bucket name', + default=None, + ) + + GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field( + description='Google Cloud storage service account json base64', + default=None, + ) diff --git a/api/configs/middleware/storage/tencent_cos_storage_config.py b/api/configs/middleware/storage/tencent_cos_storage_config.py new file mode 100644 index 00000000000000..1bcc4b7b442bff --- /dev/null +++ b/api/configs/middleware/storage/tencent_cos_storage_config.py @@ -0,0 +1,34 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class TencentCloudCOSStorageConfig(BaseModel): + """ + Tencent Cloud COS storage configs + """ + + TENCENT_COS_BUCKET_NAME: Optional[str] = Field( + description='Tencent Cloud COS bucket name', + default=None, + ) + + TENCENT_COS_REGION: Optional[str] = Field( + description='Tencent Cloud COS region', + default=None, + ) + + TENCENT_COS_SECRET_ID: Optional[str] = Field( + description='Tencent Cloud COS secret id', + default=None, + ) + + TENCENT_COS_SECRET_KEY: Optional[str] = Field( + description='Tencent Cloud COS secret key', + default=None, + ) + + TENCENT_COS_SCHEME: Optional[str] = Field( + description='Tencent Cloud COS scheme', + default=None, + ) From f67b164b0db26f7c328ea1cbf8ac09e132720892 Mon Sep 17 00:00:00 2001 From: Bowen Liang <liangbowen@gf.com.cn> Date: Sat, 22 Jun 2024 10:29:56 +0800 Subject: [PATCH 263/273] refactor: extract db configs and celery configs into dify config (#5491) --- api/config.py | 44 ------- api/configs/middleware/__init__.py | 110 +++++++++++++++++- api/migrations/README | 1 - .../unit_tests/configs/test_dify_config.py | 11 ++ 4 files changed, 119 insertions(+), 47 deletions(-) diff --git a/api/config.py b/api/config.py index 3c4721206e0874..2e722c7009e6f1 100644 --- a/api/config.py +++ b/api/config.py @@ -3,19 +3,6 @@ import dotenv DEFAULTS = { - 'DB_USERNAME': 'postgres', - 'DB_PASSWORD': '', - 'DB_HOST': 'localhost', - 'DB_PORT': '5432', - 'DB_DATABASE': 'dify', - 'DB_CHARSET': '', - 'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql', - 'SQLALCHEMY_POOL_SIZE': 30, - 'SQLALCHEMY_MAX_OVERFLOW': 10, - 'SQLALCHEMY_POOL_RECYCLE': 3600, - 'SQLALCHEMY_POOL_PRE_PING': 'False', - 'SQLALCHEMY_ECHO': 'False', - 'CELERY_BACKEND': 'database', 'HOSTED_OPENAI_QUOTA_LIMIT': 200, 'HOSTED_OPENAI_TRIAL_ENABLED': 'False', 'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003', @@ -67,37 +54,6 @@ def __init__(self): self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins( 'WEB_API_CORS_ALLOW_ORIGINS', '*') - # ------------------------ - # Database Configurations. - # ------------------------ - db_credentials = { - key: get_env(key) for key in - ['DB_USERNAME', 'DB_PASSWORD', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_CHARSET'] - } - self.SQLALCHEMY_DATABASE_URI_SCHEME = get_env('SQLALCHEMY_DATABASE_URI_SCHEME') - - db_extras = f"?client_encoding={db_credentials['DB_CHARSET']}" if db_credentials['DB_CHARSET'] else "" - - self.SQLALCHEMY_DATABASE_URI = f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/{db_credentials['DB_DATABASE']}{db_extras}" - self.SQLALCHEMY_ENGINE_OPTIONS = { - 'pool_size': int(get_env('SQLALCHEMY_POOL_SIZE')), - 'max_overflow': int(get_env('SQLALCHEMY_MAX_OVERFLOW')), - 'pool_recycle': int(get_env('SQLALCHEMY_POOL_RECYCLE')), - 'pool_pre_ping': get_bool_env('SQLALCHEMY_POOL_PRE_PING'), - 'connect_args': {'options': '-c timezone=UTC'}, - } - - self.SQLALCHEMY_ECHO = get_bool_env('SQLALCHEMY_ECHO') - - # ------------------------ - # Celery worker Configurations. - # ------------------------ - self.CELERY_BROKER_URL = get_env('CELERY_BROKER_URL') - self.CELERY_BACKEND = get_env('CELERY_BACKEND') - self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \ - if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL - self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False - # ------------------------ # Platform Configurations. # ------------------------ diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index 912e26d2a121bf..5ea557c87dbfb7 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -1,6 +1,6 @@ -from typing import Optional +from typing import Any, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt, computed_field from configs.middleware.redis_config import RedisConfig from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig @@ -49,8 +49,114 @@ class KeywordStoreConfigs(BaseModel): ) +class DatabaseConfigs: + DB_HOST: str = Field( + description='db host', + default='localhost', + ) + + DB_PORT: PositiveInt = Field( + description='db port', + default=5432, + ) + + DB_USERNAME: str = Field( + description='db username', + default='postgres', + ) + + DB_PASSWORD: str = Field( + description='db password', + default='', + ) + + DB_DATABASE: str = Field( + description='db database', + default='dify', + ) + + DB_CHARSET: str = Field( + description='db charset', + default='', + ) + + SQLALCHEMY_DATABASE_URI_SCHEME: str = Field( + description='db uri scheme', + default='postgresql', + ) + + @computed_field + @property + def SQLALCHEMY_DATABASE_URI(self) -> str: + db_extras = f"?client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else "" + return (f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://" + f"{self.DB_USERNAME}:{self.DB_PASSWORD}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}" + f"{db_extras}") + + SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field( + description='pool size of SqlAlchemy', + default=30, + ) + + SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field( + description='max overflows for SqlAlchemy', + default=10, + ) + + SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field( + description='SqlAlchemy pool recycle', + default=3600, + ) + + SQLALCHEMY_POOL_PRE_PING: bool = Field( + description='whether to enable pool pre-ping in SqlAlchemy', + default=False, + ) + + SQLALCHEMY_ECHO: bool = Field( + description='whether to enable SqlAlchemy echo', + default=False, + ) + + @computed_field + @property + def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]: + return { + 'pool_size': self.SQLALCHEMY_POOL_SIZE, + 'max_overflow': self.SQLALCHEMY_MAX_OVERFLOW, + 'pool_recycle': self.SQLALCHEMY_POOL_RECYCLE, + 'pool_pre_ping': self.SQLALCHEMY_POOL_PRE_PING, + 'connect_args': {'options': '-c timezone=UTC'}, + } + + +class CeleryConfigs(DatabaseConfigs): + CELERY_BACKEND: str = Field( + description='Celery backend, available values are `database`, `redis`', + default='database', + ) + + CELERY_BROKER_URL: Optional[str] = Field( + description='CELERY_BROKER_URL', + default=None, + ) + + @computed_field + @property + def CELERY_RESULT_BACKEND(self) -> str: + return 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \ + if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL + + @computed_field + @property + def BROKER_USE_SSL(self) -> bool: + return self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False + + class MiddlewareConfig( # place the configs in alphabet order + CeleryConfigs, + DatabaseConfigs, KeywordStoreConfigs, RedisConfig, diff --git a/api/migrations/README b/api/migrations/README index 220678df7ab06e..0e048441597444 100644 --- a/api/migrations/README +++ b/api/migrations/README @@ -1,2 +1 @@ Single-database configuration for Flask. - diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index b2e958b4d3fe21..6a6fe35f66c7b3 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -60,3 +60,14 @@ def test_flask_configs(example_env_file): assert config['CONSOLE_API_URL'] == 'https://example.com' # fallback to alias choices value as CONSOLE_API_URL assert config['FILES_URL'] == 'https://example.com' + + assert config['SQLALCHEMY_DATABASE_URI'] == 'postgresql://postgres:@localhost:5432/dify' + assert config['SQLALCHEMY_ENGINE_OPTIONS'] == { + 'connect_args': { + 'options': '-c timezone=UTC', + }, + 'max_overflow': 10, + 'pool_pre_ping': False, + 'pool_recycle': 3600, + 'pool_size': 30, + } From 3628ef7a13e5776c28a957def92f51d044337b9c Mon Sep 17 00:00:00 2001 From: takatost <takatost@gmail.com> Date: Sat, 22 Jun 2024 10:35:43 +0800 Subject: [PATCH 264/273] fix lock --- api/poetry.lock | 124 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 42 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 89140ce75ec80a..10e590e5fa8b80 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1860,13 +1860,13 @@ lxml = ["lxml (>=5.2.2)"] [[package]] name = "email-validator" -version = "2.1.2" +version = "2.2.0" description = "A robust email address syntax and deliverability validation library." optional = false python-versions = ">=3.8" files = [ - {file = "email_validator-2.1.2-py3-none-any.whl", hash = "sha256:d89f6324e13b1e39889eab7f9ca2f91dc9aebb6fa50a6d8bd4329ab50f251115"}, - {file = "email_validator-2.1.2.tar.gz", hash = "sha256:14c0f3d343c4beda37400421b39fa411bbe33a75df20825df73ad53e06a9f04c"}, + {file = "email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631"}, + {file = "email_validator-2.2.0.tar.gz", hash = "sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7"}, ] [package.dependencies] @@ -2753,13 +2753,13 @@ xai = ["tensorflow (>=2.3.0,<3.0.0dev)"] [[package]] name = "google-cloud-bigquery" -version = "3.24.0" +version = "3.25.0" description = "Google BigQuery API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.24.0.tar.gz", hash = "sha256:e95e6f6e0aa32e6c453d44e2b3298931fdd7947c309ea329a31b6ff1f939e17e"}, - {file = "google_cloud_bigquery-3.24.0-py2.py3-none-any.whl", hash = "sha256:bc08323ce99dee4e811b7c3d0cde8929f5bf0b1aeaed6bcd75fc89796dd87652"}, + {file = "google-cloud-bigquery-3.25.0.tar.gz", hash = "sha256:5b2aff3205a854481117436836ae1403f11f2594e6810a98886afd57eda28509"}, + {file = "google_cloud_bigquery-3.25.0-py2.py3-none-any.whl", hash = "sha256:7f0c371bc74d2a7fb74dacbc00ac0f90c8c2bec2289b51dd6685a275873b1ce9"}, ] [package.dependencies] @@ -3901,6 +3901,46 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "langfuse" +version = "2.36.2" +description = "A client library for accessing langfuse" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langfuse-2.36.2-py3-none-any.whl", hash = "sha256:66728feddcec0974e4eb31612151a282fcce2e333b5a61474182b5e67e78e090"}, + {file = "langfuse-2.36.2.tar.gz", hash = "sha256:3e784505d408aa2c9c2da79487b64d185d8f7fa8a855e5303bcce678454c715b"}, +] + +[package.dependencies] +backoff = ">=1.10.0" +httpx = ">=0.15.4,<1.0" +idna = ">=3.7,<4.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1.10.7,<3.0" +wrapt = ">=1.14,<2.0" + +[package.extras] +langchain = ["langchain (>=0.0.309)"] +llama-index = ["llama-index (>=0.10.12,<2.0.0)"] +openai = ["openai (>=0.27.8)"] + +[[package]] +name = "langsmith" +version = "0.1.81" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.81-py3-none-any.whl", hash = "sha256:3251d823225eef23ee541980b9d9e506367eabbb7f985a086b5d09e8f78ba7e9"}, + {file = "langsmith-0.1.81.tar.gz", hash = "sha256:585ef3a2251380bd2843a664c9a28da4a7d28432e3ee8bcebf291ffb8e1f0af0"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = ">=1,<3" +requests = ">=2,<3" + [[package]] name = "llvmlite" version = "0.43.0" @@ -4386,13 +4426,13 @@ tests = ["pytest (>=4.6)"] [[package]] name = "msal" -version = "1.28.1" +version = "1.29.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." optional = false python-versions = ">=3.7" files = [ - {file = "msal-1.28.1-py3-none-any.whl", hash = "sha256:563c2d70de77a2ca9786aab84cb4e133a38a6897e6676774edc23d610bfc9e7b"}, - {file = "msal-1.28.1.tar.gz", hash = "sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d"}, + {file = "msal-1.29.0-py3-none-any.whl", hash = "sha256:6b301e63f967481f0cc1a3a3bac0cf322b276855bc1b0955468d9deb3f33d511"}, + {file = "msal-1.29.0.tar.gz", hash = "sha256:8f6725f099752553f9b2fe84125e2a5ebe47b49f92eacca33ebedd3a9ebaae25"}, ] [package.dependencies] @@ -5205,13 +5245,13 @@ files = [ [[package]] name = "packaging" -version = "24.1" +version = "23.2" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -6951,28 +6991,28 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.4.9" +version = "0.4.10" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.4.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b262ed08d036ebe162123170b35703aaf9daffecb698cd367a8d585157732991"}, - {file = "ruff-0.4.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:98ec2775fd2d856dc405635e5ee4ff177920f2141b8e2d9eb5bd6efd50e80317"}, - {file = "ruff-0.4.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4555056049d46d8a381f746680db1c46e67ac3b00d714606304077682832998e"}, - {file = "ruff-0.4.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e91175fbe48f8a2174c9aad70438fe9cb0a5732c4159b2a10a3565fea2d94cde"}, - {file = "ruff-0.4.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e8e7b95673f22e0efd3571fb5b0cf71a5eaaa3cc8a776584f3b2cc878e46bff"}, - {file = "ruff-0.4.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2d45ddc6d82e1190ea737341326ecbc9a61447ba331b0a8962869fcada758505"}, - {file = "ruff-0.4.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78de3fdb95c4af084087628132336772b1c5044f6e710739d440fc0bccf4d321"}, - {file = "ruff-0.4.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:06b60f91bfa5514bb689b500a25ba48e897d18fea14dce14b48a0c40d1635893"}, - {file = "ruff-0.4.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88bffe9c6a454bf8529f9ab9091c99490578a593cc9f9822b7fc065ee0712a06"}, - {file = "ruff-0.4.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:673bddb893f21ab47a8334c8e0ea7fd6598ecc8e698da75bcd12a7b9d0a3206e"}, - {file = "ruff-0.4.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8c1aff58c31948cc66d0b22951aa19edb5af0a3af40c936340cd32a8b1ab7438"}, - {file = "ruff-0.4.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:784d3ec9bd6493c3b720a0b76f741e6c2d7d44f6b2be87f5eef1ae8cc1d54c84"}, - {file = "ruff-0.4.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:732dd550bfa5d85af8c3c6cbc47ba5b67c6aed8a89e2f011b908fc88f87649db"}, - {file = "ruff-0.4.9-py3-none-win32.whl", hash = "sha256:8064590fd1a50dcf4909c268b0e7c2498253273309ad3d97e4a752bb9df4f521"}, - {file = "ruff-0.4.9-py3-none-win_amd64.whl", hash = "sha256:e0a22c4157e53d006530c902107c7f550b9233e9706313ab57b892d7197d8e52"}, - {file = "ruff-0.4.9-py3-none-win_arm64.whl", hash = "sha256:5d5460f789ccf4efd43f265a58538a2c24dbce15dbf560676e430375f20a8198"}, - {file = "ruff-0.4.9.tar.gz", hash = "sha256:f1cb0828ac9533ba0135d148d214e284711ede33640465e706772645483427e3"}, + {file = "ruff-0.4.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5c2c4d0859305ac5a16310eec40e4e9a9dec5dcdfbe92697acd99624e8638dac"}, + {file = "ruff-0.4.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a79489607d1495685cdd911a323a35871abfb7a95d4f98fc6f85e799227ac46e"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1dd1681dfa90a41b8376a61af05cc4dc5ff32c8f14f5fe20dba9ff5deb80cd6"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c75c53bb79d71310dc79fb69eb4902fba804a81f374bc86a9b117a8d077a1784"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18238c80ee3d9100d3535d8eb15a59c4a0753b45cc55f8bf38f38d6a597b9739"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:d8f71885bce242da344989cae08e263de29752f094233f932d4f5cfb4ef36a81"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:330421543bd3222cdfec481e8ff3460e8702ed1e58b494cf9d9e4bf90db52b9d"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e9b6fb3a37b772628415b00c4fc892f97954275394ed611056a4b8a2631365e"}, + {file = "ruff-0.4.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f54c481b39a762d48f64d97351048e842861c6662d63ec599f67d515cb417f6"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:67fe086b433b965c22de0b4259ddfe6fa541c95bf418499bedb9ad5fb8d1c631"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:acfaaab59543382085f9eb51f8e87bac26bf96b164839955f244d07125a982ef"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3cea07079962b2941244191569cf3a05541477286f5cafea638cd3aa94b56815"}, + {file = "ruff-0.4.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:338a64ef0748f8c3a80d7f05785930f7965d71ca260904a9321d13be24b79695"}, + {file = "ruff-0.4.10-py3-none-win32.whl", hash = "sha256:ffe3cd2f89cb54561c62e5fa20e8f182c0a444934bf430515a4b422f1ab7b7ca"}, + {file = "ruff-0.4.10-py3-none-win_amd64.whl", hash = "sha256:67f67cef43c55ffc8cc59e8e0b97e9e60b4837c8f21e8ab5ffd5d66e196e25f7"}, + {file = "ruff-0.4.10-py3-none-win_arm64.whl", hash = "sha256:dd1fcee327c20addac7916ca4e2653fbbf2e8388d8a6477ce5b4e986b68ae6c0"}, + {file = "ruff-0.4.10.tar.gz", hash = "sha256:3aa4f2bc388a30d346c56524f7cacca85945ba124945fe489952aadb6b5cd804"}, ] [[package]] @@ -7579,13 +7619,13 @@ test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] name = "tencentcloud-sdk-python-common" -version = "3.0.1172" +version = "3.0.1173" description = "Tencent Cloud Common SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-common-3.0.1172.tar.gz", hash = "sha256:37b3b9f4a53caa070379afb6910ac989823eacd35169701405ddafb12ea14e9e"}, - {file = "tencentcloud_sdk_python_common-3.0.1172-py2.py3-none-any.whl", hash = "sha256:8915ddc713bcd7512e9d528ec36ad3e527990ab06f5e89f63941f2e5c23f4675"}, + {file = "tencentcloud-sdk-python-common-3.0.1173.tar.gz", hash = "sha256:867eea525efe2ce04fd2ed90ae2a6843ab4d5926415727c345e13c51a233c161"}, + {file = "tencentcloud_sdk_python_common-3.0.1173-py2.py3-none-any.whl", hash = "sha256:d257c24c355df964594b667ea43eaea1e0b1f1c38b9a4e836bda4b07b2445557"}, ] [package.dependencies] @@ -7593,17 +7633,17 @@ requests = ">=2.16.0" [[package]] name = "tencentcloud-sdk-python-hunyuan" -version = "3.0.1172" +version = "3.0.1173" description = "Tencent Cloud Hunyuan SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-hunyuan-3.0.1172.tar.gz", hash = "sha256:ae83b39c9da7302b10c4bffb7672ae95be72945b43e06a0b1ae9ac23bac2d43b"}, - {file = "tencentcloud_sdk_python_hunyuan-3.0.1172-py2.py3-none-any.whl", hash = "sha256:443908059ef1a00a798b7387f85e210d89c65b4f9db73629e53b3ec609b8528b"}, + {file = "tencentcloud-sdk-python-hunyuan-3.0.1173.tar.gz", hash = "sha256:287de951a04b3d49f07fa5075790345b1114cd84d0452185e596144e031adce5"}, + {file = "tencentcloud_sdk_python_hunyuan-3.0.1173-py2.py3-none-any.whl", hash = "sha256:da8cc5c931e2652bd66ad2d81b12be72e51b06c8353eaf06abab362e139db68e"}, ] [package.dependencies] -tencentcloud-sdk-python-common = "3.0.1172" +tencentcloud-sdk-python-common = "3.0.1173" [[package]] name = "threadpoolctl" @@ -7989,13 +8029,13 @@ typing-extensions = ">=3.7.4.3" [[package]] name = "types-requests" -version = "2.32.0.20240602" +version = "2.32.0.20240622" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" files = [ - {file = "types-requests-2.32.0.20240602.tar.gz", hash = "sha256:3f98d7bbd0dd94ebd10ff43a7fbe20c3b8528acace6d8efafef0b6a184793f06"}, - {file = "types_requests-2.32.0.20240602-py3-none-any.whl", hash = "sha256:ed3946063ea9fbc6b5fc0c44fa279188bae42d582cb63760be6cb4b9d06c3de8"}, + {file = "types-requests-2.32.0.20240622.tar.gz", hash = "sha256:ed5e8a412fcc39159d6319385c009d642845f250c63902718f605cd90faade31"}, + {file = "types_requests-2.32.0.20240622-py3-none-any.whl", hash = "sha256:97bac6b54b5bd4cf91d407e62f0932a74821bc2211f22116d9ee1dd643826caf"}, ] [package.dependencies] @@ -9040,4 +9080,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "cac196b2ddb59d7873fb3380d87b622d002613d6dc1d271a5c15e46817a38c55" +content-hash = "59a9d41baa5454de6c9032c8d9ca81d79e5a7137c654b8765034aebb8ec29793" From 3bbd75f1f255ec13c3447ac46b38def9e449c23b Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Sat, 22 Jun 2024 11:52:53 +0800 Subject: [PATCH 265/273] fix: firecrawl apikey not start with fc- (#5498) --- .../data-source-website/config-firecrawl-modal.tsx | 3 --- web/i18n/en-US/dataset-creation.ts | 3 +-- web/i18n/hi-IN/dataset-creation.ts | 3 +-- web/i18n/zh-Hans/dataset-creation.ts | 3 +-- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx index 4e196b87a86bf7..d68fc79b0d5a78 100644 --- a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx +++ b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx @@ -51,9 +51,6 @@ const ConfigFirecrawlModal: FC<Props> = ({ field: 'API Key', }) } - else if (!config.api_key.startsWith('fc-')) { - errorMsg = t(`${I18N_PREFIX}.apiKeyFormatError`) - } } if (errorMsg) { diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index cd7cd59857b350..b2090c33395819 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -13,8 +13,7 @@ const translation = { }, firecrawl: { configFirecrawl: 'Configure 🔥Firecrawl', - apiKeyPlaceholder: 'API key from firecrawl.dev, starting with "fc-"', - apiKeyFormatError: 'API key should start with "fc-"', + apiKeyPlaceholder: 'API key from firecrawl.dev', getApiKeyLinkText: 'Get your API key from firecrawl.dev', }, stepOne: { diff --git a/web/i18n/hi-IN/dataset-creation.ts b/web/i18n/hi-IN/dataset-creation.ts index 9e0ae6e60f60e8..59913c71ea6712 100644 --- a/web/i18n/hi-IN/dataset-creation.ts +++ b/web/i18n/hi-IN/dataset-creation.ts @@ -13,8 +13,7 @@ const translation = { }, firecrawl: { configFirecrawl: '🔥फायरक्रॉल को कॉन्फ़िगर करें', - apiKeyPlaceholder: 'firecrawl.dev से API कुंजी, "fc-" से शुरू होती है', - apiKeyFormatError: 'API कुंजी "fc-" से शुरू होनी चाहिए', + apiKeyPlaceholder: 'firecrawl.dev से API कुंजी', getApiKeyLinkText: 'firecrawl.dev से अपनी API कुंजी प्राप्त करें', }, stepOne: { diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index 5ae5b1d38d93cd..26e29fcbbbb641 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -13,8 +13,7 @@ const translation = { }, firecrawl: { configFirecrawl: '配置 🔥Firecrawl', - apiKeyPlaceholder: '从 firecrawl.dev 获取 API Key，以 "fc-" 开头', - apiKeyFormatError: 'API Key 应以 "fc-" 开头', + apiKeyPlaceholder: '从 firecrawl.dev 获取 API Key', getApiKeyLinkText: '从 firecrawl.dev 获取您的 API Key', }, stepOne: { From e8ad0339a3890fd7bc4a20a9517aab681f6f7b35 Mon Sep 17 00:00:00 2001 From: LXM <lxm.xupt@gmail.com> Date: Sat, 22 Jun 2024 12:25:23 +0800 Subject: [PATCH 266/273] fix: tongyi json output (#5396) --- .../model_providers/tongyi/llm/llm.py | 30 +++---- .../tongyi/test_response_format.py | 84 +++++++++++++++++++ 2 files changed, 96 insertions(+), 18 deletions(-) create mode 100644 api/tests/integration_tests/model_runtime/tongyi/test_response_format.py diff --git a/api/core/model_runtime/model_providers/tongyi/llm/llm.py b/api/core/model_runtime/model_providers/tongyi/llm/llm.py index 3d0a80144c6ddc..41d8f37aaf8477 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/llm.py +++ b/api/core/model_runtime/model_providers/tongyi/llm/llm.py @@ -18,7 +18,7 @@ ) from core.model_runtime.callbacks.base_callback import Callback -from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import ( AssistantPromptMessage, ImagePromptMessageContent, @@ -82,6 +82,7 @@ def _code_block_mode_wrapper(self, model: str, credentials: dict, <instructions> {{instructions}} </instructions> +You should also complete the text started with ``` but not tell ``` directly. """ code_block = model_parameters.get("response_format", "") @@ -113,21 +114,17 @@ def _code_block_mode_wrapper(self, model: str, credentials: dict, # insert the system message prompt_messages.insert(0, SystemPromptMessage( content=block_prompts - .replace("{{instructions}}", f"Please output a valid {code_block} object.") + .replace("{{instructions}}", f"Please output a valid {code_block} with markdown codeblocks.") )) - mode = self.get_model_mode(model, credentials) - if mode == LLMMode.CHAT: - if len(prompt_messages) > 0 and isinstance(prompt_messages[-1], UserPromptMessage): - # add ```JSON\n to the last message - prompt_messages[-1].content += f"\n```{code_block}\n" - else: - # append a user message - prompt_messages.append(UserPromptMessage( - content=f"```{code_block}\n" - )) + if len(prompt_messages) > 0 and isinstance(prompt_messages[-1], UserPromptMessage): + # add ```JSON\n to the last message + prompt_messages[-1].content += f"\n```{code_block}\n" else: - prompt_messages.append(AssistantPromptMessage(content=f"```{code_block}\n")) + # append a user message + prompt_messages.append(UserPromptMessage( + content=f"```{code_block}\n" + )) response = self._invoke( model=model, @@ -243,11 +240,8 @@ def _generate(self, model: str, credentials: dict, response = MultiModalConversation.call(**params, stream=stream) else: - if mode == LLMMode.CHAT: - params['messages'] = self._convert_prompt_messages_to_tongyi_messages(prompt_messages) - else: - params['prompt'] = prompt_messages[0].content.rstrip() - + # nothing different between chat model and completion model in tongyi + params['messages'] = self._convert_prompt_messages_to_tongyi_messages(prompt_messages) response = Generation.call(**params, result_format='message', stream=stream) diff --git a/api/tests/integration_tests/model_runtime/tongyi/test_response_format.py b/api/tests/integration_tests/model_runtime/tongyi/test_response_format.py new file mode 100644 index 00000000000000..1b0a38d5d15a00 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/tongyi/test_response_format.py @@ -0,0 +1,84 @@ +import json +import os +from collections.abc import Generator + +from core.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage +from core.model_runtime.model_providers.tongyi.llm.llm import TongyiLargeLanguageModel + + +def test_invoke_model_with_json_response(): + """ + Test the invocation of a model with JSON response. + """ + model_list = [ + "qwen-max-0403", + "qwen-max-1201", + "qwen-max-longcontext", + "qwen-max", + "qwen-plus-chat", + "qwen-plus", + "qwen-turbo-chat", + "qwen-turbo", + ] + for model_name in model_list: + print("testing model: ", model_name) + invoke_model_with_json_response(model_name) + + +def invoke_model_with_json_response(model_name="qwen-max-0403"): + """ + Method to invoke the model with JSON response format. + Args: + model_name (str): The name of the model to invoke. Defaults to "qwen-max-0403". + + Returns: + None + """ + model = TongyiLargeLanguageModel() + + response = model.invoke( + model=model_name, + credentials={ + 'dashscope_api_key': os.environ.get('TONGYI_DASHSCOPE_API_KEY') + }, + prompt_messages=[ + UserPromptMessage( + content='output json data with format `{"data": "test", "code": 200, "msg": "success"}' + ) + ], + model_parameters={ + 'temperature': 0.5, + 'max_tokens': 50, + 'response_format': 'JSON', + }, + stream=True, + user="abc-123" + ) + print("=====================================") + print(response) + assert isinstance(response, Generator) + output = "" + for chunk in response: + assert isinstance(chunk, LLMResultChunk) + assert isinstance(chunk.delta, LLMResultChunkDelta) + assert isinstance(chunk.delta.message, AssistantPromptMessage) + output += chunk.delta.message.content + assert is_json(output) + + +def is_json(s): + """ + Check if a string is a valid JSON. + + Args: + s (str): The string to check. + + Returns: + bool: True if the string is a valid JSON, False otherwise. + """ + try: + json.loads(s) + except ValueError: + return False + return True \ No newline at end of file From 48757e581ece25661a2a0827aefbb239ed2d0652 Mon Sep 17 00:00:00 2001 From: vccler <li.wei.sd.cn@gmail.com> Date: Sat, 22 Jun 2024 12:41:24 +0800 Subject: [PATCH 267/273] fix: zhipu tool calling, this PR fixes the bug described in issue #5496 (#5469) Co-authored-by: vccler <vccler@163.com> Co-authored-by: -LAN- <laipz8200@outlook.com> --- api/core/model_runtime/model_providers/zhipuai/llm/llm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index ee09b8cb742a5d..ff971964a8603e 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -10,6 +10,7 @@ PromptMessageRole, PromptMessageTool, SystemPromptMessage, + ToolPromptMessage, UserPromptMessage, ) from core.model_runtime.errors.validate import CredentialsValidateFailedError @@ -463,6 +464,8 @@ def _convert_one_message_to_text(self, message: PromptMessage) -> str: message_text = f"{ai_prompt} {content}" elif isinstance(message, SystemPromptMessage): message_text = content + elif isinstance(message, ToolPromptMessage): + message_text = content else: raise ValueError(f"Got unknown type {message}") From 57063095c13c1b140eb49d51946c3863b8be1099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= <hjlarry@163.com> Date: Sat, 22 Jun 2024 13:58:30 +0800 Subject: [PATCH 268/273] fix: summary of duckduckgo_search (#5488) --- .../builtin/duckduckgo/tools/ddgo_search.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/api/core/tools/provider/builtin/duckduckgo/tools/ddgo_search.py b/api/core/tools/provider/builtin/duckduckgo/tools/ddgo_search.py index bf963f3442fd6c..442f29f33dfcef 100644 --- a/api/core/tools/provider/builtin/duckduckgo/tools/ddgo_search.py +++ b/api/core/tools/provider/builtin/duckduckgo/tools/ddgo_search.py @@ -2,9 +2,20 @@ from duckduckgo_search import DDGS +from core.model_runtime.entities.message_entities import SystemPromptMessage from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.tool.builtin_tool import BuiltinTool +SUMMARY_PROMPT = """ +User's query: +{query} + +Here is the search engine result: +{content} + +Please summarize the result in a few sentences. +""" + class DuckDuckGoSearchTool(BuiltinTool): """ @@ -25,5 +36,12 @@ def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMe results = [res.get("body") for res in response] results = "\n".join(results) if require_summary: - results = self.summary(user_id=user_id, content=results) + results = self.summary_results(user_id=user_id, content=results, query=query) return self.create_text_message(text=results) + + def summary_results(self, user_id: str, content: str, query: str) -> str: + prompt = SUMMARY_PROMPT.format(query=query, content=content) + summary = self.invoke_model(user_id=user_id, prompt_messages=[ + SystemPromptMessage(content=prompt), + ], stop=[]) + return summary.message.content From 5217f7cf692b98cc4560fc2eec4a07b13da14179 Mon Sep 17 00:00:00 2001 From: Bowen Liang <liangbowen@gf.com.cn> Date: Sat, 22 Jun 2024 17:41:17 +0800 Subject: [PATCH 269/273] refactor: extract hosted service configs into dify config (#5504) --- api/config.py | 48 ---- api/configs/feature/__init__.py | 5 + .../feature/hosted_service/__init__.py | 208 ++++++++++++++++++ 3 files changed, 213 insertions(+), 48 deletions(-) create mode 100644 api/configs/feature/hosted_service/__init__.py diff --git a/api/config.py b/api/config.py index 2e722c7009e6f1..35e8ab5e94e192 100644 --- a/api/config.py +++ b/api/config.py @@ -3,20 +3,6 @@ import dotenv DEFAULTS = { - 'HOSTED_OPENAI_QUOTA_LIMIT': 200, - 'HOSTED_OPENAI_TRIAL_ENABLED': 'False', - 'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003', - 'HOSTED_OPENAI_PAID_ENABLED': 'False', - 'HOSTED_OPENAI_PAID_MODELS': 'gpt-4,gpt-4-turbo-preview,gpt-4-turbo-2024-04-09,gpt-4-1106-preview,gpt-4-0125-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,gpt-3.5-turbo-instruct,text-davinci-003', - 'HOSTED_AZURE_OPENAI_ENABLED': 'False', - 'HOSTED_AZURE_OPENAI_QUOTA_LIMIT': 200, - 'HOSTED_ANTHROPIC_QUOTA_LIMIT': 600000, - 'HOSTED_ANTHROPIC_TRIAL_ENABLED': 'False', - 'HOSTED_ANTHROPIC_PAID_ENABLED': 'False', - 'HOSTED_MODERATION_ENABLED': 'False', - 'HOSTED_MODERATION_PROVIDERS': '', - 'HOSTED_FETCH_APP_TEMPLATES_MODE': 'remote', - 'HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN': 'https://tmpl.dify.ai', } @@ -53,37 +39,3 @@ def __init__(self): 'CONSOLE_CORS_ALLOW_ORIGINS', get_env('CONSOLE_WEB_URL')) self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins( 'WEB_API_CORS_ALLOW_ORIGINS', '*') - - # ------------------------ - # Platform Configurations. - # ------------------------ - self.HOSTED_OPENAI_API_KEY = get_env('HOSTED_OPENAI_API_KEY') - self.HOSTED_OPENAI_API_BASE = get_env('HOSTED_OPENAI_API_BASE') - self.HOSTED_OPENAI_API_ORGANIZATION = get_env('HOSTED_OPENAI_API_ORGANIZATION') - self.HOSTED_OPENAI_TRIAL_ENABLED = get_bool_env('HOSTED_OPENAI_TRIAL_ENABLED') - self.HOSTED_OPENAI_TRIAL_MODELS = get_env('HOSTED_OPENAI_TRIAL_MODELS') - self.HOSTED_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_OPENAI_QUOTA_LIMIT')) - self.HOSTED_OPENAI_PAID_ENABLED = get_bool_env('HOSTED_OPENAI_PAID_ENABLED') - self.HOSTED_OPENAI_PAID_MODELS = get_env('HOSTED_OPENAI_PAID_MODELS') - - self.HOSTED_AZURE_OPENAI_ENABLED = get_bool_env('HOSTED_AZURE_OPENAI_ENABLED') - self.HOSTED_AZURE_OPENAI_API_KEY = get_env('HOSTED_AZURE_OPENAI_API_KEY') - self.HOSTED_AZURE_OPENAI_API_BASE = get_env('HOSTED_AZURE_OPENAI_API_BASE') - self.HOSTED_AZURE_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_AZURE_OPENAI_QUOTA_LIMIT')) - - self.HOSTED_ANTHROPIC_API_BASE = get_env('HOSTED_ANTHROPIC_API_BASE') - self.HOSTED_ANTHROPIC_API_KEY = get_env('HOSTED_ANTHROPIC_API_KEY') - self.HOSTED_ANTHROPIC_TRIAL_ENABLED = get_bool_env('HOSTED_ANTHROPIC_TRIAL_ENABLED') - self.HOSTED_ANTHROPIC_QUOTA_LIMIT = int(get_env('HOSTED_ANTHROPIC_QUOTA_LIMIT')) - self.HOSTED_ANTHROPIC_PAID_ENABLED = get_bool_env('HOSTED_ANTHROPIC_PAID_ENABLED') - - self.HOSTED_MINIMAX_ENABLED = get_bool_env('HOSTED_MINIMAX_ENABLED') - self.HOSTED_SPARK_ENABLED = get_bool_env('HOSTED_SPARK_ENABLED') - self.HOSTED_ZHIPUAI_ENABLED = get_bool_env('HOSTED_ZHIPUAI_ENABLED') - - self.HOSTED_MODERATION_ENABLED = get_bool_env('HOSTED_MODERATION_ENABLED') - self.HOSTED_MODERATION_PROVIDERS = get_env('HOSTED_MODERATION_PROVIDERS') - - # fetch app templates mode, remote, builtin, db(only for dify SaaS), default: remote - self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE') - self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN') diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 4d7d0dcd1eb350..e25a17f3b95fb8 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -2,6 +2,8 @@ from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt +from configs.feature.hosted_service import HostedServiceConfig + class SecurityConfig(BaseModel): """ @@ -421,5 +423,8 @@ class FeatureConfig( UpdateConfig, WorkflowConfig, WorkspaceConfig, + + # hosted services config + HostedServiceConfig, ): pass diff --git a/api/configs/feature/hosted_service/__init__.py b/api/configs/feature/hosted_service/__init__.py new file mode 100644 index 00000000000000..b09b6fd041ab7e --- /dev/null +++ b/api/configs/feature/hosted_service/__init__.py @@ -0,0 +1,208 @@ +from typing import Optional + +from pydantic import BaseModel, Field, NonNegativeInt + + +class HostedOpenAiConfig(BaseModel): + """ + Hosted OpenAI service config + """ + + HOSTED_OPENAI_API_KEY: Optional[str] = Field( + description='', + default=None, + ) + + HOSTED_OPENAI_API_BASE: Optional[str] = Field( + description='', + default=None, + ) + + HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field( + description='', + default=None, + ) + + HOSTED_OPENAI_TRIAL_ENABLED: bool = Field( + description='', + default=False, + ) + + HOSTED_OPENAI_TRIAL_MODELS: str = Field( + description='', + default='gpt-3.5-turbo,' + 'gpt-3.5-turbo-1106,' + 'gpt-3.5-turbo-instruct,' + 'gpt-3.5-turbo-16k,' + 'gpt-3.5-turbo-16k-0613,' + 'gpt-3.5-turbo-0613,' + 'gpt-3.5-turbo-0125,' + 'text-davinci-003', + ) + + HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field( + description='', + default=200, + ) + + HOSTED_OPENAI_PAID_ENABLED: bool = Field( + description='', + default=False, + ) + + HOSTED_OPENAI_PAID_MODELS: str = Field( + description='', + default='gpt-4,' + 'gpt-4-turbo-preview,' + 'gpt-4-turbo-2024-04-09,' + 'gpt-4-1106-preview,' + 'gpt-4-0125-preview,' + 'gpt-3.5-turbo,' + 'gpt-3.5-turbo-16k,' + 'gpt-3.5-turbo-16k-0613,' + 'gpt-3.5-turbo-1106,' + 'gpt-3.5-turbo-0613,' + 'gpt-3.5-turbo-0125,' + 'gpt-3.5-turbo-instruct,' + 'text-davinci-003', + ) + + +class HostedAzureOpenAiConfig(BaseModel): + """ + Hosted OpenAI service config + """ + + HOSTED_AZURE_OPENAI_ENABLED: bool = Field( + description='', + default=False, + ) + + HOSTED_OPENAI_API_KEY: Optional[str] = Field( + description='', + default=None, + ) + + HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field( + description='', + default=None, + ) + + HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field( + description='', + default=200, + ) + + +class HostedAnthropicConfig(BaseModel): + """ + Hosted Azure OpenAI service config + """ + + HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field( + description='', + default=None, + ) + + HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field( + description='', + default=None, + ) + + HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field( + description='', + default=False, + ) + + HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field( + description='', + default=600000, + ) + + HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field( + description='', + default=False, + ) + + +class HostedMinmaxConfig(BaseModel): + """ + Hosted Minmax service config + """ + + HOSTED_MINIMAX_ENABLED: bool = Field( + description='', + default=False, + ) + + +class HostedSparkConfig(BaseModel): + """ + Hosted Spark service config + """ + + HOSTED_SPARK_ENABLED: bool = Field( + description='', + default=False, + ) + + +class HostedZhipuAIConfig(BaseModel): + """ + Hosted Minmax service config + """ + + HOSTED_ZHIPUAI_ENABLED: bool = Field( + description='', + default=False, + ) + + +class HostedModerationConfig(BaseModel): + """ + Hosted Moderation service config + """ + + HOSTED_MODERATION_ENABLED: bool = Field( + description='', + default=False, + ) + + HOSTED_MODERATION_PROVIDERS: str = Field( + description='', + default='', + ) + + +class HostedFetchAppTemplateConfig(BaseModel): + """ + Hosted Moderation service config + """ + + HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field( + description='the mode for fetching app templates,' + ' default to remote,' + ' available values: remote, db, builtin', + default='remote', + ) + + HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field( + description='the domain for fetching remote app templates', + default='https://tmpl.dify.ai', + ) + + +class HostedServiceConfig( + # place the configs in alphabet order + HostedAnthropicConfig, + HostedAzureOpenAiConfig, + HostedFetchAppTemplateConfig, + HostedMinmaxConfig, + HostedOpenAiConfig, + HostedSparkConfig, + HostedZhipuAIConfig, + + # moderation + HostedModerationConfig, +): + pass From 29ca6815aeeb6783987a389435a62ba91ee2f19f Mon Sep 17 00:00:00 2001 From: Bowen Liang <liangbowen@gf.com.cn> Date: Sat, 22 Jun 2024 18:26:38 +0800 Subject: [PATCH 270/273] chore: use singular style in middleware config class name (#5502) --- .gitignore | 1 + api/configs/middleware/__init__.py | 68 +++++++++---------- .../{chroma_configs.py => chroma_config.py} | 2 +- .../{milvus_configs.py => milvus_config.py} | 2 +- ...search_configs.py => opensearch_config.py} | 2 +- .../{oracle_configs.py => oracle_config.py} | 2 +- ...pgvector_configs.py => pgvector_config.py} | 2 +- ...vectors_configs.py => pgvectors_config.py} | 2 +- .../{qdrant_configs.py => qdrant_config.py} | 2 +- .../vdb/{relyt_configs.py => relyt_config.py} | 2 +- ...or_configs.py => tencent_vector_config.py} | 2 +- ...ector_configs.py => tidb_vector_config.py} | 2 +- ...weaviate_configs.py => weaviate_config.py} | 2 +- 13 files changed, 46 insertions(+), 45 deletions(-) rename api/configs/middleware/vdb/{chroma_configs.py => chroma_config.py} (96%) rename api/configs/middleware/vdb/{milvus_configs.py => milvus_config.py} (96%) rename api/configs/middleware/vdb/{opensearch_configs.py => opensearch_config.py} (95%) rename api/configs/middleware/vdb/{oracle_configs.py => oracle_config.py} (95%) rename api/configs/middleware/vdb/{pgvector_configs.py => pgvector_config.py} (95%) rename api/configs/middleware/vdb/{pgvectors_configs.py => pgvectors_config.py} (95%) rename api/configs/middleware/vdb/{qdrant_configs.py => qdrant_config.py} (95%) rename api/configs/middleware/vdb/{relyt_configs.py => relyt_config.py} (95%) rename api/configs/middleware/vdb/{tencent_vector_configs.py => tencent_vector_config.py} (96%) rename api/configs/middleware/vdb/{tidb_vector_configs.py => tidb_vector_config.py} (95%) rename api/configs/middleware/vdb/{weaviate_configs.py => weaviate_config.py} (94%) diff --git a/.gitignore b/.gitignore index bfb7ad890218ec..e71b38161011a2 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,7 @@ web/.vscode/settings.json !.idea/vcs.xml !.idea/icon.png .ideaDataSources/ +*.iml api/.env api/storage/* diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index 5ea557c87dbfb7..c454f4e603de19 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -8,20 +8,20 @@ from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig -from configs.middleware.vdb.chroma_configs import ChromaConfigs -from configs.middleware.vdb.milvus_configs import MilvusConfigs -from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs -from configs.middleware.vdb.oracle_configs import OracleConfigs -from configs.middleware.vdb.pgvector_configs import PGVectorConfigs -from configs.middleware.vdb.pgvectors_configs import PGVectoRSConfigs -from configs.middleware.vdb.qdrant_configs import QdrantConfigs -from configs.middleware.vdb.relyt_configs import RelytConfigs -from configs.middleware.vdb.tencent_vector_configs import TencentVectorDBConfigs -from configs.middleware.vdb.tidb_vector_configs import TiDBVectorConfigs -from configs.middleware.vdb.weaviate_configs import WeaviateConfigs - - -class StorageConfigs(BaseModel): +from configs.middleware.vdb.chroma_config import ChromaConfig +from configs.middleware.vdb.milvus_config import MilvusConfig +from configs.middleware.vdb.opensearch_config import OpenSearchConfig +from configs.middleware.vdb.oracle_config import OracleConfig +from configs.middleware.vdb.pgvector_config import PGVectorConfig +from configs.middleware.vdb.pgvectors_config import PGVectoRSConfig +from configs.middleware.vdb.qdrant_config import QdrantConfig +from configs.middleware.vdb.relyt_config import RelytConfig +from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig +from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig +from configs.middleware.vdb.weaviate_config import WeaviateConfig + + +class StorageConfig(BaseModel): STORAGE_TYPE: str = Field( description='storage type,' ' default to `local`,' @@ -35,21 +35,21 @@ class StorageConfigs(BaseModel): ) -class VectorStoreConfigs(BaseModel): +class VectorStoreConfig(BaseModel): VECTOR_STORE: Optional[str] = Field( description='vector store type', default=None, ) -class KeywordStoreConfigs(BaseModel): +class KeywordStoreConfig(BaseModel): KEYWORD_STORE: str = Field( description='keyword store type', default='jieba', ) -class DatabaseConfigs: +class DatabaseConfig: DB_HOST: str = Field( description='db host', default='localhost', @@ -130,7 +130,7 @@ def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]: } -class CeleryConfigs(DatabaseConfigs): +class CeleryConfig(DatabaseConfig): CELERY_BACKEND: str = Field( description='Celery backend, available values are `database`, `redis`', default='database', @@ -155,13 +155,13 @@ def BROKER_USE_SSL(self) -> bool: class MiddlewareConfig( # place the configs in alphabet order - CeleryConfigs, - DatabaseConfigs, - KeywordStoreConfigs, + CeleryConfig, + DatabaseConfig, + KeywordStoreConfig, RedisConfig, # configs of storage and storage providers - StorageConfigs, + StorageConfig, AliyunOSSStorageConfig, AzureBlobStorageConfig, GoogleCloudStorageConfig, @@ -169,17 +169,17 @@ class MiddlewareConfig( S3StorageConfig, # configs of vdb and vdb providers - VectorStoreConfigs, - ChromaConfigs, - MilvusConfigs, - OpenSearchConfigs, - OracleConfigs, - PGVectorConfigs, - PGVectoRSConfigs, - QdrantConfigs, - RelytConfigs, - TencentVectorDBConfigs, - TiDBVectorConfigs, - WeaviateConfigs, + VectorStoreConfig, + ChromaConfig, + MilvusConfig, + OpenSearchConfig, + OracleConfig, + PGVectorConfig, + PGVectoRSConfig, + QdrantConfig, + RelytConfig, + TencentVectorDBConfig, + TiDBVectorConfig, + WeaviateConfig, ): pass diff --git a/api/configs/middleware/vdb/chroma_configs.py b/api/configs/middleware/vdb/chroma_config.py similarity index 96% rename from api/configs/middleware/vdb/chroma_configs.py rename to api/configs/middleware/vdb/chroma_config.py index ddb61a564d0bf2..a764ddc7968097 100644 --- a/api/configs/middleware/vdb/chroma_configs.py +++ b/api/configs/middleware/vdb/chroma_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class ChromaConfigs(BaseModel): +class ChromaConfig(BaseModel): """ Chroma configs """ diff --git a/api/configs/middleware/vdb/milvus_configs.py b/api/configs/middleware/vdb/milvus_config.py similarity index 96% rename from api/configs/middleware/vdb/milvus_configs.py rename to api/configs/middleware/vdb/milvus_config.py index f291a0e0661de5..88855db8772e51 100644 --- a/api/configs/middleware/vdb/milvus_configs.py +++ b/api/configs/middleware/vdb/milvus_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class MilvusConfigs(BaseModel): +class MilvusConfig(BaseModel): """ Milvus configs """ diff --git a/api/configs/middleware/vdb/opensearch_configs.py b/api/configs/middleware/vdb/opensearch_config.py similarity index 95% rename from api/configs/middleware/vdb/opensearch_configs.py rename to api/configs/middleware/vdb/opensearch_config.py index 4e29c9c9d51ac0..4d77e7be9468a7 100644 --- a/api/configs/middleware/vdb/opensearch_configs.py +++ b/api/configs/middleware/vdb/opensearch_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class OpenSearchConfigs(BaseModel): +class OpenSearchConfig(BaseModel): """ OpenSearch configs """ diff --git a/api/configs/middleware/vdb/oracle_configs.py b/api/configs/middleware/vdb/oracle_config.py similarity index 95% rename from api/configs/middleware/vdb/oracle_configs.py rename to api/configs/middleware/vdb/oracle_config.py index a70fd3efc2d3f9..e5c479a66fd00c 100644 --- a/api/configs/middleware/vdb/oracle_configs.py +++ b/api/configs/middleware/vdb/oracle_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class OracleConfigs(BaseModel): +class OracleConfig(BaseModel): """ ORACLE configs """ diff --git a/api/configs/middleware/vdb/pgvector_configs.py b/api/configs/middleware/vdb/pgvector_config.py similarity index 95% rename from api/configs/middleware/vdb/pgvector_configs.py rename to api/configs/middleware/vdb/pgvector_config.py index 2c6cf7486b7584..c544a84031cf50 100644 --- a/api/configs/middleware/vdb/pgvector_configs.py +++ b/api/configs/middleware/vdb/pgvector_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class PGVectorConfigs(BaseModel): +class PGVectorConfig(BaseModel): """ PGVector configs """ diff --git a/api/configs/middleware/vdb/pgvectors_configs.py b/api/configs/middleware/vdb/pgvectors_config.py similarity index 95% rename from api/configs/middleware/vdb/pgvectors_configs.py rename to api/configs/middleware/vdb/pgvectors_config.py index e2c8c1f7250fcc..78cb4e570e7b59 100644 --- a/api/configs/middleware/vdb/pgvectors_configs.py +++ b/api/configs/middleware/vdb/pgvectors_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class PGVectoRSConfigs(BaseModel): +class PGVectoRSConfig(BaseModel): """ PGVectoRS configs """ diff --git a/api/configs/middleware/vdb/qdrant_configs.py b/api/configs/middleware/vdb/qdrant_config.py similarity index 95% rename from api/configs/middleware/vdb/qdrant_configs.py rename to api/configs/middleware/vdb/qdrant_config.py index bee49921e6e659..f0223ffa1c3af7 100644 --- a/api/configs/middleware/vdb/qdrant_configs.py +++ b/api/configs/middleware/vdb/qdrant_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt -class QdrantConfigs(BaseModel): +class QdrantConfig(BaseModel): """ Qdrant configs """ diff --git a/api/configs/middleware/vdb/relyt_configs.py b/api/configs/middleware/vdb/relyt_config.py similarity index 95% rename from api/configs/middleware/vdb/relyt_configs.py rename to api/configs/middleware/vdb/relyt_config.py index 621ecfb17307bb..b550fa8e00ba01 100644 --- a/api/configs/middleware/vdb/relyt_configs.py +++ b/api/configs/middleware/vdb/relyt_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class RelytConfigs(BaseModel): +class RelytConfig(BaseModel): """ Relyt configs """ diff --git a/api/configs/middleware/vdb/tencent_vector_configs.py b/api/configs/middleware/vdb/tencent_vector_config.py similarity index 96% rename from api/configs/middleware/vdb/tencent_vector_configs.py rename to api/configs/middleware/vdb/tencent_vector_config.py index 7a5418030b566a..083f10b40b06c3 100644 --- a/api/configs/middleware/vdb/tencent_vector_configs.py +++ b/api/configs/middleware/vdb/tencent_vector_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class TencentVectorDBConfigs(BaseModel): +class TencentVectorDBConfig(BaseModel): """ Tencent Vector configs """ diff --git a/api/configs/middleware/vdb/tidb_vector_configs.py b/api/configs/middleware/vdb/tidb_vector_config.py similarity index 95% rename from api/configs/middleware/vdb/tidb_vector_configs.py rename to api/configs/middleware/vdb/tidb_vector_config.py index 5b81f8807bed90..53f985e3866e38 100644 --- a/api/configs/middleware/vdb/tidb_vector_configs.py +++ b/api/configs/middleware/vdb/tidb_vector_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class TiDBVectorConfigs(BaseModel): +class TiDBVectorConfig(BaseModel): """ TiDB Vector configs """ diff --git a/api/configs/middleware/vdb/weaviate_configs.py b/api/configs/middleware/vdb/weaviate_config.py similarity index 94% rename from api/configs/middleware/vdb/weaviate_configs.py rename to api/configs/middleware/vdb/weaviate_config.py index 369b631b6f8966..d1c9f5b5bea174 100644 --- a/api/configs/middleware/vdb/weaviate_configs.py +++ b/api/configs/middleware/vdb/weaviate_config.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, PositiveInt -class WeaviateConfigs(BaseModel): +class WeaviateConfig(BaseModel): """ Weaviate configs """ From 226cfe68063f9b1e083b6f7009949904b2f36a9e Mon Sep 17 00:00:00 2001 From: takatost <takatost@gmail.com> Date: Sat, 22 Jun 2024 19:12:14 +0800 Subject: [PATCH 271/273] feat: make show citations enable default --- api/constants/model_template.py | 6 +++--- api/core/app/apps/advanced_chat/app_generator.py | 4 ++++ api/core/app/apps/agent_chat/app_generator.py | 5 +++++ api/core/app/apps/chat/app_config_manager.py | 3 +++ api/core/app/apps/chat/app_generator.py | 5 +++++ api/models/model.py | 2 +- 6 files changed, 21 insertions(+), 4 deletions(-) diff --git a/api/constants/model_template.py b/api/constants/model_template.py index 42e182236ff793..cc5a37025479fd 100644 --- a/api/constants/model_template.py +++ b/api/constants/model_template.py @@ -22,7 +22,7 @@ 'model_config': { 'model': { "provider": "openai", - "name": "gpt-4", + "name": "gpt-4o", "mode": "chat", "completion_params": {} }, @@ -51,7 +51,7 @@ 'model_config': { 'model': { "provider": "openai", - "name": "gpt-4", + "name": "gpt-4o", "mode": "chat", "completion_params": {} } @@ -77,7 +77,7 @@ 'model_config': { 'model': { "provider": "openai", - "name": "gpt-4", + "name": "gpt-4o", "mode": "chat", "completion_params": {} } diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 3b1ee3578dea3c..2483e9f66c9330 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -84,6 +84,10 @@ def generate(self, app_model: App, workflow=workflow ) + if invoke_from == InvokeFrom.DEBUGGER: + # always enable retriever resource in debugger mode + app_config.additional_features.show_retrieve_source = True + # init application generate entity application_generate_entity = AdvancedChatAppGenerateEntity( task_id=str(uuid.uuid4()), diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index 407fb931ecb9bd..800dc1616da485 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -82,6 +82,11 @@ def generate(self, app_model: App, config=args.get('model_config') ) + # always enable retriever resource in debugger mode + override_model_config_dict["retriever_resource"] = { + "enabled": True + } + # parse files files = args['files'] if args.get('files') else [] message_file_parser = MessageFileParser(tenant_id=app_model.tenant_id, app_id=app_model.id) diff --git a/api/core/app/apps/chat/app_config_manager.py b/api/core/app/apps/chat/app_config_manager.py index 925062a66a5925..a286c349b2715b 100644 --- a/api/core/app/apps/chat/app_config_manager.py +++ b/api/core/app/apps/chat/app_config_manager.py @@ -50,6 +50,9 @@ def get_app_config(cls, app_model: App, app_model_config_dict = app_model_config.to_dict() config_dict = app_model_config_dict.copy() else: + if not override_config_dict: + raise Exception('override_config_dict is required when config_from is ARGS') + config_dict = override_config_dict app_mode = AppMode.value_of(app_model.mode) diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index 505ada09db5c90..b92eed82c36f28 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -79,6 +79,11 @@ def generate(self, app_model: App, config=args.get('model_config') ) + # always enable retriever resource in debugger mode + override_model_config_dict["retriever_resource"] = { + "enabled": True + } + # parse files files = args['files'] if args.get('files') else [] message_file_parser = MessageFileParser(tenant_id=app_model.tenant_id, app_id=app_model.id) diff --git a/api/models/model.py b/api/models/model.py index 3024be0b4c86b6..7c9fa0477fed61 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -265,7 +265,7 @@ def text_to_speech_dict(self) -> dict: @property def retriever_resource_dict(self) -> dict: return json.loads(self.retriever_resource) if self.retriever_resource \ - else {"enabled": False} + else {"enabled": True} @property def annotation_reply_dict(self) -> dict: From f1128389b3a65e59f13a1e86c931c22deee42aff Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Mon, 24 Jun 2024 14:00:09 +0800 Subject: [PATCH 272/273] fix: monitor copywrite --- .../app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx | 2 +- web/i18n/en-US/common.ts | 2 +- web/i18n/zh-Hans/common.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx index ba127f62abb11d..ba441f29aba164 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/panel.tsx @@ -29,7 +29,7 @@ const Title = ({ return ( <div className={cn(className, 'flex items-center text-lg font-semibold text-gray-900')}> - {t('appOverview.overview.title')} + {t('common.appMenus.overview')} </div> ) } diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts index 2f65e5ee0845ea..2f583dc0335e39 100644 --- a/web/i18n/en-US/common.ts +++ b/web/i18n/en-US/common.ts @@ -440,7 +440,7 @@ const translation = { latestAvailable: 'Dify {{version}} is the latest version available.', }, appMenus: { - overview: 'Observability', + overview: 'Monitoring', promptEng: 'Orchestrate', apiAccess: 'API Access', logAndAnn: 'Logs & Ann.', diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts index 9981c749088bc5..49fe6f6caded80 100644 --- a/web/i18n/zh-Hans/common.ts +++ b/web/i18n/zh-Hans/common.ts @@ -436,7 +436,7 @@ const translation = { latestAvailable: 'Dify {{version}} 已是最新版本。', }, appMenus: { - overview: '可观察性', + overview: '监测', promptEng: '编排', apiAccess: '访问 API', logAndAnn: '日志与标注', From f3397cfaffff130b203244e2fea2cc76a7a5fb31 Mon Sep 17 00:00:00 2001 From: Joel <iamjoel007@gmail.com> Date: Mon, 24 Jun 2024 14:06:44 +0800 Subject: [PATCH 273/273] chore: host required --- .../[appId]/overview/tracing/provider-config-modal.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index e4760f5d3de0e8..2411d2baa489b9 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -105,8 +105,10 @@ const ProviderConfigModal: FC<Props> = ({ const postData = config as LangFuseConfig if (!errorMessage && !postData.secret_key) errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.secretKey`) }) - if (!postData.public_key) + if (!errorMessage && !postData.public_key) errorMessage = t('common.errorMsg.fieldRequired', { field: t(`${I18N_PREFIX}.publicKey`) }) + if (!errorMessage && !postData.host) + errorMessage = t('common.errorMsg.fieldRequired', { field: 'Host' }) } return errorMessage @@ -206,6 +208,7 @@ const ProviderConfigModal: FC<Props> = ({ <Field label='Host' labelClassName='!text-sm' + isRequired value={(config as LangFuseConfig).host} onChange={handleConfigChange('host')} placeholder='https://cloud.langfuse.com'