diff --git a/api/.env.example b/api/.env.example index 40f35cff6c66f6..00b6c28a919abe 100644 --- a/api/.env.example +++ b/api/.env.example @@ -383,7 +383,7 @@ LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S LOG_TZ=UTC # Indexing configuration -INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000 +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 # Workflow runtime configuration WORKFLOW_MAX_EXECUTION_STEPS=500 @@ -413,4 +413,4 @@ RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 CREATE_TIDB_SERVICE_JOB_ENABLED=false -RETRIEVAL_TOP_N=0 \ No newline at end of file +RETRIEVAL_TOP_N=0 diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 275b16a9131add..e88eb980577121 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -647,7 +647,7 @@ class IndexingConfig(BaseSettings): INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field( description="Maximum token length for text segmentation during indexing", - default=1000, + default=4000, ) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index f20261abc227b6..de3b4f62620501 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -106,6 +106,7 @@ def get(self): # get default rules mode = DocumentService.DEFAULT_RULES["mode"] rules = DocumentService.DEFAULT_RULES["rules"] + limits = DocumentService.DEFAULT_RULES["limits"] if document_id: # get the latest process rule document = Document.query.get_or_404(document_id) @@ -132,7 +133,7 @@ def get(self): mode = dataset_process_rule.mode rules = dataset_process_rule.rules_dict - return {"mode": mode, "rules": rules} + return {"mode": mode, "rules": rules, "limits": limits} class DatasetDocumentListApi(Resource): diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index d38729f31e913e..a1014e8e0ad908 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -406,6 +406,9 @@ class DocumentService: ], "segmentation": {"delimiter": "\n", "max_tokens": 500, "chunk_overlap": 50}, }, + "limits": { + "indexing_max_segmentation_tokens_length": dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH, + }, } DOCUMENT_METADATA_SCHEMA = { diff --git a/docker-legacy/docker-compose.yaml b/docker-legacy/docker-compose.yaml index aaca3c9c12b74d..f684c9f1ac7e23 100644 --- a/docker-legacy/docker-compose.yaml +++ b/docker-legacy/docker-compose.yaml @@ -210,7 +210,7 @@ services: SSRF_PROXY_HTTP_URL: 'http://ssrf_proxy:3128' SSRF_PROXY_HTTPS_URL: 'http://ssrf_proxy:3128' # Indexing configuration - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: 1000 + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: 4000 depends_on: - db - redis diff --git a/docker/.env.example b/docker/.env.example index 3be33f6338e92a..719a025877e449 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -683,7 +683,7 @@ SMTP_OPPORTUNISTIC_TLS=false # ------------------------------ # Maximum length of segmentation tokens for indexing -INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000 +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 # Member invitation link valid time (hours), # Default: 72. diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index aa3c810d3faff0..268f77ba2442e0 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -247,7 +247,7 @@ x-shared-env: &shared-api-worker-env SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false} RESEND_API_KEY: ${RESEND_API_KEY:-your-resend-api-key} RESEND_API_URL: ${RESEND_API_URL:-https://api.resend.com} - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-1000} + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5} CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194} diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 634f031134f78f..f915c68fef0345 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -122,7 +122,8 @@ const StepTwo = ({ const setSegmentIdentifier = useCallback((value: string) => { doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER) }, []) - const [max, setMax] = useState(4000) // default chunk length + const [maxChunkLength, setMaxChunkLength] = useState(4000) // default chunk length + const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000) const [overlap, setOverlap] = useState(50) const [rules, setRules] = useState([]) const [defaultConfig, setDefaultConfig] = useState() @@ -196,7 +197,7 @@ const StepTwo = ({ const resetRules = () => { if (defaultConfig) { setSegmentIdentifier(defaultConfig.segmentation.separator) - setMax(defaultConfig.segmentation.max_tokens) + setMaxChunkLength(defaultConfig.segmentation.max_tokens) setOverlap(defaultConfig.segmentation.chunk_overlap) setRules(defaultConfig.pre_processing_rules) } @@ -212,8 +213,8 @@ const StepTwo = ({ } const confirmChangeCustomConfig = () => { - if (segmentationType === SegmentType.CUSTOM && max > 4000) { - Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') }) + if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) { + Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) }) return } setCustomFileIndexingEstimate(null) @@ -234,7 +235,7 @@ const StepTwo = ({ pre_processing_rules: rules, segmentation: { separator: unescape(segmentIdentifier), - max_tokens: max, + max_tokens: maxChunkLength, chunk_overlap: overlap, }, } @@ -339,12 +340,12 @@ const StepTwo = ({ ) const getCreationParams = () => { let params - if (segmentationType === SegmentType.CUSTOM && overlap > max) { + if (segmentationType === SegmentType.CUSTOM && overlap > maxChunkLength) { Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.overlapCheck') }) return } - if (segmentationType === SegmentType.CUSTOM && max > 4000) { - Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') }) + if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) { + Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) }) return } if (isSetting) { @@ -415,7 +416,8 @@ const StepTwo = ({ const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' }) const separator = res.rules.segmentation.separator setSegmentIdentifier(separator) - setMax(res.rules.segmentation.max_tokens) + setMaxChunkLength(res.rules.segmentation.max_tokens) + setLimitMaxChunkLength(res.limits.indexing_max_segmentation_tokens_length) setOverlap(res.rules.segmentation.chunk_overlap) setRules(res.rules.pre_processing_rules) setDefaultConfig(res.rules) @@ -432,7 +434,7 @@ const StepTwo = ({ const max = rules.segmentation.max_tokens const overlap = rules.segmentation.chunk_overlap setSegmentIdentifier(separator) - setMax(max) + setMaxChunkLength(max) setOverlap(overlap) setRules(rules.pre_processing_rules) setDefaultConfig(rules) @@ -670,10 +672,10 @@ const StepTwo = ({ type="number" className='h-9' placeholder={t('datasetCreation.stepTwo.maxLength') || ''} - value={max} - max={4000} + value={maxChunkLength} + max={limitMaxChunkLength} min={1} - onChange={e => setMax(parseInt(e.target.value.replace(/^0+/, ''), 10))} + onChange={e => setMaxChunkLength(parseInt(e.target.value.replace(/^0+/, ''), 10))} /> diff --git a/web/i18n/de-DE/dataset-creation.ts b/web/i18n/de-DE/dataset-creation.ts index 18f08814cb93ad..b27a732d1877c3 100644 --- a/web/i18n/de-DE/dataset-creation.ts +++ b/web/i18n/de-DE/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: 'Preprocess-Website', webpageUnit: 'Seiten', separatorTip: 'Ein Trennzeichen ist das Zeichen, das zum Trennen von Text verwendet wird. \\n\\n und \\n sind häufig verwendete Trennzeichen zum Trennen von Absätzen und Zeilen. In Kombination mit Kommas (\\n\\n,\\n) werden Absätze nach Zeilen segmentiert, wenn die maximale Blocklänge überschritten wird. Sie können auch spezielle, von Ihnen selbst definierte Trennzeichen verwenden (z. B. ***).', - maxLengthCheck: 'Die maximale Stücklänge sollte weniger als 4000 betragen', + maxLengthCheck: 'Die maximale Stücklänge sollte weniger als {{limit}} betragen', }, stepThree: { creationTitle: '🎉 Wissen erstellt', diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts index de885671a7bff6..6caf0056f81eb0 100644 --- a/web/i18n/en-US/dataset-creation.ts +++ b/web/i18n/en-US/dataset-creation.ts @@ -103,7 +103,7 @@ const translation = { separatorTip: 'A delimiter is the character used to separate text. \\n\\n and \\n are commonly used delimiters for separating paragraphs and lines. Combined with commas (\\n\\n,\\n), paragraphs will be segmented by lines when exceeding the maximum chunk length. You can also use special delimiters defined by yourself (e.g. ***).', separatorPlaceholder: '\\n\\n for separating paragraphs; \\n for separating lines', maxLength: 'Maximum chunk length', - maxLengthCheck: 'Maximum chunk length should be less than 4000', + maxLengthCheck: 'Maximum chunk length should be less than {{limit}}', overlap: 'Chunk overlap', overlapTip: 'Setting the chunk overlap can maintain the semantic relevance between them, enhancing the retrieve effect. It is recommended to set 10%-25% of the maximum chunk size.', overlapCheck: 'chunk overlap should not bigger than maximum chunk length', diff --git a/web/i18n/es-ES/dataset-creation.ts b/web/i18n/es-ES/dataset-creation.ts index 02415b4359e533..2b6cac1b0ee0ad 100644 --- a/web/i18n/es-ES/dataset-creation.ts +++ b/web/i18n/es-ES/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'Para cambiar el método de índice, por favor ve a la ', datasetSettingLink: 'configuración del conocimiento.', separatorTip: 'Un delimitador es el carácter que se utiliza para separar el texto. \\n\\n y \\n son delimitadores comúnmente utilizados para separar párrafos y líneas. Combinado con comas (\\n\\n,\\n), los párrafos se segmentarán por líneas cuando excedan la longitud máxima del fragmento. También puede utilizar delimitadores especiales definidos por usted mismo (por ejemplo, ***).', - maxLengthCheck: 'La longitud máxima del fragmento debe ser inferior a 4000', + maxLengthCheck: 'La longitud máxima del fragmento debe ser inferior a {{limit}}', }, stepThree: { creationTitle: '🎉 Conocimiento creado', diff --git a/web/i18n/fa-IR/dataset-creation.ts b/web/i18n/fa-IR/dataset-creation.ts index 195d968228d250..0b5f42827eb1d0 100644 --- a/web/i18n/fa-IR/dataset-creation.ts +++ b/web/i18n/fa-IR/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'برای تغییر روش شاخص، لطفاً به', datasetSettingLink: 'تنظیمات دانش بروید.', separatorTip: 'جداکننده نویسه ای است که برای جداسازی متن استفاده می شود. \\n\\n و \\n معمولا برای جداسازی پاراگراف ها و خطوط استفاده می شوند. همراه با کاما (\\n\\n,\\n)، پاراگراف ها زمانی که از حداکثر طول تکه فراتر می روند، با خطوط تقسیم بندی می شوند. همچنین می توانید از جداکننده های خاصی که توسط خودتان تعریف شده اند استفاده کنید (مثلا ***).', - maxLengthCheck: 'حداکثر طول تکه باید کمتر از 4000 باشد', + maxLengthCheck: 'حداکثر طول تکه باید کمتر از {{limit}} باشد', }, stepThree: { creationTitle: ' دانش ایجاد شد', diff --git a/web/i18n/fr-FR/dataset-creation.ts b/web/i18n/fr-FR/dataset-creation.ts index 0c866d51e6f94c..c777af4fbecaca 100644 --- a/web/i18n/fr-FR/dataset-creation.ts +++ b/web/i18n/fr-FR/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: 'Pages', websiteSource: 'Site web de prétraitement', separatorTip: 'Un délimiteur est le caractère utilisé pour séparer le texte. \\n\\n et \\n sont des délimiteurs couramment utilisés pour séparer les paragraphes et les lignes. Combiné à des virgules (\\n\\n,\\n), les paragraphes seront segmentés par des lignes lorsqu’ils dépasseront la longueur maximale des morceaux. Vous pouvez également utiliser des délimiteurs spéciaux définis par vous-même (par exemple ***).', - maxLengthCheck: 'La longueur maximale des morceaux doit être inférieure à 4000', + maxLengthCheck: 'La longueur maximale des morceaux doit être inférieure à {{limit}}', }, stepThree: { creationTitle: '🎉 Connaissance créée', diff --git a/web/i18n/hi-IN/dataset-creation.ts b/web/i18n/hi-IN/dataset-creation.ts index a242eebb24c9e3..51063dcb143f1a 100644 --- a/web/i18n/hi-IN/dataset-creation.ts +++ b/web/i18n/hi-IN/dataset-creation.ts @@ -164,7 +164,7 @@ const translation = { retrievalSettingTip: 'इंडेक्स विधि बदलने के लिए, कृपया जाएं ', datasetSettingLink: 'ज्ञान सेटिंग्स।', separatorTip: 'एक सीमांकक पाठ को अलग करने के लिए उपयोग किया जाने वाला वर्ण है। \\n\\n और \\n आमतौर पर पैराग्राफ और लाइनों को अलग करने के लिए उपयोग किए जाने वाले सीमांकक हैं। अल्पविराम (\\n\\n,\\n) के साथ संयुक्त, अधिकतम खंड लंबाई से अधिक होने पर अनुच्छेदों को पंक्तियों द्वारा खंडित किया जाएगा। आप स्वयं द्वारा परिभाषित विशेष सीमांकक का भी उपयोग कर सकते हैं (उदा. ***).', - maxLengthCheck: 'अधिकतम चंक लंबाई 4000 से कम होनी चाहिए', + maxLengthCheck: 'अधिकतम चंक लंबाई {{limit}} से कम होनी चाहिए', }, stepThree: { creationTitle: '🎉 ज्ञान बनाया गया', diff --git a/web/i18n/it-IT/dataset-creation.ts b/web/i18n/it-IT/dataset-creation.ts index ebbe34f143ef93..af174ce41f1dee 100644 --- a/web/i18n/it-IT/dataset-creation.ts +++ b/web/i18n/it-IT/dataset-creation.ts @@ -167,7 +167,7 @@ const translation = { retrievalSettingTip: 'Per cambiare il metodo di indicizzazione, vai alle ', datasetSettingLink: 'impostazioni della Conoscenza.', separatorTip: 'Un delimitatore è il carattere utilizzato per separare il testo. \\n\\n e \\n sono delimitatori comunemente usati per separare paragrafi e righe. In combinazione con le virgole (\\n\\n,\\n), i paragrafi verranno segmentati per righe quando superano la lunghezza massima del blocco. È inoltre possibile utilizzare delimitatori speciali definiti dall\'utente (ad es. ***).', - maxLengthCheck: 'La lunghezza massima del blocco deve essere inferiore a 4000', + maxLengthCheck: 'La lunghezza massima del blocco deve essere inferiore a {{limit}}', }, stepThree: { creationTitle: '🎉 Conoscenza creata', diff --git a/web/i18n/ja-JP/dataset-creation.ts b/web/i18n/ja-JP/dataset-creation.ts index 597ec5f8f1d5c5..9ae6c4e7374401 100644 --- a/web/i18n/ja-JP/dataset-creation.ts +++ b/web/i18n/ja-JP/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: '検索方法を変更するには、', datasetSettingLink: 'ナレッジ設定', separatorTip: '区切り文字は、テキストを区切るために使用される文字です。\\n\\n と \\n は、段落と行を区切るために一般的に使用される区切り記号です。カンマ (\\n\\n,\\n) と組み合わせると、最大チャンク長を超えると、段落は行で区切られます。自分で定義した特別な区切り文字を使用することもできます(例:***)。', - maxLengthCheck: 'チャンクの最大長は 4000 未満にする必要があります', + maxLengthCheck: 'チャンクの最大長は {{limit}} 未満にする必要があります', }, stepThree: { creationTitle: '🎉 ナレッジが作成されました', diff --git a/web/i18n/ko-KR/dataset-creation.ts b/web/i18n/ko-KR/dataset-creation.ts index 52f7aff75da808..ee5abb5189b408 100644 --- a/web/i18n/ko-KR/dataset-creation.ts +++ b/web/i18n/ko-KR/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: '페이지', websiteSource: '웹 사이트 전처리', separatorTip: '구분 기호는 텍스트를 구분하는 데 사용되는 문자입니다. \\n\\n 및 \\n은 단락과 줄을 구분하는 데 일반적으로 사용되는 구분 기호입니다. 쉼표(\\n\\n,\\n)와 함께 사용하면 최대 청크 길이를 초과할 경우 단락이 줄로 분할됩니다. 직접 정의한 특수 구분 기호(예: ***)를 사용할 수도 있습니다.', - maxLengthCheck: '최대 청크 길이는 4000 미만이어야 합니다.', + maxLengthCheck: '최대 청크 길이는 {{limit}} 미만이어야 합니다.', }, stepThree: { creationTitle: '🎉 지식이 생성되었습니다', diff --git a/web/i18n/pl-PL/dataset-creation.ts b/web/i18n/pl-PL/dataset-creation.ts index 81adff63367f12..c12ed662786371 100644 --- a/web/i18n/pl-PL/dataset-creation.ts +++ b/web/i18n/pl-PL/dataset-creation.ts @@ -155,7 +155,7 @@ const translation = { webpageUnit: 'Stron', websiteSource: 'Witryna internetowa przetwarzania wstępnego', separatorTip: 'Ogranicznik to znak używany do oddzielania tekstu. \\n\\n i \\n są powszechnie używanymi ogranicznikami do oddzielania akapitów i wierszy. W połączeniu z przecinkami (\\n\\n,\\n), akapity będą segmentowane wierszami po przekroczeniu maksymalnej długości fragmentu. Możesz również skorzystać ze zdefiniowanych przez siebie specjalnych ograniczników (np. ***).', - maxLengthCheck: 'Maksymalna długość porcji powinna być mniejsza niż 4000', + maxLengthCheck: 'Maksymalna długość porcji powinna być mniejsza niż {{limit}}', }, stepThree: { creationTitle: '🎉 Utworzono Wiedzę', diff --git a/web/i18n/pt-BR/dataset-creation.ts b/web/i18n/pt-BR/dataset-creation.ts index 5ab24565625fe0..8534bdce1d3de8 100644 --- a/web/i18n/pt-BR/dataset-creation.ts +++ b/web/i18n/pt-BR/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: 'Site de pré-processamento', webpageUnit: 'Páginas', separatorTip: 'Um delimitador é o caractere usado para separar o texto. \\n\\n e \\n são delimitadores comumente usados para separar parágrafos e linhas. Combinado com vírgulas (\\n\\n,\\n), os parágrafos serão segmentados por linhas ao exceder o comprimento máximo do bloco. Você também pode usar delimitadores especiais definidos por você (por exemplo, ***).', - maxLengthCheck: 'O comprimento máximo do chunk deve ser inferior a 4000', + maxLengthCheck: 'O comprimento máximo do chunk deve ser inferior a {{limit}}', }, stepThree: { creationTitle: '🎉 Conhecimento criado', diff --git a/web/i18n/ro-RO/dataset-creation.ts b/web/i18n/ro-RO/dataset-creation.ts index 2f474f440aab77..f18b2ac5c220e0 100644 --- a/web/i18n/ro-RO/dataset-creation.ts +++ b/web/i18n/ro-RO/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: 'Pagini', websiteSource: 'Site-ul web de preprocesare', separatorTip: 'Un delimitator este caracterul folosit pentru a separa textul. \\n\\n și \\n sunt delimitatori utilizați în mod obișnuit pentru separarea paragrafelor și liniilor. Combinate cu virgule (\\n\\n,\\n), paragrafele vor fi segmentate pe linii atunci când depășesc lungimea maximă a bucății. De asemenea, puteți utiliza delimitatori speciali definiți de dumneavoastră (de exemplu, ***).', - maxLengthCheck: 'Lungimea maximă a bucății trebuie să fie mai mică de 4000', + maxLengthCheck: 'Lungimea maximă a bucății trebuie să fie mai mică de {{limit}}', }, stepThree: { creationTitle: '🎉 Cunoștință creată', diff --git a/web/i18n/ru-RU/dataset-creation.ts b/web/i18n/ru-RU/dataset-creation.ts index 66d92bd2b4c832..ca88e093bc7e99 100644 --- a/web/i18n/ru-RU/dataset-creation.ts +++ b/web/i18n/ru-RU/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'Чтобы изменить метод индексации, пожалуйста, перейдите в ', datasetSettingLink: 'настройки базы знаний.', separatorTip: 'Разделитель — это символ, используемый для разделения текста. \\n\\n и \\n — это часто используемые разделители для разделения абзацев и строк. В сочетании с запятыми (\\n\\n,\\n) абзацы будут сегментированы по строкам, если максимальная длина блока превышает их. Вы также можете использовать специальные разделители, определенные вами (например, ***).', - maxLengthCheck: 'Максимальная длина блока должна быть меньше 4000', + maxLengthCheck: 'Максимальная длина блока должна быть меньше {{limit}}', }, stepThree: { creationTitle: '🎉 База знаний создана', diff --git a/web/i18n/sl-SI/dataset-creation.ts b/web/i18n/sl-SI/dataset-creation.ts index 402066ad409f82..8dcfa8c3f50cb9 100644 --- a/web/i18n/sl-SI/dataset-creation.ts +++ b/web/i18n/sl-SI/dataset-creation.ts @@ -152,7 +152,7 @@ const translation = { indexSettingTip: 'Če želite spremeniti način indeksiranja in model vdelave, pojdite na ', retrievalSettingTip: 'Če želite spremeniti nastavitve iskanja, pojdite na ', datasetSettingLink: 'nastavitve Znanja.', - maxLengthCheck: 'Največja dolžina kosa mora biti manjša od 4000', + maxLengthCheck: 'Največja dolžina kosa mora biti manjša od {{limit}}', }, stepThree: { creationTitle: '🎉 Znanje ustvarjeno', diff --git a/web/i18n/th-TH/dataset-creation.ts b/web/i18n/th-TH/dataset-creation.ts index 850477c5688fdf..16ec1fab3e988e 100644 --- a/web/i18n/th-TH/dataset-creation.ts +++ b/web/i18n/th-TH/dataset-creation.ts @@ -103,7 +103,7 @@ const translation = { separatorTip: 'ตัวคั่นคืออักขระที่ใช้ในการแยกข้อความ \\n\\n และ \\n เป็นตัวคั่นที่ใช้กันทั่วไปสําหรับการแยกย่อหน้าและบรรทัด เมื่อรวมกับเครื่องหมายจุลภาค (\\n\\n,\\n) ย่อหน้าจะถูกแบ่งตามบรรทัดเมื่อเกินความยาวของก้อนสูงสุด คุณยังสามารถใช้ตัวคั่นพิเศษที่กําหนดโดยตัวคุณเอง (เช่น ***)', separatorPlaceholder: '\\n\\n สําหรับแยกย่อหน้า \\n สําหรับแยกเส้น', maxLength: 'ความยาวก้อนสูงสุด', - maxLengthCheck: 'ความยาวก้อนสูงสุดควรน้อยกว่า 4000', + maxLengthCheck: 'ความยาวก้อนสูงสุดควรน้อยกว่า {{limit}}', overlap: 'การทับซ้อนกันของก้อน', overlapTip: 'การตั้งค่าการทับซ้อนกันของกลุ่มสามารถรักษาความเกี่ยวข้องทางความหมายระหว่างกันได้ ขอแนะนําให้ตั้งค่า 10%-25% ของขนาดก้อนสูงสุด', overlapCheck: 'การทับซ้อนกันของก้อนไม่ควรใหญ่กว่าความยาวของก้อนสูงสุด', diff --git a/web/i18n/tr-TR/dataset-creation.ts b/web/i18n/tr-TR/dataset-creation.ts index 1d5dd1963410b2..11e5789884f458 100644 --- a/web/i18n/tr-TR/dataset-creation.ts +++ b/web/i18n/tr-TR/dataset-creation.ts @@ -147,7 +147,7 @@ const translation = { retrievalSettingTip: 'Dizin yöntemini değiştirmek için, lütfen', datasetSettingLink: 'Bilgi ayarlarına gidin.', separatorTip: 'Sınırlayıcı, metni ayırmak için kullanılan karakterdir. \\n\\n ve \\n, paragrafları ve satırları ayırmak için yaygın olarak kullanılan sınırlayıcılardır. Virgüllerle (\\n\\n,\\n) birleştirildiğinde, paragraflar maksimum öbek uzunluğunu aştığında satırlarla bölünür. Kendiniz tarafından tanımlanan özel sınırlayıcıları da kullanabilirsiniz (örn.', - maxLengthCheck: 'Maksimum yığın uzunluğu 4000\'den az olmalıdır', + maxLengthCheck: 'Maksimum yığın uzunluğu {{limit}}\'den az olmalıdır', }, stepThree: { creationTitle: '🎉 Bilgi oluşturuldu', diff --git a/web/i18n/uk-UA/dataset-creation.ts b/web/i18n/uk-UA/dataset-creation.ts index e78934c9e9d660..d6ee22c2d33db4 100644 --- a/web/i18n/uk-UA/dataset-creation.ts +++ b/web/i18n/uk-UA/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { webpageUnit: 'Сторінок', websiteSource: 'Веб-сайт попередньої обробки', separatorTip: 'Роздільник – це символ, який використовується для поділу тексту. \\n\\n та \\n є часто використовуваними роздільниками для відокремлення абзаців та рядків. У поєднанні з комами (\\n\\n,\\n) абзаци будуть розділені лініями, якщо вони перевищують максимальну довжину фрагмента. Ви також можете використовувати спеціальні роздільники, визначені вами (наприклад, ***).', - maxLengthCheck: 'Максимальна довжина шматка має бути меншою за 4000', + maxLengthCheck: 'Максимальна довжина шматка має бути меншою за {{limit}}', }, stepThree: { creationTitle: '🎉 Знання створено', diff --git a/web/i18n/vi-VN/dataset-creation.ts b/web/i18n/vi-VN/dataset-creation.ts index 3dc26bee46d3b9..045f3d6ea56455 100644 --- a/web/i18n/vi-VN/dataset-creation.ts +++ b/web/i18n/vi-VN/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: 'Trang web tiền xử lý', webpageUnit: 'Trang', separatorTip: 'Dấu phân cách là ký tự được sử dụng để phân tách văn bản. \\n\\n và \\n là dấu phân cách thường được sử dụng để tách các đoạn văn và dòng. Kết hợp với dấu phẩy (\\n\\n,\\n), các đoạn văn sẽ được phân đoạn theo các dòng khi vượt quá độ dài đoạn tối đa. Bạn cũng có thể sử dụng dấu phân cách đặc biệt do chính bạn xác định (ví dụ: ***).', - maxLengthCheck: 'Chiều dài đoạn tối đa phải nhỏ hơn 4000', + maxLengthCheck: 'Chiều dài đoạn tối đa phải nhỏ hơn {{limit}}', }, stepThree: { creationTitle: '🎉 Kiến thức đã được tạo', diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts index fac809d7e23b84..e6c3aa00ba9f37 100644 --- a/web/i18n/zh-Hans/dataset-creation.ts +++ b/web/i18n/zh-Hans/dataset-creation.ts @@ -103,7 +103,7 @@ const translation = { separatorTip: '分隔符是用于分隔文本的字符。\\n\\n 和 \\n 是常用于分隔段落和行的分隔符。用逗号连接分隔符(\\n\\n,\\n),当段落超过最大块长度时,会按行进行分割。你也可以使用自定义的特殊分隔符(例如 ***)。', separatorPlaceholder: '\\n\\n 用于分段;\\n 用于分行', maxLength: '分段最大长度', - maxLengthCheck: '分段最大长度不能大于 4000', + maxLengthCheck: '分段最大长度不能大于 {{limit}}', overlap: '分段重叠长度', overlapTip: '设置分段之间的重叠长度可以保留分段之间的语义关系,提升召回效果。建议设置为最大分段长度的10%-25%', overlapCheck: '分段重叠长度不能大于分段最大长度', diff --git a/web/i18n/zh-Hant/dataset-creation.ts b/web/i18n/zh-Hant/dataset-creation.ts index 02374573cf4a43..8c5673cb3f85c3 100644 --- a/web/i18n/zh-Hant/dataset-creation.ts +++ b/web/i18n/zh-Hant/dataset-creation.ts @@ -142,7 +142,7 @@ const translation = { websiteSource: '預處理網站', webpageUnit: '頁面', separatorTip: '分隔符是用於分隔文字的字元。\\n\\n 和 \\n 是分隔段落和行的常用分隔符。與逗號 (\\n\\n,\\n) 組合使用時,當超過最大區塊長度時,段落將按行分段。您也可以使用自定義的特殊分隔符(例如 ***)。', - maxLengthCheck: '塊最大長度應小於 4000', + maxLengthCheck: '塊最大長度應小於 {{limit}}', }, stepThree: { creationTitle: '🎉 知識庫已建立', diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 95c7ce45de89fa..e2fa608bae3ae6 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -175,6 +175,7 @@ export type ProcessMode = 'automatic' | 'custom' export type ProcessRuleResponse = { mode: ProcessMode rules: Rules + limits: Limits } export type Rules = { @@ -182,6 +183,10 @@ export type Rules = { segmentation: Segmentation } +export type Limits = { + indexing_max_segmentation_tokens_length: number +} + export type PreProcessingRule = { id: string enabled: boolean