chore: add only_main_content to api

langgenius · Jun 7, 2024 · 537f7ec · 537f7ec
1 parent 6dd4aec
commit 537f7ec
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 17 deletions.
diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx
@@ -8,7 +8,7 @@ import StepOne from './step-one'
 import StepTwo from './step-two'
 import StepThree from './step-three'
 import { DataSourceType } from '@/models/datasets'
-import type { CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
+import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
 import { fetchDataSource } from '@/service/common'
 import { fetchDatasetDetail } from '@/service/datasets'
 import type { NotionPage } from '@/models/common'
@@ -19,6 +19,15 @@ type DatasetUpdateFormProps = {
   datasetId?: string
 }
 
+const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
+  crawl_sub_pages: true,
+  only_main_content: true,
+  includes: '',
+  excludes: '',
+  limit: 10,
+  max_depth: 2,
+}
+
 const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   const { t } = useTranslation()
   const { setShowAccountSettingModal } = useModalContext()
@@ -37,6 +46,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   }
 
   const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
+  const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
+
   const updateFileList = (preparedFiles: FileItem[]) => {
     setFiles(preparedFiles)
   }
@@ -127,6 +138,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
             websitePages={websitePages}
             updateWebsitePages={setWebsitePages}
             onFireCrawlJobIdChange={setFireCrawlJobId}
+            crawlOptions={crawlOptions}
+            onCrawlOptionsChange={setCrawlOptions}
           />
         </div>
         {(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
@@ -142,6 +155,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
           onStepChange={changeStep}
           updateIndexingTypeCache={updateIndexingTypeCache}
           updateResultCache={updateResultCache}
+          crawlOptions={crawlOptions}
         />}
         {step === 3 && <StepThree
           datasetId={datasetId}

diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx
@@ -9,7 +9,7 @@ import EmptyDatasetCreationModal from '../empty-dataset-creation-modal'
 import Website from '../website'
 import WebsitePreview from '../website/preview'
 import s from './index.module.css'
-import type { CrawlResultItem, FileItem } from '@/models/datasets'
+import type { CrawlOptions, CrawlResultItem, FileItem } from '@/models/datasets'
 import type { NotionPage } from '@/models/common'
 import { DataSourceType } from '@/models/datasets'
 import Button from '@/app/components/base/button'
@@ -34,6 +34,8 @@ type IStepOneProps = {
   websitePages?: CrawlResultItem[]
   updateWebsitePages: (value: CrawlResultItem[]) => void
   onFireCrawlJobIdChange: (jobId: string) => void
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
 }
 
 type NotionConnectorProps = {
@@ -68,6 +70,8 @@ const StepOne = ({
   websitePages = [],
   updateWebsitePages,
   onFireCrawlJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
 }: IStepOneProps) => {
   const { dataset } = useDatasetDetailContext()
   const [showModal, setShowModal] = useState(false)
@@ -221,6 +225,8 @@ const StepOne = ({
                   checkedCrawlResult={websitePages}
                   onCheckedCrawlResultChange={updateWebsitePages}
                   onJobIdChange={onFireCrawlJobIdChange}
+                  crawlOptions={crawlOptions}
+                  onCrawlOptionsChange={onCrawlOptionsChange}
                 />
               </div>
               {isShowVectorSpaceFull && (

diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx
@@ -12,7 +12,7 @@ import RetrievalMethodInfo from '../../common/retrieval-method-info'
 import PreviewItem, { PreviewType } from './preview-item'
 import LanguageSelect from './language-select'
 import s from './index.module.css'
-import type { CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
+import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
 import {
   createDocument,
   createFirstDocument,
@@ -58,6 +58,7 @@ type StepTwoProps = {
   files: CustomFile[]
   notionPages?: NotionPage[]
   websitePages?: CrawlResultItem[]
+  crawlOptions?: CrawlOptions
   fireCrawlJobId?: string
   onStepChange?: (delta: number) => void
   updateIndexingTypeCache?: (type: string) => void
@@ -86,6 +87,7 @@ const StepTwo = ({
   files,
   notionPages = [],
   websitePages = [],
+  crawlOptions,
   fireCrawlJobId = '',
   onStepChange,
   updateIndexingTypeCache,
@@ -252,6 +254,7 @@ const StepTwo = ({
       provider: 'firecrawl',
       job_id: fireCrawlJobId,
       urls: websitePages.map(page => page.source_url),
+      only_main_content: crawlOptions?.only_main_content,
     }
   }
 

diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
@@ -61,7 +61,7 @@ const CrawledResult: FC<Props> = ({
         />
         <div>{t(`${I18N_PREFIX}.scrapTimeInfo`, {
           total: list.length,
-          time: '12.4 seconds',
+          time: '12.4 seconds', // TODO toFixed(1)
         })}</div>
       </div>
       <div className='p-2'>

diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -23,15 +23,8 @@ type Props = {
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
-}
-
-const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
-  crawl_sub_pages: true,
-  only_main_content: true,
-  includes: '',
-  excludes: '',
-  limit: 10,
-  max_depth: 2,
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
 }
 
 enum Step {
@@ -45,6 +38,8 @@ const FireCrawl: FC<Props> = ({
   checkedCrawlResult,
   onCheckedCrawlResultChange,
   onJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
 }) => {
   const { t } = useTranslation()
   const [step, setStep] = useState<Step>(Step.init)
@@ -55,7 +50,6 @@ const FireCrawl: FC<Props> = ({
     })
   }, [setShowAccountSettingModal])
 
-  const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
   const checkValid = useCallback((url: string) => {
     let errorMsg = ''
     if (!url) {
@@ -172,7 +166,7 @@ const FireCrawl: FC<Props> = ({
           isFilledFull={!isInit}
           hasError={isCrawlFinished && crawlHasError}
         >
-          {isInit && <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />}
+          {isInit && <Options className='mt-2' payload={crawlOptions} onChange={onCrawlOptionsChange} />}
           {isRunning
             && <Crawling
               className='mt-2'

diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx
@@ -4,7 +4,7 @@ import React, { useCallback, useEffect, useState } from 'react'
 import NoData from './no-data'
 import Firecrawl from './firecrawl'
 import { useModalContext } from '@/context/modal-context'
-import type { CrawlResultItem } from '@/models/datasets'
+import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import { fetchFirecrawlApiKey } from '@/service/datasets'
 import { type DataSourceWebsiteItem, WebsiteProvider } from '@/models/common'
 
@@ -13,13 +13,17 @@ type Props = {
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
 }
 
 const Website: FC<Props> = ({
   onPreview,
   checkedCrawlResult,
   onCheckedCrawlResultChange,
   onJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
 }) => {
   const { setShowAccountSettingModal } = useModalContext()
   const [isLoaded, setIsLoaded] = useState(false)
@@ -55,6 +59,8 @@ const Website: FC<Props> = ({
             checkedCrawlResult={checkedCrawlResult}
             onCheckedCrawlResultChange={onCheckedCrawlResultChange}
             onJobIdChange={onJobIdChange}
+            crawlOptions={crawlOptions}
+            onCrawlOptionsChange={onCrawlOptionsChange}
           />
         )
         : (

diff --git a/web/models/datasets.ts b/web/models/datasets.ts
@@ -5,7 +5,7 @@ import type { Tag } from '@/app/components/base/tag-management/constant'
 export enum DataSourceType {
   FILE = 'upload_file',
   NOTION = 'notion_import',
-  WEB = 'website',
+  WEB = 'website_crawl',
 }
 
 export type DataSet = {