Skip to content

Commit

Permalink
chore(web): strong type (#1259)
Browse files Browse the repository at this point in the history
  • Loading branch information
@JunIce
JunIce authored Oct 7, 2023
1 parent 657fa80 commit 52bec63
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 36 deletions.
2 changes: 1 addition & 1 deletion web/app/components/datasets/create/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
{(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
hasSetAPIKEY={!!embeddingsDefaultModel}
onSetting={showSetAPIKey}
indexingType={detail?.indexing_technique || ''}
indexingType={detail?.indexing_technique}
datasetId={datasetId}
dataSourceType={dataSourceType}
files={fileList.map(file => file.file)}
Expand Down
66 changes: 34 additions & 32 deletions web/app/components/datasets/create/step-two/index.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
/* eslint-disable no-mixed-operators */
'use client'
import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
Expand All @@ -11,7 +10,7 @@ import { groupBy } from 'lodash-es'
import PreviewItem, { PreviewType } from './preview-item'
import LanguageSelect from './language-select'
import s from './index.module.css'
import type { CreateDocumentReq, CustomFile, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
import type { CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
import {
createDocument,
createFirstDocument,
Expand All @@ -33,13 +32,14 @@ import { useDatasetDetailContext } from '@/context/dataset-detail'
import I18n from '@/context/i18n'
import { IS_CE_EDITION } from '@/config'

type ValueOf<T> = T[keyof T]
type StepTwoProps = {
isSetting?: boolean
documentDetail?: FullDocumentDetail
hasSetAPIKEY: boolean
onSetting: () => void
datasetId?: string
indexingType?: string
indexingType?: ValueOf<IndexingType>
dataSourceType: DataSourceType
files: CustomFile[]
notionPages?: NotionPage[]
Expand Down Expand Up @@ -89,21 +89,23 @@ const StepTwo = ({
const [rules, setRules] = useState<PreProcessingRule[]>([])
const [defaultConfig, setDefaultConfig] = useState<Rules>()
const hasSetIndexType = !!indexingType
const [indexType, setIndexType] = useState<IndexingType>(
indexingType
|| hasSetAPIKEY
const [indexType, setIndexType] = useState<ValueOf<IndexingType>>(
(indexingType
|| hasSetAPIKEY)
? IndexingType.QUALIFIED
: IndexingType.ECONOMICAL,
)
const [docForm, setDocForm] = useState<DocForm | string>(
datasetId && documentDetail ? documentDetail.doc_form : DocForm.TEXT,
(datasetId && documentDetail) ? documentDetail.doc_form : DocForm.TEXT,
)
const [docLanguage, setDocLanguage] = useState<string>(locale === 'en' ? 'English' : 'Chinese')
const [QATipHide, setQATipHide] = useState(false)
const [previewSwitched, setPreviewSwitched] = useState(false)
const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
const [automaticFileIndexingEstimate, setAutomaticFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<FileIndexingEstimateResponse | null>(null)
const [automaticFileIndexingEstimate, setAutomaticFileIndexingEstimate] = useState<FileIndexingEstimateResponse | null>(null)
const [estimateTokes, setEstimateTokes] = useState<Pick<IndexingEstimateResponse, 'tokens' | 'total_price'> | null>(null)

const fileIndexingEstimate = (() => {
return segmentationType === SegmentType.AUTO ? automaticFileIndexingEstimate : customFileIndexingEstimate
})()
Expand Down Expand Up @@ -153,20 +155,22 @@ const StepTwo = ({
}
const resetRules = () => {
if (defaultConfig) {
setSegmentIdentifier(defaultConfig.segmentation.separator === '\n' ? '\\n' : defaultConfig.segmentation.separator || '\\n')
setSegmentIdentifier((defaultConfig.segmentation.separator === '\n' ? '\\n' : defaultConfig.segmentation.separator) || '\\n')
setMax(defaultConfig.segmentation.max_tokens)
setRules(defaultConfig.pre_processing_rules)
}
}

const fetchFileIndexingEstimate = async (docForm = DocForm.TEXT) => {
// eslint-disable-next-line @typescript-eslint/no-use-before-define
const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm))
if (segmentationType === SegmentType.CUSTOM)
const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm)!)
if (segmentationType === SegmentType.CUSTOM) {
setCustomFileIndexingEstimate(res)

else
}
else {
setAutomaticFileIndexingEstimate(res)
indexType === IndexingType.QUALIFIED && setEstimateTokes({ tokens: res.tokens, total_price: res.total_price })
}
}

const confirmChangeCustomConfig = () => {
Expand All @@ -179,8 +183,8 @@ const StepTwo = ({
const getIndexing_technique = () => indexingType || indexType

const getProcessRule = () => {
const processRule: any = {
rules: {}, // api will check this. It will be removed after api refactored.
const processRule: ProcessRule = {
rules: {} as any, // api will check this. It will be removed after api refactored.
mode: segmentationType,
}
if (segmentationType === SegmentType.CUSTOM) {
Expand Down Expand Up @@ -220,37 +224,35 @@ const StepTwo = ({
}) as NotionInfo[]
}

const getFileIndexingEstimateParams = (docForm: DocForm) => {
let params
const getFileIndexingEstimateParams = (docForm: DocForm): IndexingEstimateParams | undefined => {
if (dataSourceType === DataSourceType.FILE) {
params = {
return {
info_list: {
data_source_type: dataSourceType,
file_info_list: {
file_ids: files.map(file => file.id),
file_ids: files.map(file => file.id) as string[],
},
},
indexing_technique: getIndexing_technique(),
indexing_technique: getIndexing_technique() as string,
process_rule: getProcessRule(),
doc_form: docForm,
doc_language: docLanguage,
dataset_id: datasetId,
dataset_id: datasetId as string,
}
}
if (dataSourceType === DataSourceType.NOTION) {
params = {
return {
info_list: {
data_source_type: dataSourceType,
notion_info_list: getNotionInfo(),
},
indexing_technique: getIndexing_technique(),
indexing_technique: getIndexing_technique() as string,
process_rule: getProcessRule(),
doc_form: docForm,
doc_language: docLanguage,
dataset_id: datasetId,
dataset_id: datasetId as string,
}
}
return params
}

const getCreationParams = () => {
Expand Down Expand Up @@ -291,7 +293,7 @@ const StepTwo = ({
try {
const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' })
const separator = res.rules.segmentation.separator
setSegmentIdentifier(separator === '\n' ? '\\n' : separator || '\\n')
setSegmentIdentifier((separator === '\n' ? '\\n' : separator) || '\\n')
setMax(res.rules.segmentation.max_tokens)
setRules(res.rules.pre_processing_rules)
setDefaultConfig(res.rules)
Expand All @@ -306,7 +308,7 @@ const StepTwo = ({
const rules = documentDetail.dataset_process_rule.rules
const separator = rules.segmentation.separator
const max = rules.segmentation.max_tokens
setSegmentIdentifier(separator === '\n' ? '\\n' : separator || '\\n')
setSegmentIdentifier((separator === '\n' ? '\\n' : separator) || '\\n')
setMax(max)
setRules(rules.pre_processing_rules)
setDefaultConfig(rules)
Expand All @@ -330,15 +332,15 @@ const StepTwo = ({
res = await createFirstDocument({
body: params,
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(res)
}
else {
res = await createDocument({
datasetId,
body: params,
})
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
updateIndexingTypeCache && updateIndexingTypeCache(indexType as string)
updateResultCache && updateResultCache(res)
}
if (mutateDatasetRes)
Expand Down Expand Up @@ -549,9 +551,9 @@ const StepTwo = ({
<div className={s.tip}>{t('datasetCreation.stepTwo.qualifiedTip')}</div>
<div className='pb-0.5 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateCost')}</div>
{
fileIndexingEstimate
estimateTokes
? (
<div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.tokens)} tokens(<span className='text-yellow-500'>${formatNumber(fileIndexingEstimate.total_price)}</span>)</div>
<div className='text-xs font-medium text-gray-800'>{formatNumber(estimateTokes.tokens)} tokens(<span className='text-yellow-500'>${formatNumber(estimateTokes.total_price)}</span>)</div>
)
: (
<div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
Expand Down
11 changes: 9 additions & 2 deletions web/models/datasets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,15 +183,22 @@ export type DocumentListResponse = {
limit: number
}

export type CreateDocumentReq = {
export type DocumentReq = {
original_document_id?: string
indexing_technique?: string
doc_form: 'text_model' | 'qa_model'
doc_language: string
data_source: DataSource
process_rule: ProcessRule
}

export type CreateDocumentReq = DocumentReq & {
data_source: DataSource
}

export type IndexingEstimateParams = DocumentReq & Partial<DataSource> & {
dataset_id: string
}

export type DataSource = {
type: DataSourceType
info_list: {
Expand Down
3 changes: 2 additions & 1 deletion web/service/datasets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type {
FileIndexingEstimateResponse,
HitTestingRecordsResponse,
HitTestingResponse,
IndexingEstimateParams,
IndexingEstimateResponse,
IndexingStatusBatchResponse,
IndexingStatusResponse,
Expand Down Expand Up @@ -189,7 +190,7 @@ export const fetchTestingRecords: Fetcher<HitTestingRecordsResponse, { datasetId
return get<HitTestingRecordsResponse>(`/datasets/${datasetId}/queries`, { params })
}

export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, any> = (body: any) => {
export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, IndexingEstimateParams> = (body: IndexingEstimateParams) => {
return post<FileIndexingEstimateResponse>('/datasets/indexing-estimate', { body })
}

Expand Down

0 comments on commit 52bec63

Please sign in to comment.