From 1646c5f9423b4d8d953820139f038021918e9973 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:54:45 -0700 Subject: [PATCH] Add standalone form state for source data (#445) (#449) (cherry picked from commit a197021eb7809fd2e238bf8e7e60bf28c5814d5a) Signed-off-by: Tyler Ohlsen Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- common/constants.ts | 5 + common/interfaces.ts | 10 +- .../ingest_inputs/source_data.tsx | 221 +----------- .../ingest_inputs/source_data_modal.tsx | 323 ++++++++++++++++++ .../input_fields/json_field.tsx | 4 +- .../input_fields/model_field.tsx | 4 +- .../input_fields/number_field.tsx | 4 +- .../input_fields/select_field.tsx | 4 +- .../select_with_custom_options.tsx | 4 +- .../input_fields/text_field.tsx | 4 +- public/utils/config_to_schema_utils.ts | 2 +- 11 files changed, 361 insertions(+), 224 deletions(-) create mode 100644 public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx diff --git a/common/constants.ts b/common/constants.ts index 03c9e175..5e8062de 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -484,3 +484,8 @@ export enum CONFIG_STEP { INGEST = 'Ingestion pipeline', SEARCH = 'Search pipeline', } +export enum SOURCE_OPTIONS { + MANUAL = 'manual', + UPLOAD = 'upload', + EXISTING_INDEX = 'existing_index', +} diff --git a/common/interfaces.ts b/common/interfaces.ts index b7eb7781..6f13aa71 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -67,9 +67,6 @@ export type IndexConfig = { settings: IConfigField; }; -// TODO: may expand to just IndexConfig (including mappings/settings info) -// if we want to persist this for users using some existing index, -// and want to pass that index config around. export type SearchIndexConfig = { name: IConfigField; }; @@ -113,6 +110,13 @@ export type WorkflowSchemaObj = { }; export type WorkflowSchema = ObjectSchema; +// Form / schema interfaces for the ingest docs sub-form +export type IngestDocsFormValues = { + docs: FormikValues; +}; +export type IngestDocsSchemaObj = WorkflowSchemaObj; +export type IngestDocsSchema = WorkflowSchema; + /** ********** WORKSPACE TYPES/INTERFACES ********** */ diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index c905ddec..b5a0e735 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -4,47 +4,26 @@ */ import React, { useEffect, useState } from 'react'; -import { useSelector } from 'react-redux'; import { getIn, useFormikContext } from 'formik'; import { EuiSmallButton, - EuiCompressedFilePicker, EuiFlexGroup, EuiFlexItem, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, EuiSpacer, EuiText, - EuiFilterGroup, - EuiSmallFilterButton, - EuiSuperSelectOption, - EuiCompressedSuperSelect, EuiCodeBlock, EuiSmallButtonEmpty, } from '@elastic/eui'; -import { JsonField } from '../input_fields'; import { - FETCH_ALL_QUERY, - IndexMappings, MapEntry, - SearchHit, + SOURCE_OPTIONS, Workflow, WorkflowConfig, - WorkspaceFormValues, - customStringify, + WorkflowFormValues, isVectorSearchUseCase, toFormattedDate, } from '../../../../../common'; -import { - AppState, - getMappings, - searchIndex, - useAppDispatch, -} from '../../../../store'; -import { getDataSourceId } from '../../../../utils'; +import { SourceDataModal } from './source_data_modal'; interface SourceDataProps { workflow: Workflow | undefined; @@ -53,20 +32,11 @@ interface SourceDataProps { lastIngested: number | undefined; } -enum SOURCE_OPTIONS { - MANUAL = 'manual', - UPLOAD = 'upload', - EXISTING_INDEX = 'existing_index', -} - /** * Input component for configuring the source data for ingest. */ export function SourceData(props: SourceDataProps) { - const dispatch = useAppDispatch(); - const dataSourceId = getDataSourceId(); - const { values, setFieldValue } = useFormikContext(); - const indices = useSelector((state: AppState) => state.opensearch.indices); + const { values, setFieldValue } = useFormikContext(); // empty/populated docs state let docs = []; @@ -83,74 +53,6 @@ export function SourceData(props: SourceDataProps) { // edit modal state const [isEditModalOpen, setIsEditModalOpen] = useState(false); - // files state. when a file is read, update the form value. - const fileReader = new FileReader(); - fileReader.onload = (e) => { - if (e.target) { - setFieldValue('ingest.docs', e.target.result); - } - }; - - // selected index state. when an index is selected, update several form values (if vector search) - const [selectedIndex, setSelectedIndex] = useState( - undefined - ); - useEffect(() => { - if (selectedIndex !== undefined) { - // 1. fetch and set sample docs - dispatch( - searchIndex({ - apiBody: { - index: selectedIndex, - body: FETCH_ALL_QUERY, - searchPipeline: '_none', - }, - dataSourceId, - }) - ) - .unwrap() - .then((resp) => { - const docObjs = resp.hits?.hits - ?.slice(0, 5) - ?.map((hit: SearchHit) => hit?._source); - setFieldValue('ingest.docs', customStringify(docObjs)); - }); - - // 2. fetch index mappings, and try to set defaults for the ML processor configs, if applicable - if (isVectorSearchUseCase(props.workflow)) { - dispatch(getMappings({ index: selectedIndex, dataSourceId })) - .unwrap() - .then((resp: IndexMappings) => { - const { processorId, inputMapEntry } = getProcessorInfo( - props.uiConfig, - values - ); - if (processorId !== undefined && inputMapEntry !== undefined) { - // set/overwrite default text field for the input map. may be empty. - if (inputMapEntry !== undefined) { - const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; - const curTextField = getIn(values, textFieldFormPath) as string; - if (!Object.keys(resp.properties).includes(curTextField)) { - const defaultTextField = - Object.keys(resp.properties).find((fieldName) => { - return resp.properties[fieldName]?.type === 'text'; - }) || ''; - setFieldValue(textFieldFormPath, defaultTextField); - } - } - } - }); - } - } - }, [selectedIndex]); - - // hook to clear out the selected index when switching options - useEffect(() => { - if (selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX) { - setSelectedIndex(undefined); - } - }, [selectedOption]); - // hook to listen when the docs form value changes. useEffect(() => { if (values?.ingest?.docs) { @@ -188,110 +90,13 @@ export function SourceData(props: SourceDataProps) { return ( <> {isEditModalOpen && ( - setIsEditModalOpen(false)} - style={{ width: '70vw' }} - > - - -

{`Import data`}

- - - - <> - - setSelectedOption(SOURCE_OPTIONS.MANUAL)} - data-testid="manualEditSourceDataButton" - > - Manual - - setSelectedOption(SOURCE_OPTIONS.UPLOAD)} - data-testid="uploadSourceDataButton" - > - Upload - - - setSelectedOption(SOURCE_OPTIONS.EXISTING_INDEX) - } - data-testid="selectIndexSourceDataButton" - > - Existing index - - - - {selectedOption === SOURCE_OPTIONS.UPLOAD && ( - <> - { - if (files && files.length > 0) { - fileReader.readAsText(files[0]); - } - }} - display="default" - /> - - - )} - {selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && ( - <> - - Up to 5 sample documents will be automatically populated. - - - - ({ - value: option.name, - inputDisplay: ( - {option.name} - ), - disabled: false, - } as EuiSuperSelectOption) - )} - valueOfSelected={selectedIndex} - onChange={(option) => { - setSelectedIndex(option); - }} - isInvalid={false} - /> - - - )} - - - - - setIsEditModalOpen(false)} - fill={false} - color="primary" - data-testid="closeSourceDataButton" - > - Close - - - + )} @@ -361,9 +166,9 @@ export function SourceData(props: SourceDataProps) { // helper fn to parse out some useful info from the ML ingest processor config, if applicable // takes on the assumption the first processor is an ML inference processor, and should // only be executed for workflows coming from preset vector search use cases. -function getProcessorInfo( +export function getProcessorInfo( uiConfig: WorkflowConfig, - values: WorkspaceFormValues + values: WorkflowFormValues ): { processorId: string | undefined; inputMapEntry: MapEntry | undefined; diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx new file mode 100644 index 00000000..f871ce1b --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx @@ -0,0 +1,323 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; +import { Formik, getIn, useFormikContext } from 'formik'; +import { isEmpty } from 'lodash'; +import * as yup from 'yup'; +import { + EuiSmallButton, + EuiCompressedFilePicker, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSpacer, + EuiText, + EuiFilterGroup, + EuiSmallFilterButton, + EuiSuperSelectOption, + EuiCompressedSuperSelect, +} from '@elastic/eui'; +import { JsonField } from '../input_fields'; +import { + customStringify, + FETCH_ALL_QUERY, + IConfigField, + IndexMappings, + IngestDocsFormValues, + isVectorSearchUseCase, + SearchHit, + SOURCE_OPTIONS, + Workflow, + WorkflowConfig, + WorkflowFormValues, +} from '../../../../../common'; +import { + AppState, + getMappings, + searchIndex, + useAppDispatch, +} from '../../../../store'; +import { + getDataSourceId, + getFieldSchema, + getInitialValue, +} from '../../../../utils'; +import { getProcessorInfo } from './source_data'; + +interface SourceDataProps { + workflow: Workflow | undefined; + uiConfig: WorkflowConfig; + selectedOption: SOURCE_OPTIONS; + setSelectedOption: (option: SOURCE_OPTIONS) => void; + setIsModalOpen: (isOpen: boolean) => void; +} + +/** + * Modal for configuring the source data for ingest. Maintains standalone form state, and only updates + * parent form if the user explicitly clicks "Update", and there are no validation errors. + */ +export function SourceDataModal(props: SourceDataProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); + const { values, setFieldValue } = useFormikContext(); + const indices = useSelector((state: AppState) => state.opensearch.indices); + + // sub-form values/schema + const docsFormValues = { + docs: getInitialValue('jsonArray'), + } as IngestDocsFormValues; + const docsFormSchema = yup.object({ + docs: getFieldSchema({ + type: 'jsonArray', + } as IConfigField), + }); + + // persist standalone values. update / initialize when it is first opened + const [tempDocs, setTempDocs] = useState('[]'); + const [tempErrors, setTempErrors] = useState(false); + + // button updating state + const [isUpdating, setIsUpdating] = useState(false); + + // selected index state + const [selectedIndex, setSelectedIndex] = useState( + undefined + ); + + // hook to clear out the selected index when switching options + useEffect(() => { + if (props.selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX) { + setSelectedIndex(undefined); + } + }, [props.selectedOption]); + + function onClose() { + props.setIsModalOpen(false); + } + + function onUpdate() { + setIsUpdating(true); + // 1. Update the form with the temp docs + setFieldValue('ingest.docs', tempDocs); + + // 2. Update several form values if an index is selected (and if vector search) + if (selectedIndex !== undefined) { + if (isVectorSearchUseCase(props.workflow)) { + dispatch(getMappings({ index: selectedIndex, dataSourceId })) + .unwrap() + .then((resp: IndexMappings) => { + const { processorId, inputMapEntry } = getProcessorInfo( + props.uiConfig, + values + ); + if (processorId !== undefined && inputMapEntry !== undefined) { + // set/overwrite default text field for the input map. may be empty. + if (inputMapEntry !== undefined) { + const textFieldFormPath = `ingest.enrich.${processorId}.input_map.0.0.value`; + const curTextField = getIn(values, textFieldFormPath) as string; + if (!Object.keys(resp.properties).includes(curTextField)) { + const defaultTextField = + Object.keys(resp.properties).find((fieldName) => { + return resp.properties[fieldName]?.type === 'text'; + }) || ''; + setFieldValue(textFieldFormPath, defaultTextField); + setIsUpdating(false); + } + } + } + }); + } + } else { + setIsUpdating(false); + } + props.setIsModalOpen(false); + } + + return ( + {}} + validate={(values) => {}} + > + {(formikProps) => { + // override to parent form value when changes detected + useEffect(() => { + formikProps.setFieldValue('docs', getIn(values, 'ingest.docs')); + }, [getIn(values, 'ingest.docs')]); + + // update tempDocs when form changes are detected + useEffect(() => { + setTempDocs(getIn(formikProps.values, 'docs')); + }, [getIn(formikProps.values, 'docs')]); + + // fetch & populate sample documents if an existing index is chosen + useEffect(() => { + if (selectedIndex !== undefined) { + dispatch( + searchIndex({ + apiBody: { + index: selectedIndex, + body: FETCH_ALL_QUERY, + searchPipeline: '_none', + }, + dataSourceId, + }) + ) + .unwrap() + .then((resp) => { + const docObjs = resp.hits?.hits + ?.slice(0, 5) + ?.map((hit: SearchHit) => hit?._source); + formikProps.setFieldValue('docs', customStringify(docObjs)); + }); + } + }, [selectedIndex]); + + // update tempErrors if errors detected + useEffect(() => { + setTempErrors(!isEmpty(formikProps.errors)); + }, [formikProps.errors]); + + return ( + onClose()} style={{ width: '70vw' }}> + + +

{`Import data`}

+ + + + <> + + + props.setSelectedOption(SOURCE_OPTIONS.MANUAL) + } + data-testid="manualEditSourceDataButton" + > + Manual + + + props.setSelectedOption(SOURCE_OPTIONS.UPLOAD) + } + data-testid="uploadSourceDataButton" + > + Upload + + + props.setSelectedOption(SOURCE_OPTIONS.EXISTING_INDEX) + } + data-testid="selectIndexSourceDataButton" + > + Existing index + + + + {props.selectedOption === SOURCE_OPTIONS.UPLOAD && ( + <> + { + if (files && files.length > 0) { + // create a custom filereader to update form with file values + const fileReader = new FileReader(); + fileReader.onload = (e) => { + if (e.target) { + formikProps.setFieldValue( + 'docs', + e.target.result as string + ); + } + }; + fileReader.readAsText(files[0]); + } + }} + display="default" + /> + + + )} + {props.selectedOption === SOURCE_OPTIONS.EXISTING_INDEX && ( + <> + + Up to 5 sample documents will be automatically populated. + + + + ({ + value: option.name, + inputDisplay: ( + {option.name} + ), + disabled: false, + } as EuiSuperSelectOption) + )} + valueOfSelected={selectedIndex} + onChange={(option) => { + setSelectedIndex(option); + }} + isInvalid={false} + /> + + + )} + + + + + onClose()} + fill={false} + color="primary" + data-testid="closeSourceDataButton" + > + Cancel + + onUpdate()} + isLoading={isUpdating} + isDisabled={tempErrors} // blocking update until valid input is given + fill={true} + color="primary" + data-testid="updateSourceDataButton" + > + Update + + + + ); + }} + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/json_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/json_field.tsx index ca49bd97..88c61e8d 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/json_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/json_field.tsx @@ -11,7 +11,7 @@ import { EuiLink, EuiText, } from '@elastic/eui'; -import { WorkspaceFormValues, customStringify } from '../../../../../common'; +import { WorkflowFormValues, customStringify } from '../../../../../common'; import { camelCaseToTitleString } from '../../../../utils'; interface JsonFieldProps { @@ -31,7 +31,7 @@ interface JsonFieldProps { export function JsonField(props: JsonFieldProps) { const validate = props.validate !== undefined ? props.validate : true; - const { errors, touched, values } = useFormikContext(); + const { errors, touched, values } = useFormikContext(); // temp input state. only format when users click out of the code editor const [jsonStr, setJsonStr] = useState('{}'); diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx index b87ca877..cb8ec809 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx @@ -18,7 +18,7 @@ import { } from '@elastic/eui'; import { MODEL_STATE, - WorkspaceFormValues, + WorkflowFormValues, ModelFormValue, IConfigField, ML_CHOOSE_MODEL_LINK, @@ -46,7 +46,7 @@ export function ModelField(props: ModelFieldProps) { // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows const models = useSelector((state: AppState) => state.ml.models); - const { errors, touched, values } = useFormikContext(); + const { errors, touched, values } = useFormikContext(); // Deployed models state const [deployedModels, setDeployedModels] = useState([]); diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/number_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/number_field.tsx index d847dfd9..c628c5de 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/number_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/number_field.tsx @@ -11,7 +11,7 @@ import { EuiText, EuiCompressedFieldNumber, } from '@elastic/eui'; -import { WorkspaceFormValues } from '../../../../../common'; +import { WorkflowFormValues } from '../../../../../common'; import { camelCaseToTitleString, getInitialValue } from '../../../../utils'; interface NumberFieldProps { @@ -27,7 +27,7 @@ interface NumberFieldProps { * An input field for a component where users input numbers */ export function NumberField(props: NumberFieldProps) { - const { errors, touched } = useFormikContext(); + const { errors, touched } = useFormikContext(); return ( diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx index 3146a944..e27f9d43 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/select_field.tsx @@ -11,7 +11,7 @@ import { EuiSuperSelectOption, EuiText, } from '@elastic/eui'; -import { WorkspaceFormValues, IConfigField } from '../../../../../common'; +import { WorkflowFormValues, IConfigField } from '../../../../../common'; import { camelCaseToTitleString } from '../../../../utils'; interface SelectFieldProps { @@ -24,7 +24,7 @@ interface SelectFieldProps { * A generic select field from a list of preconfigured options */ export function SelectField(props: SelectFieldProps) { - const { errors, touched } = useFormikContext(); + const { errors, touched } = useFormikContext(); return ( diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx index ac698805..6f0067ec 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx @@ -7,7 +7,7 @@ import React, { useEffect, useState } from 'react'; import { getIn, useFormikContext } from 'formik'; import { get, isEmpty } from 'lodash'; import { EuiComboBox, EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; -import { WorkspaceFormValues } from '../../../../../common'; +import { WorkflowFormValues } from '../../../../../common'; interface SelectWithCustomOptionsProps { fieldPath: string; @@ -20,7 +20,7 @@ interface SelectWithCustomOptionsProps { */ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { const { values, setFieldTouched, setFieldValue } = useFormikContext< - WorkspaceFormValues + WorkflowFormValues >(); // selected option state diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/text_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/text_field.tsx index e8194663..0dd0b5a3 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/text_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/text_field.tsx @@ -11,7 +11,7 @@ import { EuiLink, EuiText, } from '@elastic/eui'; -import { WorkspaceFormValues } from '../../../../../common'; +import { WorkflowFormValues } from '../../../../../common'; import { getInitialValue } from '../../../../utils'; interface TextFieldProps { @@ -28,7 +28,7 @@ interface TextFieldProps { * An input field for a component where users input plaintext */ export function TextField(props: TextFieldProps) { - const { errors, touched } = useFormikContext(); + const { errors, touched } = useFormikContext(); return ( {({ field, form }: FieldProps) => { diff --git a/public/utils/config_to_schema_utils.ts b/public/utils/config_to_schema_utils.ts index b441cab7..dd5a23d2 100644 --- a/public/utils/config_to_schema_utils.ts +++ b/public/utils/config_to_schema_utils.ts @@ -103,7 +103,7 @@ function processorsConfigToSchema(processorsConfig: ProcessorsConfig): Schema { **************** Yup (validation) utils ********************** */ -function getFieldSchema( +export function getFieldSchema( field: IConfigField, optional: boolean = false ): Schema {