From e2fa399882eb26b8ad636e62f4fbd61465bb1000 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 11 Jul 2024 04:35:13 -0700 Subject: [PATCH 01/59] initial implementation for dataset dropdown Signed-off-by: Sean Li --- .../data/common/index_patterns/types.ts | 4 + .../datasource_selectable.tsx | 5 + .../data_sources/datasource_selector/types.ts | 1 + src/plugins/data/public/index.ts | 2 + .../dataset_navigator/dataset_navigator.tsx | 64 +++++++++ .../public/ui/dataset_navigator/index.tsx | 7 + .../index_patterns_selectable.tsx | 135 ++++++++++++++++++ src/plugins/data/public/ui/index.ts | 1 + .../public/components/sidebar/index.tsx | 65 ++++++++- 9 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/index.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx diff --git a/src/plugins/data/common/index_patterns/types.ts b/src/plugins/data/common/index_patterns/types.ts index 108a93a3725b..0a8f44e5383c 100644 --- a/src/plugins/data/common/index_patterns/types.ts +++ b/src/plugins/data/common/index_patterns/types.ts @@ -51,6 +51,10 @@ export interface IIndexPattern { ) => FieldFormat; } +// export interface IDataSet { +// dataSetType: opensearch / s3 / index / indexpattern +// } + export interface IndexPatternAttributes { type: string; fields: string; diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index e826cbe18af1..c9099035144f 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -20,6 +20,8 @@ const getAndFormatDataSetFromDataSource = async ( ds: DataSource ): Promise> => { const { dataSets } = await ds.getDataSet(); + console.log('ds:', ds); + console.log('dataSets:', dataSets); return { ds, list: dataSets } as DataSetWithDataSource; }; @@ -125,6 +127,8 @@ export const DataSourceSelectable = ({ onRefresh, ...comboBoxProps }: DataSourceSelectableProps) => { + console.log('dataSources:', dataSources); + console.log('dataSources filter:', dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource)); // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { Promise.all( @@ -149,6 +153,7 @@ export const DataSourceSelectable = ({ ); const memorizedDataSourceOptionList = useMemo(() => { + console.log('dataSourceOptionList:', dataSourceOptionList); return dataSourceOptionList.map((dsGroup: DataSourceGroup) => { return { ...dsGroup, diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index fe1e4360e961..e3547046aec3 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -24,6 +24,7 @@ export interface DataSourceOption { value: string; type: string; ds: DataSource; + checked?: boolean; } export interface DataSourceSelectableProps extends Pick, 'fullWidth'> { diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index f1ac419e9ec1..7ddfef61db5d 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -445,6 +445,8 @@ export { QueryEditorTopRow, // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, + DataSetNavigator, + IndexPatternSelectable } from './ui'; /** diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx new file mode 100644 index 000000000000..31ce6c4e1f55 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState } from 'react'; + +import { EuiButton, EuiContextMenu, EuiPopover } from '@elastic/eui'; +// import { DataSourceSelectable } from '../../data_sources/datasource_selector' + +export const DataSetNavigator = ({ indexPatternSelectable, dataConnectionsRef }) => { + const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); + + const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); + const closePopover = () => setIsDataSetNavigatorOpen(false); + + const panels = [ + { + id: 0, + title: 'DATA SOURCE', + items: [ + { + name: 'Index Patterns', + panel: 1, + }, + { + name: 'Clusters', + panel: 2, + }, + { + name: '...', + onClick: () => console.log('clicked ...'), + }, + ], + }, + { + id: 1, + title: 'Index Patterns', + content:
{indexPatternSelectable}
, + }, + { + id: 2, + title: 'Clusters', + content:
{dataConnectionsRef?.current}
, + }, + ]; + + const button = ( + + Datasets + + ); + + return ( + + + + ); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx new file mode 100644 index 000000000000..fca1eb55afe1 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSetNavigator } from './dataset_navigator'; +export { IndexPatternSelectable } from './index_patterns_selectable'; diff --git a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx new file mode 100644 index 000000000000..3f61e91ab8cb --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { Fragment, useCallback, useEffect, useMemo } from 'react'; +import { EuiSelectable, EuiSpacer } from '@elastic/eui'; +import { + DataSetWithDataSource, + DataSource, + IndexPatternOption, +} from '../../data_sources/datasource'; +import { DataSourceGroup, DataSourceOption } from '../../data_sources/datasource_selector/types'; +import { isIndexPatterns } from '../../data_sources/datasource_selector/datasource_selectable'; + +const getAndFormatIndexPatternsFromDataSource = async ( + ds: DataSource +): Promise> => { + const { dataSets } = await ds.getDataSet(); + return { ds, list: dataSets } as DataSetWithDataSource; +}; + +const getAllIndexPatterns = (dataSources: DataSource[]) => + dataSources.map((ds) => getAndFormatIndexPatternsFromDataSource(ds)); + +const mapToOption = ( + dataSource: DataSource, + dataSet: DataSetWithDataSource | undefined = undefined +): DataSourceOption => { + const baseOption = { + type: dataSource.getType(), + name: dataSource.getName(), + ds: dataSource, + }; + if (dataSet && 'title' in dataSet && 'id' in dataSet && isIndexPatterns(dataSet)) { + return { + ...baseOption, + label: dataSet.title as string, + value: dataSet.id as string, + key: dataSet.id as string, + }; + } + return { + ...baseOption, + label: dataSource.getName(), + value: dataSource.getName(), + key: dataSource.getId(), + }; +}; + +function consolidate( + dataSets: DataSetWithDataSource[], + selectedSources: DataSourceOption[] +): DataSourceOption[] { + const result: DataSourceOption[] = []; + console.log('selectedSources:', selectedSources); + + dataSets.forEach((dataSet) => { + dataSet.list.forEach((indexPatternOption) => { + const dataSourceOption: DataSourceOption = { + type: 'indexPattern', + name: dataSet.ds.getName(), + ds: dataSet.ds, + key: indexPatternOption.id as string, + label: indexPatternOption.title as string, + value: indexPatternOption.id as string, + }; + + if (selectedSources.length !== 0 && selectedSources[0].value === dataSourceOption.value) { + dataSourceOption.checked = true; + } + + result.push(dataSourceOption); + }); + }); + + return result; +} + +interface IndexPatternSelectableProps { + dataSources: DataSource[]; + indexPatternOptionList: any; + selectedSources: any; + setIndexPatternOptionList: any; + handleSourceSelection: any; +} + +export const IndexPatternSelectable = ({ + dataSources, + indexPatternOptionList, + selectedSources, + // onIndexPatternSelect, + setIndexPatternOptionList, + handleSourceSelection, +}: IndexPatternSelectableProps) => { + useEffect(() => { + Promise.all( + getAllIndexPatterns( + dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) + ) + ).then((dataSetResults) => { + setIndexPatternOptionList(consolidate(dataSetResults, selectedSources)); + }); + }, [dataSources, setIndexPatternOptionList]); + + const handleSourceChange = useCallback( + (selectedOptions: any) => handleSourceSelection(selectedOptions), + [handleSourceSelection] + ); + + // const memoizedIndexPatternOptionList = useMemo(() => { + // return indexPatternOptionList; + // }, [indexPatternOptionList]); + + return ( +
+ { + setIndexPatternOptionList(newOptions); + handleSourceChange(newOptions.filter((option) => option?.checked)); + }} + singleSelection="always" + > + {(list, search) => ( + + {search} + {list} + + )} + +
+ ); +}; diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index 5483b540d5bf..d3163f9f0010 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -49,3 +49,4 @@ export { } from './query_editor'; export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; export { SuggestionsComponent } from './typeahead'; +export { DataSetNavigator, IndexPatternSelectable } from './dataset_navigator'; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index eea1860dc950..13f3bfbb36f4 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -6,7 +6,13 @@ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; +import { + DataSource, + DataSourceGroup, + DataSourceSelectable, + DataSetNavigator, + IndexPatternSelectable, +} from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; @@ -17,10 +23,12 @@ export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); const dispatch = useTypedDispatch(); const [selectedSources, setSelectedSources] = useState([]); + const [indexPatternOptionList, setIndexPatternOptionList] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); const [activeDataSources, setActiveDataSources] = useState([]); const [isEnhancementsEnabled, setIsEnhancementsEnabled] = useState(false); const containerRef = useRef(null); + const connectionsRef = useRef(null); const { services: { @@ -46,6 +54,10 @@ export const Sidebar: FC = ({ children }) => { uiContainerRef.appendChild(containerRef.current); }, []); + const setConnectionsRef = useCallback((uiConnectionsRef) => { + uiConnectionsRef.appendChild(connectionsRef.current); + }, []); + useEffect(() => { if (!isEnhancementsEnabled) return; const subscriptions = ui.container$.subscribe((container) => { @@ -55,10 +67,26 @@ export const Sidebar: FC = ({ children }) => { } }); + const connectionsSubscriptions = ui.dataSourceContainer$.subscribe((container) => { + if (container === null) return; + if (connectionsRef.current) { + setConnectionsRef(container); + } + }); + return () => { subscriptions.unsubscribe(); + connectionsSubscriptions.unsubscribe(); }; - }, [ui.container$, containerRef, setContainerRef, isEnhancementsEnabled]); + }, [ + ui.container$, + containerRef, + setContainerRef, + ui.dataSourceContainer$, + connectionsRef, + setConnectionsRef, + isEnhancementsEnabled, + ]); useEffect(() => { let isMounted = true; @@ -91,6 +119,17 @@ export const Sidebar: FC = ({ children }) => { } }, [indexPatternId, activeDataSources, dataSourceOptionList]); + const getMatchedIndexPattern = (indexPatternList: DataSourceOption[], ipId: string) => { + return indexPatternList.find((indexPattern) => indexPattern.value === ipId); + }; + + useEffect(() => { + if (indexPatternId) { + const option = getMatchedIndexPattern(indexPatternOptionList, indexPatternId); + setSelectedSources(option ? [option] : []); + } + }, [indexPatternId, activeDataSources, indexPatternOptionList]); + const redirectToLogExplorer = useCallback( (dsName: string, dsType: string) => { return application.navigateToUrl( @@ -147,6 +186,23 @@ export const Sidebar: FC = ({ children }) => { /> ); + const indexPatternSelectable = ( + + ); + + const dataSetNavigator = ( + + ); + return ( { containerRef.current = node; }} > - {dataSourceSelector} + {dataSetNavigator} + {/* {indexPatternSelectable} */} )} {!isEnhancementsEnabled && ( @@ -171,7 +228,7 @@ export const Sidebar: FC = ({ children }) => { color="transparent" className="deSidebar_dataSource" > - {dataSourceSelector} + {dataSetNavigator} )} From 7fc9d4beedc7f20d74080541fdcf6225c05be660 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 11 Jul 2024 12:04:05 -0700 Subject: [PATCH 02/59] initial metadata commit Signed-off-by: Sean Li --- .../index_patterns_selectable.tsx | 1 + .../utils/state_management/metadata_slice.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx index 3f61e91ab8cb..d9bd5dee422d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx @@ -116,6 +116,7 @@ export const IndexPatternSelectable = ({
{ setIndexPatternOptionList(newOptions); diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index e9fe84713120..d79ba2af6d7a 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -6,10 +6,26 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import { DataExplorerServices } from '../../types'; +interface DataSourceMeta { + ref: string // MDS ID + dsName?: string // flint datasource +} + +export interface DataSet { + id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table + datasource?: DataSourceMeta; + meta?: { + timestampField: string; + mapping?: any; + } + type?: 'dataset' | 'temporary'; +} + export interface MetadataState { indexPattern?: string; originatingApp?: string; view?: string; + dataset?: DataSet; } const initialState: MetadataState = {}; @@ -28,6 +44,9 @@ export const getPreloadedState = async ({ ...initialState, originatingApp, indexPattern: defaultIndexPattern?.id, + dataset: { + id: defaultIndexPattern?.id + } }; return preloadedState; From d0d5ec92b275dade6aea0a22d55b706b8bf5f2ac Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 11 Jul 2024 15:42:17 -0700 Subject: [PATCH 03/59] bug fixes Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 12 +++++++-- .../index_patterns_selectable.tsx | 1 + .../public/components/sidebar/index.tsx | 25 +++++++++++++------ .../utils/state_management/metadata_slice.ts | 5 +++- .../public/utils/state_management/store.ts | 2 +- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 31ce6c4e1f55..647e4ea256ea 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -8,7 +8,15 @@ import React, { useState } from 'react'; import { EuiButton, EuiContextMenu, EuiPopover } from '@elastic/eui'; // import { DataSourceSelectable } from '../../data_sources/datasource_selector' -export const DataSetNavigator = ({ indexPatternSelectable, dataConnectionsRef }) => { +interface DataSetNavigatorProps { + indexPatternSelectable: any; + dataConnectionsRef: HTMLDivElement | null; +} + +export const DataSetNavigator = ({ + indexPatternSelectable, + dataConnectionsRef, +}: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); @@ -41,7 +49,7 @@ export const DataSetNavigator = ({ indexPatternSelectable, dataConnectionsRef }) { id: 2, title: 'Clusters', - content:
{dataConnectionsRef?.current}
, + content:
{dataConnectionsRef?.[0]}
}, ]; diff --git a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx index d9bd5dee422d..78f22962019a 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx @@ -73,6 +73,7 @@ function consolidate( result.push(dataSourceOption); }); }); + console.log('result:', result); return result; } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 13f3bfbb36f4..2fffb14b3991 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -16,7 +16,12 @@ import { import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; -import { setIndexPattern, useTypedDispatch, useTypedSelector } from '../../utils/state_management'; +import { + setIndexPattern, + setDataset, + useTypedDispatch, + useTypedSelector, +} from '../../utils/state_management'; import './index.scss'; export const Sidebar: FC = ({ children }) => { @@ -69,9 +74,7 @@ export const Sidebar: FC = ({ children }) => { const connectionsSubscriptions = ui.dataSourceContainer$.subscribe((container) => { if (container === null) return; - if (connectionsRef.current) { - setConnectionsRef(container); - } + connectionsRef.current = container; }); return () => { @@ -84,7 +87,6 @@ export const Sidebar: FC = ({ children }) => { setContainerRef, ui.dataSourceContainer$, connectionsRef, - setConnectionsRef, isEnhancementsEnabled, ]); @@ -153,6 +155,12 @@ export const Sidebar: FC = ({ children }) => { } setSelectedSources(selectedDataSources); dispatch(setIndexPattern(selectedDataSources[0].value)); + dispatch( + setDataset({ + id: selectedDataSources[0].value, + datasource: { ref: selectedDataSources[0].ds.getId() }, + }) + ); }, [dispatch, redirectToLogExplorer, setSelectedSources] ); @@ -199,7 +207,7 @@ export const Sidebar: FC = ({ children }) => { const dataSetNavigator = ( ); @@ -217,7 +225,10 @@ export const Sidebar: FC = ({ children }) => { containerRef.current = node; }} > - {dataSetNavigator} + {/* {indexPatternSelectable} */} )} diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index d79ba2af6d7a..628f313cad0a 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -59,6 +59,9 @@ export const slice = createSlice({ setIndexPattern: (state, action: PayloadAction) => { state.indexPattern = action.payload; }, + setDataset: (state, action: PayloadAction) => { + state.dataset = action.payload; + }, setOriginatingApp: (state, action: PayloadAction) => { state.originatingApp = action.payload; }, @@ -72,4 +75,4 @@ export const slice = createSlice({ }); export const { reducer } = slice; -export const { setIndexPattern, setOriginatingApp, setView, setState } = slice.actions; +export const { setIndexPattern, setDataset, setOriginatingApp, setView, setState } = slice.actions; diff --git a/src/plugins/data_explorer/public/utils/state_management/store.ts b/src/plugins/data_explorer/public/utils/state_management/store.ts index daf0b3d7e369..3683d7d201ce 100644 --- a/src/plugins/data_explorer/public/utils/state_management/store.ts +++ b/src/plugins/data_explorer/public/utils/state_management/store.ts @@ -116,4 +116,4 @@ export type RenderState = Omit; // Remaining state after export type Store = ReturnType; export type AppDispatch = Store['dispatch']; -export { MetadataState, setIndexPattern, setOriginatingApp } from './metadata_slice'; +export { MetadataState, setIndexPattern, setDataset, setOriginatingApp } from './metadata_slice'; From d60ead0c434bef5818cdc069c30506cf88a18287 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 11 Jul 2024 16:11:41 -0700 Subject: [PATCH 04/59] hiding datasource cluster selctor and other bug fixes Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 19 ++++++++++++++----- .../index_patterns_selectable.tsx | 4 +++- .../public/ui/query_editor/query_editor.tsx | 6 ------ .../public/components/sidebar/index.tsx | 4 +++- .../utils/create_extension.tsx | 3 ++- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 647e4ea256ea..e3a2f1d4a622 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -5,17 +5,19 @@ import React, { useState } from 'react'; -import { EuiButton, EuiContextMenu, EuiPopover } from '@elastic/eui'; +import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; // import { DataSourceSelectable } from '../../data_sources/datasource_selector' interface DataSetNavigatorProps { indexPatternSelectable: any; dataConnectionsRef: HTMLDivElement | null; + children: any; } export const DataSetNavigator = ({ indexPatternSelectable, dataConnectionsRef, + children }: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); @@ -49,14 +51,21 @@ export const DataSetNavigator = ({ { id: 2, title: 'Clusters', - content:
{dataConnectionsRef?.[0]}
+ content:
{dataConnectionsRef?.[0]}
, }, ]; const button = ( - - Datasets - + + {children && children.length > 0 ? children[0].label : 'Datasets'} + ); return ( diff --git a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx index 78f22962019a..100a580d597f 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx @@ -105,7 +105,9 @@ export const IndexPatternSelectable = ({ }, [dataSources, setIndexPatternOptionList]); const handleSourceChange = useCallback( - (selectedOptions: any) => handleSourceSelection(selectedOptions), + (selectedOptions: any) => { + handleSourceSelection(selectedOptions); + }, [handleSourceSelection] ); diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 69d332c65c04..c76575fdc25d 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -385,12 +385,6 @@ export default class QueryEditorUI extends Component { isCollapsed={!this.state.isCollapsed} /> - {this.state.isDataSourcesVisible && ( - -
- - )} - {this.state.isDataSetsVisible && (
diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 2fffb14b3991..2a1faa986623 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -228,7 +228,9 @@ export const Sidebar: FC = ({ children }) => { + > + {selectedSources} + {/* {indexPatternSelectable} */} )} diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx index e5822c4b378e..5b3a603103bf 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx @@ -9,6 +9,7 @@ import { QueryEditorExtensionConfig } from '../../../../data/public'; import { ConfigSchema } from '../../../common/config'; import { ConnectionsBar } from '../components'; import { ConnectionsService } from '../services'; +import { of } from 'rxjs'; export const createDataSourceConnectionExtension = ( connectionsService: ConnectionsService, @@ -19,7 +20,7 @@ export const createDataSourceConnectionExtension = ( id: 'data-source-connection', order: 2000, isEnabled$: (dependencies) => { - return connectionsService.getIsDataSourceEnabled$(); + return of(false); }, getComponent: (dependencies) => { return ( From 8506e88005669b358d02b245465ab0274dc15fad Mon Sep 17 00:00:00 2001 From: Sean Li Date: Mon, 15 Jul 2024 10:27:08 -0700 Subject: [PATCH 05/59] refactoring and fixing datasource selector button Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 120 +++++++++++---- .../public/ui/dataset_navigator/index.tsx | 2 +- .../index_pattern_selectable.tsx | 48 ++++++ .../index_patterns_selectable.tsx | 139 ------------------ .../public/components/sidebar/index.tsx | 40 +---- 5 files changed, 145 insertions(+), 204 deletions(-) create mode 100644 src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx delete mode 100644 src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index e3a2f1d4a622..b27a447509a1 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -3,43 +3,120 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; -// import { DataSourceSelectable } from '../../data_sources/datasource_selector' +import { + DataSetWithDataSource, + DataSource, + IndexPatternOption, +} from '../../data_sources/datasource'; +import { DataSourceOption } from '../../data_sources/datasource_selector/types'; +import { IndexPatternSelectable } from './index_pattern_selectable'; + +const getAndFormatIndexPatternsFromDataSource = async ( + ds: DataSource +): Promise> => { + const { dataSets } = await ds.getDataSet(); + return { ds, list: dataSets } as DataSetWithDataSource; +}; + +const getAllIndexPatterns = (dataSources: DataSource[]) => + dataSources.map((ds) => getAndFormatIndexPatternsFromDataSource(ds)); + +const consolidateIndexPatternList = ( + dataSets: DataSetWithDataSource[], + selectedSources: DataSourceOption[] +): DataSourceOption[] => { + const result: DataSourceOption[] = []; + + dataSets.forEach((dataSet) => { + dataSet.list.forEach((indexPatternOption) => { + const dataSourceOption: DataSourceOption = { + type: 'indexPattern', + name: dataSet.ds.getName(), + ds: dataSet.ds, + key: indexPatternOption.id as string, + label: indexPatternOption.title as string, + value: indexPatternOption.id as string, + }; + + if (selectedSources.length !== 0 && selectedSources[0].value === dataSourceOption.value) { + dataSourceOption.checked = true; + } + + result.push(dataSourceOption); + }); + }); + + return result; +}; interface DataSetNavigatorProps { - indexPatternSelectable: any; - dataConnectionsRef: HTMLDivElement | null; - children: any; + dataSources: DataSource[]; + indexPatternOptionList: any; + selectedSources: DataSourceOption[]; + setIndexPatternOptionList: any; + handleSourceSelection: any; } export const DataSetNavigator = ({ - indexPatternSelectable, - dataConnectionsRef, - children + dataSources, + indexPatternOptionList, + selectedSources, + setIndexPatternOptionList, + handleSourceSelection, }: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); - const panels = [ + useEffect(() => { + Promise.all( + getAllIndexPatterns( + dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) + ) + ).then((dataSetResults) => { + setIndexPatternOptionList(consolidateIndexPatternList(dataSetResults, selectedSources)); + }); + }, [dataSources, selectedSources, setIndexPatternOptionList]); + + const indexPatternSelectable = ( + + ); + + const dataSetButton = ( + + {selectedSources.length > 0 ? selectedSources[0].label : 'Datasets'} + + ); + + const dataSetNavigatorPanels = [ { id: 0, - title: 'DATA SOURCE', + title: 'DATA SETS', items: [ { name: 'Index Patterns', panel: 1, }, { - name: 'Clusters', + name: 'Connected Data Sources', panel: 2, }, { name: '...', - onClick: () => console.log('clicked ...'), }, ], }, @@ -51,31 +128,18 @@ export const DataSetNavigator = ({ { id: 2, title: 'Clusters', - content:
{dataConnectionsRef?.[0]}
, + content:
, }, ]; - const button = ( - - {children && children.length > 0 ? children[0].label : 'Datasets'} - - ); - return ( - + ); }; diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx index fca1eb55afe1..a50eed767b4d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -4,4 +4,4 @@ */ export { DataSetNavigator } from './dataset_navigator'; -export { IndexPatternSelectable } from './index_patterns_selectable'; +export { IndexPatternSelectable } from './index_pattern_selectable'; diff --git a/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx new file mode 100644 index 000000000000..d9f73ded5308 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { Fragment, useCallback } from 'react'; +import { EuiSelectable } from '@elastic/eui'; + +interface IndexPatternSelectableProps { + indexPatternOptionList: any; + setIndexPatternOptionList: any; + handleSourceSelection: any; +} + +export const IndexPatternSelectable = ({ + indexPatternOptionList, + setIndexPatternOptionList, + handleSourceSelection, +}: IndexPatternSelectableProps) => { + const handleSourceChange = useCallback( + (selectedOptions: any) => { + handleSourceSelection(selectedOptions); + }, + [handleSourceSelection] + ); + + return ( +
+ { + setIndexPatternOptionList(newOptions); + handleSourceChange(newOptions.filter((option) => option?.checked)); + }} + singleSelection="always" + > + {(list, search) => ( + + {search} + {list} + + )} + +
+ ); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx deleted file mode 100644 index 100a580d597f..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/index_patterns_selectable.tsx +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { Fragment, useCallback, useEffect, useMemo } from 'react'; -import { EuiSelectable, EuiSpacer } from '@elastic/eui'; -import { - DataSetWithDataSource, - DataSource, - IndexPatternOption, -} from '../../data_sources/datasource'; -import { DataSourceGroup, DataSourceOption } from '../../data_sources/datasource_selector/types'; -import { isIndexPatterns } from '../../data_sources/datasource_selector/datasource_selectable'; - -const getAndFormatIndexPatternsFromDataSource = async ( - ds: DataSource -): Promise> => { - const { dataSets } = await ds.getDataSet(); - return { ds, list: dataSets } as DataSetWithDataSource; -}; - -const getAllIndexPatterns = (dataSources: DataSource[]) => - dataSources.map((ds) => getAndFormatIndexPatternsFromDataSource(ds)); - -const mapToOption = ( - dataSource: DataSource, - dataSet: DataSetWithDataSource | undefined = undefined -): DataSourceOption => { - const baseOption = { - type: dataSource.getType(), - name: dataSource.getName(), - ds: dataSource, - }; - if (dataSet && 'title' in dataSet && 'id' in dataSet && isIndexPatterns(dataSet)) { - return { - ...baseOption, - label: dataSet.title as string, - value: dataSet.id as string, - key: dataSet.id as string, - }; - } - return { - ...baseOption, - label: dataSource.getName(), - value: dataSource.getName(), - key: dataSource.getId(), - }; -}; - -function consolidate( - dataSets: DataSetWithDataSource[], - selectedSources: DataSourceOption[] -): DataSourceOption[] { - const result: DataSourceOption[] = []; - console.log('selectedSources:', selectedSources); - - dataSets.forEach((dataSet) => { - dataSet.list.forEach((indexPatternOption) => { - const dataSourceOption: DataSourceOption = { - type: 'indexPattern', - name: dataSet.ds.getName(), - ds: dataSet.ds, - key: indexPatternOption.id as string, - label: indexPatternOption.title as string, - value: indexPatternOption.id as string, - }; - - if (selectedSources.length !== 0 && selectedSources[0].value === dataSourceOption.value) { - dataSourceOption.checked = true; - } - - result.push(dataSourceOption); - }); - }); - console.log('result:', result); - - return result; -} - -interface IndexPatternSelectableProps { - dataSources: DataSource[]; - indexPatternOptionList: any; - selectedSources: any; - setIndexPatternOptionList: any; - handleSourceSelection: any; -} - -export const IndexPatternSelectable = ({ - dataSources, - indexPatternOptionList, - selectedSources, - // onIndexPatternSelect, - setIndexPatternOptionList, - handleSourceSelection, -}: IndexPatternSelectableProps) => { - useEffect(() => { - Promise.all( - getAllIndexPatterns( - dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) - ) - ).then((dataSetResults) => { - setIndexPatternOptionList(consolidate(dataSetResults, selectedSources)); - }); - }, [dataSources, setIndexPatternOptionList]); - - const handleSourceChange = useCallback( - (selectedOptions: any) => { - handleSourceSelection(selectedOptions); - }, - [handleSourceSelection] - ); - - // const memoizedIndexPatternOptionList = useMemo(() => { - // return indexPatternOptionList; - // }, [indexPatternOptionList]); - - return ( -
- { - setIndexPatternOptionList(newOptions); - handleSourceChange(newOptions.filter((option) => option?.checked)); - }} - singleSelection="always" - > - {(list, search) => ( - - {search} - {list} - - )} - -
- ); -}; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 2a1faa986623..849486f066ed 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -6,13 +6,7 @@ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { - DataSource, - DataSourceGroup, - DataSourceSelectable, - DataSetNavigator, - IndexPatternSelectable, -} from '../../../../data/public'; +import { DataSource, DataSourceGroup, DataSetNavigator } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; @@ -181,21 +175,8 @@ export const Sidebar: FC = ({ children }) => { dataSources.dataSourceService.reload(); }, [dataSources.dataSourceService]); - const dataSourceSelector = ( - - ); - - const indexPatternSelectable = ( - { /> ); - const dataSetNavigator = ( - - ); - return ( { containerRef.current = node; }} > - - {selectedSources} - - {/* {indexPatternSelectable} */} + {dataSetNavigator} )} {!isEnhancementsEnabled && ( From 32bfcb51e3ae4469867e85bcaa3588273671e916 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Mon, 15 Jul 2024 10:35:36 -0700 Subject: [PATCH 06/59] fix breaking loop Signed-off-by: Sean Li --- .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index b27a447509a1..f4debecc7399 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -80,7 +80,7 @@ export const DataSetNavigator = ({ ).then((dataSetResults) => { setIndexPatternOptionList(consolidateIndexPatternList(dataSetResults, selectedSources)); }); - }, [dataSources, selectedSources, setIndexPatternOptionList]); + }, [dataSources, setIndexPatternOptionList]); const indexPatternSelectable = ( Date: Tue, 16 Jul 2024 01:40:42 -0700 Subject: [PATCH 07/59] last working commit Signed-off-by: Sean Li --- .../datasource_selectable.tsx | 2 - .../dataset_navigator/dataset_navigator.tsx | 116 ++++++++++++------ .../public/components/sidebar/index.tsx | 3 + .../public/components/utils.ts | 1 + 4 files changed, 82 insertions(+), 40 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index c9099035144f..90f7ea6b23c1 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -20,8 +20,6 @@ const getAndFormatDataSetFromDataSource = async ( ds: DataSource ): Promise> => { const { dataSets } = await ds.getDataSet(); - console.log('ds:', ds); - console.log('dataSets:', dataSets); return { ds, list: dataSets } as DataSetWithDataSource; }; diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index f4debecc7399..ebc44e9173b6 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -5,7 +5,12 @@ import React, { useEffect, useState } from 'react'; -import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; +import { + EuiButtonEmpty, + EuiContextMenu, + EuiContextMenuPanelDescriptor, + EuiPopover, +} from '@elastic/eui'; import { DataSetWithDataSource, DataSource, @@ -13,6 +18,8 @@ import { } from '../../data_sources/datasource'; import { DataSourceOption } from '../../data_sources/datasource_selector/types'; import { IndexPatternSelectable } from './index_pattern_selectable'; +import { SavedObjectsClientContract, SimpleSavedObject } from 'opensearch-dashboards/public'; +import { DataSourceAttributes } from '../../../../data_source_management/public/types'; const getAndFormatIndexPatternsFromDataSource = async ( ds: DataSource @@ -52,7 +59,15 @@ const consolidateIndexPatternList = ( return result; }; +const getClusters = async (savedObjectsClient: SavedObjectsClientContract) => { + return await savedObjectsClient.find({ + type: 'data-source', + perPage: 10000, + }); +}; + interface DataSetNavigatorProps { + savedObjectsClient: SavedObjectsClientContract; dataSources: DataSource[]; indexPatternOptionList: any; selectedSources: DataSourceOption[]; @@ -61,6 +76,7 @@ interface DataSetNavigatorProps { } export const DataSetNavigator = ({ + savedObjectsClient, dataSources, indexPatternOptionList, selectedSources, @@ -68,41 +84,10 @@ export const DataSetNavigator = ({ handleSourceSelection, }: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); - - const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); - const closePopover = () => setIsDataSetNavigatorOpen(false); - - useEffect(() => { - Promise.all( - getAllIndexPatterns( - dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) - ) - ).then((dataSetResults) => { - setIndexPatternOptionList(consolidateIndexPatternList(dataSetResults, selectedSources)); - }); - }, [dataSources, setIndexPatternOptionList]); - - const indexPatternSelectable = ( - - ); - - const dataSetButton = ( - - {selectedSources.length > 0 ? selectedSources[0].label : 'Datasets'} - - ); - - const dataSetNavigatorPanels = [ + const [clusterList, setClusterList] = useState[]>([]); + const [dataSetNavigatorPanels, setDataSetNavigatorPanels] = useState< + EuiContextMenuPanelDescriptor[] + >([ { id: 0, title: 'DATA SETS', @@ -117,20 +102,75 @@ export const DataSetNavigator = ({ }, { name: '...', + panel: 2, + onClick: () => console.log('clicked ..') }, ], }, { id: 1, title: 'Index Patterns', - content:
{indexPatternSelectable}
, + content: ( + + ), }, { id: 2, title: 'Clusters', content:
, }, - ]; + ]); + + const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); + const closePopover = () => setIsDataSetNavigatorOpen(false); + + useEffect(() => { + Promise.all( + getAllIndexPatterns( + dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) + ) + ).then((dataSetResults) => { + setIndexPatternOptionList(consolidateIndexPatternList(dataSetResults, selectedSources)); + }); + }, [dataSources, setIndexPatternOptionList]); + + useEffect(() => { + Promise.all([getClusters(savedObjectsClient)]).then((res) => { + setClusterList(res.length > 0 ? res?.[0].savedObjects : []); + }); + }, [savedObjectsClient]); + + useEffect(() => { + console.log('cluster list:', clusterList); + }, [clusterList]); + + // useEffect(() => { + // setDataSetNavigatorPanels((prevPanels: EuiContextMenuPanelDescriptor[]) => + // prevPanels.map(panel => { + // const initialItems = panel.items?.filter(item => item?.panel === 1); + // const clusterPanels = clusterList.map((cluster) => ({ + // name: cluster.attributes.title, + // })) + // return panel; + // }) + // ) + // }, [clusterList]) + + const dataSetButton = ( + + {selectedSources.length > 0 ? selectedSources[0].label : 'Datasets'} + + ); return ( { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); const dispatch = useTypedDispatch(); const [selectedSources, setSelectedSources] = useState([]); + const [selectedCluster, setSelectedCluster] = useState(); const [indexPatternOptionList, setIndexPatternOptionList] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); const [activeDataSources, setActiveDataSources] = useState([]); @@ -34,6 +35,7 @@ export const Sidebar: FC = ({ children }) => { data: { indexPatterns, dataSources, ui }, notifications: { toasts }, application, + savedObjects, }, } = useOpenSearchDashboards(); @@ -177,6 +179,7 @@ export const Sidebar: FC = ({ children }) => { const dataSetNavigator = ( >, filter = (ds: SavedObject) => true ): DataSourceOption[] { + console.log('dataSources:', dataSources); return dataSources .filter((ds) => filter!(ds)) .map((ds) => ({ From d3a4519d726040e5322b36f872a5da1e1ba350ce Mon Sep 17 00:00:00 2001 From: Sean Li Date: Wed, 17 Jul 2024 11:39:03 -0700 Subject: [PATCH 08/59] WIP indices Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 242 ++++++++++++++---- .../field_selector_panel.tsx | 79 ++++++ .../index_pattern_selectable.tsx | 2 + .../public/components/sidebar/index.tsx | 29 ++- .../lib/get_indices.ts | 1 + .../query_enhancements/public/plugin.tsx | 4 +- 6 files changed, 289 insertions(+), 68 deletions(-) create mode 100644 src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index ebc44e9173b6..b8b82fdc8389 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -8,6 +8,7 @@ import React, { useEffect, useState } from 'react'; import { EuiButtonEmpty, EuiContextMenu, + EuiContextMenuItem, EuiContextMenuPanelDescriptor, EuiPopover, } from '@elastic/eui'; @@ -18,8 +19,15 @@ import { } from '../../data_sources/datasource'; import { DataSourceOption } from '../../data_sources/datasource_selector/types'; import { IndexPatternSelectable } from './index_pattern_selectable'; -import { SavedObjectsClientContract, SimpleSavedObject } from 'opensearch-dashboards/public'; +import { + HttpSetup, + SavedObjectsClientContract, + SimpleSavedObject, +} from 'opensearch-dashboards/public'; import { DataSourceAttributes } from '../../../../data_source_management/public/types'; +import { ISearchStart } from '../../search/types'; +import { map, scan } from 'rxjs/operators'; +import { IndexPatternsContract } from '../..'; const getAndFormatIndexPatternsFromDataSource = async ( ds: DataSource @@ -55,6 +63,7 @@ const consolidateIndexPatternList = ( result.push(dataSourceOption); }); }); + console.log('results:', result); return result; }; @@ -66,64 +75,97 @@ const getClusters = async (savedObjectsClient: SavedObjectsClientContract) => { }); }; +export const searchResponseToArray = (showAllIndices: boolean) => (response) => { + const { rawResponse } = response; + if (!rawResponse.aggregations) { + return []; + } else { + return rawResponse.aggregations.indices.buckets + .map((bucket: { key: string }) => { + return bucket.key; + }) + .filter((indexName: string) => { + if (showAllIndices) { + return true; + } else { + return !indexName.startsWith('.'); + } + }) + .map((indexName: string) => { + return { + name: indexName, + // item: {}, + }; + }); + } +}; + +const buildSearchRequest = (showAllIndices: boolean, pattern: string, dataSourceId?: string) => { + const request = { + params: { + ignoreUnavailable: true, + expand_wildcards: showAllIndices ? 'all' : 'open', + index: pattern, + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: 100, + }, + }, + }, + }, + }, + dataSourceId: dataSourceId, + }; + + return request; +}; + +const getIndices = async (search: ISearchStart, dataSourceId: string) => { + const request = buildSearchRequest(true, '*', dataSourceId); + return search + .search(request) + .pipe(map(searchResponseToArray(true))) + .pipe(scan((accumulator = [], value) => accumulator.join(value))) + .toPromise() + .catch(() => []); +}; + interface DataSetNavigatorProps { + http: HttpSetup; + search: ISearchStart; savedObjectsClient: SavedObjectsClientContract; + indexPatterns: IndexPatternsContract; dataSources: DataSource[]; indexPatternOptionList: any; selectedSources: DataSourceOption[]; + selectedCluster: any; setIndexPatternOptionList: any; + setSelectedCluster: any; handleSourceSelection: any; } export const DataSetNavigator = ({ + http, + search, savedObjectsClient, + indexPatterns, dataSources, indexPatternOptionList, selectedSources, + selectedCluster, setIndexPatternOptionList, + setSelectedCluster, handleSourceSelection, }: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); const [clusterList, setClusterList] = useState[]>([]); - const [dataSetNavigatorPanels, setDataSetNavigatorPanels] = useState< - EuiContextMenuPanelDescriptor[] - >([ - { - id: 0, - title: 'DATA SETS', - items: [ - { - name: 'Index Patterns', - panel: 1, - }, - { - name: 'Connected Data Sources', - panel: 2, - }, - { - name: '...', - panel: 2, - onClick: () => console.log('clicked ..') - }, - ], - }, - { - id: 1, - title: 'Index Patterns', - content: ( - - ), - }, - { - id: 2, - title: 'Clusters', - content:
, - }, - ]); + const [indexList, setIndexList] = useState([]); + const [selectedIndex, setSelectedIndex] = useState(''); + const [selectedField, setSelectedField] = useState(''); const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); @@ -145,20 +187,53 @@ export const DataSetNavigator = ({ }, [savedObjectsClient]); useEffect(() => { - console.log('cluster list:', clusterList); - }, [clusterList]); - - // useEffect(() => { - // setDataSetNavigatorPanels((prevPanels: EuiContextMenuPanelDescriptor[]) => - // prevPanels.map(panel => { - // const initialItems = panel.items?.filter(item => item?.panel === 1); - // const clusterPanels = clusterList.map((cluster) => ({ - // name: cluster.attributes.title, - // })) - // return panel; - // }) - // ) - // }, [clusterList]) + if (selectedCluster) { + // Get all indexes + Promise.all([getIndices(search, selectedCluster.id)]).then((res) => { + if (res && res.length > 0) { + console.log('res', Object.values(res?.[0])); + setIndexList( + Object.values(res?.[0]).map((index: any) => ({ + ...index, + // panel: 4, + onClick: async () => { + setSelectedIndex(index.name); + const existingIndexPattern = indexPatterns.getByTitle(index.name, true); + const dataSet = await indexPatterns.create( + { id: index.name, title: index.name }, + !existingIndexPattern?.id + ); + // save to cache by title because the id is not unique for temporary index pattern created + indexPatterns.saveToCache(dataSet.title, dataSet); + handleSourceSelection([ + { + type: 'indexPattern', + name: 'OpenSearch Default', + ds: selectedCluster, + key: index.name, + label: index.name, + value: index.name, + }, + ]); + }, + })) + ); + } + }); + // Update panels related to cluster + } + }, [selectedCluster, setIndexList, setSelectedIndex]); + + useEffect(() => { + if (selectedIndex) { + Promise.all([ + indexPatterns.getFieldsForWildcard({ + pattern: selectedIndex, + dataSourceId: selectedCluster.id, + }), + ]).then((res) => console.log(res)); + } + }, [selectedIndex]); const dataSetButton = ( - + ({ + name: cluster.attributes.title, + panel: 2, + onClick: () => { + setSelectedCluster(cluster); + }, + })) + : []), + ], + }, + { + id: 1, + title: 'Index Patterns', + content: ( + + ), + }, + { + id: 2, + title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', + items: [ + { + name: 'Indexes', + panel: 3, + }, + { + name: 'Connected Data Sources', + }, + ], + }, + { + id: 3, + title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', + items: indexList, + }, + { + id: 4, + title: 'clicked', + }, + ]} + /> ); }; diff --git a/src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx b/src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx new file mode 100644 index 000000000000..dc1bff8210a2 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { Fragment, useCallback, useEffect, useState } from 'react'; +import { EuiSelectable, EuiContextMenuPanel, EuiComboBox, EuiSelect } from '@elastic/eui'; +import { IFieldType, IndexPatternsContract } from '../..'; +import { i18n } from '@osd/i18n'; + +export function extractTimeFields(fields: IFieldType[]) { + const dateFields = fields.filter((field) => field.type === 'date'); + const label = i18n.translate( + 'indexPatternManagement.createIndexPattern.stepTime.noTimeFieldsLabel', + { + defaultMessage: "The indices which match this index pattern don't contain any time fields.", + } + ); + + if (dateFields.length === 0) { + return [ + { + display: label, + }, + ]; + } + + const disabledDividerOption = { + isDisabled: true, + display: '───', + fieldName: '', + }; + const noTimeFieldLabel = i18n.translate( + 'indexPatternManagement.createIndexPattern.stepTime.noTimeFieldOptionLabel', + { + defaultMessage: "I don't want to use the time filter", + } + ); + const noTimeFieldOption = { + display: noTimeFieldLabel, + fieldName: undefined, + }; + + return [ + ...dateFields.map((field) => ({ + display: field.name, + fieldName: field.name, + })), + disabledDividerOption, + noTimeFieldOption, + ]; +} + + +export const FieldSelctorPanel = ({ + index, + dataSourceId, + indexPatterns, +}: { + index: string; + dataSourceId: string; + indexPatterns: IndexPatternsContract; +}) => { + const [timeStampFields, setTimeStampFields] = useState([]); + + const getTimeStampFields = async () => { + const fields = await indexPatterns.getFieldsForWildcard({ pattern: index, dataSourceId}); + const timeFields = extractTimeFields(fields); + setTimeStampFields(timeFields); + }; + + const dropdown = ; + + return { + id: 3, + title: index, + content:
, + }; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx index d9f73ded5308..0b5cfaac051e 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx @@ -24,6 +24,8 @@ export const IndexPatternSelectable = ({ [handleSourceSelection] ); + console.log('option list:', indexPatternOptionList); + return (
{ const { services: { - data: { indexPatterns, dataSources, ui }, + data: { indexPatterns, dataSources, ui, search }, notifications: { toasts }, application, + http, savedObjects, }, } = useOpenSearchDashboards(); @@ -143,20 +144,21 @@ export const Sidebar: FC = ({ children }) => { setSelectedSources(selectedDataSources); return; } + console.log('selectedDataSources:', selectedDataSources); // Temporary redirection solution for 2.11, where clicking non-index-pattern data sources // will prompt users with modal explaining they are being redirected to Observability log explorer - if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { - redirectToLogExplorer(selectedDataSources[0].label, selectedDataSources[0].type); - return; - } + // if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { + // redirectToLogExplorer(selectedDataSources[0].label, selectedDataSources[0].type); + // return; + // } setSelectedSources(selectedDataSources); dispatch(setIndexPattern(selectedDataSources[0].value)); - dispatch( - setDataset({ - id: selectedDataSources[0].value, - datasource: { ref: selectedDataSources[0].ds.getId() }, - }) - ); + // dispatch( + // setDataset({ + // id: selectedDataSources[0].value, + // datasource: { ref: selectedDataSources[0]?.ds?.getId() }, + // }) + // ); }, [dispatch, redirectToLogExplorer, setSelectedSources] ); @@ -179,11 +181,16 @@ export const Sidebar: FC = ({ children }) => { const dataSetNavigator = ( ); diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts index 0130bbac2df2..149d59eeb9ce 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts @@ -122,6 +122,7 @@ export const getIndicesViaResolve = async ({ query, }) .then((response) => { + console.log('response in get indices:', response); if (!response) { return []; } else { diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index 0ea557db8ce2..29a83e1c4c64 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -89,7 +89,7 @@ export class QueryEnhancementsPlugin initialTo: moment().add(2, 'days').toISOString(), }, showFilterBar: false, - showDataSetsSelector: false, + showDataSetsSelector: true, showDataSourcesSelector: true, }, fields: { @@ -109,7 +109,7 @@ export class QueryEnhancementsPlugin searchBar: { showDatePicker: false, showFilterBar: false, - showDataSetsSelector: false, + showDataSetsSelector: true, showDataSourcesSelector: true, queryStringInput: { initialValue: 'SELECT * FROM ' }, }, From 4d09c35e0300e3a939bf9e8fa1046359ac96cdcb Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 02:52:37 -0700 Subject: [PATCH 09/59] refactor to query editor Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 169 +++++++++--------- .../ui/query_editor/dataset_navigator.tsx | 0 .../public/ui/query_editor/query_editor.tsx | 16 ++ .../data/public/ui/settings/settings.ts | 9 + .../public/components/sidebar/index.tsx | 59 ++---- src/plugins/data_explorer/public/index.ts | 1 + .../utils/state_management/metadata_slice.ts | 1 + .../utils/state_management/index.ts | 3 +- .../utils/use_index_pattern.ts | 20 ++- 9 files changed, 141 insertions(+), 137 deletions(-) create mode 100644 src/plugins/data/public/ui/query_editor/dataset_navigator.tsx diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index b8b82fdc8389..a3a6ee073e03 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -28,44 +28,11 @@ import { DataSourceAttributes } from '../../../../data_source_management/public/ import { ISearchStart } from '../../search/types'; import { map, scan } from 'rxjs/operators'; import { IndexPatternsContract } from '../..'; +import { useTypedDispatch, useTypedSelector } from '../../../../data_explorer/public'; +import { getUiService, getIndexPatterns } from '../../services'; -const getAndFormatIndexPatternsFromDataSource = async ( - ds: DataSource -): Promise> => { - const { dataSets } = await ds.getDataSet(); - return { ds, list: dataSets } as DataSetWithDataSource; -}; - -const getAllIndexPatterns = (dataSources: DataSource[]) => - dataSources.map((ds) => getAndFormatIndexPatternsFromDataSource(ds)); - -const consolidateIndexPatternList = ( - dataSets: DataSetWithDataSource[], - selectedSources: DataSourceOption[] -): DataSourceOption[] => { - const result: DataSourceOption[] = []; - - dataSets.forEach((dataSet) => { - dataSet.list.forEach((indexPatternOption) => { - const dataSourceOption: DataSourceOption = { - type: 'indexPattern', - name: dataSet.ds.getName(), - ds: dataSet.ds, - key: indexPatternOption.id as string, - label: indexPatternOption.title as string, - value: indexPatternOption.id as string, - }; - - if (selectedSources.length !== 0 && selectedSources[0].value === dataSourceOption.value) { - dataSourceOption.checked = true; - } - - result.push(dataSourceOption); - }); - }); - console.log('results:', result); - - return result; +const getAllIndexPatterns = async (indexPatternsService: IndexPatternsContract) => { + return indexPatternsService.getIdsWithTitle(); }; const getClusters = async (savedObjectsClient: SavedObjectsClientContract) => { @@ -134,17 +101,17 @@ const getIndices = async (search: ISearchStart, dataSourceId: string) => { .catch(() => []); }; +interface DataSetOption { + id: string; + name: string; + dataSourceRef?: string; +} + interface DataSetNavigatorProps { http: HttpSetup; search: ISearchStart; savedObjectsClient: SavedObjectsClientContract; - indexPatterns: IndexPatternsContract; - dataSources: DataSource[]; - indexPatternOptionList: any; - selectedSources: DataSourceOption[]; - selectedCluster: any; - setIndexPatternOptionList: any; - setSelectedCluster: any; + indexPatterns: any; handleSourceSelection: any; } @@ -153,32 +120,60 @@ export const DataSetNavigator = ({ search, savedObjectsClient, indexPatterns, - dataSources, - indexPatternOptionList, - selectedSources, - selectedCluster, - setIndexPatternOptionList, - setSelectedCluster, handleSourceSelection, }: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); const [clusterList, setClusterList] = useState[]>([]); - const [indexList, setIndexList] = useState([]); + const [indexList, setIndexList] = useState([]); + const [selectedCluster, setSelectedCluster] = useState(); const [selectedIndex, setSelectedIndex] = useState(''); const [selectedField, setSelectedField] = useState(''); + const [selectedDataSet, setSelectedDataSet] = useState(); + const [indexPatternList, setIndexPatternList] = useState([]); + const uiService = getUiService(); + const indexPatternsService = getIndexPatterns(); const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); + const onDataSetClick = async (ds: DataSetOption) => { + const existingIndexPattern = indexPatterns.getByTitle(ds.id, true); + const dataSet = await indexPatterns.create( + { id: ds.id, title: ds.name }, + !existingIndexPattern?.id + ); + // save to cache by title because the id is not unique for temporary index pattern created + indexPatterns.saveToCache(dataSet.title, dataSet); + uiService.Settings.setSelectedDataSet({ + id: dataSet.id, + name: dataSet.title, + dataSourceRef: selectedCluster?.id ?? undefined, + }); + setSelectedDataSet(ds); + closePopover(); + }; + useEffect(() => { - Promise.all( - getAllIndexPatterns( - dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource) - ) - ).then((dataSetResults) => { - setIndexPatternOptionList(consolidateIndexPatternList(dataSetResults, selectedSources)); + const subscription = uiService.Settings.getSelectedDataSet$().subscribe((dataSet) => { + console.log('dataset in subscription:', dataSet); + if (dataSet) { + setSelectedDataSet(dataSet); + } }); - }, [dataSources, setIndexPatternOptionList]); + return () => subscription.unsubscribe(); + }, [uiService]); + + // get all index patterns + useEffect(() => { + indexPatternsService.getIdsWithTitle().then((res) => + setIndexPatternList( + res.map((indexPattern: { id: string; title: string }) => ({ + id: indexPattern.id, + name: indexPattern.title, + })) + ) + ); + }, [indexPatternsService]); useEffect(() => { Promise.all([getClusters(savedObjectsClient)]).then((res) => { @@ -191,36 +186,26 @@ export const DataSetNavigator = ({ // Get all indexes Promise.all([getIndices(search, selectedCluster.id)]).then((res) => { if (res && res.length > 0) { + console.log(res); console.log('res', Object.values(res?.[0])); setIndexList( Object.values(res?.[0]).map((index: any) => ({ ...index, // panel: 4, - onClick: async () => { - setSelectedIndex(index.name); - const existingIndexPattern = indexPatterns.getByTitle(index.name, true); - const dataSet = await indexPatterns.create( - { id: index.name, title: index.name }, - !existingIndexPattern?.id - ); - // save to cache by title because the id is not unique for temporary index pattern created - indexPatterns.saveToCache(dataSet.title, dataSet); - handleSourceSelection([ - { - type: 'indexPattern', - name: 'OpenSearch Default', - ds: selectedCluster, - key: index.name, - label: index.name, - value: index.name, - }, - ]); - }, + onClick: () => onDataSetClick({ name: index.name, id: index.name }), })) ); } }); - // Update panels related to cluster + getIndices(search, selectedCluster.id).then((res) => { + setIndexList( + Object.values(res?.[0]).map((index: any) => ({ + name: index.name, + id: index.name, + dataSourceRef: selectedCluster.id, + })) + ); + }); } }, [selectedCluster, setIndexList, setSelectedIndex]); @@ -243,7 +228,7 @@ export const DataSetNavigator = ({ iconSide="right" onClick={onButtonClick} > - {selectedSources.length > 0 ? selectedSources[0].label : 'Datasets'} + {selectedDataSet ? selectedDataSet.name : 'Datasets'} ); @@ -280,13 +265,14 @@ export const DataSetNavigator = ({ { id: 1, title: 'Index Patterns', - content: ( - - ), + items: [ + ...(indexPatternList + ? indexPatternList.map((indexPattern) => ({ + name: indexPattern.name, + onClick: () => onDataSetClick(indexPattern), + })) + : []), + ], }, { id: 2, @@ -304,7 +290,14 @@ export const DataSetNavigator = ({ { id: 3, title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', - items: indexList, + items: [ + ...(indexList + ? indexList.map((index) => ({ + name: index.name, + onClick: () => onDataSetClick(index), + })) + : []), + ], }, { id: 4, diff --git a/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx b/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index c76575fdc25d..0b73436450f4 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -13,6 +13,7 @@ import { } from '@elastic/eui'; import classNames from 'classnames'; import { isEqual } from 'lodash'; +<<<<<<< HEAD import React, { Component, createRef, RefObject } from 'react'; import { monaco } from '@osd/monaco'; import { Settings } from '..'; @@ -24,6 +25,11 @@ import { Query, TimeRange, } from '../..'; +======= +import React, { Component, createRef, RefObject, useCallback } from 'react'; +import { DataSetNavigator, Settings } from '..'; +import { DataSource, IDataPluginServices, IIndexPattern, Query, TimeRange } from '../..'; +>>>>>>> 939a9cb0ac... refactor to query editor import { CodeEditor, OpenSearchDashboardsReactContextValue, @@ -35,7 +41,17 @@ import { DataSettings } from '../types'; import { fetchIndexPatterns } from './fetch_index_patterns'; import { QueryLanguageSelector } from './language_selector'; import { QueryEditorExtensions } from './query_editor_extensions'; +<<<<<<< HEAD import { QueryEditorBtnCollapse } from './query_editor_btn_collapse'; +======= +import { + setIndexPattern, + setDataset, + useTypedDispatch, + useTypedSelector, +} from '../../../../data_explorer/public/utils/state_management'; + +>>>>>>> 939a9cb0ac... refactor to query editor export interface QueryEditorProps { indexPatterns: Array; dataSource?: DataSource; diff --git a/src/plugins/data/public/ui/settings/settings.ts b/src/plugins/data/public/ui/settings/settings.ts index df04d3dc6e7d..79306fc551e8 100644 --- a/src/plugins/data/public/ui/settings/settings.ts +++ b/src/plugins/data/public/ui/settings/settings.ts @@ -27,6 +27,7 @@ export class Settings { private isEnabled = false; private enabledQueryEnhancementsUpdated$ = new BehaviorSubject(this.isEnabled); private enhancedAppNames: string[] = []; + private selectedDataSet$ = new BehaviorSubject(null); constructor( private readonly config: ConfigSchema['enhancements'], @@ -40,6 +41,14 @@ export class Settings { this.enhancedAppNames = this.isEnabled ? this.config.supportedAppNames : []; } + setSelectedDataSet = (dataSet: any) => { + this.selectedDataSet$.next(dataSet); + } + + getSelectedDataSet$ = () => { + return this.selectedDataSet$.asObservable(); + } + supportsEnhancementsEnabled(appName: string) { return this.enhancedAppNames.includes(appName); } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 38afb4703371..db2ac781bbfb 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -6,7 +6,7 @@ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSource, DataSourceGroup, DataSetNavigator } from '../../../../data/public'; +import { DataSource, DataSourceGroup, DataSetNavigator, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; @@ -28,15 +28,12 @@ export const Sidebar: FC = ({ children }) => { const [activeDataSources, setActiveDataSources] = useState([]); const [isEnhancementsEnabled, setIsEnhancementsEnabled] = useState(false); const containerRef = useRef(null); - const connectionsRef = useRef(null); const { services: { - data: { indexPatterns, dataSources, ui, search }, + data: { indexPatterns, dataSources, ui }, notifications: { toasts }, application, - http, - savedObjects, }, } = useOpenSearchDashboards(); @@ -56,10 +53,6 @@ export const Sidebar: FC = ({ children }) => { uiContainerRef.appendChild(containerRef.current); }, []); - const setConnectionsRef = useCallback((uiConnectionsRef) => { - uiConnectionsRef.appendChild(connectionsRef.current); - }, []); - useEffect(() => { if (!isEnhancementsEnabled) return; const subscriptions = ui.container$.subscribe((container) => { @@ -69,21 +62,14 @@ export const Sidebar: FC = ({ children }) => { } }); - const connectionsSubscriptions = ui.dataSourceContainer$.subscribe((container) => { - if (container === null) return; - connectionsRef.current = container; - }); - return () => { subscriptions.unsubscribe(); - connectionsSubscriptions.unsubscribe(); }; }, [ ui.container$, containerRef, setContainerRef, ui.dataSourceContainer$, - connectionsRef, isEnhancementsEnabled, ]); @@ -114,7 +100,7 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (indexPatternId) { const option = getMatchedOption(dataSourceOptionList, indexPatternId); - setSelectedSources(option ? [option] : []); + setSelectedSources((prev) => option ? [option] : prev); } }, [indexPatternId, activeDataSources, dataSourceOptionList]); @@ -125,7 +111,7 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (indexPatternId) { const option = getMatchedIndexPattern(indexPatternOptionList, indexPatternId); - setSelectedSources(option ? [option] : []); + setSelectedSources((prev) => option ? [option] : prev); } }, [indexPatternId, activeDataSources, indexPatternOptionList]); @@ -144,13 +130,6 @@ export const Sidebar: FC = ({ children }) => { setSelectedSources(selectedDataSources); return; } - console.log('selectedDataSources:', selectedDataSources); - // Temporary redirection solution for 2.11, where clicking non-index-pattern data sources - // will prompt users with modal explaining they are being redirected to Observability log explorer - // if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { - // redirectToLogExplorer(selectedDataSources[0].label, selectedDataSources[0].type); - // return; - // } setSelectedSources(selectedDataSources); dispatch(setIndexPattern(selectedDataSources[0].value)); // dispatch( @@ -179,19 +158,16 @@ export const Sidebar: FC = ({ children }) => { dataSources.dataSourceService.reload(); }, [dataSources.dataSourceService]); - const dataSetNavigator = ( - ); @@ -203,15 +179,6 @@ export const Sidebar: FC = ({ children }) => { borderRadius="none" color="transparent" > - {isEnhancementsEnabled && ( - { - containerRef.current = node; - }} - > - {dataSetNavigator} - - )} {!isEnhancementsEnabled && ( { color="transparent" className="deSidebar_dataSource" > - {dataSetNavigator} + {dataSourceSelector} )} diff --git a/src/plugins/data_explorer/public/index.ts b/src/plugins/data_explorer/public/index.ts index f8adda434ced..6825e1d82e53 100644 --- a/src/plugins/data_explorer/public/index.ts +++ b/src/plugins/data_explorer/public/index.ts @@ -18,4 +18,5 @@ export { useTypedSelector, useTypedDispatch, setIndexPattern, + setDataset, } from './utils/state_management'; diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index 628f313cad0a..22004c3b8837 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -40,6 +40,7 @@ export const getPreloadedState = async ({ .getStateTransfer(scopedHistory) .getIncomingEditorState({ keysToRemoveAfterFetch: ['id', 'input'] }) || {}; const defaultIndexPattern = await data.indexPatterns.getDefault(); + data.ui.Settings.setSelectedDataSet({ id: defaultIndexPattern?.id, name: defaultIndexPattern?.title }); const preloadedState: MetadataState = { ...initialState, originatingApp, diff --git a/src/plugins/discover/public/application/utils/state_management/index.ts b/src/plugins/discover/public/application/utils/state_management/index.ts index 989b2662f0d4..a8b732b1c583 100644 --- a/src/plugins/discover/public/application/utils/state_management/index.ts +++ b/src/plugins/discover/public/application/utils/state_management/index.ts @@ -7,6 +7,7 @@ import { TypedUseSelectorHook } from 'react-redux'; import { RootState, setIndexPattern as updateIndexPattern, + setDataset as updateDataSet, useTypedDispatch, useTypedSelector, } from '../../../../../data_explorer/public'; @@ -20,4 +21,4 @@ export interface DiscoverRootState extends RootState { export const useSelector: TypedUseSelectorHook = useTypedSelector; export const useDispatch = useTypedDispatch; -export { updateIndexPattern }; +export { updateIndexPattern, updateDataSet }; diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index e8a81234278e..4bf0d351a7fa 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -6,9 +6,11 @@ import { useEffect, useState } from 'react'; import { i18n } from '@osd/i18n'; import { IndexPattern } from '../../../../../data/public'; -import { useSelector, updateIndexPattern } from '../../utils/state_management'; +import { updateDataSet, useSelector, updateIndexPattern } from '../../utils/state_management'; import { DiscoverViewServices } from '../../../build_services'; import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; +import { Subscription } from 'rxjs'; +import { distinctUntilChanged, first } from 'rxjs/operators'; /** * Custom hook to fetch and manage the index pattern based on the provided services. @@ -28,6 +30,7 @@ export const useIndexPattern = (services: DiscoverViewServices) => { const indexPatternIdFromState = useSelector((state) => state.metadata.indexPattern); const [indexPattern, setIndexPattern] = useState(undefined); const { data, toastNotifications, uiSettings: config, store } = services; + const subscription = new Subscription(); useEffect(() => { let isMounted = true; @@ -58,6 +61,14 @@ export const useIndexPattern = (services: DiscoverViewServices) => { }); }; + data.ui.Settings.getSelectedDataSet$().subscribe((dataSet) => { + console.log('subscribing to THIS'); + if (dataSet) { + store!.dispatch(updateDataSet(dataSet)); + fetchIndexPatternDetails(dataSet.id); + } + }); + if (!indexPatternIdFromState) { data.indexPatterns.getCache().then((indexPatternList) => { const newId = getIndexPatternId('', indexPatternList, config.get('defaultIndex')); @@ -70,8 +81,13 @@ export const useIndexPattern = (services: DiscoverViewServices) => { return () => { isMounted = false; + subscription.unsubscribe(); }; - }, [indexPatternIdFromState, data.indexPatterns, toastNotifications, config, store]); + }, [indexPatternIdFromState, data.indexPatterns, toastNotifications, config, store, data.ui.Settings]); + + useEffect(() => { + + }, ) return indexPattern; }; From de7818f680e50b444acacbdb7f0673ab2249801b Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 04:17:05 -0700 Subject: [PATCH 10/59] cleanup dataset dropdown Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 102 ++++-------------- .../ui/query_editor/_dataset_navigator.scss | 8 ++ .../data/public/ui/query_editor/_index.scss | 1 + .../public/ui/query_editor/_query_editor.scss | 6 ++ .../public/ui/query_editor/query_editor.tsx | 18 +--- .../data/public/ui/settings/settings.ts | 10 +- .../utils/state_management/metadata_slice.ts | 11 +- .../utils/use_index_pattern.ts | 24 +++-- .../opensearch_dashboards.json | 2 +- .../components/connections_bar.tsx | 16 +-- .../routes/data_source_connection/routes.ts | 5 +- .../query_enhancements/server/types.ts | 2 +- 12 files changed, 71 insertions(+), 134 deletions(-) create mode 100644 src/plugins/data/public/ui/query_editor/_dataset_navigator.scss diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index a3a6ee073e03..1afdbd324e3d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -5,38 +5,15 @@ import React, { useEffect, useState } from 'react'; -import { - EuiButtonEmpty, - EuiContextMenu, - EuiContextMenuItem, - EuiContextMenuPanelDescriptor, - EuiPopover, -} from '@elastic/eui'; -import { - DataSetWithDataSource, - DataSource, - IndexPatternOption, -} from '../../data_sources/datasource'; -import { DataSourceOption } from '../../data_sources/datasource_selector/types'; -import { IndexPatternSelectable } from './index_pattern_selectable'; -import { - HttpSetup, - SavedObjectsClientContract, - SimpleSavedObject, -} from 'opensearch-dashboards/public'; -import { DataSourceAttributes } from '../../../../data_source_management/public/types'; -import { ISearchStart } from '../../search/types'; +import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; +import { SavedObjectsClientContract, SimpleSavedObject } from 'opensearch-dashboards/public'; import { map, scan } from 'rxjs/operators'; -import { IndexPatternsContract } from '../..'; -import { useTypedDispatch, useTypedSelector } from '../../../../data_explorer/public'; -import { getUiService, getIndexPatterns } from '../../services'; - -const getAllIndexPatterns = async (indexPatternsService: IndexPatternsContract) => { - return indexPatternsService.getIdsWithTitle(); -}; +import { ISearchStart } from '../../search/types'; +import { IIndexPattern, IndexPatternsContract } from '../..'; +import { getUiService, getIndexPatterns, getSearchService } from '../../services'; const getClusters = async (savedObjectsClient: SavedObjectsClientContract) => { - return await savedObjectsClient.find({ + return await savedObjectsClient.find({ type: 'data-source', perPage: 10000, }); @@ -85,7 +62,7 @@ const buildSearchRequest = (showAllIndices: boolean, pattern: string, dataSource }, }, }, - dataSourceId: dataSourceId, + dataSourceId, }; return request; @@ -108,28 +85,21 @@ interface DataSetOption { } interface DataSetNavigatorProps { - http: HttpSetup; - search: ISearchStart; savedObjectsClient: SavedObjectsClientContract; - indexPatterns: any; - handleSourceSelection: any; + indexPatterns: Array; } -export const DataSetNavigator = ({ - http, - search, - savedObjectsClient, - indexPatterns, - handleSourceSelection, -}: DataSetNavigatorProps) => { +export const DataSetNavigator = ({ savedObjectsClient, indexPatterns }: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); - const [clusterList, setClusterList] = useState[]>([]); + const [clusterList, setClusterList] = useState([]); const [indexList, setIndexList] = useState([]); const [selectedCluster, setSelectedCluster] = useState(); - const [selectedIndex, setSelectedIndex] = useState(''); - const [selectedField, setSelectedField] = useState(''); - const [selectedDataSet, setSelectedDataSet] = useState(); + const [selectedDataSet, setSelectedDataSet] = useState({ + id: indexPatterns[0]?.id, + name: indexPatterns[0]?.title, + }); const [indexPatternList, setIndexPatternList] = useState([]); + const search = getSearchService(); const uiService = getUiService(); const indexPatternsService = getIndexPatterns(); @@ -137,13 +107,13 @@ export const DataSetNavigator = ({ const closePopover = () => setIsDataSetNavigatorOpen(false); const onDataSetClick = async (ds: DataSetOption) => { - const existingIndexPattern = indexPatterns.getByTitle(ds.id, true); - const dataSet = await indexPatterns.create( + const existingIndexPattern = indexPatternsService.getByTitle(ds.id, true); + const dataSet = await indexPatternsService.create( { id: ds.id, title: ds.name }, !existingIndexPattern?.id ); // save to cache by title because the id is not unique for temporary index pattern created - indexPatterns.saveToCache(dataSet.title, dataSet); + indexPatternsService.saveToCache(dataSet.title, dataSet); uiService.Settings.setSelectedDataSet({ id: dataSet.id, name: dataSet.title, @@ -155,7 +125,6 @@ export const DataSetNavigator = ({ useEffect(() => { const subscription = uiService.Settings.getSelectedDataSet$().subscribe((dataSet) => { - console.log('dataset in subscription:', dataSet); if (dataSet) { setSelectedDataSet(dataSet); } @@ -184,22 +153,9 @@ export const DataSetNavigator = ({ useEffect(() => { if (selectedCluster) { // Get all indexes - Promise.all([getIndices(search, selectedCluster.id)]).then((res) => { - if (res && res.length > 0) { - console.log(res); - console.log('res', Object.values(res?.[0])); - setIndexList( - Object.values(res?.[0]).map((index: any) => ({ - ...index, - // panel: 4, - onClick: () => onDataSetClick({ name: index.name, id: index.name }), - })) - ); - } - }); getIndices(search, selectedCluster.id).then((res) => { setIndexList( - Object.values(res?.[0]).map((index: any) => ({ + res.map((index: { name: string }) => ({ name: index.name, id: index.name, dataSourceRef: selectedCluster.id, @@ -207,18 +163,7 @@ export const DataSetNavigator = ({ ); }); } - }, [selectedCluster, setIndexList, setSelectedIndex]); - - useEffect(() => { - if (selectedIndex) { - Promise.all([ - indexPatterns.getFieldsForWildcard({ - pattern: selectedIndex, - dataSourceId: selectedCluster.id, - }), - ]).then((res) => console.log(res)); - } - }, [selectedIndex]); + }, [search, selectedCluster, setIndexList]); const dataSetButton = ( ({ name: cluster.attributes.title, @@ -282,9 +227,6 @@ export const DataSetNavigator = ({ name: 'Indexes', panel: 3, }, - { - name: 'Connected Data Sources', - }, ], }, { diff --git a/src/plugins/data/public/ui/query_editor/_dataset_navigator.scss b/src/plugins/data/public/ui/query_editor/_dataset_navigator.scss new file mode 100644 index 000000000000..568d5532bc4a --- /dev/null +++ b/src/plugins/data/public/ui/query_editor/_dataset_navigator.scss @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +.datasetNavigator { + min-width: 350px; + border-bottom: $euiBorderThin !important; +} diff --git a/src/plugins/data/public/ui/query_editor/_index.scss b/src/plugins/data/public/ui/query_editor/_index.scss index 64fb0056cb71..f51d45b8245c 100644 --- a/src/plugins/data/public/ui/query_editor/_index.scss +++ b/src/plugins/data/public/ui/query_editor/_index.scss @@ -1,2 +1,3 @@ @import "./language_selector"; +@import "./dataset_navigator"; @import "./query_editor"; diff --git a/src/plugins/data/public/ui/query_editor/_query_editor.scss b/src/plugins/data/public/ui/query_editor/_query_editor.scss index 8fc81308b533..ac411b38ab88 100644 --- a/src/plugins/data/public/ui/query_editor/_query_editor.scss +++ b/src/plugins/data/public/ui/query_editor/_query_editor.scss @@ -86,6 +86,12 @@ } } +.osdQueryEditor__dataSetNavigatorWrapper { + :first-child { + border-bottom: $euiBorderThin !important; + } +} + @include euiBreakpoint("xs", "s") { .osdQueryEditor--withDatePicker { > :first-child { diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 0b73436450f4..85c32b2bc276 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -13,7 +13,6 @@ import { } from '@elastic/eui'; import classNames from 'classnames'; import { isEqual } from 'lodash'; -<<<<<<< HEAD import React, { Component, createRef, RefObject } from 'react'; import { monaco } from '@osd/monaco'; import { Settings } from '..'; @@ -25,11 +24,6 @@ import { Query, TimeRange, } from '../..'; -======= -import React, { Component, createRef, RefObject, useCallback } from 'react'; -import { DataSetNavigator, Settings } from '..'; -import { DataSource, IDataPluginServices, IIndexPattern, Query, TimeRange } from '../..'; ->>>>>>> 939a9cb0ac... refactor to query editor import { CodeEditor, OpenSearchDashboardsReactContextValue, @@ -41,17 +35,8 @@ import { DataSettings } from '../types'; import { fetchIndexPatterns } from './fetch_index_patterns'; import { QueryLanguageSelector } from './language_selector'; import { QueryEditorExtensions } from './query_editor_extensions'; -<<<<<<< HEAD import { QueryEditorBtnCollapse } from './query_editor_btn_collapse'; -======= -import { - setIndexPattern, - setDataset, - useTypedDispatch, - useTypedSelector, -} from '../../../../data_explorer/public/utils/state_management'; ->>>>>>> 939a9cb0ac... refactor to query editor export interface QueryEditorProps { indexPatterns: Array; dataSource?: DataSource; @@ -384,10 +369,13 @@ export default class QueryEditorUI extends Component { const className = classNames(this.props.className); const headerClassName = classNames('osdQueryEditorHeader', this.props.headerClassName); const bannerClassName = classNames('osdQueryEditorBanner', this.props.bannerClassName); +<<<<<<< HEAD const footerClassName = classNames('osdQueryEditorFooter', this.props.footerClassName); const useQueryEditor = this.props.query.language !== 'kuery' && this.props.query.language !== 'lucene'; +======= +>>>>>>> 0ae2098677... cleanup dataset dropdown return (
diff --git a/src/plugins/data/public/ui/settings/settings.ts b/src/plugins/data/public/ui/settings/settings.ts index 79306fc551e8..d7f7cd104c83 100644 --- a/src/plugins/data/public/ui/settings/settings.ts +++ b/src/plugins/data/public/ui/settings/settings.ts @@ -39,15 +39,21 @@ export class Settings { this.isEnabled = true; this.setUserQueryEnhancementsEnabled(this.isEnabled); this.enhancedAppNames = this.isEnabled ? this.config.supportedAppNames : []; + this.setSelectedDataSet(this.getSelectedDataSet()); } setSelectedDataSet = (dataSet: any) => { + this.storage.set('opensearchDashboards.userQueryDataSet', dataSet); this.selectedDataSet$.next(dataSet); - } + }; getSelectedDataSet$ = () => { return this.selectedDataSet$.asObservable(); - } + }; + + getSelectedDataSet = () => { + return this.storage.get('opensearchDashboards.userQueryDataSet'); + }; supportsEnhancementsEnabled(appName: string) { return this.enhancedAppNames.includes(appName); diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index 22004c3b8837..51b492c5dec5 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -7,8 +7,8 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import { DataExplorerServices } from '../../types'; interface DataSourceMeta { - ref: string // MDS ID - dsName?: string // flint datasource + ref: string; // MDS ID + dsName?: string; // flint datasource } export interface DataSet { @@ -17,7 +17,7 @@ export interface DataSet { meta?: { timestampField: string; mapping?: any; - } + }; type?: 'dataset' | 'temporary'; } @@ -40,14 +40,13 @@ export const getPreloadedState = async ({ .getStateTransfer(scopedHistory) .getIncomingEditorState({ keysToRemoveAfterFetch: ['id', 'input'] }) || {}; const defaultIndexPattern = await data.indexPatterns.getDefault(); - data.ui.Settings.setSelectedDataSet({ id: defaultIndexPattern?.id, name: defaultIndexPattern?.title }); const preloadedState: MetadataState = { ...initialState, originatingApp, indexPattern: defaultIndexPattern?.id, dataset: { - id: defaultIndexPattern?.id - } + id: defaultIndexPattern?.id, + }, }; return preloadedState; diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index 4bf0d351a7fa..74f5165cf425 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -5,12 +5,11 @@ import { useEffect, useState } from 'react'; import { i18n } from '@osd/i18n'; +import { batch } from 'react-redux'; import { IndexPattern } from '../../../../../data/public'; import { updateDataSet, useSelector, updateIndexPattern } from '../../utils/state_management'; import { DiscoverViewServices } from '../../../build_services'; import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; -import { Subscription } from 'rxjs'; -import { distinctUntilChanged, first } from 'rxjs/operators'; /** * Custom hook to fetch and manage the index pattern based on the provided services. @@ -30,7 +29,6 @@ export const useIndexPattern = (services: DiscoverViewServices) => { const indexPatternIdFromState = useSelector((state) => state.metadata.indexPattern); const [indexPattern, setIndexPattern] = useState(undefined); const { data, toastNotifications, uiSettings: config, store } = services; - const subscription = new Subscription(); useEffect(() => { let isMounted = true; @@ -62,9 +60,11 @@ export const useIndexPattern = (services: DiscoverViewServices) => { }; data.ui.Settings.getSelectedDataSet$().subscribe((dataSet) => { - console.log('subscribing to THIS'); if (dataSet) { - store!.dispatch(updateDataSet(dataSet)); + batch(() => { + store!.dispatch(updateDataSet(dataSet)); + store!.dispatch(updateIndexPattern(dataSet.id)); + }); fetchIndexPatternDetails(dataSet.id); } }); @@ -81,13 +81,15 @@ export const useIndexPattern = (services: DiscoverViewServices) => { return () => { isMounted = false; - subscription.unsubscribe(); }; - }, [indexPatternIdFromState, data.indexPatterns, toastNotifications, config, store, data.ui.Settings]); - - useEffect(() => { - - }, ) + }, [ + indexPatternIdFromState, + data.indexPatterns, + toastNotifications, + config, + store, + data.ui.Settings, + ]); return indexPattern; }; diff --git a/src/plugins/query_enhancements/opensearch_dashboards.json b/src/plugins/query_enhancements/opensearch_dashboards.json index b09494aab0ca..69d8fd3bd667 100644 --- a/src/plugins/query_enhancements/opensearch_dashboards.json +++ b/src/plugins/query_enhancements/opensearch_dashboards.json @@ -3,7 +3,7 @@ "version": "opensearchDashboards", "server": true, "ui": true, - "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "dataSourceManagement", "savedObjects", "uiActions"], + "requiredPlugins": ["data", "opensearchDashboardsReact", "opensearchDashboardsUtils", "savedObjects", "uiActions"], "optionalPlugins": ["dataSource"] } diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx b/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx index 3fd592e50b31..196cb5eb4011 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx +++ b/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx @@ -8,7 +8,6 @@ import { EuiPortal } from '@elastic/eui'; import { distinctUntilChanged } from 'rxjs/operators'; import { ToastsSetup } from 'opensearch-dashboards/public'; import { DataPublicPluginStart, QueryEditorExtensionDependencies } from '../../../../data/public'; -import { DataSourceSelector } from '../../../../data_source_management/public'; import { ConnectionsService } from '../services'; interface ConnectionsProps { @@ -75,20 +74,7 @@ export const ConnectionsBar: React.FC = ({ connectionsService, containerRef.current = node; }} > -
- - handleSelectedConnection(dataSource[0]?.id || undefined) - } - /> -
+
); }; diff --git a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts index f4fe42779dae..162cc7e8f103 100644 --- a/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts +++ b/src/plugins/query_enhancements/server/routes/data_source_connection/routes.ts @@ -5,7 +5,6 @@ import { schema } from '@osd/config-schema'; import { IRouter } from 'opensearch-dashboards/server'; -import { DataSourceAttributes } from '../../../../data_source/common/data_sources'; import { API } from '../../../common'; export function registerDataSourceConnectionsRoutes(router: IRouter) { @@ -18,7 +17,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, async (context, request, response) => { const fields = ['id', 'title', 'auth.type']; - const resp = await context.core.savedObjects.client.find({ + const resp = await context.core.savedObjects.client.find({ type: 'data-source', fields, perPage: 10000, @@ -38,7 +37,7 @@ export function registerDataSourceConnectionsRoutes(router: IRouter) { }, }, async (context, request, response) => { - const resp = await context.core.savedObjects.client.get( + const resp = await context.core.savedObjects.client.get( 'data-source', request.params.dataSourceId ); diff --git a/src/plugins/query_enhancements/server/types.ts b/src/plugins/query_enhancements/server/types.ts index 2ab716b98928..dd264568386d 100644 --- a/src/plugins/query_enhancements/server/types.ts +++ b/src/plugins/query_enhancements/server/types.ts @@ -4,7 +4,7 @@ */ import { PluginSetup } from 'src/plugins/data/server'; -import { DataSourcePluginSetup } from '../../data_source/server'; +import { DataSourcePluginSetup } from 'src/plugins/data_source/server'; import { Logger } from '../../../core/server'; import { ConfigSchema } from '../common/config'; From 5779503e86e235559ff482091f805c74c24eee21 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 04:24:06 -0700 Subject: [PATCH 11/59] running linter Signed-off-by: Sean Li --- .../data/common/index_patterns/types.ts | 4 - .../datasource_selectable.tsx | 3 - .../field_selector_panel.tsx | 79 ------------------- .../index_pattern_selectable.tsx | 50 ------------ .../public/components/utils.ts | 1 - .../lib/get_indices.ts | 1 - 6 files changed, 138 deletions(-) delete mode 100644 src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx delete mode 100644 src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx diff --git a/src/plugins/data/common/index_patterns/types.ts b/src/plugins/data/common/index_patterns/types.ts index 0a8f44e5383c..108a93a3725b 100644 --- a/src/plugins/data/common/index_patterns/types.ts +++ b/src/plugins/data/common/index_patterns/types.ts @@ -51,10 +51,6 @@ export interface IIndexPattern { ) => FieldFormat; } -// export interface IDataSet { -// dataSetType: opensearch / s3 / index / indexpattern -// } - export interface IndexPatternAttributes { type: string; fields: string; diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 90f7ea6b23c1..e826cbe18af1 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -125,8 +125,6 @@ export const DataSourceSelectable = ({ onRefresh, ...comboBoxProps }: DataSourceSelectableProps) => { - console.log('dataSources:', dataSources); - console.log('dataSources filter:', dataSources.filter((ds) => ds.getMetadata().ui.selector.displayDatasetsAsSource)); // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { Promise.all( @@ -151,7 +149,6 @@ export const DataSourceSelectable = ({ ); const memorizedDataSourceOptionList = useMemo(() => { - console.log('dataSourceOptionList:', dataSourceOptionList); return dataSourceOptionList.map((dsGroup: DataSourceGroup) => { return { ...dsGroup, diff --git a/src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx b/src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx deleted file mode 100644 index dc1bff8210a2..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/field_selector_panel.tsx +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { Fragment, useCallback, useEffect, useState } from 'react'; -import { EuiSelectable, EuiContextMenuPanel, EuiComboBox, EuiSelect } from '@elastic/eui'; -import { IFieldType, IndexPatternsContract } from '../..'; -import { i18n } from '@osd/i18n'; - -export function extractTimeFields(fields: IFieldType[]) { - const dateFields = fields.filter((field) => field.type === 'date'); - const label = i18n.translate( - 'indexPatternManagement.createIndexPattern.stepTime.noTimeFieldsLabel', - { - defaultMessage: "The indices which match this index pattern don't contain any time fields.", - } - ); - - if (dateFields.length === 0) { - return [ - { - display: label, - }, - ]; - } - - const disabledDividerOption = { - isDisabled: true, - display: '───', - fieldName: '', - }; - const noTimeFieldLabel = i18n.translate( - 'indexPatternManagement.createIndexPattern.stepTime.noTimeFieldOptionLabel', - { - defaultMessage: "I don't want to use the time filter", - } - ); - const noTimeFieldOption = { - display: noTimeFieldLabel, - fieldName: undefined, - }; - - return [ - ...dateFields.map((field) => ({ - display: field.name, - fieldName: field.name, - })), - disabledDividerOption, - noTimeFieldOption, - ]; -} - - -export const FieldSelctorPanel = ({ - index, - dataSourceId, - indexPatterns, -}: { - index: string; - dataSourceId: string; - indexPatterns: IndexPatternsContract; -}) => { - const [timeStampFields, setTimeStampFields] = useState([]); - - const getTimeStampFields = async () => { - const fields = await indexPatterns.getFieldsForWildcard({ pattern: index, dataSourceId}); - const timeFields = extractTimeFields(fields); - setTimeStampFields(timeFields); - }; - - const dropdown = ; - - return { - id: 3, - title: index, - content:
, - }; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx b/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx deleted file mode 100644 index 0b5cfaac051e..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/index_pattern_selectable.tsx +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { Fragment, useCallback } from 'react'; -import { EuiSelectable } from '@elastic/eui'; - -interface IndexPatternSelectableProps { - indexPatternOptionList: any; - setIndexPatternOptionList: any; - handleSourceSelection: any; -} - -export const IndexPatternSelectable = ({ - indexPatternOptionList, - setIndexPatternOptionList, - handleSourceSelection, -}: IndexPatternSelectableProps) => { - const handleSourceChange = useCallback( - (selectedOptions: any) => { - handleSourceSelection(selectedOptions); - }, - [handleSourceSelection] - ); - - console.log('option list:', indexPatternOptionList); - - return ( -
- { - setIndexPatternOptionList(newOptions); - handleSourceChange(newOptions.filter((option) => option?.checked)); - }} - singleSelection="always" - > - {(list, search) => ( - - {search} - {list} - - )} - -
- ); -}; diff --git a/src/plugins/data_source_management/public/components/utils.ts b/src/plugins/data_source_management/public/components/utils.ts index 9167f4a49e42..aa54f8b352fe 100644 --- a/src/plugins/data_source_management/public/components/utils.ts +++ b/src/plugins/data_source_management/public/components/utils.ts @@ -132,7 +132,6 @@ export function getFilteredDataSources( dataSources: Array>, filter = (ds: SavedObject) => true ): DataSourceOption[] { - console.log('dataSources:', dataSources); return dataSources .filter((ds) => filter!(ds)) .map((ds) => ({ diff --git a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts index 149d59eeb9ce..0130bbac2df2 100644 --- a/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts +++ b/src/plugins/index_pattern_management/public/components/create_index_pattern_wizard/lib/get_indices.ts @@ -122,7 +122,6 @@ export const getIndicesViaResolve = async ({ query, }) .then((response) => { - console.log('response in get indices:', response); if (!response) { return []; } else { From 7296be56c5c25040e392a93dda6e7ccc54e44005 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 04:27:24 -0700 Subject: [PATCH 12/59] removing exports for deleted components Signed-off-by: Sean Li --- src/plugins/data/public/index.ts | 1 - .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 2 +- src/plugins/data/public/ui/dataset_navigator/index.tsx | 1 - src/plugins/data/public/ui/index.ts | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 7ddfef61db5d..208359352e4b 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -446,7 +446,6 @@ export { // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, DataSetNavigator, - IndexPatternSelectable } from './ui'; /** diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 1afdbd324e3d..ef6be6b0294d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -9,7 +9,7 @@ import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; import { SavedObjectsClientContract, SimpleSavedObject } from 'opensearch-dashboards/public'; import { map, scan } from 'rxjs/operators'; import { ISearchStart } from '../../search/types'; -import { IIndexPattern, IndexPatternsContract } from '../..'; +import { IIndexPattern } from '../..'; import { getUiService, getIndexPatterns, getSearchService } from '../../services'; const getClusters = async (savedObjectsClient: SavedObjectsClientContract) => { diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx index a50eed767b4d..ea04b4d206dd 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -4,4 +4,3 @@ */ export { DataSetNavigator } from './dataset_navigator'; -export { IndexPatternSelectable } from './index_pattern_selectable'; diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index d3163f9f0010..400887e51d57 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -49,4 +49,4 @@ export { } from './query_editor'; export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; export { SuggestionsComponent } from './typeahead'; -export { DataSetNavigator, IndexPatternSelectable } from './dataset_navigator'; +export { DataSetNavigator } from './dataset_navigator'; From dc81d51814486a971b782b4ce1e37aec9e6c7d88 Mon Sep 17 00:00:00 2001 From: "opensearch-changeset-bot[bot]" <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:34:51 +0000 Subject: [PATCH 13/59] Changeset file for PR #7289 created/updated --- changelogs/fragments/7289.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/7289.yml diff --git a/changelogs/fragments/7289.yml b/changelogs/fragments/7289.yml new file mode 100644 index 000000000000..b181433dbeca --- /dev/null +++ b/changelogs/fragments/7289.yml @@ -0,0 +1,2 @@ +feat: +- Add DataSet dropdown with index patterns and indices ([#7289](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7289)) \ No newline at end of file From d406b87009c21c2b2940a2efad487081d0dbdf45 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 11:27:16 -0700 Subject: [PATCH 14/59] running linter Signed-off-by: Sean Li --- .../ui/query_editor/dataset_navigator.tsx | 4 ++ .../public/components/sidebar/index.tsx | 38 +++++-------------- .../utils/create_extension.tsx | 2 +- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx b/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx index e69de29bb2d1..a850c1690e3b 100644 --- a/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx @@ -0,0 +1,4 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index db2ac781bbfb..f4c628a7d70f 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -4,26 +4,19 @@ */ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; -import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; +import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { DataSource, DataSourceGroup, DataSetNavigator, DataSourceSelectable } from '../../../../data/public'; +import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; -import { - setIndexPattern, - setDataset, - useTypedDispatch, - useTypedSelector, -} from '../../utils/state_management'; +import { setIndexPattern, useTypedDispatch, useTypedSelector } from '../../utils/state_management'; import './index.scss'; export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); const dispatch = useTypedDispatch(); const [selectedSources, setSelectedSources] = useState([]); - const [selectedCluster, setSelectedCluster] = useState(); - const [indexPatternOptionList, setIndexPatternOptionList] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); const [activeDataSources, setActiveDataSources] = useState([]); const [isEnhancementsEnabled, setIsEnhancementsEnabled] = useState(false); @@ -100,21 +93,10 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (indexPatternId) { const option = getMatchedOption(dataSourceOptionList, indexPatternId); - setSelectedSources((prev) => option ? [option] : prev); + setSelectedSources((prev) => (option ? [option] : prev)); } }, [indexPatternId, activeDataSources, dataSourceOptionList]); - const getMatchedIndexPattern = (indexPatternList: DataSourceOption[], ipId: string) => { - return indexPatternList.find((indexPattern) => indexPattern.value === ipId); - }; - - useEffect(() => { - if (indexPatternId) { - const option = getMatchedIndexPattern(indexPatternOptionList, indexPatternId); - setSelectedSources((prev) => option ? [option] : prev); - } - }, [indexPatternId, activeDataSources, indexPatternOptionList]); - const redirectToLogExplorer = useCallback( (dsName: string, dsType: string) => { return application.navigateToUrl( @@ -130,14 +112,14 @@ export const Sidebar: FC = ({ children }) => { setSelectedSources(selectedDataSources); return; } + // Temporary redirection solution for 2.11, where clicking non-index-pattern data sources + // will prompt users with modal explaining they are being redirected to Observability log explorer + if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { + redirectToLogExplorer(selectedDataSources[0].label, selectedDataSources[0].type); + return; + } setSelectedSources(selectedDataSources); dispatch(setIndexPattern(selectedDataSources[0].value)); - // dispatch( - // setDataset({ - // id: selectedDataSources[0].value, - // datasource: { ref: selectedDataSources[0]?.ds?.getId() }, - // }) - // ); }, [dispatch, redirectToLogExplorer, setSelectedSources] ); diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx index 5b3a603103bf..0250c196d12b 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx @@ -5,11 +5,11 @@ import React from 'react'; import { ToastsSetup } from 'opensearch-dashboards/public'; +import { of } from 'rxjs'; import { QueryEditorExtensionConfig } from '../../../../data/public'; import { ConfigSchema } from '../../../common/config'; import { ConnectionsBar } from '../components'; import { ConnectionsService } from '../services'; -import { of } from 'rxjs'; export const createDataSourceConnectionExtension = ( connectionsService: ConnectionsService, From 1a0b4133802d12461648b81ca9152dd1d593c30d Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 12:37:00 -0700 Subject: [PATCH 15/59] removing connections bar component Signed-off-by: Sean Li --- .../data/public/ui/settings/settings.ts | 6 ++ src/plugins/data/public/ui/types.ts | 3 + .../utils/use_index_pattern.ts | 1 - .../components/connections_bar.tsx | 80 ------------------- .../components/index.ts | 6 -- .../public/data_source_connection/index.ts | 7 -- .../data_source_connection/services/index.ts | 6 -- .../utils/create_extension.tsx | 35 -------- .../data_source_connection/utils/index.ts | 6 -- .../query_enhancements/public/plugin.tsx | 13 +-- .../query_enhancements/public/services.ts | 11 --- .../services/connections_service.ts | 4 +- .../public/services/index.ts | 13 +++ 13 files changed, 25 insertions(+), 166 deletions(-) delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/components/index.ts delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/index.ts delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/services/index.ts delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx delete mode 100644 src/plugins/query_enhancements/public/data_source_connection/utils/index.ts delete mode 100644 src/plugins/query_enhancements/public/services.ts rename src/plugins/query_enhancements/public/{data_source_connection => }/services/connections_service.ts (95%) create mode 100644 src/plugins/query_enhancements/public/services/index.ts diff --git a/src/plugins/data/public/ui/settings/settings.ts b/src/plugins/data/public/ui/settings/settings.ts index d7f7cd104c83..40541d87ebdd 100644 --- a/src/plugins/data/public/ui/settings/settings.ts +++ b/src/plugins/data/public/ui/settings/settings.ts @@ -42,11 +42,17 @@ export class Settings { this.setSelectedDataSet(this.getSelectedDataSet()); } + /** + * @experimental - Sets the dataset BehaviorSubject + */ setSelectedDataSet = (dataSet: any) => { this.storage.set('opensearchDashboards.userQueryDataSet', dataSet); this.selectedDataSet$.next(dataSet); }; + /** + * @experimental - Gets the dataset Observable + */ getSelectedDataSet$ = () => { return this.selectedDataSet$.asObservable(); }; diff --git a/src/plugins/data/public/ui/types.ts b/src/plugins/data/public/ui/types.ts index afa0c8130504..d3074163c7bc 100644 --- a/src/plugins/data/public/ui/types.ts +++ b/src/plugins/data/public/ui/types.ts @@ -66,6 +66,9 @@ export interface IUiStart { IndexPatternSelect: React.ComponentType; SearchBar: React.ComponentType; SuggestionsComponent: React.ComponentType; + /** + * @experimental - Subject to change + */ Settings: Settings; dataSourceContainer$: Observable; container$: Observable; diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index 74f5165cf425..076f33fc2662 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -58,7 +58,6 @@ export const useIndexPattern = (services: DiscoverViewServices) => { } }); }; - data.ui.Settings.getSelectedDataSet$().subscribe((dataSet) => { if (dataSet) { batch(() => { diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx b/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx deleted file mode 100644 index 196cb5eb4011..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/components/connections_bar.tsx +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useEffect, useRef, useState } from 'react'; -import { EuiPortal } from '@elastic/eui'; -import { distinctUntilChanged } from 'rxjs/operators'; -import { ToastsSetup } from 'opensearch-dashboards/public'; -import { DataPublicPluginStart, QueryEditorExtensionDependencies } from '../../../../data/public'; -import { ConnectionsService } from '../services'; - -interface ConnectionsProps { - dependencies: QueryEditorExtensionDependencies; - toasts: ToastsSetup; - connectionsService: ConnectionsService; -} - -export const ConnectionsBar: React.FC = ({ connectionsService, toasts }) => { - const [isDataSourceEnabled, setIsDataSourceEnabled] = useState(false); - const [uiService, setUiService] = useState(undefined); - const containerRef = useRef(null); - - useEffect(() => { - const uiServiceSubscription = connectionsService.getUiService().subscribe(setUiService); - const dataSourceEnabledSubscription = connectionsService - .getIsDataSourceEnabled$() - .subscribe(setIsDataSourceEnabled); - - return () => { - uiServiceSubscription.unsubscribe(); - dataSourceEnabledSubscription.unsubscribe(); - }; - }, [connectionsService]); - - useEffect(() => { - if (!uiService || !isDataSourceEnabled || !containerRef.current) return; - const subscriptions = uiService.dataSourceContainer$.subscribe((container) => { - if (container && containerRef.current) { - container.append(containerRef.current); - } - }); - - return () => subscriptions.unsubscribe(); - }, [uiService, isDataSourceEnabled]); - - useEffect(() => { - const selectedConnectionSubscription = connectionsService - .getSelectedConnection$() - .pipe(distinctUntilChanged()) - .subscribe((connection) => { - if (connection) { - // Assuming setSelectedConnection$ is meant to update some state or perform an action outside this component - connectionsService.setSelectedConnection$(connection); - } - }); - - return () => selectedConnectionSubscription.unsubscribe(); - }, [connectionsService]); - - const handleSelectedConnection = (id: string | undefined) => { - if (!id) { - connectionsService.setSelectedConnection$(undefined); - return; - } - connectionsService.getConnectionById(id).subscribe((connection) => { - connectionsService.setSelectedConnection$(connection); - }); - }; - - return ( - { - containerRef.current = node; - }} - > -
- - ); -}; diff --git a/src/plugins/query_enhancements/public/data_source_connection/components/index.ts b/src/plugins/query_enhancements/public/data_source_connection/components/index.ts deleted file mode 100644 index 1ee969a1d079..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/components/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { ConnectionsBar } from './connections_bar'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/index.ts b/src/plugins/query_enhancements/public/data_source_connection/index.ts deleted file mode 100644 index e334163d91d4..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { createDataSourceConnectionExtension } from './utils'; -export * from './services'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/services/index.ts b/src/plugins/query_enhancements/public/data_source_connection/services/index.ts deleted file mode 100644 index 08eeda5a7aa1..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/services/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { ConnectionsService } from './connections_service'; diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx b/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx deleted file mode 100644 index 0250c196d12b..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/create_extension.tsx +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { ToastsSetup } from 'opensearch-dashboards/public'; -import { of } from 'rxjs'; -import { QueryEditorExtensionConfig } from '../../../../data/public'; -import { ConfigSchema } from '../../../common/config'; -import { ConnectionsBar } from '../components'; -import { ConnectionsService } from '../services'; - -export const createDataSourceConnectionExtension = ( - connectionsService: ConnectionsService, - toasts: ToastsSetup, - config: ConfigSchema -): QueryEditorExtensionConfig => { - return { - id: 'data-source-connection', - order: 2000, - isEnabled$: (dependencies) => { - return of(false); - }, - getComponent: (dependencies) => { - return ( - - ); - }, - }; -}; diff --git a/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts b/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts deleted file mode 100644 index 9eccc9e6f35a..000000000000 --- a/src/plugins/query_enhancements/public/data_source_connection/utils/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export * from './create_extension'; diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index 29a83e1c4c64..9baec694f1a9 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -7,10 +7,9 @@ import moment from 'moment'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '../../../core/public'; import { IStorageWrapper, Storage } from '../../opensearch_dashboards_utils/public'; import { ConfigSchema } from '../common/config'; -import { ConnectionsService, createDataSourceConnectionExtension } from './data_source_connection'; +import { ConnectionsService, setData, setStorage } from './services'; import { createQueryAssistExtension } from './query_assist'; import { PPLSearchInterceptor, SQLAsyncSearchInterceptor, SQLSearchInterceptor } from './search'; -import { setData, setStorage } from './services'; import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, @@ -155,16 +154,6 @@ export class QueryEnhancementsPlugin }, }); - data.__enhance({ - ui: { - queryEditorExtension: createDataSourceConnectionExtension( - this.connectionsService, - core.notifications.toasts, - this.config - ), - }, - }); - return {}; } diff --git a/src/plugins/query_enhancements/public/services.ts b/src/plugins/query_enhancements/public/services.ts deleted file mode 100644 index d11233be2dca..000000000000 --- a/src/plugins/query_enhancements/public/services.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { createGetterSetter } from '../../opensearch_dashboards_utils/common'; -import { IStorageWrapper } from '../../opensearch_dashboards_utils/public'; -import { DataPublicPluginStart } from '../../data/public'; - -export const [getStorage, setStorage] = createGetterSetter('storage'); -export const [getData, setData] = createGetterSetter('data'); diff --git a/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts b/src/plugins/query_enhancements/public/services/connections_service.ts similarity index 95% rename from src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts rename to src/plugins/query_enhancements/public/services/connections_service.ts index 6afec4b51a99..97a59c2cd94a 100644 --- a/src/plugins/query_enhancements/public/data_source_connection/services/connections_service.ts +++ b/src/plugins/query_enhancements/public/services/connections_service.ts @@ -6,8 +6,8 @@ import { BehaviorSubject, Observable, from } from 'rxjs'; import { DataPublicPluginStart } from 'src/plugins/data/public'; import { CoreStart } from 'opensearch-dashboards/public'; -import { API } from '../../../common'; -import { Connection, ConnectionsServiceDeps } from '../../types'; +import { API } from '../../common'; +import { Connection, ConnectionsServiceDeps } from '../types'; export class ConnectionsService { protected http!: ConnectionsServiceDeps['http']; diff --git a/src/plugins/query_enhancements/public/services/index.ts b/src/plugins/query_enhancements/public/services/index.ts new file mode 100644 index 000000000000..bb0284408faa --- /dev/null +++ b/src/plugins/query_enhancements/public/services/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createGetterSetter } from '../../../opensearch_dashboards_utils/common'; +import { IStorageWrapper } from '../../../opensearch_dashboards_utils/public'; +import { DataPublicPluginStart } from '../../../data/public'; + +export const [getStorage, setStorage] = createGetterSetter('storage'); +export const [getData, setData] = createGetterSetter('data'); + +export { ConnectionsService } from './connections_service'; From e2f80be30a41de2cd8d8d3d44b97d19d2efacee7 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 12:39:47 -0700 Subject: [PATCH 16/59] remove unused property Signed-off-by: Sean Li --- .../data/public/data_sources/datasource_selector/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index e3547046aec3..fe1e4360e961 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -24,7 +24,6 @@ export interface DataSourceOption { value: string; type: string; ds: DataSource; - checked?: boolean; } export interface DataSourceSelectableProps extends Pick, 'fullWidth'> { From 6b61983677910386b1b1210c0e76bd43c63ee285 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 12:55:34 -0700 Subject: [PATCH 17/59] use default search interceptor for get indices Signed-off-by: Sean Li --- .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index ef6be6b0294d..ce81f8c8e577 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -71,6 +71,7 @@ const buildSearchRequest = (showAllIndices: boolean, pattern: string, dataSource const getIndices = async (search: ISearchStart, dataSourceId: string) => { const request = buildSearchRequest(true, '*', dataSourceId); return search + .getDefaultSearchInterceptor() .search(request) .pipe(map(searchResponseToArray(true))) .pipe(scan((accumulator = [], value) => accumulator.join(value))) From a23b6b90761430191ccf50edb24bf95cb68ea3a5 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 14:22:24 -0700 Subject: [PATCH 18/59] removing unused file Signed-off-by: Sean Li --- src/plugins/data/public/ui/query_editor/dataset_navigator.tsx | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/plugins/data/public/ui/query_editor/dataset_navigator.tsx diff --git a/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx b/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx deleted file mode 100644 index a850c1690e3b..000000000000 --- a/src/plugins/data/public/ui/query_editor/dataset_navigator.tsx +++ /dev/null @@ -1,4 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ From 43a44b635a7f71371dc7fc344973a4b3025a0624 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Thu, 18 Jul 2024 22:37:27 -0700 Subject: [PATCH 19/59] adding onselect for dataset navigator Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 4 ++- .../public/ui/query_editor/query_editor.tsx | 34 +++++++++++++++++-- .../ui/query_editor/query_editor_top_row.tsx | 19 +++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index ce81f8c8e577..fa34acbd26d1 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -88,9 +88,10 @@ interface DataSetOption { interface DataSetNavigatorProps { savedObjectsClient: SavedObjectsClientContract; indexPatterns: Array; + onSubmit: any; } -export const DataSetNavigator = ({ savedObjectsClient, indexPatterns }: DataSetNavigatorProps) => { +export const DataSetNavigator = ({ savedObjectsClient, indexPatterns, onSubmit }: DataSetNavigatorProps) => { const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); const [clusterList, setClusterList] = useState([]); const [indexList, setIndexList] = useState([]); @@ -121,6 +122,7 @@ export const DataSetNavigator = ({ savedObjectsClient, indexPatterns }: DataSetN dataSourceRef: selectedCluster?.id ?? undefined, }); setSelectedDataSet(ds); + onSubmit(ds); closePopover(); }; diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 85c32b2bc276..0c5d092eeb39 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -58,6 +58,7 @@ export interface QueryEditorProps { onChangeQueryEditorFocus?: (isFocused: boolean) => void; onSubmit?: (query: Query, dateRange?: TimeRange) => void; getQueryStringInitialValue?: (language: string) => string; + getQueryStringInitialValueByDataSet?: (language: string, dataSet: any) => string; dataTestSubj?: string; size?: SuggestionsListSize; className?: string; @@ -215,6 +216,36 @@ export default class QueryEditorUI extends Component { } }; + private onSelectDataSet = (dataSet: any) => { + const newQuery = { + query: this.props.getQueryStringInitialValueByDataSet?.(this.props.query.language, dataSet) ?? '', + language: this.props.query.language, + }; + + const enhancement = this.props.settings.getQueryEnhancements(newQuery.language); + const fields = enhancement?.fields; + const newSettings: DataSettings = { + userQueryLanguage: newQuery.language, + userQueryString: newQuery.query, + ...(fields && { uiOverrides: { fields } }), + }; + this.props.settings?.updateSettings(newSettings); + + const dateRangeEnhancement = enhancement?.searchBar?.dateRange; + const dateRange = dateRangeEnhancement + ? { + from: dateRangeEnhancement.initialFrom!, + to: dateRangeEnhancement.initialTo!, + } + : undefined; + this.onChange(newQuery, dateRange); + this.onSubmit(newQuery, dateRange); + this.setState({ + isDataSourcesVisible: enhancement?.searchBar?.showDataSourcesSelector ?? true, + isDataSetsVisible: enhancement?.searchBar?.showDataSetsSelector ?? true, + }); + } + // TODO: MQL consider moving language select language of setting search source here private onSelectLanguage = (language: string) => { // Send telemetry info every time the user opts in or out of kuery @@ -369,13 +400,10 @@ export default class QueryEditorUI extends Component { const className = classNames(this.props.className); const headerClassName = classNames('osdQueryEditorHeader', this.props.headerClassName); const bannerClassName = classNames('osdQueryEditorBanner', this.props.bannerClassName); -<<<<<<< HEAD const footerClassName = classNames('osdQueryEditorFooter', this.props.footerClassName); const useQueryEditor = this.props.query.language !== 'kuery' && this.props.query.language !== 'lucene'; -======= ->>>>>>> 0ae2098677... cleanup dataset dropdown return (
diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index 8304fdc252ee..7c8bab597121 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -213,6 +213,24 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { return input.replace('', dataSource); } + function getQueryStringInitialValueByDataSet(language: string, dataSet: any) { + const { indexPatterns, settings } = props; + const input = settings?.getQueryEnhancements(language)?.searchBar?.queryStringInput + ?.initialValue; + + if ( + !indexPatterns || + (!Array.isArray(indexPatterns) && compact(indexPatterns).length > 0) || + !input + ) + return ''; + + // const defaultDataSource = indexPatterns[0]; + const dataSource = dataSet.name; + + return input.replace('', dataSource); + } + function renderQueryEditor() { if (!shouldRenderQueryEditor()) return; return ( @@ -231,6 +249,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { onChangeQueryEditorFocus={onChangeQueryEditorFocus} onSubmit={onInputSubmit} getQueryStringInitialValue={getQueryStringInitialValue} + getQueryStringInitialValueByDataSet={getQueryStringInitialValueByDataSet} persistedLog={persistedLog} className="osdQueryEditor" dataTestSubj={props.dataTestSubj} From f43621a9e2671dfb63af5807704a5255ca5e1daa Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Fri, 19 Jul 2024 12:43:02 +0000 Subject: [PATCH 20/59] trying to make a commponetto call Signed-off-by: Kawika Avilla --- src/plugins/data/public/index.ts | 1 + .../create_dataset_navigator.tsx | 17 ++ .../dataset_navigator/dataset_navigator.tsx | 215 ++++++++++-------- .../ui/dataset_navigator/fetch_clusters.ts | 13 ++ .../dataset_navigator/fetch_index_patterns.ts | 21 ++ .../ui/dataset_navigator/fetch_indices.ts | 67 ++++++ .../public/ui/dataset_navigator/index.tsx | 5 +- src/plugins/data/public/ui/index.ts | 2 +- .../public/ui/query_editor/query_editor.tsx | 21 +- .../ui/query_editor/query_editor_top_row.tsx | 28 +-- .../data/public/ui/settings/settings.ts | 4 +- src/plugins/data/public/ui/ui_service.ts | 3 +- .../application/helpers/get_data_set.ts | 21 +- .../utils/use_index_pattern.ts | 18 +- 14 files changed, 275 insertions(+), 161 deletions(-) create mode 100644 src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/fetch_clusters.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/fetch_index_patterns.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/fetch_indices.ts diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 208359352e4b..46390b819821 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -446,6 +446,7 @@ export { // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, DataSetNavigator, + DataSetOption, } from './ui'; /** diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx new file mode 100644 index 000000000000..d04a5464a486 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import _ from 'lodash'; +import React from 'react'; + +import { SavedObjectsClientContract } from 'src/core/public'; +import { DataSetNavigator, DataSetNavigatorProps } from './'; + +// Takes in stateful runtime dependencies and pre-wires them to the component +export function createDataSetNavigator(savedObjectsClient: SavedObjectsClientContract) { + return (props: Omit) => ( + + ); +} diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index fa34acbd26d1..d8647e082f5c 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -3,95 +3,143 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useEffect, useState } from 'react'; +import React, { Component, useEffect, useState } from 'react'; import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; import { SavedObjectsClientContract, SimpleSavedObject } from 'opensearch-dashboards/public'; -import { map, scan } from 'rxjs/operators'; -import { ISearchStart } from '../../search/types'; import { IIndexPattern } from '../..'; -import { getUiService, getIndexPatterns, getSearchService } from '../../services'; +import { getUiService, getIndexPatterns, getSearchService, getQueryService } from '../../services'; +import { fetchClusters } from './fetch_clusters'; +import { fetchIndices } from './fetch_indices'; +import _ from 'lodash'; +import { fetchIndexPatterns } from './fetch_index_patterns'; -const getClusters = async (savedObjectsClient: SavedObjectsClientContract) => { - return await savedObjectsClient.find({ - type: 'data-source', - perPage: 10000, - }); -}; - -export const searchResponseToArray = (showAllIndices: boolean) => (response) => { - const { rawResponse } = response; - if (!rawResponse.aggregations) { - return []; - } else { - return rawResponse.aggregations.indices.buckets - .map((bucket: { key: string }) => { - return bucket.key; - }) - .filter((indexName: string) => { - if (showAllIndices) { - return true; - } else { - return !indexName.startsWith('.'); - } - }) - .map((indexName: string) => { - return { - name: indexName, - // item: {}, - }; - }); - } -}; - -const buildSearchRequest = (showAllIndices: boolean, pattern: string, dataSourceId?: string) => { - const request = { - params: { - ignoreUnavailable: true, - expand_wildcards: showAllIndices ? 'all' : 'open', - index: pattern, - body: { - size: 0, // no hits - aggs: { - indices: { - terms: { - field: '_index', - size: 100, - }, - }, - }, - }, - }, - dataSourceId, - }; - - return request; -}; - -const getIndices = async (search: ISearchStart, dataSourceId: string) => { - const request = buildSearchRequest(true, '*', dataSourceId); - return search - .getDefaultSearchInterceptor() - .search(request) - .pipe(map(searchResponseToArray(true))) - .pipe(scan((accumulator = [], value) => accumulator.join(value))) - .toPromise() - .catch(() => []); -}; - -interface DataSetOption { +export interface DataSetOption { id: string; name: string; dataSourceRef?: string; } -interface DataSetNavigatorProps { +export interface DataSetNavigatorProps { savedObjectsClient: SavedObjectsClientContract; indexPatterns: Array; - onSubmit: any; + dataSetId: string; +} + +interface DataSetNavigatorState { + isLoading: boolean; + options: []; + selectedDataSet: DataSetOption | undefined; + searchValue: string | undefined; + dataSourceIdToTitle: Map; } -export const DataSetNavigator = ({ savedObjectsClient, indexPatterns, onSubmit }: DataSetNavigatorProps) => { +// eslint-disable-next-line import/no-default-export +export default class DataSetNavigator extends Component { + private isMounted: boolean = false; + state: DataSetNavigatorState; + + + constructor(props: DataSetNavigatorProps) { + super(props); + + this.state = { + isLoading: false, + options: [], + selectedDataSet: undefined, + searchValue: undefined, + dataSourceIdToTitle: new Map(), + }; + } + + debouncedFetch = _.debounce(async (searchValue: string) => { + const { savedObjectsClient } = this.props; + + const savedObjectFields = ['title']; + let savedObjects = await fetchIndexPatterns(savedObjectsClient, searchValue, savedObjectFields); + + + + if (!this.isMounted) { + return; + } + + // We need this check to handle the case where search results come back in a different + // order than they were sent out. Only load results for the most recent search. + if (searchValue === this.state.searchValue) { + const dataSourcesToFetch: Array<{ type: string; id: string }> = []; + const dataSourceIdSet = new Set(); + savedObjects.map((indexPatternSavedObject: SimpleSavedObject) => { + const dataSourceReference = getDataSourceReference(indexPatternSavedObject.references); + if ( + dataSourceReference && + !this.state.dataSourceIdToTitle.has(dataSourceReference.id) && + !dataSourceIdSet.has(dataSourceReference.id) + ) { + dataSourceIdSet.add(dataSourceReference.id); + dataSourcesToFetch.push({ type: 'data-source', id: dataSourceReference.id }); + } + }); + + const dataSourceIdToTitleToUpdate = new Map(); + + if (dataSourcesToFetch.length > 0) { + const resp = await savedObjectsClient.bulkGet(dataSourcesToFetch); + resp.savedObjects.map((dataSourceSavedObject: SimpleSavedObject) => { + dataSourceIdToTitleToUpdate.set( + dataSourceSavedObject.id, + dataSourceSavedObject.attributes.title + ); + }); + } + + const options = savedObjects.map((indexPatternSavedObject: SimpleSavedObject) => { + const dataSourceReference = getDataSourceReference(indexPatternSavedObject.references); + if (dataSourceReference) { + const dataSourceTitle = + this.state.dataSourceIdToTitle.get(dataSourceReference.id) || + dataSourceIdToTitleToUpdate.get(dataSourceReference.id) || + dataSourceReference.id; + return { + label: `${concatDataSourceWithIndexPattern( + dataSourceTitle, + indexPatternSavedObject.attributes.title + )}`, + value: indexPatternSavedObject.id, + }; + } + return { + label: indexPatternSavedObject.attributes.title, + value: indexPatternSavedObject.id, + }; + }); + + if (dataSourceIdToTitleToUpdate.size > 0) { + const mergedDataSourceIdToTitle = new Map(); + this.state.dataSourceIdToTitle.forEach((k, v) => { + mergedDataSourceIdToTitle.set(k, v); + }); + dataSourceIdToTitleToUpdate.forEach((k, v) => { + mergedDataSourceIdToTitle.set(k, v); + }); + this.setState({ + dataSourceIdToTitle: mergedDataSourceIdToTitle, + isLoading: false, + options, + }); + } else { + this.setState({ + isLoading: false, + options, + }); + } + + if (onNoIndexPatterns && searchValue === '' && options.length === 0) { + onNoIndexPatterns(); + } + } + }, 300); + const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); const [clusterList, setClusterList] = useState([]); const [indexList, setIndexList] = useState([]); @@ -103,24 +151,13 @@ export const DataSetNavigator = ({ savedObjectsClient, indexPatterns, onSubmit } const [indexPatternList, setIndexPatternList] = useState([]); const search = getSearchService(); const uiService = getUiService(); + const queryService = getQueryService(); const indexPatternsService = getIndexPatterns(); const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); const onDataSetClick = async (ds: DataSetOption) => { - const existingIndexPattern = indexPatternsService.getByTitle(ds.id, true); - const dataSet = await indexPatternsService.create( - { id: ds.id, title: ds.name }, - !existingIndexPattern?.id - ); - // save to cache by title because the id is not unique for temporary index pattern created - indexPatternsService.saveToCache(dataSet.title, dataSet); - uiService.Settings.setSelectedDataSet({ - id: dataSet.id, - name: dataSet.title, - dataSourceRef: selectedCluster?.id ?? undefined, - }); setSelectedDataSet(ds); onSubmit(ds); closePopover(); @@ -148,7 +185,7 @@ export const DataSetNavigator = ({ savedObjectsClient, indexPatterns, onSubmit } }, [indexPatternsService]); useEffect(() => { - Promise.all([getClusters(savedObjectsClient)]).then((res) => { + Promise.all([fetchClusters(savedObjectsClient)]).then((res) => { setClusterList(res.length > 0 ? res?.[0].savedObjects : []); }); }, [savedObjectsClient]); @@ -156,7 +193,7 @@ export const DataSetNavigator = ({ savedObjectsClient, indexPatterns, onSubmit } useEffect(() => { if (selectedCluster) { // Get all indexes - getIndices(search, selectedCluster.id).then((res) => { + fetchIndices(search, selectedCluster.id).then((res) => { setIndexList( res.map((index: { name: string }) => ({ name: index.name, diff --git a/src/plugins/data/public/ui/dataset_navigator/fetch_clusters.ts b/src/plugins/data/public/ui/dataset_navigator/fetch_clusters.ts new file mode 100644 index 000000000000..54519d062fe1 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/fetch_clusters.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; + +export const fetchClusters = async (savedObjectsClient: SavedObjectsClientContract) => { + return await savedObjectsClient.find({ + type: 'data-source', + perPage: 10000, + }); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/fetch_index_patterns.ts new file mode 100644 index 000000000000..fd1d893565e7 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/fetch_index_patterns.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; + +export const fetchIndexPatterns = async ( + client: SavedObjectsClientContract, + search: string, + fields: string[] +) => { + const resp = await client.find({ + type: 'index-pattern', + fields, + search: `${search}*`, + searchFields: ['title'], + perPage: 100, + }); + return resp.savedObjects; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/fetch_indices.ts new file mode 100644 index 000000000000..db08a3f96253 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/fetch_indices.ts @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { map, scan } from 'rxjs/operators'; +import { ISearchStart } from '../../search'; + +export const fetchIndices = async (search: ISearchStart, dataSourceId: string) => { + const request = buildSearchRequest(true, '*', dataSourceId); + return search + .getDefaultSearchInterceptor() + .search(request) + .pipe(map(searchResponseToArray(true))) + .pipe(scan((accumulator = [], value) => accumulator.join(value))) + .toPromise() + .catch(() => []); +}; + +const searchResponseToArray = (showAllIndices: boolean) => (response) => { + const { rawResponse } = response; + if (!rawResponse.aggregations) { + return []; + } else { + return rawResponse.aggregations.indices.buckets + .map((bucket: { key: string }) => { + return bucket.key; + }) + .filter((indexName: string) => { + if (showAllIndices) { + return true; + } else { + return !indexName.startsWith('.'); + } + }) + .map((indexName: string) => { + return { + name: indexName, + // item: {}, + }; + }); + } +}; + +const buildSearchRequest = (showAllIndices: boolean, pattern: string, dataSourceId?: string) => { + const request = { + params: { + ignoreUnavailable: true, + expand_wildcards: showAllIndices ? 'all' : 'open', + index: pattern, + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: 100, + }, + }, + }, + }, + }, + dataSourceId, + }; + + return request; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx index ea04b4d206dd..fdded3c18a68 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -3,4 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { DataSetNavigator } from './dataset_navigator'; +export { DataSetNavigator, DataSetNavigatorProps, DataSetOption } from './dataset_navigator'; +export { fetchClusters } from './fetch_clusters'; +export { fetchIndexPatterns } from './fetch_index_patterns'; +export { fetchIndices } from './fetch_indices'; diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index 400887e51d57..f0824978966f 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -49,4 +49,4 @@ export { } from './query_editor'; export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; export { SuggestionsComponent } from './typeahead'; -export { DataSetNavigator } from './dataset_navigator'; +export { DataSetNavigator, DataSetOption } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 0c5d092eeb39..5e4531ca746e 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -57,8 +57,7 @@ export interface QueryEditorProps { onChange?: (query: Query, dateRange?: TimeRange) => void; onChangeQueryEditorFocus?: (isFocused: boolean) => void; onSubmit?: (query: Query, dateRange?: TimeRange) => void; - getQueryStringInitialValue?: (language: string) => string; - getQueryStringInitialValueByDataSet?: (language: string, dataSet: any) => string; + getQueryStringInitialValue?: (language: string, dataSetName?: string) => string; dataTestSubj?: string; size?: SuggestionsListSize; className?: string; @@ -216,9 +215,9 @@ export default class QueryEditorUI extends Component { } }; - private onSelectDataSet = (dataSet: any) => { + private onSelectDataSet = (dataSetName: string) => { const newQuery = { - query: this.props.getQueryStringInitialValueByDataSet?.(this.props.query.language, dataSet) ?? '', + query: this.props.getQueryStringInitialValue?.(this.props.query.language, dataSetName) ?? '', language: this.props.query.language, }; @@ -240,11 +239,7 @@ export default class QueryEditorUI extends Component { : undefined; this.onChange(newQuery, dateRange); this.onSubmit(newQuery, dateRange); - this.setState({ - isDataSourcesVisible: enhancement?.searchBar?.showDataSourcesSelector ?? true, - isDataSetsVisible: enhancement?.searchBar?.showDataSetsSelector ?? true, - }); - } + }; // TODO: MQL consider moving language select language of setting search source here private onSelectLanguage = (language: string) => { @@ -278,10 +273,6 @@ export default class QueryEditorUI extends Component { : undefined; this.onChange(newQuery, dateRange); this.onSubmit(newQuery, dateRange); - this.setState({ - isDataSourcesVisible: enhancement?.searchBar?.showDataSourcesSelector ?? true, - isDataSetsVisible: enhancement?.searchBar?.showDataSetsSelector ?? true, - }); }; private initPersistedLog = () => { @@ -319,10 +310,6 @@ export default class QueryEditorUI extends Component { this.initPersistedLog(); // this.fetchIndexPatterns().then(this.updateSuggestions); - this.setState({ - isDataSourcesVisible: this.initDataSourcesVisibility() || true, - isDataSetsVisible: this.initDataSetsVisibility() || true, - }); } public componentDidUpdate(prevProps: Props) { diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index 7c8bab597121..eca09935fc3b 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -194,7 +194,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { if (query && query.query) return true; } - function getQueryStringInitialValue(language: string) { + function getQueryStringInitialValue(language: string, dataSetName?: string) { const { indexPatterns, settings } = props; const input = settings?.getQueryEnhancements(language)?.searchBar?.queryStringInput ?.initialValue; @@ -206,29 +206,10 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ) return ''; - const defaultDataSource = indexPatterns[0]; - const dataSource = - typeof defaultDataSource === 'string' ? defaultDataSource : defaultDataSource.title; + const defaultDataSet = dataSetName ?? indexPatterns[0]; + const dataSet = typeof defaultDataSet === 'string' ? defaultDataSet : defaultDataSet.title; - return input.replace('', dataSource); - } - - function getQueryStringInitialValueByDataSet(language: string, dataSet: any) { - const { indexPatterns, settings } = props; - const input = settings?.getQueryEnhancements(language)?.searchBar?.queryStringInput - ?.initialValue; - - if ( - !indexPatterns || - (!Array.isArray(indexPatterns) && compact(indexPatterns).length > 0) || - !input - ) - return ''; - - // const defaultDataSource = indexPatterns[0]; - const dataSource = dataSet.name; - - return input.replace('', dataSource); + return input.replace('', dataSet); } function renderQueryEditor() { @@ -249,7 +230,6 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { onChangeQueryEditorFocus={onChangeQueryEditorFocus} onSubmit={onInputSubmit} getQueryStringInitialValue={getQueryStringInitialValue} - getQueryStringInitialValueByDataSet={getQueryStringInitialValueByDataSet} persistedLog={persistedLog} className="osdQueryEditor" dataTestSubj={props.dataTestSubj} diff --git a/src/plugins/data/public/ui/settings/settings.ts b/src/plugins/data/public/ui/settings/settings.ts index 40541d87ebdd..c2b1ae24197a 100644 --- a/src/plugins/data/public/ui/settings/settings.ts +++ b/src/plugins/data/public/ui/settings/settings.ts @@ -109,8 +109,10 @@ export class Settings { } setUserQueryLanguage(language: string) { + if (language !== this.getUserQueryLanguage()) { + this.search.df.clear(); + } this.storage.set('opensearchDashboards.userQueryLanguage', language); - this.search.df.clear(); const queryEnhancement = this.queryEnhancements.get(language); this.search.__enhance({ searchInterceptor: queryEnhancement diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 1e0e6be8b78c..b0cfb4852d77 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -81,11 +81,10 @@ export class UiService implements Plugin { return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), + DataSetNavigator: createDataSetNavigator(core.savedObjects.client), SearchBar, SuggestionsComponent, Settings, - dataSourceContainer$: this.dataSourceContainer$, - container$: this.container$, }; } diff --git a/src/plugins/discover/public/application/helpers/get_data_set.ts b/src/plugins/discover/public/application/helpers/get_data_set.ts index b0431ac31c1e..762274f5bf00 100644 --- a/src/plugins/discover/public/application/helpers/get_data_set.ts +++ b/src/plugins/discover/public/application/helpers/get_data_set.ts @@ -3,23 +3,26 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IndexPattern, IndexPatternsContract } from '../../../../data/public'; +import { DataSetOption, IndexPattern, IndexPatternsContract } from '../../../../data/public'; import { SearchData } from '../view_components/utils/use_search'; function getDataSet( - indexPattern: IndexPattern | undefined, + dataSet: IndexPattern | DataSetOption | undefined, state: SearchData, indexPatternsService: IndexPatternsContract ) { - if (!indexPattern) { + if (!dataSet) { return; } - return ( - (state.title && - state.title !== indexPattern?.title && - indexPatternsService.getByTitle(state.title!, true)) || - indexPattern - ); + if (dataSet instanceof IndexPattern) { + return ( + (state.title && + state.title !== dataSet?.title && + indexPatternsService.getByTitle(state.title!, true)) || + dataSet + ); + } + return dataSet; } export { getDataSet }; diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index 076f33fc2662..05b7cc648ecd 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -58,15 +58,6 @@ export const useIndexPattern = (services: DiscoverViewServices) => { } }); }; - data.ui.Settings.getSelectedDataSet$().subscribe((dataSet) => { - if (dataSet) { - batch(() => { - store!.dispatch(updateDataSet(dataSet)); - store!.dispatch(updateIndexPattern(dataSet.id)); - }); - fetchIndexPatternDetails(dataSet.id); - } - }); if (!indexPatternIdFromState) { data.indexPatterns.getCache().then((indexPatternList) => { @@ -81,14 +72,7 @@ export const useIndexPattern = (services: DiscoverViewServices) => { return () => { isMounted = false; }; - }, [ - indexPatternIdFromState, - data.indexPatterns, - toastNotifications, - config, - store, - data.ui.Settings, - ]); + }, [indexPatternIdFromState, data.indexPatterns, toastNotifications, config, store]); return indexPattern; }; From f61ca0738a6bb040bde46fb6fbde34a152154836 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Fri, 19 Jul 2024 13:18:21 +0000 Subject: [PATCH 21/59] no clue if it works Signed-off-by: Kawika Avilla --- .../create_dataset_navigator.tsx | 23 +- .../dataset_navigator/dataset_navigator.tsx | 302 +++++------------- .../public/ui/query_editor/query_editor.tsx | 5 + src/plugins/data/public/ui/types.ts | 3 +- src/plugins/data/public/ui/ui_service.ts | 8 +- .../public/components/sidebar/index.tsx | 23 +- 6 files changed, 126 insertions(+), 238 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx index d04a5464a486..11f0ff41e073 100644 --- a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx @@ -3,15 +3,26 @@ * SPDX-License-Identifier: Apache-2.0 */ -import _ from 'lodash'; import React from 'react'; - import { SavedObjectsClientContract } from 'src/core/public'; +import { IndexPatternsContract } from 'src/plugins/data/public'; import { DataSetNavigator, DataSetNavigatorProps } from './'; -// Takes in stateful runtime dependencies and pre-wires them to the component -export function createDataSetNavigator(savedObjectsClient: SavedObjectsClientContract) { - return (props: Omit) => ( - +// Updated function signature to include additional dependencies +export function createDataSetNavigator( + savedObjectsClient: SavedObjectsClientContract, + indexPatternsService: IndexPatternsContract, + search: any +) { + // Return a function that takes props, omitting the dependencies from the props type + return ( + props: Omit + ) => ( + ); } diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index d8647e082f5c..c24e0b7d8e99 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -3,16 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { Component, useEffect, useState } from 'react'; - +import React, { useEffect, useState } from 'react'; import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; -import { SavedObjectsClientContract, SimpleSavedObject } from 'opensearch-dashboards/public'; +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import _ from 'lodash'; import { IIndexPattern } from '../..'; -import { getUiService, getIndexPatterns, getSearchService, getQueryService } from '../../services'; import { fetchClusters } from './fetch_clusters'; import { fetchIndices } from './fetch_indices'; -import _ from 'lodash'; -import { fetchIndexPatterns } from './fetch_index_patterns'; export interface DataSetOption { id: string; @@ -24,186 +21,60 @@ export interface DataSetNavigatorProps { savedObjectsClient: SavedObjectsClientContract; indexPatterns: Array; dataSetId: string; + onDataSetSelected: (dataSet: DataSetOption) => void; } interface DataSetNavigatorState { isLoading: boolean; - options: []; + isOpen: boolean; + clusters: DataSetOption[]; + indices: DataSetOption[]; + indexPatterns: DataSetOption[]; selectedDataSet: DataSetOption | undefined; + selectedCluster: DataSetOption | undefined; searchValue: string | undefined; dataSourceIdToTitle: Map; } -// eslint-disable-next-line import/no-default-export -export default class DataSetNavigator extends Component { - private isMounted: boolean = false; - state: DataSetNavigatorState; - - - constructor(props: DataSetNavigatorProps) { - super(props); - - this.state = { - isLoading: false, - options: [], - selectedDataSet: undefined, - searchValue: undefined, - dataSourceIdToTitle: new Map(), - }; - } - - debouncedFetch = _.debounce(async (searchValue: string) => { - const { savedObjectsClient } = this.props; - - const savedObjectFields = ['title']; - let savedObjects = await fetchIndexPatterns(savedObjectsClient, searchValue, savedObjectFields); - - - - if (!this.isMounted) { - return; - } - - // We need this check to handle the case where search results come back in a different - // order than they were sent out. Only load results for the most recent search. - if (searchValue === this.state.searchValue) { - const dataSourcesToFetch: Array<{ type: string; id: string }> = []; - const dataSourceIdSet = new Set(); - savedObjects.map((indexPatternSavedObject: SimpleSavedObject) => { - const dataSourceReference = getDataSourceReference(indexPatternSavedObject.references); - if ( - dataSourceReference && - !this.state.dataSourceIdToTitle.has(dataSourceReference.id) && - !dataSourceIdSet.has(dataSourceReference.id) - ) { - dataSourceIdSet.add(dataSourceReference.id); - dataSourcesToFetch.push({ type: 'data-source', id: dataSourceReference.id }); - } - }); - - const dataSourceIdToTitleToUpdate = new Map(); - - if (dataSourcesToFetch.length > 0) { - const resp = await savedObjectsClient.bulkGet(dataSourcesToFetch); - resp.savedObjects.map((dataSourceSavedObject: SimpleSavedObject) => { - dataSourceIdToTitleToUpdate.set( - dataSourceSavedObject.id, - dataSourceSavedObject.attributes.title - ); - }); - } - - const options = savedObjects.map((indexPatternSavedObject: SimpleSavedObject) => { - const dataSourceReference = getDataSourceReference(indexPatternSavedObject.references); - if (dataSourceReference) { - const dataSourceTitle = - this.state.dataSourceIdToTitle.get(dataSourceReference.id) || - dataSourceIdToTitleToUpdate.get(dataSourceReference.id) || - dataSourceReference.id; - return { - label: `${concatDataSourceWithIndexPattern( - dataSourceTitle, - indexPatternSavedObject.attributes.title - )}`, - value: indexPatternSavedObject.id, - }; - } - return { - label: indexPatternSavedObject.attributes.title, - value: indexPatternSavedObject.id, - }; - }); - - if (dataSourceIdToTitleToUpdate.size > 0) { - const mergedDataSourceIdToTitle = new Map(); - this.state.dataSourceIdToTitle.forEach((k, v) => { - mergedDataSourceIdToTitle.set(k, v); - }); - dataSourceIdToTitleToUpdate.forEach((k, v) => { - mergedDataSourceIdToTitle.set(k, v); - }); - this.setState({ - dataSourceIdToTitle: mergedDataSourceIdToTitle, - isLoading: false, - options, - }); - } else { - this.setState({ - isLoading: false, - options, - }); - } - - if (onNoIndexPatterns && searchValue === '' && options.length === 0) { - onNoIndexPatterns(); - } - } - }, 300); - +export const DataSetNavigator = ({ indexPatternsService, savedObjectsClient, search }) => { + const [indexPatternList, setIndexPatternList] = useState([]); + const [clusterList, setClusterList] = useState([]); + const [indexList, setIndexList] = useState([]); + const [selectedCluster, setSelectedCluster] = useState(null); + const [selectedDataSet, setSelectedDataSet] = useState(null); const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); - const [clusterList, setClusterList] = useState([]); - const [indexList, setIndexList] = useState([]); - const [selectedCluster, setSelectedCluster] = useState(); - const [selectedDataSet, setSelectedDataSet] = useState({ - id: indexPatterns[0]?.id, - name: indexPatterns[0]?.title, - }); - const [indexPatternList, setIndexPatternList] = useState([]); - const search = getSearchService(); - const uiService = getUiService(); - const queryService = getQueryService(); - const indexPatternsService = getIndexPatterns(); - const onButtonClick = () => setIsDataSetNavigatorOpen((isOpen) => !isOpen); + const onButtonClick = () => setIsDataSetNavigatorOpen(!isDataSetNavigatorOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); - - const onDataSetClick = async (ds: DataSetOption) => { - setSelectedDataSet(ds); - onSubmit(ds); + const onDataSetClick = (dataSet) => { + setSelectedDataSet(dataSet); closePopover(); }; useEffect(() => { - const subscription = uiService.Settings.getSelectedDataSet$().subscribe((dataSet) => { - if (dataSet) { - setSelectedDataSet(dataSet); - } + // Fetch index patterns + indexPatternsService.getIdsWithTitle().then((res) => { + setIndexPatternList(res.map(({ id, title }) => ({ id, name: title }))); }); - return () => subscription.unsubscribe(); - }, [uiService]); - // get all index patterns - useEffect(() => { - indexPatternsService.getIdsWithTitle().then((res) => - setIndexPatternList( - res.map((indexPattern: { id: string; title: string }) => ({ - id: indexPattern.id, - name: indexPattern.title, - })) - ) - ); - }, [indexPatternsService]); - - useEffect(() => { - Promise.all([fetchClusters(savedObjectsClient)]).then((res) => { - setClusterList(res.length > 0 ? res?.[0].savedObjects : []); + // Fetch clusters + fetchClusters(savedObjectsClient).then((res) => { + setClusterList(res.savedObjects); }); - }, [savedObjectsClient]); - useEffect(() => { + // Fetch indices if a cluster is selected if (selectedCluster) { - // Get all indexes fetchIndices(search, selectedCluster.id).then((res) => { setIndexList( - res.map((index: { name: string }) => ({ - name: index.name, - id: index.name, + res.map(({ name }) => ({ + name, + id: name, dataSourceRef: selectedCluster.id, })) ); }); } - }, [search, selectedCluster, setIndexList]); + }, [indexPatternsService, savedObjectsClient, search, selectedCluster]); const dataSetButton = ( { ); + const contextMenuPanels = [ + { + id: 0, + title: 'DATA', + items: [ + { + name: 'Index Patterns', + panel: 1, + }, + ...clusterList.map((cluster) => ({ + name: cluster.attributes.title, + panel: 2, + onClick: () => setSelectedCluster(cluster), + })), + ], + }, + { + id: 1, + title: 'Index Patterns', + items: indexPatternList.map((indexPattern) => ({ + name: indexPattern.name, + onClick: () => onDataSetClick(indexPattern), + })), + }, + { + id: 2, + title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', + items: [ + { + name: 'Indexes', + panel: 3, + }, + ], + }, + { + id: 3, + title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', + items: indexList.map((index) => ({ + name: index.name, + onClick: () => onDataSetClick(index), + })), + }, + { + id: 4, + title: 'clicked', + }, + ]; + return ( { closePopover={closePopover} anchorPosition="downLeft" > - ({ - name: cluster.attributes.title, - panel: 2, - onClick: () => { - setSelectedCluster(cluster); - }, - })) - : []), - ], - }, - { - id: 1, - title: 'Index Patterns', - items: [ - ...(indexPatternList - ? indexPatternList.map((indexPattern) => ({ - name: indexPattern.name, - onClick: () => onDataSetClick(indexPattern), - })) - : []), - ], - }, - { - id: 2, - title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', - items: [ - { - name: 'Indexes', - panel: 3, - }, - ], - }, - { - id: 3, - title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', - items: [ - ...(indexList - ? indexList.map((index) => ({ - name: index.name, - onClick: () => onDataSetClick(index), - })) - : []), - ], - }, - { - id: 4, - title: 'clicked', - }, - ]} - /> + ); }; + +// eslint-disable-next-line import/no-default-export +export default DataSetNavigator; diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 5e4531ca746e..d2464d49b580 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -467,6 +467,11 @@ export default class QueryEditorUI extends Component { > {this.props.prepend} + {this.state.isDataSetsVisible && ( + +
+ + )} diff --git a/src/plugins/data/public/ui/types.ts b/src/plugins/data/public/ui/types.ts index d3074163c7bc..73726327c683 100644 --- a/src/plugins/data/public/ui/types.ts +++ b/src/plugins/data/public/ui/types.ts @@ -5,6 +5,7 @@ import { Observable } from 'rxjs'; import { SearchInterceptor } from '../search'; +import { DataSetNavigatorProps } from './dataset_navigator'; import { IndexPatternSelectProps } from './index_pattern_select'; import { StatefulSearchBarProps } from './search_bar'; import { QueryEditorExtensionConfig } from './query_editor/query_editor_extensions'; @@ -70,6 +71,6 @@ export interface IUiStart { * @experimental - Subject to change */ Settings: Settings; - dataSourceContainer$: Observable; + DataSetNavigator: React.ComponentType; container$: Observable; } diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index b0cfb4852d77..228c202eb50c 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -14,6 +14,7 @@ import { createSearchBar } from './search_bar/create_search_bar'; import { createSettings } from './settings'; import { SuggestionsComponent } from './typeahead'; import { IUiSetup, IUiStart, QueryEnhancement, UiEnhancements } from './types'; +import { createDataSetNavigator } from './dataset_navigator/create_dataset_navigator'; /** @internal */ // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -81,10 +82,15 @@ export class UiService implements Plugin { return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), - DataSetNavigator: createDataSetNavigator(core.savedObjects.client), + DataSetNavigator: createDataSetNavigator( + core.savedObjects.client, + dataServices.indexPatterns, + dataServices.search + ), SearchBar, SuggestionsComponent, Settings, + container$: this.container$, }; } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index f4c628a7d70f..da7497a07d7c 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -4,7 +4,7 @@ */ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; -import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; +import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; @@ -58,13 +58,7 @@ export const Sidebar: FC = ({ children }) => { return () => { subscriptions.unsubscribe(); }; - }, [ - ui.container$, - containerRef, - setContainerRef, - ui.dataSourceContainer$, - isEnhancementsEnabled, - ]); + }, [ui.container$, containerRef, setContainerRef, isEnhancementsEnabled]); useEffect(() => { let isMounted = true; @@ -93,7 +87,7 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (indexPatternId) { const option = getMatchedOption(dataSourceOptionList, indexPatternId); - setSelectedSources((prev) => (option ? [option] : prev)); + setSelectedSources(option ? [option] : []); } }, [indexPatternId, activeDataSources, dataSourceOptionList]); @@ -153,6 +147,8 @@ export const Sidebar: FC = ({ children }) => { /> ); + const dataSetNavigator = ui.DataSetNavigator; + return ( { borderRadius="none" color="transparent" > + {isEnhancementsEnabled && ( + { + containerRef.current = node; + }} + > + {dataSetNavigator} + + )} {!isEnhancementsEnabled && ( Date: Fri, 19 Jul 2024 13:55:19 -0700 Subject: [PATCH 22/59] trying out changes Signed-off-by: Sean Li --- .../create_dataset_navigator.tsx | 7 ++- .../dataset_navigator/dataset_navigator.tsx | 15 ++++-- .../public/ui/query_editor/query_editor.tsx | 50 +++++++++---------- .../ui/query_editor/query_editor_top_row.tsx | 5 +- .../data/public/ui/settings/settings.ts | 1 + src/plugins/data/public/ui/types.ts | 2 +- src/plugins/data/public/ui/ui_service.ts | 6 ++- .../public/components/sidebar/index.tsx | 42 +++++++++++++--- .../utils/state_management/metadata_slice.ts | 4 +- .../public/utils/state_management/store.ts | 2 +- 10 files changed, 90 insertions(+), 44 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx index 11f0ff41e073..552411067c90 100644 --- a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx @@ -7,12 +7,15 @@ import React from 'react'; import { SavedObjectsClientContract } from 'src/core/public'; import { IndexPatternsContract } from 'src/plugins/data/public'; import { DataSetNavigator, DataSetNavigatorProps } from './'; +import { Settings } from '../settings'; // Updated function signature to include additional dependencies export function createDataSetNavigator( + settings: Settings, savedObjectsClient: SavedObjectsClientContract, indexPatternsService: IndexPatternsContract, - search: any + search: any, + onDataSetSelected: any, ) { // Return a function that takes props, omitting the dependencies from the props type return ( @@ -20,9 +23,11 @@ export function createDataSetNavigator( ) => ( ); } diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index c24e0b7d8e99..c8e1e5abd185 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -10,6 +10,7 @@ import _ from 'lodash'; import { IIndexPattern } from '../..'; import { fetchClusters } from './fetch_clusters'; import { fetchIndices } from './fetch_indices'; +import { Settings } from '../settings'; export interface DataSetOption { id: string; @@ -18,10 +19,13 @@ export interface DataSetOption { } export interface DataSetNavigatorProps { + settings: Settings; savedObjectsClient: SavedObjectsClientContract; - indexPatterns: Array; - dataSetId: string; + indexPattern?: Array; + dataSetId?: string; onDataSetSelected: (dataSet: DataSetOption) => void; + indexPatternsService: any; + search: any; } interface DataSetNavigatorState { @@ -36,7 +40,8 @@ interface DataSetNavigatorState { dataSourceIdToTitle: Map; } -export const DataSetNavigator = ({ indexPatternsService, savedObjectsClient, search }) => { +export const DataSetNavigator = (props: DataSetNavigatorProps) => { + const { settings, indexPatternsService, savedObjectsClient, search, onDataSetSelected } = props; const [indexPatternList, setIndexPatternList] = useState([]); const [clusterList, setClusterList] = useState([]); const [indexList, setIndexList] = useState([]); @@ -46,8 +51,10 @@ export const DataSetNavigator = ({ indexPatternsService, savedObjectsClient, sea const onButtonClick = () => setIsDataSetNavigatorOpen(!isDataSetNavigatorOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); - const onDataSetClick = (dataSet) => { + const onDataSetClick = async (dataSet) => { setSelectedDataSet(dataSet); + onDataSetSelected(dataSet); + settings.setSelectedDataSet(dataSet); closePopover(); }; diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index d2464d49b580..5ed400377edc 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -215,31 +215,31 @@ export default class QueryEditorUI extends Component { } }; - private onSelectDataSet = (dataSetName: string) => { - const newQuery = { - query: this.props.getQueryStringInitialValue?.(this.props.query.language, dataSetName) ?? '', - language: this.props.query.language, - }; - - const enhancement = this.props.settings.getQueryEnhancements(newQuery.language); - const fields = enhancement?.fields; - const newSettings: DataSettings = { - userQueryLanguage: newQuery.language, - userQueryString: newQuery.query, - ...(fields && { uiOverrides: { fields } }), - }; - this.props.settings?.updateSettings(newSettings); - - const dateRangeEnhancement = enhancement?.searchBar?.dateRange; - const dateRange = dateRangeEnhancement - ? { - from: dateRangeEnhancement.initialFrom!, - to: dateRangeEnhancement.initialTo!, - } - : undefined; - this.onChange(newQuery, dateRange); - this.onSubmit(newQuery, dateRange); - }; + // private onSelectDataSet = (dataSetName: string) => { + // const newQuery = { + // query: this.props.getQueryStringInitialValue?.(this.props.query.language, dataSetName) ?? '', + // language: this.props.query.language, + // }; + + // const enhancement = this.props.settings.getQueryEnhancements(newQuery.language); + // const fields = enhancement?.fields; + // const newSettings: DataSettings = { + // userQueryLanguage: newQuery.language, + // userQueryString: newQuery.query, + // ...(fields && { uiOverrides: { fields } }), + // }; + // this.props.settings?.updateSettings(newSettings); + + // const dateRangeEnhancement = enhancement?.searchBar?.dateRange; + // const dateRange = dateRangeEnhancement + // ? { + // from: dateRangeEnhancement.initialFrom!, + // to: dateRangeEnhancement.initialTo!, + // } + // : undefined; + // this.onChange(newQuery, dateRange); + // this.onSubmit(newQuery, dateRange); + // }; // TODO: MQL consider moving language select language of setting search source here private onSelectLanguage = (language: string) => { diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index eca09935fc3b..f59155ffd24f 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -194,7 +194,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { if (query && query.query) return true; } - function getQueryStringInitialValue(language: string, dataSetName?: string) { + function getQueryStringInitialValue(language: string) { const { indexPatterns, settings } = props; const input = settings?.getQueryEnhancements(language)?.searchBar?.queryStringInput ?.initialValue; @@ -206,7 +206,8 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ) return ''; - const defaultDataSet = dataSetName ?? indexPatterns[0]; + console.log('settings ds:', settings?.getSelectedDataSet()); + const defaultDataSet = settings?.getSelectedDataSet() ?? indexPatterns[0]; const dataSet = typeof defaultDataSet === 'string' ? defaultDataSet : defaultDataSet.title; return input.replace('', dataSet); diff --git a/src/plugins/data/public/ui/settings/settings.ts b/src/plugins/data/public/ui/settings/settings.ts index c2b1ae24197a..ebdde370db26 100644 --- a/src/plugins/data/public/ui/settings/settings.ts +++ b/src/plugins/data/public/ui/settings/settings.ts @@ -46,6 +46,7 @@ export class Settings { * @experimental - Sets the dataset BehaviorSubject */ setSelectedDataSet = (dataSet: any) => { + console.log('dataSet in settings:', dataSet); this.storage.set('opensearchDashboards.userQueryDataSet', dataSet); this.selectedDataSet$.next(dataSet); }; diff --git a/src/plugins/data/public/ui/types.ts b/src/plugins/data/public/ui/types.ts index 73726327c683..a5938e6aa129 100644 --- a/src/plugins/data/public/ui/types.ts +++ b/src/plugins/data/public/ui/types.ts @@ -71,6 +71,6 @@ export interface IUiStart { * @experimental - Subject to change */ Settings: Settings; - DataSetNavigator: React.ComponentType; + DataSetNavigator: (onDataSetSelected: any) => React.ComponentType; container$: Observable; } diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 228c202eb50c..9d7d8455ef3a 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -82,10 +82,12 @@ export class UiService implements Plugin { return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), - DataSetNavigator: createDataSetNavigator( + DataSetNavigator: (onSelectedDataSet) => createDataSetNavigator( + Settings, core.savedObjects.client, dataServices.indexPatterns, - dataServices.search + dataServices.search, + onSelectedDataSet ), SearchBar, SuggestionsComponent, diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index da7497a07d7c..413293ca6e41 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -3,15 +3,22 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; +import React, { ComponentType, FC, useCallback, useEffect, useRef, useState } from 'react'; import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; -import { setIndexPattern, useTypedDispatch, useTypedSelector } from '../../utils/state_management'; +import { + setIndexPattern, + setDataSet, + useTypedDispatch, + useTypedSelector, +} from '../../utils/state_management'; import './index.scss'; +import { DataSetNavigatorProps } from '../../../../data/public/ui/dataset_navigator'; +import { batch } from 'react-redux'; export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); @@ -20,6 +27,8 @@ export const Sidebar: FC = ({ children }) => { const [dataSourceOptionList, setDataSourceOptionList] = useState([]); const [activeDataSources, setActiveDataSources] = useState([]); const [isEnhancementsEnabled, setIsEnhancementsEnabled] = useState(false); + const [DataSetNavigator, setDataSetNavigator] = useState(); + const [selectedDataSet, setSelectedDataSet] = useState(); const containerRef = useRef(null); const { @@ -30,17 +39,40 @@ export const Sidebar: FC = ({ children }) => { }, } = useOpenSearchDashboards(); + const handleDataSetSelection = useCallback( + (selectedDataSet: any) => { + setSelectedDataSet(selectedDataSet); + setSelectedSources([ + { + key: selectedDataSet.id, + name: selectedDataSet.name, + label: selectedDataSet.name, + value: selectedDataSet.id, + type: 'OpenSearch Default', + ds: activeDataSources[0], + }, + ]); + batch(() => { + dispatch(setIndexPattern(selectedDataSet.id)); + dispatch(setDataSet(selectedDataSet)); + }); + }, + [dispatch] + ); + useEffect(() => { const subscriptions = ui.Settings.getEnabledQueryEnhancementsUpdated$().subscribe( (enabledQueryEnhancements) => { setIsEnhancementsEnabled(enabledQueryEnhancements); + if (enabledQueryEnhancements) + setDataSetNavigator(ui.DataSetNavigator(handleDataSetSelection)); } ); return () => { subscriptions.unsubscribe(); }; - }, [ui.Settings]); + }, [ui.Settings, handleDataSetSelection]); const setContainerRef = useCallback((uiContainerRef) => { uiContainerRef.appendChild(containerRef.current); @@ -147,8 +179,6 @@ export const Sidebar: FC = ({ children }) => { /> ); - const dataSetNavigator = ui.DataSetNavigator; - return ( { containerRef.current = node; }} > - {dataSetNavigator} + {DataSetNavigator} )} {!isEnhancementsEnabled && ( diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index 51b492c5dec5..a2842916a78b 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -59,7 +59,7 @@ export const slice = createSlice({ setIndexPattern: (state, action: PayloadAction) => { state.indexPattern = action.payload; }, - setDataset: (state, action: PayloadAction) => { + setDataSet: (state, action: PayloadAction) => { state.dataset = action.payload; }, setOriginatingApp: (state, action: PayloadAction) => { @@ -75,4 +75,4 @@ export const slice = createSlice({ }); export const { reducer } = slice; -export const { setIndexPattern, setDataset, setOriginatingApp, setView, setState } = slice.actions; +export const { setIndexPattern, setDataSet, setOriginatingApp, setView, setState } = slice.actions; diff --git a/src/plugins/data_explorer/public/utils/state_management/store.ts b/src/plugins/data_explorer/public/utils/state_management/store.ts index 3683d7d201ce..9d320de4b54b 100644 --- a/src/plugins/data_explorer/public/utils/state_management/store.ts +++ b/src/plugins/data_explorer/public/utils/state_management/store.ts @@ -116,4 +116,4 @@ export type RenderState = Omit; // Remaining state after export type Store = ReturnType; export type AppDispatch = Store['dispatch']; -export { MetadataState, setIndexPattern, setDataset, setOriginatingApp } from './metadata_slice'; +export { MetadataState, setIndexPattern, setDataSet, setOriginatingApp } from './metadata_slice'; From b287f60c84accd238f0c3c4dc44229c406613e57 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Sun, 21 Jul 2024 18:09:32 -0700 Subject: [PATCH 23/59] initial commit for external datasources work Signed-off-by: Sean Li --- .../create_dataset_navigator.tsx | 6 +- .../dataset_navigator/dataset_navigator.tsx | 283 +++++++++-- .../catalog_cache/cache_intercept.ts | 21 + .../framework/catalog_cache/cache_loader.tsx | 441 ++++++++++++++++++ .../framework/catalog_cache/cache_manager.ts | 307 ++++++++++++ .../dataset_navigator/framework/constants.ts | 99 ++++ .../framework/hooks/direct_query_hook.tsx | 99 ++++ .../framework/requests/sql.ts | 60 +++ .../ui/dataset_navigator/framework/types.tsx | 292 ++++++++++++ .../framework/utils/query_session_utils.ts | 16 + .../framework/utils/shared.ts | 353 ++++++++++++++ .../framework/utils/use_polling.ts | 137 ++++++ .../public/ui/dataset_navigator/index.tsx | 6 +- .../{ => utils}/fetch_clusters.ts | 0 .../utils/fetch_external_data_sources.ts | 25 + .../{ => utils}/fetch_index_patterns.ts | 0 .../{ => utils}/fetch_indices.ts | 2 +- .../ui/dataset_navigator/utils/utils.ts | 18 + src/plugins/data/public/ui/ui_service.ts | 4 +- 19 files changed, 2134 insertions(+), 35 deletions(-) create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/constants.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/hooks/direct_query_hook.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/requests/sql.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/types.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/utils/query_session_utils.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/utils/shared.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/framework/utils/use_polling.ts rename src/plugins/data/public/ui/dataset_navigator/{ => utils}/fetch_clusters.ts (100%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts rename src/plugins/data/public/ui/dataset_navigator/{ => utils}/fetch_index_patterns.ts (100%) rename src/plugins/data/public/ui/dataset_navigator/{ => utils}/fetch_indices.ts (97%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/utils/utils.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx index 552411067c90..6ba50ad526f8 100644 --- a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx @@ -4,7 +4,7 @@ */ import React from 'react'; -import { SavedObjectsClientContract } from 'src/core/public'; +import { HttpStart, NotificationsStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { IndexPatternsContract } from 'src/plugins/data/public'; import { DataSetNavigator, DataSetNavigatorProps } from './'; import { Settings } from '../settings'; @@ -16,6 +16,8 @@ export function createDataSetNavigator( indexPatternsService: IndexPatternsContract, search: any, onDataSetSelected: any, + http: HttpStart, + notifications: NotificationsStart, ) { // Return a function that takes props, omitting the dependencies from the props type return ( @@ -28,6 +30,8 @@ export function createDataSetNavigator( indexPatternsService={indexPatternsService} search={search} onDataSetSelected={onDataSetSelected} + http={http} + notifications={notifications} /> ); } diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index c8e1e5abd185..1bb9785f39a8 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -4,13 +4,32 @@ */ import React, { useEffect, useState } from 'react'; -import { EuiButtonEmpty, EuiContextMenu, EuiPopover } from '@elastic/eui'; -import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { + EuiButtonEmpty, + EuiButtonIcon, + EuiContextMenu, + EuiLoadingSpinner, + EuiPopover, + EuiText, +} from '@elastic/eui'; +import { + HttpStart, + NotificationsStart, + SavedObjectsClientContract, +} from 'opensearch-dashboards/public'; import _ from 'lodash'; import { IIndexPattern } from '../..'; -import { fetchClusters } from './fetch_clusters'; -import { fetchIndices } from './fetch_indices'; +import { fetchClusters } from './utils/fetch_clusters'; +import { fetchIndices } from './utils/fetch_indices'; +import { fetchExternalDataSources } from './utils/fetch_external_data_sources'; import { Settings } from '../settings'; +import { + useLoadDatabasesToCache, + useLoadTablesToCache, +} from './framework/catalog_cache/cache_loader'; +import { CatalogCacheManager } from './framework/catalog_cache/cache_manager'; +import { CachedDataSourceStatus, DirectQueryLoadingStatus } from './framework/types'; +import { isCatalogCacheFetching } from './framework/utils/shared'; export interface DataSetOption { id: string; @@ -18,6 +37,12 @@ export interface DataSetOption { dataSourceRef?: string; } +export interface ExternalDataSource { + name: string; + dataSourceRef: string; + status: DirectQueryLoadingStatus; +} + export interface DataSetNavigatorProps { settings: Settings; savedObjectsClient: SavedObjectsClientContract; @@ -26,28 +51,45 @@ export interface DataSetNavigatorProps { onDataSetSelected: (dataSet: DataSetOption) => void; indexPatternsService: any; search: any; -} - -interface DataSetNavigatorState { - isLoading: boolean; - isOpen: boolean; - clusters: DataSetOption[]; - indices: DataSetOption[]; - indexPatterns: DataSetOption[]; - selectedDataSet: DataSetOption | undefined; - selectedCluster: DataSetOption | undefined; - searchValue: string | undefined; - dataSourceIdToTitle: Map; + http: HttpStart; + notifications: NotificationsStart; } export const DataSetNavigator = (props: DataSetNavigatorProps) => { - const { settings, indexPatternsService, savedObjectsClient, search, onDataSetSelected } = props; + const { + settings, + indexPatternsService, + savedObjectsClient, + search, + onDataSetSelected, + http, + notifications, + } = props; + const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); + const [selectedDataSet, setSelectedDataSet] = useState(null); const [indexPatternList, setIndexPatternList] = useState([]); const [clusterList, setClusterList] = useState([]); - const [indexList, setIndexList] = useState([]); const [selectedCluster, setSelectedCluster] = useState(null); - const [selectedDataSet, setSelectedDataSet] = useState(null); - const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); + const [indexList, setIndexList] = useState([]); + const [externalDataSourceList, setExternalDataSourceList] = useState([]); + const [selectedExternalDataSource, setSelectedExternalDataSource] = useState< + ExternalDataSource + >(); + const [selectedDatabase, setSelectedDatabase] = useState(); + const [cacheRefreshable, setCacheRefreshable] = useState(false); + const [cachedDatabases, setCachedDatabases] = useState([]); + const [cachedTables, setCachedTables] = useState([]); + const [loading, setLoading] = useState(false); + const [failed, setFailed] = useState(false); + + const { + loadStatus: databasesLoadStatus, + startLoading: startLoadingDatabases, + } = useLoadDatabasesToCache(http, notifications); + const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache( + http, + notifications + ); const onButtonClick = () => setIsDataSetNavigatorOpen(!isDataSetNavigatorOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); @@ -57,8 +99,20 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { settings.setSelectedDataSet(dataSet); closePopover(); }; + const handleRefresh = () => { + if ( + !isCatalogCacheFetching(databasesLoadStatus, tablesLoadStatus) && + selectedExternalDataSource + ) { + startLoadingDatabases({ + dataSourceName: selectedExternalDataSource.name, + dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + }); + } + }; useEffect(() => { + setLoading(true); // Fetch index patterns indexPatternsService.getIdsWithTitle().then((res) => { setIndexPatternList(res.map(({ id, title }) => ({ id, name: title }))); @@ -81,8 +135,113 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { ); }); } + setLoading(false); }, [indexPatternsService, savedObjectsClient, search, selectedCluster]); + // Start loading databases for datasource + useEffect(() => { + if (selectedExternalDataSource) { + const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( + selectedExternalDataSource.name, + selectedExternalDataSource.dataSourceRef + ); + if ( + (dataSourceCache.status === CachedDataSourceStatus.Empty || + dataSourceCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(databasesLoadStatus) + ) { + // setLoading(true); + startLoadingDatabases({ + dataSourceName: selectedExternalDataSource.name, + dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + }); + } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { + setCachedDatabases(dataSourceCache.databases); + } + } + }, [selectedExternalDataSource]); + + // Retrieve databases from cache upon success + useEffect(() => { + const status = databasesLoadStatus.toLowerCase(); + if (selectedExternalDataSource) { + const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( + selectedExternalDataSource?.name, + selectedExternalDataSource?.dataSourceRef + ); + if (status === DirectQueryLoadingStatus.SUCCESS) { + setCachedDatabases(dataSourceCache.databases); + } else if ( + status === DirectQueryLoadingStatus.CANCELED || + status === DirectQueryLoadingStatus.FAILED + ) { + setFailed(true); + } + } + }, [selectedExternalDataSource, databasesLoadStatus]); + + // Start loading tables for selected database + useEffect(() => { + if (selectedExternalDataSource && selectedDatabase) { + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase( + selectedExternalDataSource.name, + selectedDatabase, + selectedExternalDataSource.dataSourceRef + ); + } catch (error) { + console.error(error); + return; + } + if ( + databaseCache.status === CachedDataSourceStatus.Empty || + (databaseCache.status === CachedDataSourceStatus.Failed && + !isCatalogCacheFetching(tablesLoadStatus)) + ) { + startLoadingTables({ + dataSourceName: selectedExternalDataSource.name, + databaseName: selectedDatabase, + dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + }); + } else if (databaseCache.status === CachedDataSourceStatus.Updated) { + setCachedTables(databaseCache.tables); + } + } + }, [selectedExternalDataSource, selectedDatabase]); + + // Retrieve tables from cache upon success + useEffect(() => { + if (selectedExternalDataSource && selectedDatabase) { + const tablesStatus = tablesLoadStatus.toLowerCase(); + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase( + selectedExternalDataSource.name, + selectedDatabase, + selectedExternalDataSource.dataSourceRef + ); + } catch (error) { + console.error(error); + return; + } + if (tablesStatus === DirectQueryLoadingStatus.SUCCESS) { + setCachedTables(databaseCache.tables); + } else if ( + tablesStatus === DirectQueryLoadingStatus.CANCELED || + tablesStatus === DirectQueryLoadingStatus.FAILED + ) { + setFailed(true); + } + } + }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus]); + + useEffect(() => { + console.log('selectedExternalDataSource:', selectedExternalDataSource); + console.log('cachedDatabases:', cachedDatabases); + console.log('cachedTables:', cachedTables); + }, [selectedExternalDataSource, cachedDatabases, cachedTables]); + const dataSetButton = ( { ); + const RefreshButton = ( + + ); + + const LoadingSpinner = ; + const contextMenuPanels = [ { id: 0, - title: 'DATA', + // title: (Data), + title: 'Data', items: [ { name: 'Index Patterns', panel: 1, }, - ...clusterList.map((cluster) => ({ - name: cluster.attributes.title, + { + name: 'Indexes', panel: 2, - onClick: () => setSelectedCluster(cluster), - })), + }, + { + name: 'S3', + panel: 5, + onClick: () => + fetchExternalDataSources( + http, + clusterList.map((cluster) => cluster.id), + setExternalDataSourceList, + setLoading + ), + }, ], + content:
hello
, }, { id: 1, @@ -118,28 +299,72 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { name: indexPattern.name, onClick: () => onDataSetClick(indexPattern), })), + content:
{loading && LoadingSpinner}
, }, { id: 2, + title: 'Clusters', + items: [ + ...clusterList.map((cluster) => ({ + name: cluster.attributes.title, + panel: 3, + onClick: () => setSelectedCluster(cluster), + })), + ], + content:
{loading && LoadingSpinner}
, + }, + { + id: 3, title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', items: [ { name: 'Indexes', - panel: 3, + panel: 4, }, ], }, { - id: 3, + id: 4, title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', items: indexList.map((index) => ({ name: index.name, onClick: () => onDataSetClick(index), })), + content:
{loading && LoadingSpinner}
, }, { - id: 4, - title: 'clicked', + id: 5, + title:
S3 Connections {selectedExternalDataSource && RefreshButton}
, + items: [ + ...externalDataSourceList.map((ds) => ({ + name: ds.name, + onClick: () => setSelectedExternalDataSource(ds), + panel: 6, + })), + ], + content:
{loading && LoadingSpinner}
, + }, + { + id: 6, + title: selectedExternalDataSource ? selectedExternalDataSource.name : 'Databases', + items: [ + ...cachedDatabases.map((db) => ({ + name: db.name, + onClick: () => setSelectedDatabase(db.name), + panel: 7, + })), + ], + content:
{isCatalogCacheFetching(databasesLoadStatus) && LoadingSpinner}
, + }, + { + id: 7, + title: selectedDatabase ? selectedDatabase : 'Tables', + items: [ + ...cachedTables.map((table) => ({ + name: table.name, + })), + ], + content:
{isCatalogCacheFetching(tablesLoadStatus) && LoadingSpinner}
, }, ]; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts new file mode 100644 index 000000000000..cbf165035cb2 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpFetchOptionsWithPath, IHttpInterceptController } from 'opensearch-dashboards/public'; +import { SECURITY_DASHBOARDS_LOGOUT_URL } from '../constants'; +import { CatalogCacheManager } from './cache_manager'; + +export function catalogRequestIntercept(): any { + return ( + fetchOptions: Readonly, + _controller: IHttpInterceptController + ) => { + if (fetchOptions.path.includes(SECURITY_DASHBOARDS_LOGOUT_URL)) { + // Clears all user catalog cache details + CatalogCacheManager.clearDataSourceCache(); + CatalogCacheManager.clearAccelerationsCache(); + } + }; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx new file mode 100644 index 000000000000..26d6a43e8ebe --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx @@ -0,0 +1,441 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useRef, useState } from 'react'; +import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; +import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants'; +import { + AsyncPollingResult, + CachedAccelerations, + CachedColumn, + CachedDataSourceStatus, + CachedTable, + LoadCacheType, + StartLoadingParams, + DirectQueryLoadingStatus, + DirectQueryRequest, +} from '../types'; +import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; +import { + addBackticksIfNeeded, + combineSchemaAndDatarows, + get as getObjValue, + formatError, +} from '../utils/shared'; +import { usePolling } from '../utils/use_polling'; +import { SQLService } from '../requests/sql'; +import { CatalogCacheManager } from './cache_manager'; + +export const updateDatabasesToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + const cachedDataSource = CatalogCacheManager.getOrCreateDataSource( + dataSourceName, + dataSourceMDSId + ); + + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.addOrUpdateDataSource( + { + ...cachedDataSource, + databases: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + ...(dataSourceMDSId && { dataSourceMDSId }), + }, + dataSourceMDSId + ); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newDatabases = combinedData.map((row: any) => ({ + name: row.namespace, + tables: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + })); + + CatalogCacheManager.addOrUpdateDataSource( + { + ...cachedDataSource, + databases: newDatabases, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + ...(dataSourceMDSId && { dataSourceMDSId }), + }, + dataSourceMDSId + ); +}; + +export const updateTablesToCache = ( + dataSourceName: string, + databaseName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + try { + const cachedDatabase = CatalogCacheManager.getDatabase( + dataSourceName, + databaseName, + dataSourceMDSId + ); + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + }, + dataSourceMDSId + ); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + const newTables = combinedData + .filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information)) + .map((row: any) => ({ + name: row.tableName, + })); + + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: newTables, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }, + dataSourceMDSId + ); + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } +}; + +export const updateAccelerationsToCache = ( + dataSourceName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + const currentTime = new Date().toUTCString(); + + if (!pollingResult) { + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, + accelerations: [], + lastUpdated: currentTime, + status: CachedDataSourceStatus.Failed, + ...(dataSourceMDSId && { dataSourceMDSId }), + }); + return; + } + + const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); + + const newAccelerations: CachedAccelerations[] = combinedData.map((row: any) => ({ + flintIndexName: row.flint_index_name, + type: row.kind === 'mv' ? 'materialized' : row.kind, + database: row.database, + table: row.table, + indexName: row.index_name, + autoRefresh: row.auto_refresh, + status: row.status, + })); + + CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ + name: dataSourceName, + accelerations: newAccelerations, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + ...(dataSourceMDSId && { dataSourceMDSId }), + }); +}; + +export const updateTableColumnsToCache = ( + dataSourceName: string, + databaseName: string, + tableName: string, + pollingResult: AsyncPollingResult, + dataSourceMDSId?: string +) => { + try { + if (!pollingResult) { + return; + } + const cachedDatabase = CatalogCacheManager.getDatabase( + dataSourceName, + databaseName, + dataSourceMDSId + ); + const currentTime = new Date().toUTCString(); + + const combinedData: Array<{ col_name: string; data_type: string }> = combineSchemaAndDatarows( + pollingResult.schema, + pollingResult.datarows + ); + + const tableColumns: CachedColumn[] = []; + for (const row of combinedData) { + if (row.col_name === SPARK_PARTITION_INFO) { + break; + } + tableColumns.push({ + fieldName: row.col_name, + dataType: row.data_type, + }); + } + + const newTables: CachedTable[] = cachedDatabase.tables.map((ts) => + ts.name === tableName ? { ...ts, columns: tableColumns } : { ...ts } + ); + + if (cachedDatabase.status === CachedDataSourceStatus.Updated) { + CatalogCacheManager.updateDatabase( + dataSourceName, + { + ...cachedDatabase, + tables: newTables, + lastUpdated: currentTime, + status: CachedDataSourceStatus.Updated, + }, + dataSourceMDSId + ); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } +}; + +export const updateToCache = ( + pollResults: any, + loadCacheType: LoadCacheType, + dataSourceName: string, + databaseName?: string, + tableName?: string, + dataSourceMDSId?: string +) => { + switch (loadCacheType) { + case 'databases': + updateDatabasesToCache(dataSourceName, pollResults, dataSourceMDSId); + break; + case 'tables': + updateTablesToCache(dataSourceName, databaseName!, pollResults, dataSourceMDSId); + break; + case 'accelerations': + updateAccelerationsToCache(dataSourceName, pollResults, dataSourceMDSId); + break; + case 'tableColumns': + updateTableColumnsToCache( + dataSourceName, + databaseName!, + tableName!, + pollResults, + dataSourceMDSId + ); + default: + break; + } +}; + +export const createLoadQuery = ( + loadCacheType: LoadCacheType, + dataSourceName: string, + databaseName?: string, + tableName?: string +) => { + let query; + switch (loadCacheType) { + case 'databases': + query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; + break; + case 'tables': + query = `SHOW TABLE EXTENDED IN ${addBackticksIfNeeded( + dataSourceName + )}.${addBackticksIfNeeded(databaseName!)} LIKE '*'`; + break; + case 'accelerations': + query = `SHOW FLINT INDEX in ${addBackticksIfNeeded(dataSourceName)}`; + break; + case 'tableColumns': + query = `DESC ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( + databaseName! + )}.${addBackticksIfNeeded(tableName!)}`; + break; + default: + query = ''; + break; + } + return query; +}; + +export const useLoadToCache = ( + loadCacheType: LoadCacheType, + http: HttpStart, + notifications: NotificationsStart +) => { + const sqlService = new SQLService(http); + const [currentDataSourceName, setCurrentDataSourceName] = useState(''); + const [currentDatabaseName, setCurrentDatabaseName] = useState(''); + const [currentTableName, setCurrentTableName] = useState(''); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.INITIAL + ); + const dataSourceMDSClientId = useRef(''); + + const { + data: pollingResult, + loading: _pollingLoading, + error: pollingError, + startPolling, + stopPolling: stopLoading, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params, dataSourceMDSClientId.current); + }, ASYNC_POLLING_INTERVAL); + + const onLoadingFailed = () => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + updateToCache( + null, + loadCacheType, + currentDataSourceName, + currentDatabaseName, + currentTableName, + dataSourceMDSClientId.current + ); + }; + + const startLoading = ({ + dataSourceName, + dataSourceMDSId, + databaseName, + tableName, + }: StartLoadingParams) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + setCurrentDataSourceName(dataSourceName); + setCurrentDatabaseName(databaseName); + setCurrentTableName(tableName); + dataSourceMDSClientId.current = dataSourceMDSId || ''; + + let requestPayload: DirectQueryRequest = { + lang: 'sql', + query: createLoadQuery(loadCacheType, dataSourceName, databaseName, tableName), + datasource: dataSourceName, + }; + + const sessionId = getAsyncSessionId(dataSourceName); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } + sqlService + .fetch(requestPayload, dataSourceMDSId) + .then((result) => { + setAsyncSessionId(dataSourceName, getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + // eslint-disable-next-line no-console + console.error('No query id found in response'); + onLoadingFailed(); + } + }) + .catch((e) => { + onLoadingFailed(); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body?.message + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + // eslint-disable-next-line no-console + console.error(e); + }); + }; + + useEffect(() => { + // cancel direct query + if (!pollingResult) return; + const { status: anyCaseStatus, datarows, error } = pollingResult; + const status = anyCaseStatus?.toLowerCase(); + + if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { + setLoadStatus(status); + stopLoading(); + updateToCache( + pollingResult, + loadCacheType, + currentDataSourceName, + currentDatabaseName, + currentTableName, + dataSourceMDSClientId.current + ); + } else if (status === DirectQueryLoadingStatus.FAILED) { + onLoadingFailed(); + stopLoading(); + + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + error + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + } else { + setLoadStatus(status); + } + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [pollingResult, pollingError]); + + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadDatabasesToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'databases', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadTablesToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache('tables', http, notifications); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadTableColumnsToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'tableColumns', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; + +export const useLoadAccelerationsToCache = (http: HttpStart, notifications: NotificationsStart) => { + const { loadStatus, startLoading, stopLoading } = useLoadToCache( + 'accelerations', + http, + notifications + ); + return { loadStatus, startLoading, stopLoading }; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts new file mode 100644 index 000000000000..53f7a5e01e7f --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts @@ -0,0 +1,307 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CATALOG_CACHE_VERSION } from '../constants'; +import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE } from '../utils/shared'; +import { + AccelerationsCacheData, + CachedAccelerationByDataSource, + CachedDataSource, + CachedDataSourceStatus, + CachedDatabase, + CachedTable, + DataSourceCacheData, +} from '../types'; + +/** + * Manages caching for catalog data including data sources and accelerations. + */ +export class CatalogCacheManager { + /** + * Key for the data source cache in local storage. + */ + private static readonly datasourceCacheKey = ASYNC_QUERY_DATASOURCE_CACHE; + + /** + * Key for the accelerations cache in local storage. + */ + private static readonly accelerationsCacheKey = ASYNC_QUERY_ACCELERATIONS_CACHE; + + /** + * Saves data source cache to local storage. + * @param {DataSourceCacheData} cacheData - The data source cache data to save. + */ + static saveDataSourceCache(cacheData: DataSourceCacheData): void { + sessionStorage.setItem(this.datasourceCacheKey, JSON.stringify(cacheData)); + } + + /** + * Retrieves data source cache from local storage. + * @returns {DataSourceCacheData} The retrieved data source cache. + */ + static getDataSourceCache(): DataSourceCacheData { + const catalogData = sessionStorage.getItem(this.datasourceCacheKey); + + if (catalogData) { + return JSON.parse(catalogData); + } else { + const defaultCacheObject = { version: CATALOG_CACHE_VERSION, dataSources: [] }; + this.saveDataSourceCache(defaultCacheObject); + return defaultCacheObject; + } + } + + /** + * Saves accelerations cache to local storage. + * @param {AccelerationsCacheData} cacheData - The accelerations cache data to save. + */ + static saveAccelerationsCache(cacheData: AccelerationsCacheData): void { + sessionStorage.setItem(this.accelerationsCacheKey, JSON.stringify(cacheData)); + } + + /** + * Retrieves accelerations cache from local storage. + * @returns {AccelerationsCacheData} The retrieved accelerations cache. + */ + static getAccelerationsCache(): AccelerationsCacheData { + const accelerationCacheData = sessionStorage.getItem(this.accelerationsCacheKey); + + if (accelerationCacheData) { + return JSON.parse(accelerationCacheData); + } else { + const defaultCacheObject = { + version: CATALOG_CACHE_VERSION, + dataSources: [], + }; + this.saveAccelerationsCache(defaultCacheObject); + return defaultCacheObject; + } + } + + /** + * Adds or updates a data source in the accelerations cache. + * @param {CachedAccelerationByDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateAccelerationsByDataSource( + dataSource: CachedAccelerationByDataSource, + dataSourceMDSId?: string + ): void { + let index = -1; + const accCacheData = this.getAccelerationsCache(); + if (dataSourceMDSId) { + index = accCacheData.dataSources.findIndex( + (ds: CachedAccelerationByDataSource) => + ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId + ); + } else { + index = accCacheData.dataSources.findIndex( + (ds: CachedAccelerationByDataSource) => ds.name === dataSource.name + ); + } + if (index !== -1) { + accCacheData.dataSources[index] = dataSource; + } else { + accCacheData.dataSources.push(dataSource); + } + this.saveAccelerationsCache(accCacheData); + } + + /** + * Retrieves accelerations cache from local storage by the datasource name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedAccelerationByDataSource} The retrieved accelerations by datasource in cache. + * @throws {Error} If the data source is not found. + */ + static getOrCreateAccelerationsByDataSource( + dataSourceName: string, + dataSourceMDSId?: string + ): CachedAccelerationByDataSource { + const accCacheData = this.getAccelerationsCache(); + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = accCacheData.dataSources.find( + (ds) => ds.name === dataSourceName && ds.dataSourceMDSId === dataSourceMDSId + ); + } else { + cachedDataSource = accCacheData.dataSources.find((ds) => ds.name === dataSourceName); + } + if (cachedDataSource) return cachedDataSource; + else { + let defaultDataSourceObject: CachedAccelerationByDataSource = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + accelerations: [], + }; + + if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { + defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; + } + this.addOrUpdateAccelerationsByDataSource(defaultDataSourceObject, dataSourceMDSId); + return defaultDataSourceObject; + } + } + + /** + * Adds or updates a data source in the cache. + * @param {CachedDataSource} dataSource - The data source to add or update. + */ + static addOrUpdateDataSource(dataSource: CachedDataSource, dataSourceMDSId?: string): void { + const cacheData = this.getDataSourceCache(); + let index; + if (dataSourceMDSId) { + index = cacheData.dataSources.findIndex( + (ds: CachedDataSource) => + ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId + ); + } + index = cacheData.dataSources.findIndex((ds: CachedDataSource) => ds.name === dataSource.name); + if (index !== -1) { + cacheData.dataSources[index] = dataSource; + } else { + cacheData.dataSources.push(dataSource); + } + this.saveDataSourceCache(cacheData); + } + + /** + * Retrieves or creates a data source with the specified name. + * @param {string} dataSourceName - The name of the data source. + * @returns {CachedDataSource} The retrieved or created data source. + */ + static getOrCreateDataSource(dataSourceName: string, dataSourceMDSId?: string): CachedDataSource { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + if (cachedDataSource) { + return cachedDataSource; + } else { + let defaultDataSourceObject: CachedDataSource = { + name: dataSourceName, + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + databases: [], + }; + if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { + defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; + } + this.addOrUpdateDataSource(defaultDataSourceObject, dataSourceMDSId); + return defaultDataSourceObject; + } + } + + /** + * Retrieves a database from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @returns {CachedDatabase} The retrieved database. + * @throws {Error} If the data source or database is not found. + */ + static getDatabase( + dataSourceName: string, + databaseName: string, + dataSourceMDSId?: string + ): CachedDatabase { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const cachedDatabase = cachedDataSource.databases.find((db) => db.name === databaseName); + if (!cachedDatabase) { + throw new Error('Database not found exception: ' + databaseName); + } + + return cachedDatabase; + } + + /** + * Retrieves a table from the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {string} databaseName - The name of the database. + * @param {string} tableName - The name of the database. + * @returns {Cachedtable} The retrieved database. + * @throws {Error} If the data source, database or table is not found. + */ + static getTable( + dataSourceName: string, + databaseName: string, + tableName: string, + dataSourceMDSId?: string + ): CachedTable { + const cachedDatabase = this.getDatabase(dataSourceName, databaseName, dataSourceMDSId); + + const cachedTable = cachedDatabase.tables!.find((table) => table.name === tableName); + if (!cachedTable) { + throw new Error('Table not found exception: ' + tableName); + } + return cachedTable; + } + + /** + * Updates a database in the cache. + * @param {string} dataSourceName - The name of the data source containing the database. + * @param {CachedDatabase} database - The database to be updated. + * @throws {Error} If the data source or database is not found. + */ + static updateDatabase( + dataSourceName: string, + database: CachedDatabase, + dataSourceMDSId?: string + ): void { + let cachedDataSource; + if (dataSourceMDSId) { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName + ); + } else { + cachedDataSource = this.getDataSourceCache().dataSources.find( + (ds) => ds.name === dataSourceName + ); + } + + if (!cachedDataSource) { + throw new Error('DataSource not found exception: ' + dataSourceName); + } + + const index = cachedDataSource.databases.findIndex((db) => db.name === database.name); + if (index !== -1) { + cachedDataSource.databases[index] = database; + this.addOrUpdateDataSource(cachedDataSource, dataSourceMDSId); + } else { + throw new Error('Database not found exception: ' + database.name); + } + } + + /** + * Clears the data source cache from local storage. + */ + static clearDataSourceCache(): void { + sessionStorage.removeItem(this.datasourceCacheKey); + } + + /** + * Clears the accelerations cache from local storage. + */ + static clearAccelerationsCache(): void { + sessionStorage.removeItem(this.accelerationsCacheKey); + } +} diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts b/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts new file mode 100644 index 000000000000..785e17bbaa95 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; + +export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; +export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; +export const OLLY_QUESTION_URL_PARAM_KEY = 'olly_q'; +export const INDEX_URL_PARAM_KEY = 'indexPattern'; +export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATA_SOURCE_NAME = 'Default cluster'; +export const DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME = 'OpenSearch'; +export const DEFAULT_DATA_SOURCE_TYPE_NAME = 'Default Group'; +export const enum QUERY_LANGUAGE { + PPL = 'PPL', + SQL = 'SQL', + DQL = 'DQL', +} +export enum DATA_SOURCE_TYPES { + DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE, + SPARK = 'spark', + S3Glue = 's3glue', +} +export const ASYNC_POLLING_INTERVAL = 2000; + +export const CATALOG_CACHE_VERSION = '1.0'; +export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping'; +export const ACCELERATION_TIME_INTERVAL = [ + { text: 'millisecond(s)', value: 'millisecond' }, + { text: 'second(s)', value: 'second' }, + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; +export const ACCELERATION_REFRESH_TIME_INTERVAL = [ + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; + +export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; +export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; +export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; +export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; +export const SANITIZE_QUERY_REGEX = /\s+/g; +export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; +export const SPARK_STRING_DATATYPE = 'string'; + +export const ACCELERATION_INDEX_TYPES = [ + { label: 'Skipping Index', value: 'skipping' }, + { label: 'Covering Index', value: 'covering' }, + { label: 'Materialized View', value: 'materialized' }, +]; + +export const ACC_INDEX_TYPE_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; +export const ACC_CHECKPOINT_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; + +export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. +##### Skipping Index +- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. + - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. +##### Covering Index +- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. + - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. +##### Materialized View Index +- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. + - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. +##### Note: +- All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. + `; + +export const SKIPPING_INDEX_ACCELERATION_METHODS = [ + { value: 'PARTITION', text: 'Partition' }, + { value: 'VALUE_SET', text: 'Value Set' }, + { value: 'MIN_MAX', text: 'Min Max' }, + { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, +]; + +export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ + { label: 'window.start' }, + { label: 'count' }, + { label: 'sum' }, + { label: 'avg' }, + { label: 'max' }, + { label: 'min' }, +]; + +export const SPARK_PARTITION_INFO = `# Partition Information`; +export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector +export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector +export const SECURITY_DASHBOARDS_LOGOUT_URL = '/logout'; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/hooks/direct_query_hook.tsx b/src/plugins/data/public/ui/dataset_navigator/framework/hooks/direct_query_hook.tsx new file mode 100644 index 000000000000..a2b05f47e9ee --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/hooks/direct_query_hook.tsx @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useState } from 'react'; +import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; +import { ASYNC_POLLING_INTERVAL } from '../constants'; +import { DirectQueryLoadingStatus, DirectQueryRequest } from '../types'; +import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; +import { get as getObjValue, formatError } from '../utils/shared'; +import { usePolling } from '../utils/use_polling'; +import { SQLService } from '../requests/sql'; + +export const useDirectQuery = ( + http: HttpStart, + notifications: NotificationsStart, + dataSourceMDSId?: string +) => { + const sqlService = new SQLService(http); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.SCHEDULED + ); + + const { + data: pollingResult, + loading: _pollingLoading, + error: pollingError, + startPolling, + stopPolling: stopLoading, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params, dataSourceMDSId || ''); + }, ASYNC_POLLING_INTERVAL); + + const startLoading = (requestPayload: DirectQueryRequest) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + + const sessionId = getAsyncSessionId(requestPayload.datasource); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } + + sqlService + .fetch(requestPayload, dataSourceMDSId) + .then((result) => { + setAsyncSessionId(requestPayload.datasource, getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + // eslint-disable-next-line no-console + console.error('No query id found in response'); + setLoadStatus(DirectQueryLoadingStatus.FAILED); + } + }) + .catch((e) => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body?.message + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + // eslint-disable-next-line no-console + console.error(e); + }); + }; + + useEffect(() => { + // cancel direct query + if (!pollingResult) return; + const { status: anyCaseStatus, datarows, error } = pollingResult; + const status = anyCaseStatus?.toLowerCase(); + + if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { + setLoadStatus(status); + stopLoading(); + } else if (status === DirectQueryLoadingStatus.FAILED) { + setLoadStatus(status); + stopLoading(); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + error + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + } else { + setLoadStatus(status); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [pollingResult, pollingError, stopLoading]); + + return { loadStatus, startLoading, stopLoading, pollingResult }; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/requests/sql.ts b/src/plugins/data/public/ui/dataset_navigator/framework/requests/sql.ts new file mode 100644 index 000000000000..f2c9c30c79b9 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/requests/sql.ts @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart } from 'opensearch-dashboards/public'; +import { DirectQueryRequest } from '../types'; + +export class SQLService { + private http: HttpStart; + + constructor(http: HttpStart) { + this.http = http; + } + + fetch = async ( + params: DirectQueryRequest, + dataSourceMDSId?: string, + errorHandler?: (error: any) => void + ) => { + const query = { + dataSourceMDSId, + }; + return this.http + .post('/api/observability/query/jobs', { + body: JSON.stringify(params), + query, + }) + .catch((error) => { + // eslint-disable-next-line no-console + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + fetchWithJobId = async ( + params: { queryId: string }, + dataSourceMDSId?: string, + errorHandler?: (error: any) => void + ) => { + return this.http + .get(`/api/observability/query/jobs/${params.queryId}/${dataSourceMDSId ?? ''}`) + .catch((error) => { + // eslint-disable-next-line no-console + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + deleteWithJobId = async (params: { queryId: string }, errorHandler?: (error: any) => void) => { + return this.http.delete(`/api/observability/query/jobs/${params.queryId}`).catch((error) => { + // eslint-disable-next-line no-console + console.error('delete error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx b/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx new file mode 100644 index 000000000000..b556f412c2cd --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx @@ -0,0 +1,292 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiComboBoxOptionOption } from '@elastic/eui'; + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} + +export interface DirectQueryRequest { + query: string; + lang: string; + datasource: string; + sessionId?: string; +} + +export type AccelerationStatus = 'ACTIVE' | 'INACTIVE'; + +export interface PermissionsConfigurationProps { + roles: Role[]; + selectedRoles: Role[]; + setSelectedRoles: React.Dispatch>; + layout: 'horizontal' | 'vertical'; + hasSecurityAccess: boolean; +} + +export interface TableColumn { + name: string; + dataType: string; +} + +export interface Acceleration { + name: string; + status: AccelerationStatus; + type: string; + database: string; + table: string; + destination: string; + dateCreated: number; + dateUpdated: number; + index: string; + sql: string; +} + +export interface AssociatedObject { + tableName: string; + datasource: string; + id: string; + name: string; + database: string; + type: AssociatedObjectIndexType; + accelerations: CachedAcceleration[] | AssociatedObject; + columns?: CachedColumn[]; +} + +export type Role = EuiComboBoxOptionOption; + +export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; + +export interface S3GlueProperties { + 'glue.indexstore.opensearch.uri': string; + 'glue.indexstore.opensearch.region': string; +} + +export interface PrometheusProperties { + 'prometheus.uri': string; +} + +export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; + +export interface DatasourceDetails { + allowedRoles: string[]; + name: string; + connector: DatasourceType; + description: string; + properties: S3GlueProperties | PrometheusProperties; + status: DatasourceStatus; +} + +interface AsyncApiDataResponse { + status: string; + schema?: Array<{ name: string; type: string }>; + datarows?: any; + total?: number; + size?: number; + error?: string; +} + +export interface AsyncApiResponse { + data: { + ok: boolean; + resp: AsyncApiDataResponse; + }; +} + +export type PollingCallback = (statusObj: AsyncApiResponse) => void; + +export type AssociatedObjectIndexType = AccelerationIndexType | 'table'; + +export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; + +export type LoadCacheType = 'databases' | 'tables' | 'accelerations' | 'tableColumns'; + +export enum CachedDataSourceStatus { + Updated = 'Updated', + Failed = 'Failed', + Empty = 'Empty', +} + +export interface CachedColumn { + fieldName: string; + dataType: string; +} + +export interface CachedTable { + name: string; + columns?: CachedColumn[]; +} + +export interface CachedDatabase { + name: string; + tables: CachedTable[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; +} + +export interface CachedDataSource { + name: string; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + databases: CachedDatabase[]; + dataSourceMDSId?: string; +} + +export interface DataSourceCacheData { + version: string; + dataSources: CachedDataSource[]; +} + +export interface CachedAcceleration { + flintIndexName: string; + type: AccelerationIndexType; + database: string; + table: string; + indexName: string; + autoRefresh: boolean; + status: string; +} + +export interface CachedAccelerationByDataSource { + name: string; + accelerations: CachedAcceleration[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + dataSourceMDSId?: string; +} + +export interface AccelerationsCacheData { + version: string; + dataSources: CachedAccelerationByDataSource[]; +} + +export interface PollingSuccessResult { + schema: Array<{ name: string; type: string }>; + datarows: Array>; +} + +export type AsyncPollingResult = PollingSuccessResult | null; + +export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min' | 'window.start'; + +export interface MaterializedViewColumn { + id: string; + functionName: AggregationFunctionType; + functionParam?: string; + fieldAlias?: string; +} + +export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX' | 'BLOOM_FILTER'; + +export interface SkippingIndexRowType { + id: string; + fieldName: string; + dataType: string; + accelerationMethod: SkippingIndexAccMethodType; +} + +export interface DataTableFieldsType { + id: string; + fieldName: string; + dataType: string; +} + +export interface RefreshIntervalType { + refreshWindow: number; + refreshInterval: string; +} + +export interface WatermarkDelayType { + delayWindow: number; + delayInterval: string; +} + +export interface GroupByTumbleType { + timeField: string; + tumbleWindow: number; + tumbleInterval: string; +} + +export interface MaterializedViewQueryType { + columnsValues: MaterializedViewColumn[]; + groupByTumbleValue: GroupByTumbleType; +} + +export interface FormErrorsType { + dataSourceError: string[]; + databaseError: string[]; + dataTableError: string[]; + skippingIndexError: string[]; + coveringIndexError: string[]; + materializedViewError: string[]; + indexNameError: string[]; + primaryShardsError: string[]; + replicaShardsError: string[]; + refreshIntervalError: string[]; + checkpointLocationError: string[]; + watermarkDelayError: string[]; +} + +export type AccelerationRefreshType = 'autoInterval' | 'manual' | 'manualIncrement'; + +export interface CreateAccelerationForm { + dataSource: string; + database: string; + dataTable: string; + dataTableFields: DataTableFieldsType[]; + accelerationIndexType: AccelerationIndexType; + skippingIndexQueryData: SkippingIndexRowType[]; + coveringIndexQueryData: string[]; + materializedViewQueryData: MaterializedViewQueryType; + accelerationIndexName: string; + primaryShardsCount: number; + replicaShardsCount: number; + refreshType: AccelerationRefreshType; + checkpointLocation: string | undefined; + watermarkDelay: WatermarkDelayType; + refreshIntervalOptions: RefreshIntervalType; + formErrors: FormErrorsType; +} + +export interface LoadCachehookOutput { + loadStatus: DirectQueryLoadingStatus; + startLoading: (params: StartLoadingParams) => void; + stopLoading: () => void; +} + +export interface StartLoadingParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; +} + +export interface RenderAccelerationFlyoutParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; + handleRefresh?: () => void; +} + +export interface RenderAssociatedObjectsDetailsFlyoutParams { + tableDetail: AssociatedObject; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} + +export interface RenderAccelerationDetailsFlyoutParams { + acceleration: CachedAcceleration; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/utils/query_session_utils.ts b/src/plugins/data/public/ui/dataset_navigator/framework/utils/query_session_utils.ts new file mode 100644 index 000000000000..beabcb48c197 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/utils/query_session_utils.ts @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ASYNC_QUERY_SESSION_ID } from '../constants'; + +export const setAsyncSessionId = (dataSource: string, value: string | null) => { + if (value !== null) { + sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value); + } +}; + +export const getAsyncSessionId = (dataSource: string) => { + return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/utils/shared.ts b/src/plugins/data/public/ui/dataset_navigator/framework/utils/shared.ts new file mode 100644 index 000000000000..a7bafd7f4092 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/utils/shared.ts @@ -0,0 +1,353 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * TODO making this method type-safe is nontrivial: if you just define + * `Nested = { [k: string]: Nested | T }` then you can't accumulate because `T` is not `Nested` + * There might be a way to define a recursive type that accumulates cleanly but it's probably not + * worth the effort. + */ + +export function get(obj: Record, path: string, defaultValue?: T): T { + return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; +} + +export function addBackticksIfNeeded(input: string): string { + if (input === undefined) { + return ''; + } + // Check if the string already has backticks + if (input.startsWith('`') && input.endsWith('`')) { + return input; // Return the string as it is + } else { + // Add backticks to the string + return '`' + input + '`'; + } +} + +export function combineSchemaAndDatarows( + schema: Array<{ name: string; type: string }>, + datarows: Array> +): object[] { + const combinedData: object[] = []; + + datarows.forEach((row) => { + const rowData: { [key: string]: string | number | boolean } = {}; + schema.forEach((field, index) => { + rowData[field.name] = row[index]; + }); + combinedData.push(rowData); + }); + + return combinedData; +} + +export const formatError = (name: string, message: string, details: string) => { + return { + name, + message, + body: { + attributes: { + error: { + caused_by: { + type: '', + reason: details, + }, + }, + }, + }, + }; +}; + +// TODO: relocate to a more appropriate location +// Client route +export const PPL_BASE = '/api/ppl'; +export const PPL_SEARCH = '/search'; +export const DSL_BASE = '/api/dsl'; +export const DSL_SEARCH = '/search'; +export const DSL_CAT = '/cat.indices'; +export const DSL_MAPPING = '/indices.getFieldMapping'; +export const DSL_SETTINGS = '/indices.getFieldSettings'; +export const OBSERVABILITY_BASE = '/api/observability'; +export const INTEGRATIONS_BASE = '/api/integrations'; +export const JOBS_BASE = '/query/jobs'; +export const DATACONNECTIONS_BASE = '/api/dataconnections'; +export const EDIT = '/edit'; +export const DATACONNECTIONS_UPDATE_STATUS = '/status'; +export const SECURITY_ROLES = '/api/v1/configuration/roles'; +export const EVENT_ANALYTICS = '/event_analytics'; +export const SAVED_OBJECTS = '/saved_objects'; +export const SAVED_QUERY = '/query'; +export const SAVED_VISUALIZATION = '/vis'; +export const CONSOLE_PROXY = '/api/console/proxy'; +export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; + +// Server route +export const PPL_ENDPOINT = '/_plugins/_ppl'; +export const SQL_ENDPOINT = '/_plugins/_sql'; +export const DSL_ENDPOINT = '/_plugins/_dsl'; +export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources'; +export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; +export const JOB_RESULT_ENDPOINT = '/result'; + +export const observabilityID = 'observability-logs'; +export const observabilityTitle = 'Observability'; +export const observabilityPluginOrder = 1500; + +export const observabilityApplicationsID = 'observability-applications'; +export const observabilityApplicationsTitle = 'Applications'; +export const observabilityApplicationsPluginOrder = 5090; + +export const observabilityLogsID = 'observability-logs'; +export const observabilityLogsTitle = 'Logs'; +export const observabilityLogsPluginOrder = 5091; + +export const observabilityMetricsID = 'observability-metrics'; +export const observabilityMetricsTitle = 'Metrics'; +export const observabilityMetricsPluginOrder = 5092; + +export const observabilityTracesID = 'observability-traces'; +export const observabilityTracesTitle = 'Traces'; +export const observabilityTracesPluginOrder = 5093; + +export const observabilityNotebookID = 'observability-notebooks'; +export const observabilityNotebookTitle = 'Notebooks'; +export const observabilityNotebookPluginOrder = 5094; + +export const observabilityPanelsID = 'observability-dashboards'; +export const observabilityPanelsTitle = 'Dashboards'; +export const observabilityPanelsPluginOrder = 5095; + +export const observabilityIntegrationsID = 'integrations'; +export const observabilityIntegrationsTitle = 'Integrations'; +export const observabilityIntegrationsPluginOrder = 9020; + +export const observabilityDataConnectionsID = 'datasources'; +export const observabilityDataConnectionsTitle = 'Data sources'; +export const observabilityDataConnectionsPluginOrder = 9030; + +export const queryWorkbenchPluginID = 'opensearch-query-workbench'; +export const queryWorkbenchPluginCheck = 'plugin:queryWorkbenchDashboards'; + +// Shared Constants +export const SQL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/index/'; +export const PPL_DOCUMENTATION_URL = + 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index'; +export const PPL_PATTERNS_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/cmd/patterns.rst#description'; +export const UI_DATE_FORMAT = 'MM/DD/YYYY hh:mm A'; +export const PPL_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss.SSSSSS'; +export const OTEL_DATE_FORMAT = 'YYYY-MM-DDTHH:mm:ss'; +export const SPAN_REGEX = /span/; + +export const PROMQL_METRIC_SUBTYPE = 'promqlmetric'; +export const OTEL_METRIC_SUBTYPE = 'openTelemetryMetric'; +export const PPL_METRIC_SUBTYPE = 'metric'; + +export const PPL_SPAN_REGEX = /by\s*span/i; +export const PPL_STATS_REGEX = /\|\s*stats/i; +export const PPL_INDEX_INSERT_POINT_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)(.*)/i; +export const PPL_INDEX_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)/i; +export const PPL_WHERE_CLAUSE_REGEX = /\s*where\s+/i; +export const PPL_NEWLINE_REGEX = /[\n\r]+/g; +export const PPL_DESCRIBE_INDEX_REGEX = /(describe)\s+([^|\s]+)/i; + +// Observability plugin URI +const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; +const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; +export const OPENSEARCH_PANELS_API = { + OBJECT: `${BASE_OBSERVABILITY_URI}/object`, +}; +export const OPENSEARCH_DATACONNECTIONS_API = { + DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, +}; + +// Saved Objects +export const SAVED_OBJECT = '/object'; + +// Color Constants +export const PLOTLY_COLOR = [ + '#3CA1C7', + '#54B399', + '#DB748A', + '#F2BE4B', + '#68CCC2', + '#2A7866', + '#843769', + '#374FB8', + '#BD6F26', + '#4C636F', +]; + +export const LONG_CHART_COLOR = PLOTLY_COLOR[1]; + +export const pageStyles: CSS.Properties = { + float: 'left', + width: '100%', + maxWidth: '1130px', +}; + +export enum VIS_CHART_TYPES { + Bar = 'bar', + HorizontalBar = 'horizontal_bar', + Line = 'line', + Pie = 'pie', + HeatMap = 'heatmap', + Text = 'text', + Histogram = 'histogram', +} + +export const NUMERICAL_FIELDS = ['short', 'integer', 'long', 'float', 'double']; + +export const ENABLED_VIS_TYPES = [ + VIS_CHART_TYPES.Bar, + VIS_CHART_TYPES.HorizontalBar, + VIS_CHART_TYPES.Line, + VIS_CHART_TYPES.Pie, + VIS_CHART_TYPES.HeatMap, + VIS_CHART_TYPES.Text, +]; + +// Live tail constants +export const LIVE_OPTIONS = [ + { + label: '5s', + startTime: 'now-5s', + delayTime: 5000, + }, + { + label: '10s', + startTime: 'now-10s', + delayTime: 10000, + }, + { + label: '30s', + startTime: 'now-30s', + delayTime: 30000, + }, + { + label: '1m', + startTime: 'now-1m', + delayTime: 60000, + }, + { + label: '5m', + startTime: 'now-5m', + delayTime: 60000 * 5, + }, + { + label: '15m', + startTime: 'now-15m', + delayTime: 60000 * 15, + }, + { + label: '30m', + startTime: 'now-30m', + delayTime: 60000 * 30, + }, + { + label: '1h', + startTime: 'now-1h', + delayTime: 60000 * 60, + }, + { + label: '2h', + startTime: 'now-2h', + delayTime: 60000 * 120, + }, +]; + +export const LIVE_END_TIME = 'now'; + +export interface DefaultChartStylesProps { + DefaultModeLine: string; + Interpolation: string; + LineWidth: number; + FillOpacity: number; + MarkerSize: number; + ShowLegend: string; + LegendPosition: string; + LabelAngle: number; + DefaultSortSectors: string; + DefaultModeScatter: string; +} + +export const DEFAULT_CHART_STYLES: DefaultChartStylesProps = { + DefaultModeLine: 'lines+markers', + Interpolation: 'spline', + LineWidth: 0, + FillOpacity: 100, + MarkerSize: 25, + ShowLegend: 'show', + LegendPosition: 'v', + LabelAngle: 0, + DefaultSortSectors: 'largest_to_smallest', + DefaultModeScatter: 'markers', +}; + +export const FILLOPACITY_DIV_FACTOR = 200; +export const SLIDER_MIN_VALUE = 0; +export const SLIDER_MAX_VALUE = 100; +export const SLIDER_STEP = 1; +export const THRESHOLD_LINE_WIDTH = 3; +export const THRESHOLD_LINE_OPACITY = 0.7; +export const MAX_BUCKET_LENGTH = 16; + +export enum BarOrientation { + horizontal = 'h', + vertical = 'v', +} + +export const PLOT_MARGIN = { + l: 30, + r: 5, + b: 30, + t: 50, + pad: 4, +}; + +export const WAITING_TIME_ON_USER_ACTIONS = 300; + +export const VISUALIZATION_ERROR = { + NO_DATA: 'No data found.', + INVALID_DATA: 'Invalid visualization data', + NO_SERIES: 'Add a field to start', + NO_METRIC: 'Invalid Metric MetaData', +}; + +export const S3_DATA_SOURCE_TYPE = 's3glue'; + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; +export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; +export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; + +export const DIRECT_DUMMY_QUERY = 'select 1'; + +export const DEFAULT_START_TIME = 'now-15m'; +export const QUERY_ASSIST_START_TIME = 'now-40y'; +export const QUERY_ASSIST_END_TIME = 'now'; + +export const TIMESTAMP_DATETIME_TYPES = ['date', 'date_nanos']; + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} +const catalogCacheFetchingStatus = [ + DirectQueryLoadingStatus.RUNNING, + DirectQueryLoadingStatus.WAITING, + DirectQueryLoadingStatus.SCHEDULED, +]; + +export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { + return statuses.some((status: DirectQueryLoadingStatus) => + catalogCacheFetchingStatus.includes(status) + ); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/utils/use_polling.ts b/src/plugins/data/public/ui/dataset_navigator/framework/utils/use_polling.ts new file mode 100644 index 000000000000..74fedd6cf110 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/framework/utils/use_polling.ts @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useRef, useState } from 'react'; + +type FetchFunction = (params?: P) => Promise; + +export interface PollingConfigurations { + tabId: string; +} + +export class UsePolling { + public data: T | null = null; + public error: Error | null = null; + public loading: boolean = true; + private shouldPoll: boolean = false; + private intervalRef?: NodeJS.Timeout; + + constructor( + private fetchFunction: FetchFunction, + private interval: number = 5000, + private onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + private onPollingError?: (error: Error) => boolean, + private configurations?: PollingConfigurations + ) {} + + async fetchData(params?: P) { + this.loading = true; + try { + const result = await this.fetchFunction(params); + this.data = result; + this.loading = false; + + if (this.onPollingSuccess && this.onPollingSuccess(result, this.configurations!)) { + this.stopPolling(); + } + } catch (err) { + this.error = err as Error; + this.loading = false; + + if (this.onPollingError && this.onPollingError(this.error)) { + this.stopPolling(); + } + } + } + + startPolling(params?: P) { + this.shouldPoll = true; + if (!this.intervalRef) { + this.intervalRef = setInterval(() => { + if (this.shouldPoll) { + this.fetchData(params); + } + }, this.interval); + } + } + + stopPolling() { + this.shouldPoll = false; + if (this.intervalRef) { + clearInterval(this.intervalRef); + this.intervalRef = undefined; + } + } +} + +interface UsePollingReturn { + data: T | null; + loading: boolean; + error: Error | null; + startPolling: (params?: any) => void; + stopPolling: () => void; +} + +export function usePolling( + fetchFunction: FetchFunction, + interval: number = 5000, + onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + onPollingError?: (error: Error) => boolean, + configurations?: PollingConfigurations +): UsePollingReturn { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const intervalRef = useRef(undefined); + const unmounted = useRef(false); + + const shouldPoll = useRef(false); + + const startPolling = (params?: P) => { + shouldPoll.current = true; + const intervalId = setInterval(() => { + if (shouldPoll.current) { + fetchData(params); + } + }, interval); + intervalRef.current = intervalId; + if (unmounted.current) { + clearInterval(intervalId); + } + }; + + const stopPolling = () => { + shouldPoll.current = false; + clearInterval(intervalRef.current); + }; + + const fetchData = async (params?: P) => { + try { + const result = await fetchFunction(params); + setData(result); + // Check the success condition and stop polling if it's met + if (onPollingSuccess && onPollingSuccess(result, configurations)) { + stopPolling(); + } + } catch (err: unknown) { + setError(err as Error); + + // Check the error condition and stop polling if it's met + if (onPollingError && onPollingError(err as Error)) { + stopPolling(); + } + } finally { + setLoading(false); + } + }; + + useEffect(() => { + return () => { + unmounted.current = true; + }; + }, []); + + return { data, loading, error, startPolling, stopPolling }; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx index fdded3c18a68..44ac162dc22d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -4,6 +4,6 @@ */ export { DataSetNavigator, DataSetNavigatorProps, DataSetOption } from './dataset_navigator'; -export { fetchClusters } from './fetch_clusters'; -export { fetchIndexPatterns } from './fetch_index_patterns'; -export { fetchIndices } from './fetch_indices'; +export { fetchClusters } from './utils/fetch_clusters'; +export { fetchIndexPatterns } from './utils/fetch_index_patterns'; +export { fetchIndices } from './utils/fetch_indices'; diff --git a/src/plugins/data/public/ui/dataset_navigator/fetch_clusters.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_clusters.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/fetch_clusters.ts rename to src/plugins/data/public/ui/dataset_navigator/utils/fetch_clusters.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts new file mode 100644 index 000000000000..7b375538a4c1 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart } from 'opensearch-dashboards/public'; + +export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[], setExternalDataSources: any, setLoading: any) => { + setLoading(true); + const results = await Promise.all(connectedClusters.map(async (cluster) => { + const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`); + return dataSources + .filter(dataSource => dataSource.connector === 'S3GLUE') + .map(dataSource => ({ + name: dataSource.name, + status: dataSource.status, + dataSourceRef: cluster, + })); + })); + + const flattenedResults = results.flat(); + console.log('results:', flattenedResults); + setExternalDataSources(flattenedResults); + setLoading(false); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_index_patterns.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/fetch_index_patterns.ts rename to src/plugins/data/public/ui/dataset_navigator/utils/fetch_index_patterns.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_indices.ts similarity index 97% rename from src/plugins/data/public/ui/dataset_navigator/fetch_indices.ts rename to src/plugins/data/public/ui/dataset_navigator/utils/fetch_indices.ts index db08a3f96253..bbc1f685854b 100644 --- a/src/plugins/data/public/ui/dataset_navigator/fetch_indices.ts +++ b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_indices.ts @@ -4,7 +4,7 @@ */ import { map, scan } from 'rxjs/operators'; -import { ISearchStart } from '../../search'; +import { ISearchStart } from '../../../search'; export const fetchIndices = async (search: ISearchStart, dataSourceId: string) => { const request = buildSearchRequest(true, '*', dataSourceId); diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/utils.ts b/src/plugins/data/public/ui/dataset_navigator/utils/utils.ts new file mode 100644 index 000000000000..b74e3738ba03 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/utils/utils.ts @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DirectQueryLoadingStatus } from '../framework/types'; + +const catalogCacheFetchingStatus = [ + DirectQueryLoadingStatus.RUNNING, + DirectQueryLoadingStatus.WAITING, + DirectQueryLoadingStatus.SCHEDULED, +]; + +export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { + return statuses.some((status: DirectQueryLoadingStatus) => + catalogCacheFetchingStatus.includes(status) + ); +}; diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 9d7d8455ef3a..d4020210eeca 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -87,7 +87,9 @@ export class UiService implements Plugin { core.savedObjects.client, dataServices.indexPatterns, dataServices.search, - onSelectedDataSet + onSelectedDataSet, + core.http, + core.notifications, ), SearchBar, SuggestionsComponent, From 1fbeb3a0f6b3e064428bf8f7fd35d76f59286d70 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Mon, 22 Jul 2024 04:08:35 -0700 Subject: [PATCH 24/59] caching recently used and external datasources Signed-off-by: Sean Li --- .../dataset_navigator/dataset_navigator.tsx | 144 +++++++++++------- .../catalog_cache/cache_intercept.ts | 2 + .../framework/catalog_cache/cache_loader.tsx | 30 ++++ .../framework/catalog_cache/cache_manager.ts | 111 +++++++++++++- .../dataset_navigator/framework/constants.ts | 2 + .../ui/dataset_navigator/framework/types.tsx | 24 +++ .../utils/fetch_external_data_sources.ts | 29 ++-- 7 files changed, 268 insertions(+), 74 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 1bb9785f39a8..9b64d44d3e7f 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -25,24 +25,18 @@ import { fetchExternalDataSources } from './utils/fetch_external_data_sources'; import { Settings } from '../settings'; import { useLoadDatabasesToCache, + useLoadExternalDataSourcesToCache, useLoadTablesToCache, } from './framework/catalog_cache/cache_loader'; import { CatalogCacheManager } from './framework/catalog_cache/cache_manager'; -import { CachedDataSourceStatus, DirectQueryLoadingStatus } from './framework/types'; +import { + CachedDataSourceStatus, + DataSetOption, + DirectQueryLoadingStatus, + ExternalDataSource, +} from './framework/types'; import { isCatalogCacheFetching } from './framework/utils/shared'; -export interface DataSetOption { - id: string; - name: string; - dataSourceRef?: string; -} - -export interface ExternalDataSource { - name: string; - dataSourceRef: string; - status: DirectQueryLoadingStatus; -} - export interface DataSetNavigatorProps { settings: Settings; savedObjectsClient: SavedObjectsClientContract; @@ -66,7 +60,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { notifications, } = props; const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); - const [selectedDataSet, setSelectedDataSet] = useState(null); + const [selectedDataSet, setSelectedDataSet] = useState(); const [indexPatternList, setIndexPatternList] = useState([]); const [clusterList, setClusterList] = useState([]); const [selectedCluster, setSelectedCluster] = useState(null); @@ -82,6 +76,10 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const [loading, setLoading] = useState(false); const [failed, setFailed] = useState(false); + const { + loadStatus: dataSourcesLoadStatus, + loadExternalDataSources: startLoadingDataSources, + } = useLoadExternalDataSourcesToCache(http, notifications); const { loadStatus: databasesLoadStatus, startLoading: startLoadingDatabases, @@ -93,21 +91,27 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const onButtonClick = () => setIsDataSetNavigatorOpen(!isDataSetNavigatorOpen); const closePopover = () => setIsDataSetNavigatorOpen(false); - const onDataSetClick = async (dataSet) => { + const onDataSetClick = async (dataSet: DataSetOption) => { setSelectedDataSet(dataSet); onDataSetSelected(dataSet); settings.setSelectedDataSet(dataSet); + CatalogCacheManager.addRecentDataSet(dataSet); closePopover(); }; - const handleRefresh = () => { - if ( - !isCatalogCacheFetching(databasesLoadStatus, tablesLoadStatus) && - selectedExternalDataSource - ) { - startLoadingDatabases({ - dataSourceName: selectedExternalDataSource.name, - dataSourceMDSId: selectedExternalDataSource.dataSourceRef, - }); + // const handleRefresh = () => { + // if ( + // !isCatalogCacheFetching(databasesLoadStatus, tablesLoadStatus) && + // selectedExternalDataSource + // ) { + // startLoadingDatabases({ + // dataSourceName: selectedExternalDataSource.name, + // dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + // }); + // } + // }; + const handleExternalDataSourcesRefresh = () => { + if (!isCatalogCacheFetching(dataSourcesLoadStatus) && clusterList.length > 0) { + startLoadingDataSources(clusterList.map((cluster) => cluster.id)); } }; @@ -138,6 +142,20 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { setLoading(false); }, [indexPatternsService, savedObjectsClient, search, selectedCluster]); + // Retrieve external datasources from cache upon success + useEffect(() => { + const status = dataSourcesLoadStatus.toLowerCase(); + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if (status === DirectQueryLoadingStatus.SUCCESS) { + setExternalDataSourceList(externalDataSourcesCache.externalDataSources); + } else if ( + status === DirectQueryLoadingStatus.CANCELED || + status === DirectQueryLoadingStatus.FAILED + ) { + setFailed(true); + } + }, [dataSourcesLoadStatus]); + // Start loading databases for datasource useEffect(() => { if (selectedExternalDataSource) { @@ -150,7 +168,6 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { dataSourceCache.status === CachedDataSourceStatus.Failed) && !isCatalogCacheFetching(databasesLoadStatus) ) { - // setLoading(true); startLoadingDatabases({ dataSourceName: selectedExternalDataSource.name, dataSourceMDSId: selectedExternalDataSource.dataSourceRef, @@ -236,12 +253,6 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { } }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus]); - useEffect(() => { - console.log('selectedExternalDataSource:', selectedExternalDataSource); - console.log('cachedDatabases:', cachedDatabases); - console.log('cachedTables:', cachedTables); - }, [selectedExternalDataSource, cachedDatabases, cachedTables]); - const dataSetButton = ( { const RefreshButton = ( ); @@ -270,6 +281,14 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { // title: (Data), title: 'Data', items: [ + ...(CatalogCacheManager.getRecentDataSets().length > 0 + ? [ + { + name: 'Recently Used', + panel: 7, + }, + ] + : []), { name: 'Index Patterns', panel: 1, @@ -280,17 +299,22 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { }, { name: 'S3', - panel: 5, - onClick: () => - fetchExternalDataSources( - http, - clusterList.map((cluster) => cluster.id), - setExternalDataSourceList, - setLoading - ), + panel: 4, + onClick: () => { + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if ( + (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || + externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(dataSourcesLoadStatus) && + clusterList.length > 0 + ) { + startLoadingDataSources(clusterList.map((cluster) => cluster.id)); + } else if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) { + setExternalDataSourceList(externalDataSourcesCache.externalDataSources); + } + }, }, ], - content:
hello
, }, { id: 1, @@ -316,16 +340,6 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { { id: 3, title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', - items: [ - { - name: 'Indexes', - panel: 4, - }, - ], - }, - { - id: 4, - title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', items: indexList.map((index) => ({ name: index.name, onClick: () => onDataSetClick(index), @@ -333,31 +347,37 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { content:
{loading && LoadingSpinner}
, }, { - id: 5, - title:
S3 Connections {selectedExternalDataSource && RefreshButton}
, + id: 4, + title: ( +
+ S3 Connections{' '} + {CatalogCacheManager.getExternalDataSourcesCache().status === + CachedDataSourceStatus.Updated && RefreshButton} +
+ ), items: [ ...externalDataSourceList.map((ds) => ({ name: ds.name, onClick: () => setSelectedExternalDataSource(ds), - panel: 6, + panel: 5, })), ], - content:
{loading && LoadingSpinner}
, + content:
{dataSourcesLoadStatus && LoadingSpinner}
, }, { - id: 6, + id: 5, title: selectedExternalDataSource ? selectedExternalDataSource.name : 'Databases', items: [ ...cachedDatabases.map((db) => ({ name: db.name, onClick: () => setSelectedDatabase(db.name), - panel: 7, + panel: 6, })), ], content:
{isCatalogCacheFetching(databasesLoadStatus) && LoadingSpinner}
, }, { - id: 7, + id: 6, title: selectedDatabase ? selectedDatabase : 'Tables', items: [ ...cachedTables.map((table) => ({ @@ -366,6 +386,14 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { ], content:
{isCatalogCacheFetching(tablesLoadStatus) && LoadingSpinner}
, }, + { + id: 7, + title: 'Recently Used', + items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ + name: ds.name, + onClick: () => onDataSetClick(ds), + })), + }, ]; return ( diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts index cbf165035cb2..0526cfd51212 100644 --- a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts +++ b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts @@ -16,6 +16,8 @@ export function catalogRequestIntercept(): any { // Clears all user catalog cache details CatalogCacheManager.clearDataSourceCache(); CatalogCacheManager.clearAccelerationsCache(); + CatalogCacheManager.clearExternalDataSourcesCache(); + CatalogCacheManager.clearRecentDataSetsCache(); } }; } diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx index 26d6a43e8ebe..f1e28a2b7fcb 100644 --- a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx @@ -27,6 +27,7 @@ import { import { usePolling } from '../utils/use_polling'; import { SQLService } from '../requests/sql'; import { CatalogCacheManager } from './cache_manager'; +import { fetchExternalDataSources } from '../../utils/fetch_external_data_sources'; export const updateDatabasesToCache = ( dataSourceName: string, @@ -439,3 +440,32 @@ export const useLoadAccelerationsToCache = (http: HttpStart, notifications: Noti ); return { loadStatus, startLoading, stopLoading }; }; + +export const useLoadExternalDataSourcesToCache = ( + http: HttpStart, + notifications: NotificationsStart +) => { + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.INITIAL + ); + + const loadExternalDataSources = async (connectedClusters: string[]) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty); + + try { + const externalDataSources = await fetchExternalDataSources(http, connectedClusters); + CatalogCacheManager.updateExternalDataSources(externalDataSources); + setLoadStatus(DirectQueryLoadingStatus.SUCCESS); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated); + } catch (error) { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed); + notifications.toasts.addError(error, { + title: 'Failed to load external datasources', + }); + } + }; + + return { loadStatus, loadExternalDataSources }; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts index 53f7a5e01e7f..a55130e419fd 100644 --- a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts +++ b/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts @@ -3,7 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { CATALOG_CACHE_VERSION } from '../constants'; +import { + ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE, + CATALOG_CACHE_VERSION, + RECENT_DATASET_OPTIONS_CACHE, +} from '../constants'; import { ASYNC_QUERY_ACCELERATIONS_CACHE, ASYNC_QUERY_DATASOURCE_CACHE } from '../utils/shared'; import { AccelerationsCacheData, @@ -12,7 +16,11 @@ import { CachedDataSourceStatus, CachedDatabase, CachedTable, + DataSetOption, DataSourceCacheData, + ExternalDataSource, + ExternalDataSourcesCacheData, + RecentDataSetOptionsCacheData, } from '../types'; /** @@ -29,6 +37,19 @@ export class CatalogCacheManager { */ private static readonly accelerationsCacheKey = ASYNC_QUERY_ACCELERATIONS_CACHE; + /** + * Key for external datasources cache in local storage + */ + private static readonly externalDataSourcesCacheKey = ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE; + + /** + * Key for recently selected datasets in local storage + */ + private static readonly recentDataSetCacheKey = RECENT_DATASET_OPTIONS_CACHE; + + // TODO: make this an advanced setting + private static readonly maxRecentDataSet = 4; + /** * Saves data source cache to local storage. * @param {DataSourceCacheData} cacheData - The data source cache data to save. @@ -296,6 +317,7 @@ export class CatalogCacheManager { */ static clearDataSourceCache(): void { sessionStorage.removeItem(this.datasourceCacheKey); + this.clearExternalDataSourcesCache(); } /** @@ -304,4 +326,91 @@ export class CatalogCacheManager { static clearAccelerationsCache(): void { sessionStorage.removeItem(this.accelerationsCacheKey); } + + static saveExternalDataSourcesCache(cacheData: ExternalDataSourcesCacheData): void { + sessionStorage.setItem(this.externalDataSourcesCacheKey, JSON.stringify(cacheData)); + } + + static getExternalDataSourcesCache(): ExternalDataSourcesCacheData { + const externalDataSourcesData = sessionStorage.getItem(this.externalDataSourcesCacheKey); + + if (externalDataSourcesData) { + return JSON.parse(externalDataSourcesData); + } else { + const defaultCacheObject: ExternalDataSourcesCacheData = { + version: CATALOG_CACHE_VERSION, + externalDataSources: [], + lastUpdated: '', + status: CachedDataSourceStatus.Empty, + }; + this.saveExternalDataSourcesCache(defaultCacheObject); + return defaultCacheObject; + } + } + + static updateExternalDataSources(externalDataSources: ExternalDataSource[]): void { + const currentTime = new Date().toUTCString(); + const cacheData = this.getExternalDataSourcesCache(); + cacheData.externalDataSources = externalDataSources; + cacheData.lastUpdated = currentTime; + cacheData.status = CachedDataSourceStatus.Updated; + this.saveExternalDataSourcesCache(cacheData); + } + + static getExternalDataSources(): ExternalDataSourcesCacheData { + return this.getExternalDataSourcesCache(); + } + + static clearExternalDataSourcesCache(): void { + sessionStorage.removeItem(this.externalDataSourcesCacheKey); + } + + static setExternalDataSourcesLoadingStatus(status: CachedDataSourceStatus): void { + const cacheData = this.getExternalDataSourcesCache(); + cacheData.status = status; + this.saveExternalDataSourcesCache(cacheData); + } + + static saveRecentDataSetsCache(cacheData: RecentDataSetOptionsCacheData): void { + sessionStorage.setItem(this.recentDataSetCacheKey, JSON.stringify(cacheData)); + } + + static getRecentDataSetsCache(): RecentDataSetOptionsCacheData { + const recentDataSetOptionsData = sessionStorage.getItem(this.recentDataSetCacheKey); + + if (recentDataSetOptionsData) { + return JSON.parse(recentDataSetOptionsData); + } else { + const defaultCacheObject: RecentDataSetOptionsCacheData = { + version: CATALOG_CACHE_VERSION, + recentDataSets: [], + }; + this.saveRecentDataSetsCache(defaultCacheObject); + return defaultCacheObject; + } + } + + static addRecentDataSet(dataSetOption: DataSetOption): void { + const cacheData = this.getRecentDataSetsCache(); + + cacheData.recentDataSets = cacheData.recentDataSets.filter( + (option) => option.id !== dataSetOption.id + ); + + cacheData.recentDataSets.push(dataSetOption); + + if (cacheData.recentDataSets.length > this.maxRecentDataSet) { + cacheData.recentDataSets.shift(); + } + + this.saveRecentDataSetsCache(cacheData); + } + + static getRecentDataSets(): DataSetOption[] { + return this.getRecentDataSetsCache().recentDataSets; + } + + static clearRecentDataSetsCache(): void { + sessionStorage.removeItem(this.recentDataSetCacheKey); + } } diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts b/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts index 785e17bbaa95..e22da95ff4c6 100644 --- a/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts +++ b/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts @@ -4,6 +4,8 @@ */ export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; +export const ASYNC_QUERY_EXTERNAL_DATASOURCES_CACHE = 'async_query_external_datasources_cache'; +export const RECENT_DATASET_OPTIONS_CACHE = 'recent_dataset_options_cache'; export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx b/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx index b556f412c2cd..a51870b4e74f 100644 --- a/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx @@ -290,3 +290,27 @@ export interface RenderAccelerationDetailsFlyoutParams { handleRefresh?: () => void; dataSourceMDSId?: string; } + +export interface DataSetOption { + id: string; + name: string; + dataSourceRef?: string; +} + +export interface RecentDataSetOptionsCacheData { + version: string; + recentDataSets: DataSetOption[]; +} + +export interface ExternalDataSource { + name: string; + status: string; + dataSourceRef: string; +} + +export interface ExternalDataSourcesCacheData { + version: string; + externalDataSources: ExternalDataSource[]; + lastUpdated: string; + status: CachedDataSourceStatus; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts index 7b375538a4c1..8f104c6288d9 100644 --- a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts +++ b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts @@ -5,21 +5,20 @@ import { HttpStart } from 'opensearch-dashboards/public'; -export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[], setExternalDataSources: any, setLoading: any) => { - setLoading(true); - const results = await Promise.all(connectedClusters.map(async (cluster) => { - const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`); - return dataSources - .filter(dataSource => dataSource.connector === 'S3GLUE') - .map(dataSource => ({ - name: dataSource.name, - status: dataSource.status, - dataSourceRef: cluster, - })); - })); +export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[]) => { + const results = await Promise.all( + connectedClusters.map(async (cluster) => { + const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`); + return dataSources + .filter((dataSource) => dataSource.connector === 'S3GLUE') + .map((dataSource) => ({ + name: dataSource.name, + status: dataSource.status, + dataSourceRef: cluster, + })); + }) + ); const flattenedResults = results.flat(); - console.log('results:', flattenedResults); - setExternalDataSources(flattenedResults); - setLoading(false); + return flattenedResults; }; From e0d4931aa5608f961bd4f0781045200af3b62a2a Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 21:21:00 +0000 Subject: [PATCH 25/59] still porting over Signed-off-by: Kawika Avilla --- src/plugins/data/common/data_sets/index.ts | 6 + src/plugins/data/common/data_sets/types.ts | 36 ++ src/plugins/data/common/index.ts | 1 + src/plugins/data/common/types.ts | 1 + .../dataset_navigator/dataset_navigator.tsx | 561 +++++++++++------- .../public/ui/dataset_navigator/index.tsx | 5 +- .../catalog_cache/cache_intercept.ts | 0 .../catalog_cache/cache_loader.tsx | 0 .../catalog_cache/cache_manager.ts | 0 .../{framework => lib}/constants.ts | 0 .../hooks/direct_query_hook.tsx | 0 .../public/ui/dataset_navigator/lib/index.tsx | 6 + .../{framework => lib}/requests/sql.ts | 0 .../{framework => lib}/types.tsx | 0 .../utils/fetch_catalog_cache_status.ts} | 10 +- .../lib/utils/fetch_data_sources.ts | 19 + .../utils/fetch_external_data_sources.ts | 0 .../lib/utils/fetch_index_patterns.ts | 56 ++ .../lib/utils/fetch_indices.ts | 46 ++ .../ui/dataset_navigator/lib/utils/index.ts | 13 + .../utils/query_session_utils.ts | 0 .../{framework => lib}/utils/shared.ts | 21 - .../{framework => lib}/utils/use_polling.ts | 0 .../dataset_navigator/utils/fetch_clusters.ts | 13 - .../utils/fetch_index_patterns.ts | 21 - .../dataset_navigator/utils/fetch_indices.ts | 67 --- 26 files changed, 539 insertions(+), 343 deletions(-) create mode 100644 src/plugins/data/common/data_sets/index.ts create mode 100644 src/plugins/data/common/data_sets/types.ts rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/catalog_cache/cache_intercept.ts (100%) rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/catalog_cache/cache_loader.tsx (100%) rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/catalog_cache/cache_manager.ts (100%) rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/constants.ts (100%) rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/hooks/direct_query_hook.tsx (100%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/index.tsx rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/requests/sql.ts (100%) rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/types.tsx (100%) rename src/plugins/data/public/ui/dataset_navigator/{utils/utils.ts => lib/utils/fetch_catalog_cache_status.ts} (68%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts rename src/plugins/data/public/ui/dataset_navigator/{ => lib}/utils/fetch_external_data_sources.ts (100%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/utils/query_session_utils.ts (100%) rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/utils/shared.ts (94%) rename src/plugins/data/public/ui/dataset_navigator/{framework => lib}/utils/use_polling.ts (100%) delete mode 100644 src/plugins/data/public/ui/dataset_navigator/utils/fetch_clusters.ts delete mode 100644 src/plugins/data/public/ui/dataset_navigator/utils/fetch_index_patterns.ts delete mode 100644 src/plugins/data/public/ui/dataset_navigator/utils/fetch_indices.ts diff --git a/src/plugins/data/common/data_sets/index.ts b/src/plugins/data/common/data_sets/index.ts new file mode 100644 index 000000000000..9f269633f307 --- /dev/null +++ b/src/plugins/data/common/data_sets/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './types'; diff --git a/src/plugins/data/common/data_sets/types.ts b/src/plugins/data/common/data_sets/types.ts new file mode 100644 index 000000000000..7db48f4f718c --- /dev/null +++ b/src/plugins/data/common/data_sets/types.ts @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** @public **/ +export enum SIMPLE_DATA_SOURCE_TYPES { + DEFAULT = 'data-source', + EXTERNAL = 'external-source', +} + +/** @public **/ +export enum SIMPLE_DATA_SET_TYPES { + INDEX_PATTERN = 'index-pattern', + TEMPORARY = 'temporary', +} + +export interface SimpleObject { + id: string; + title: string; + dataSourceRef?: SimpleDataSource; +} + +export interface SimpleDataSource { + id: string; + name: string; + indices?: SimpleObject[]; + type: SIMPLE_DATA_SOURCE_TYPES; +} + +export interface SimpleDataSet extends SimpleObject { + fields?: any[]; + timeFieldName?: string; + timeFields?: any[]; + type: SIMPLE_DATA_SET_TYPES; +} diff --git a/src/plugins/data/common/index.ts b/src/plugins/data/common/index.ts index d7b7e56e2280..0250a6ec2e01 100644 --- a/src/plugins/data/common/index.ts +++ b/src/plugins/data/common/index.ts @@ -31,6 +31,7 @@ export * from './constants'; export * from './opensearch_query'; export * from './data_frames'; +export * from './data_sets'; export * from './field_formats'; export * from './field_mapping'; export * from './index_patterns'; diff --git a/src/plugins/data/common/types.ts b/src/plugins/data/common/types.ts index 6a1f6e5a99d3..1670fbf72d5d 100644 --- a/src/plugins/data/common/types.ts +++ b/src/plugins/data/common/types.ts @@ -35,6 +35,7 @@ export * from './query/types'; export * from './osd_field_types/types'; export * from './index_patterns/types'; export * from './data_frames/types'; +export * from './data_sets/types'; /** * If a service is being shared on both the client and the server, and diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 9b64d44d3e7f..daa2198b802f 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -5,11 +5,16 @@ import React, { useEffect, useState } from 'react'; import { + EuiButton, EuiButtonEmpty, EuiButtonIcon, EuiContextMenu, + EuiContextMenuPanelItemDescriptor, + EuiForm, + EuiFormRow, EuiLoadingSpinner, EuiPopover, + EuiSelect, EuiText, } from '@elastic/eui'; import { @@ -18,57 +23,53 @@ import { SavedObjectsClientContract, } from 'opensearch-dashboards/public'; import _ from 'lodash'; +import { + SIMPLE_DATA_SET_TYPES, + SimpleDataSet, + SimpleDataSource, + SimpleObject, +} from 'src/plugins/data/common'; +import { i18n } from '@osd/i18n'; import { IIndexPattern } from '../..'; -import { fetchClusters } from './utils/fetch_clusters'; -import { fetchIndices } from './utils/fetch_indices'; -import { fetchExternalDataSources } from './utils/fetch_external_data_sources'; -import { Settings } from '../settings'; import { useLoadDatabasesToCache, useLoadExternalDataSourcesToCache, useLoadTablesToCache, -} from './framework/catalog_cache/cache_loader'; -import { CatalogCacheManager } from './framework/catalog_cache/cache_manager'; +} from './lib/catalog_cache/cache_loader'; +import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; import { CachedDataSourceStatus, DataSetOption, DirectQueryLoadingStatus, ExternalDataSource, -} from './framework/types'; -import { isCatalogCacheFetching } from './framework/utils/shared'; +} from './lib/types'; +import { getIndexPatterns, getQueryService, getSearchService, getUiService } from '../../services'; +import { fetchDataSources, fetchIndexPatterns, fetchIndices, isCatalogCacheFetching } from './lib'; export interface DataSetNavigatorProps { - settings: Settings; - savedObjectsClient: SavedObjectsClientContract; - indexPattern?: Array; - dataSetId?: string; - onDataSetSelected: (dataSet: DataSetOption) => void; - indexPatternsService: any; - search: any; - http: HttpStart; - notifications: NotificationsStart; + dataSetId: string | undefined; + savedObjectsClient?: SavedObjectsClientContract; + onSelectDataSet: (dataSet: SimpleDataSet) => void; } export const DataSetNavigator = (props: DataSetNavigatorProps) => { - const { - settings, - indexPatternsService, - savedObjectsClient, - search, - onDataSetSelected, - http, - notifications, - } = props; - const [isDataSetNavigatorOpen, setIsDataSetNavigatorOpen] = useState(false); - const [selectedDataSet, setSelectedDataSet] = useState(); - const [indexPatternList, setIndexPatternList] = useState([]); - const [clusterList, setClusterList] = useState([]); - const [selectedCluster, setSelectedCluster] = useState(null); - const [indexList, setIndexList] = useState([]); - const [externalDataSourceList, setExternalDataSourceList] = useState([]); - const [selectedExternalDataSource, setSelectedExternalDataSource] = useState< - ExternalDataSource - >(); + const searchService = getSearchService(); + const queryService = getQueryService(); + const uiService = getUiService(); + const indexPatternsService = getIndexPatterns(); + + const [isOpen, setIsOpen] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [selectedDataSet, setSelectedDataSet] = useState(); + const [selectedObject, setSelectedObject] = useState(); + const [selectedDataSource, setSelectedDataSource] = useState(); + const [selectedDataSourceObjects, setSelectedDataSourceObjects] = useState([]); + const [selectedExternalDataSource, setSelectedExternalDataSource] = useState(); + const [dataSources, setDataSources] = useState([]); + const [externalDataSources, setExternalDataSources] = useState([]); + // TODO iindexpattern + const [indexPatterns, setIndexPatterns] = useState([]); + const [selectedDatabase, setSelectedDatabase] = useState(); const [cacheRefreshable, setCacheRefreshable] = useState(false); const [cachedDatabases, setCachedDatabases] = useState([]); @@ -89,8 +90,9 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { notifications ); - const onButtonClick = () => setIsDataSetNavigatorOpen(!isDataSetNavigatorOpen); - const closePopover = () => setIsDataSetNavigatorOpen(false); + const onButtonClick = () => setIsOpen(!isOpen); + const closePopover = () => setIsOpen(false); + const onDataSetClick = async (dataSet: DataSetOption) => { setSelectedDataSet(dataSet); onDataSetSelected(dataSet); @@ -98,56 +100,108 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { CatalogCacheManager.addRecentDataSet(dataSet); closePopover(); }; - // const handleRefresh = () => { - // if ( - // !isCatalogCacheFetching(databasesLoadStatus, tablesLoadStatus) && - // selectedExternalDataSource - // ) { - // startLoadingDatabases({ - // dataSourceName: selectedExternalDataSource.name, - // dataSourceMDSId: selectedExternalDataSource.dataSourceRef, - // }); - // } - // }; + const handleExternalDataSourcesRefresh = () => { if (!isCatalogCacheFetching(dataSourcesLoadStatus) && clusterList.length > 0) { startLoadingDataSources(clusterList.map((cluster) => cluster.id)); } }; + const getInitialQuery = (dataSet: SimpleDataSet) => { + const language = uiService.Settings.getUserQueryLanguage(); + const input = uiService.Settings.getQueryEnhancements(language)?.searchBar?.queryStringInput + ?.initialValue; + + if (!dataSet || !input) + return { + query: '', + language, + }; + + return { + query: input.replace('', dataSet.title), + language, + }; + }; + useEffect(() => { setLoading(true); - // Fetch index patterns - indexPatternsService.getIdsWithTitle().then((res) => { - setIndexPatternList(res.map(({ id, title }) => ({ id, name: title }))); - }); - // Fetch clusters - fetchClusters(savedObjectsClient).then((res) => { - setClusterList(res.savedObjects); - }); + Promise.all([ + fetchIndexPatterns(props.savedObjectsClient!, ''), + fetchDataSources(props.savedObjectsClient!), + ]) + .then(([defaultIndexPatterns, defaultDataSources]) => { + setIndexPatterns(defaultIndexPatterns); + setDataSources(defaultDataSources); - // Fetch indices if a cluster is selected - if (selectedCluster) { - fetchIndices(search, selectedCluster.id).then((res) => { - setIndexList( - res.map(({ name }) => ({ - name, - id: name, - dataSourceRef: selectedCluster.id, - })) - ); + if (!selectedDataSet && props.dataSetId) { + const selectedPattern = defaultIndexPatterns.find( + (pattern) => pattern.id === props.dataSetId + ); + if (selectedPattern) { + setSelectedDataSet({ + id: selectedPattern.id ?? selectedPattern.title, + title: selectedPattern.title, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + }); + } + } + }) + .finally(() => { + setLoading(false); }); + }, [indexPatternsService, props.dataSetId, props.savedObjectsClient, selectedDataSet]); + + useEffect(() => { + if (selectedDataSource) { + setLoading(true); + fetchIndices(searchService, selectedDataSource.id).then((indices) => { + const objects = indices.map(({ indexName }: { indexName: string }) => ({ + id: indexName, + title: indexName, + dataSourceRef: { + id: selectedDataSource.id, + name: selectedDataSource.name, + type: selectedDataSource.type, + }, + })); + setSelectedDataSourceObjects(objects); + setLoading(false); + }); + } + }, [searchService, selectedDataSource]); + + useEffect(() => { + const getFieldsForWildcard = async (object: SimpleObject | SimpleDataSet) => { + const fields = await indexPatternsService.getFieldsForWildcard({ + pattern: object.title, + dataSourceId: object.dataSourceRef?.id, + }); + + const timeFields = fields.filter((field: any) => field.type === 'date'); + + setSelectedObject({ + id: object.id, + title: object.title, + fields, + timeFields, + ...(timeFields[0]?.name ? { timeFieldName: timeFields[0].name } : {}), + dataSourceRef: object.dataSourceRef, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY, + }); + }; + + if (selectedObject) { + getFieldsForWildcard(selectedObject); } - setLoading(false); - }, [indexPatternsService, savedObjectsClient, search, selectedCluster]); + }, [indexPatternsService, searchService, selectedObject]); - // Retrieve external datasources from cache upon success useEffect(() => { const status = dataSourcesLoadStatus.toLowerCase(); const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); if (status === DirectQueryLoadingStatus.SUCCESS) { - setExternalDataSourceList(externalDataSourcesCache.externalDataSources); + setExternalDataSources(externalDataSourcesCache.externalDataSources); } else if ( status === DirectQueryLoadingStatus.CANCELED || status === DirectQueryLoadingStatus.FAILED @@ -161,7 +215,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { if (selectedExternalDataSource) { const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( selectedExternalDataSource.name, - selectedExternalDataSource.dataSourceRef + selectedExternalDataSource.id ); if ( (dataSourceCache.status === CachedDataSourceStatus.Empty || @@ -170,13 +224,13 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { ) { startLoadingDatabases({ dataSourceName: selectedExternalDataSource.name, - dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + dataSourceMDSId: selectedExternalDataSource.id, }); } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { setCachedDatabases(dataSourceCache.databases); } } - }, [selectedExternalDataSource]); + }, [databasesLoadStatus, selectedExternalDataSource, startLoadingDatabases]); // Retrieve databases from cache upon success useEffect(() => { @@ -184,7 +238,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { if (selectedExternalDataSource) { const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( selectedExternalDataSource?.name, - selectedExternalDataSource?.dataSourceRef + selectedExternalDataSource?.id ); if (status === DirectQueryLoadingStatus.SUCCESS) { setCachedDatabases(dataSourceCache.databases); @@ -205,10 +259,9 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { databaseCache = CatalogCacheManager.getDatabase( selectedExternalDataSource.name, selectedDatabase, - selectedExternalDataSource.dataSourceRef + selectedExternalDataSource.id ); } catch (error) { - console.error(error); return; } if ( @@ -219,13 +272,13 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { startLoadingTables({ dataSourceName: selectedExternalDataSource.name, databaseName: selectedDatabase, - dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + dataSourceMDSId: selectedExternalDataSource.id, }); } else if (databaseCache.status === CachedDataSourceStatus.Updated) { setCachedTables(databaseCache.tables); } } - }, [selectedExternalDataSource, selectedDatabase]); + }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus, startLoadingTables]); // Retrieve tables from cache upon success useEffect(() => { @@ -236,10 +289,9 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { databaseCache = CatalogCacheManager.getDatabase( selectedExternalDataSource.name, selectedDatabase, - selectedExternalDataSource.dataSourceRef + selectedExternalDataSource.id ); } catch (error) { - console.error(error); return; } if (tablesStatus === DirectQueryLoadingStatus.SUCCESS) { @@ -253,18 +305,6 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { } }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus]); - const dataSetButton = ( - - {selectedDataSet ? selectedDataSet.name : 'Datasets'} - - ); - const RefreshButton = ( { const LoadingSpinner = ; - const contextMenuPanels = [ - { - id: 0, - // title: (Data), - title: 'Data', - items: [ - ...(CatalogCacheManager.getRecentDataSets().length > 0 - ? [ - { - name: 'Recently Used', - panel: 7, - }, - ] - : []), - { - name: 'Index Patterns', - panel: 1, - }, - { - name: 'Indexes', - panel: 2, - }, - { - name: 'S3', - panel: 4, - onClick: () => { - const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); - if ( - (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || - externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && - !isCatalogCacheFetching(dataSourcesLoadStatus) && - clusterList.length > 0 - ) { - startLoadingDataSources(clusterList.map((cluster) => cluster.id)); - } else if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) { - setExternalDataSourceList(externalDataSourcesCache.externalDataSources); - } - }, - }, - ], - }, - { - id: 1, - title: 'Index Patterns', - items: indexPatternList.map((indexPattern) => ({ - name: indexPattern.name, - onClick: () => onDataSetClick(indexPattern), - })), - content:
{loading && LoadingSpinner}
, - }, - { - id: 2, - title: 'Clusters', - items: [ - ...clusterList.map((cluster) => ({ - name: cluster.attributes.title, - panel: 3, - onClick: () => setSelectedCluster(cluster), - })), - ], - content:
{loading && LoadingSpinner}
, - }, - { - id: 3, - title: selectedCluster ? selectedCluster.attributes.title : 'Cluster', - items: indexList.map((index) => ({ - name: index.name, - onClick: () => onDataSetClick(index), - })), - content:
{loading && LoadingSpinner}
, - }, - { - id: 4, - title: ( -
- S3 Connections{' '} - {CatalogCacheManager.getExternalDataSourcesCache().status === - CachedDataSourceStatus.Updated && RefreshButton} -
- ), - items: [ - ...externalDataSourceList.map((ds) => ({ - name: ds.name, - onClick: () => setSelectedExternalDataSource(ds), - panel: 5, - })), - ], - content:
{dataSourcesLoadStatus && LoadingSpinner}
, - }, - { - id: 5, - title: selectedExternalDataSource ? selectedExternalDataSource.name : 'Databases', - items: [ - ...cachedDatabases.map((db) => ({ - name: db.name, - onClick: () => setSelectedDatabase(db.name), - panel: 6, - })), - ], - content:
{isCatalogCacheFetching(databasesLoadStatus) && LoadingSpinner}
, - }, - { - id: 6, - title: selectedDatabase ? selectedDatabase : 'Tables', - items: [ - ...cachedTables.map((table) => ({ - name: table.name, - })), - ], - content:
{isCatalogCacheFetching(tablesLoadStatus) && LoadingSpinner}
, - }, - { - id: 7, - title: 'Recently Used', - items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ - name: ds.name, - onClick: () => onDataSetClick(ds), - })), - }, - ]; + const indexPatternsLabel = i18n.translate('data.query.dataSetNavigator.indexPatternsName', { + defaultMessage: 'Index patterns', + }); + const indicesLabel = i18n.translate('data.query.dataSetNavigator.indicesName', { + defaultMessage: 'Indexes', + }); return ( setIsOpen(!isOpen)} + > + {`${selectedDataSet?.dataSourceRef ? `${selectedDataSet.dataSourceRef.name}::` : ''}${ + selectedDataSet?.title ?? + i18n.translate('data.query.dataSetNavigator.selectDataSet', { + defaultMessage: 'Select data set', + }) + }`} + + } + isOpen={isOpen} closePopover={closePopover} anchorPosition="downLeft" + panelPaddingSize="none" > - + 0 + ? [ + { + name: 'Recently Used', + panel: 7, + }, + ] + : []), + { + name: indexPatternsLabel, + panel: 1, + }, + { + name: indicesLabel, + panel: 3, + }, + { + name: 'Connected data sources', + panel: 4, + onClick: () => { + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if ( + (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || + externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(dataSourcesLoadStatus) && + dataSources.length > 0 + ) { + startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); + } else if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) { + setExternalDataSources(externalDataSourcesCache.externalDataSources); + } + }, + }, + ], + }, + { + id: 1, + title: indexPatternsLabel, + items: indexPatterns.flatMap((indexPattern, indexNum, arr) => [ + { + name: indexPattern.title, + onClick: () => { + setSelectedDataSet({ + id: indexPattern.id ?? indexPattern.title, + title: indexPattern.title, + fields: indexPattern.fields, + timeFieldName: indexPattern.timeFieldName, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + }); + }, + }, + ...(indexNum < arr.length - 1 ? [{ isSeparator: true }] : []), + ]) as EuiContextMenuPanelItemDescriptor[], + content:
{loading && LoadingSpinner}
, + }, + { + id: 2, + items: [ + ...dataSources.map((dataSource) => ({ + name: dataSource.name, + panel: 3, + onClick: () => setSelectedDataSource(dataSource), + })), + ], + content:
{loading && LoadingSpinner}
, + }, + { + id: 3, + title: selectedDataSource?.name ?? indicesLabel, + items: selectedDataSourceObjects.map((object) => ({ + name: object.title, + onClick: () => + setSelectedObject({ + ...object, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY, + }), + })), + content:
{loading && LoadingSpinner}
, + }, + { + id: 4, + title: ( +
+ Connected data sources + {CatalogCacheManager.getExternalDataSourcesCache().status === + CachedDataSourceStatus.Updated && RefreshButton} +
+ ), + items: [ + ...externalDataSources.map((ds) => ({ + name: ds.name, + onClick: () => setSelectedExternalDataSource(ds), + panel: 5, + })), + ], + content:
{dataSourcesLoadStatus && LoadingSpinner}
, + }, + { + id: 5, + title: selectedExternalDataSource ? selectedExternalDataSource.name : 'Databases', + items: [ + ...cachedDatabases.map((db) => ({ + name: db.name, + onClick: () => setSelectedDatabase(db.name), + panel: 6, + })), + ], + content:
{isCatalogCacheFetching(databasesLoadStatus) && LoadingSpinner}
, + }, + { + id: 6, + title: selectedDatabase ? selectedDatabase : 'Tables', + items: [ + ...cachedTables.map((table) => ({ + name: table.name, + })), + ], + content:
{isCatalogCacheFetching(tablesLoadStatus) && LoadingSpinner}
, + }, + { + id: 8, + title: selectedObject?.title, + content: + loading && !selectedObject ? ( +
{LoadingSpinner}
+ ) : ( + + + 0 + ? [ + ...selectedObject!.timeFields.map((field: any) => ({ + value: field.name, + text: field.name, + })), + ] + : []), + { value: 'no-time-filter', text: "I don't want to use a time filter" }, + ]} + onChange={(event) => { + setSelectedObject({ + ...selectedObject, + timeFieldName: + event.target.value !== 'no-time-filter' + ? event.target.value + : undefined, + } as SimpleDataSet); + }} + aria-label="Select a date field" + /> + + { + setSelectedDataSet(selectedObject); + }} + > + Select + + + ), + }, + { + id: 9, + title: 'Recently Used', + items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ + name: ds.name, + onClick: () => onDataSetClick(ds), + })), + }, + ]} + /> ); }; diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx index 44ac162dc22d..238f2c704e44 100644 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/index.tsx @@ -3,7 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { DataSetNavigator, DataSetNavigatorProps, DataSetOption } from './dataset_navigator'; -export { fetchClusters } from './utils/fetch_clusters'; -export { fetchIndexPatterns } from './utils/fetch_index_patterns'; -export { fetchIndices } from './utils/fetch_indices'; +export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_intercept.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_loader.tsx rename to src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/catalog_cache/cache_manager.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/constants.ts b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/constants.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/constants.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/hooks/direct_query_hook.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/hooks/direct_query_hook.tsx rename to src/plugins/data/public/ui/dataset_navigator/lib/hooks/direct_query_hook.tsx diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx new file mode 100644 index 000000000000..079132ce99d2 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './utils'; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/requests/sql.ts b/src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/requests/sql.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/requests/sql.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/types.tsx rename to src/plugins/data/public/ui/dataset_navigator/lib/types.tsx diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/utils.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts similarity index 68% rename from src/plugins/data/public/ui/dataset_navigator/utils/utils.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts index b74e3738ba03..697852fdd772 100644 --- a/src/plugins/data/public/ui/dataset_navigator/utils/utils.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts @@ -3,7 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { DirectQueryLoadingStatus } from '../framework/types'; +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} const catalogCacheFetchingStatus = [ DirectQueryLoadingStatus.RUNNING, diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts new file mode 100644 index 000000000000..9d0a47f6ee26 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { SimpleDataSource } from 'src/plugins/data/common'; + +export const fetchDataSources = async (client: SavedObjectsClientContract) => { + const resp = await client.find({ + type: 'data-source', + perPage: 10000, + }); + return resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + name: savedObject.attributes.title, + type: 'data-source', + })) as SimpleDataSource[]; +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/utils/fetch_external_data_sources.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts new file mode 100644 index 000000000000..85d491df1518 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { IIndexPattern } from '../.././../..'; + +export const fetchIndexPatterns = async (client: SavedObjectsClientContract, search: string) => { + const resp = await client.find({ + type: 'index-pattern', + fields: ['title'], + search: `${search}*`, + searchFields: ['title'], + perPage: 100, + }); + return resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + dataSourceId: savedObject.references[0]?.id, + })); +}; + +// export async function fetchIndexPatterns( +// savedObjectsClient: SavedObjectsClientContract, +// indexPatternStrings: string[], +// uiSettings: IUiSettingsClient +// ) { +// if (!indexPatternStrings || isEmpty(indexPatternStrings)) { +// return []; +// } + +// const searchString = indexPatternStrings.map((string) => `"${string}"`).join(' | '); +// const indexPatternsFromSavedObjects = await savedObjectsClient.find({ +// type: 'index-pattern', +// fields: ['title', 'fields'], +// search: searchString, +// searchFields: ['title'], +// }); + +// const exactMatches = indexPatternsFromSavedObjects.savedObjects.filter((savedObject) => { +// return indexPatternStrings.includes(savedObject.attributes.title); +// }); + +// const defaultIndex = uiSettings.get('defaultIndex'); + +// const allMatches = +// exactMatches.length === indexPatternStrings.length +// ? exactMatches +// : [ +// ...exactMatches, +// await savedObjectsClient.get('index-pattern', defaultIndex), +// ]; + +// return allMatches.map(indexPatterns.getFromSavedObject); +// } diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts new file mode 100644 index 000000000000..ef10c72bc08c --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { map } from 'rxjs/operators'; +import { ISearchStart } from '../../../../search'; + +export const fetchIndices = async (search: ISearchStart, dataSourceId?: string) => { + const buildSearchRequest = () => { + const request = { + params: { + ignoreUnavailable: true, + expand_wildcards: 'all', + index: '*', + body: { + size: 0, // no hits + aggs: { + indices: { + terms: { + field: '_index', + size: 100, + }, + }, + }, + }, + }, + dataSourceId, + }; + + return request; + }; + + const searchResponseToArray = (response: any) => { + const { rawResponse } = response; + return rawResponse.aggregations + ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key) + : []; + }; + + return search + .getDefaultSearchInterceptor() + .search(buildSearchRequest()) + .pipe(map(searchResponseToArray)) + .toPromise(); +}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts new file mode 100644 index 000000000000..7dbe7ec2d4f4 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './fetch_catalog_cache_status'; +export * from './fetch_data_sources'; +export * from './fetch_external_data_sources'; +export * from './fetch_index_patterns'; +export * from './fetch_indices'; +export * from './query_session_utils'; +export * from './shared'; +export * from './use_polling'; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/utils/query_session_utils.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/utils/query_session_utils.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/utils/query_session_utils.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/utils/shared.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts similarity index 94% rename from src/plugins/data/public/ui/dataset_navigator/framework/utils/shared.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts index a7bafd7f4092..3e4afc94e80b 100644 --- a/src/plugins/data/public/ui/dataset_navigator/framework/utils/shared.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts @@ -330,24 +330,3 @@ export const QUERY_ASSIST_START_TIME = 'now-40y'; export const QUERY_ASSIST_END_TIME = 'now'; export const TIMESTAMP_DATETIME_TYPES = ['date', 'date_nanos']; - -export enum DirectQueryLoadingStatus { - SUCCESS = 'success', - FAILED = 'failed', - RUNNING = 'running', - SCHEDULED = 'scheduled', - CANCELED = 'canceled', - WAITING = 'waiting', - INITIAL = 'initial', -} -const catalogCacheFetchingStatus = [ - DirectQueryLoadingStatus.RUNNING, - DirectQueryLoadingStatus.WAITING, - DirectQueryLoadingStatus.SCHEDULED, -]; - -export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { - return statuses.some((status: DirectQueryLoadingStatus) => - catalogCacheFetchingStatus.includes(status) - ); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/framework/utils/use_polling.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts similarity index 100% rename from src/plugins/data/public/ui/dataset_navigator/framework/utils/use_polling.ts rename to src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_clusters.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_clusters.ts deleted file mode 100644 index 54519d062fe1..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_clusters.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; - -export const fetchClusters = async (savedObjectsClient: SavedObjectsClientContract) => { - return await savedObjectsClient.find({ - type: 'data-source', - perPage: 10000, - }); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_index_patterns.ts deleted file mode 100644 index fd1d893565e7..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_index_patterns.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; - -export const fetchIndexPatterns = async ( - client: SavedObjectsClientContract, - search: string, - fields: string[] -) => { - const resp = await client.find({ - type: 'index-pattern', - fields, - search: `${search}*`, - searchFields: ['title'], - perPage: 100, - }); - return resp.savedObjects; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/utils/fetch_indices.ts deleted file mode 100644 index bbc1f685854b..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/utils/fetch_indices.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { map, scan } from 'rxjs/operators'; -import { ISearchStart } from '../../../search'; - -export const fetchIndices = async (search: ISearchStart, dataSourceId: string) => { - const request = buildSearchRequest(true, '*', dataSourceId); - return search - .getDefaultSearchInterceptor() - .search(request) - .pipe(map(searchResponseToArray(true))) - .pipe(scan((accumulator = [], value) => accumulator.join(value))) - .toPromise() - .catch(() => []); -}; - -const searchResponseToArray = (showAllIndices: boolean) => (response) => { - const { rawResponse } = response; - if (!rawResponse.aggregations) { - return []; - } else { - return rawResponse.aggregations.indices.buckets - .map((bucket: { key: string }) => { - return bucket.key; - }) - .filter((indexName: string) => { - if (showAllIndices) { - return true; - } else { - return !indexName.startsWith('.'); - } - }) - .map((indexName: string) => { - return { - name: indexName, - // item: {}, - }; - }); - } -}; - -const buildSearchRequest = (showAllIndices: boolean, pattern: string, dataSourceId?: string) => { - const request = { - params: { - ignoreUnavailable: true, - expand_wildcards: showAllIndices ? 'all' : 'open', - index: pattern, - body: { - size: 0, // no hits - aggs: { - indices: { - terms: { - field: '_index', - size: 100, - }, - }, - }, - }, - }, - dataSourceId, - }; - - return request; -}; From 8cb69055eb506bd4d0a16bd87f59b9d588051858 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 21:52:23 +0000 Subject: [PATCH 26/59] more ported over Signed-off-by: Kawika Avilla --- .../create_dataset_navigator.tsx | 26 +-- .../dataset_navigator/dataset_navigator.tsx | 157 ++++++++++-------- src/plugins/data/public/ui/types.ts | 2 +- src/plugins/data/public/ui/ui_service.ts | 10 +- 4 files changed, 98 insertions(+), 97 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx index 6ba50ad526f8..6d19cd20d551 100644 --- a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx @@ -4,34 +4,16 @@ */ import React from 'react'; -import { HttpStart, NotificationsStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { IndexPatternsContract } from 'src/plugins/data/public'; +import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { DataSetNavigator, DataSetNavigatorProps } from './'; -import { Settings } from '../settings'; // Updated function signature to include additional dependencies export function createDataSetNavigator( - settings: Settings, savedObjectsClient: SavedObjectsClientContract, - indexPatternsService: IndexPatternsContract, - search: any, - onDataSetSelected: any, - http: HttpStart, - notifications: NotificationsStart, + http: HttpStart ) { // Return a function that takes props, omitting the dependencies from the props type - return ( - props: Omit - ) => ( - + return (props: Omit) => ( + ); } diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index daa2198b802f..cc6755538c64 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -7,7 +7,6 @@ import React, { useEffect, useState } from 'react'; import { EuiButton, EuiButtonEmpty, - EuiButtonIcon, EuiContextMenu, EuiContextMenuPanelItemDescriptor, EuiForm, @@ -15,13 +14,8 @@ import { EuiLoadingSpinner, EuiPopover, EuiSelect, - EuiText, } from '@elastic/eui'; -import { - HttpStart, - NotificationsStart, - SavedObjectsClientContract, -} from 'opensearch-dashboards/public'; +import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; import _ from 'lodash'; import { SIMPLE_DATA_SET_TYPES, @@ -30,25 +24,26 @@ import { SimpleObject, } from 'src/plugins/data/common'; import { i18n } from '@osd/i18n'; -import { IIndexPattern } from '../..'; import { useLoadDatabasesToCache, useLoadExternalDataSourcesToCache, useLoadTablesToCache, } from './lib/catalog_cache/cache_loader'; import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; +import { CachedDataSourceStatus, DirectQueryLoadingStatus, ExternalDataSource } from './lib/types'; import { - CachedDataSourceStatus, - DataSetOption, - DirectQueryLoadingStatus, - ExternalDataSource, -} from './lib/types'; -import { getIndexPatterns, getQueryService, getSearchService, getUiService } from '../../services'; + getIndexPatterns, + getNotifications, + getQueryService, + getSearchService, + getUiService, +} from '../../services'; import { fetchDataSources, fetchIndexPatterns, fetchIndices, isCatalogCacheFetching } from './lib'; export interface DataSetNavigatorProps { dataSetId: string | undefined; savedObjectsClient?: SavedObjectsClientContract; + http?: HttpStart; onSelectDataSet: (dataSet: SimpleDataSet) => void; } @@ -57,6 +52,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const queryService = getQueryService(); const uiService = getUiService(); const indexPatternsService = getIndexPatterns(); + const notifications = getNotifications(); const [isOpen, setIsOpen] = useState(false); const [isLoading, setIsLoading] = useState(false); @@ -64,9 +60,11 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const [selectedObject, setSelectedObject] = useState(); const [selectedDataSource, setSelectedDataSource] = useState(); const [selectedDataSourceObjects, setSelectedDataSourceObjects] = useState([]); - const [selectedExternalDataSource, setSelectedExternalDataSource] = useState(); + const [selectedExternalDataSource, setSelectedExternalDataSource] = useState< + ExternalDataSource + >(); const [dataSources, setDataSources] = useState([]); - const [externalDataSources, setExternalDataSources] = useState([]); + const [externalDataSources, setExternalDataSources] = useState([]); // TODO iindexpattern const [indexPatterns, setIndexPatterns] = useState([]); @@ -74,58 +72,31 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const [cacheRefreshable, setCacheRefreshable] = useState(false); const [cachedDatabases, setCachedDatabases] = useState([]); const [cachedTables, setCachedTables] = useState([]); - const [loading, setLoading] = useState(false); const [failed, setFailed] = useState(false); const { loadStatus: dataSourcesLoadStatus, loadExternalDataSources: startLoadingDataSources, - } = useLoadExternalDataSourcesToCache(http, notifications); + } = useLoadExternalDataSourcesToCache(props.http!, notifications); const { loadStatus: databasesLoadStatus, startLoading: startLoadingDatabases, - } = useLoadDatabasesToCache(http, notifications); + } = useLoadDatabasesToCache(props.http!, notifications); const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache( - http, + props.http!, notifications ); - const onButtonClick = () => setIsOpen(!isOpen); const closePopover = () => setIsOpen(false); - const onDataSetClick = async (dataSet: DataSetOption) => { - setSelectedDataSet(dataSet); - onDataSetSelected(dataSet); - settings.setSelectedDataSet(dataSet); - CatalogCacheManager.addRecentDataSet(dataSet); - closePopover(); - }; - const handleExternalDataSourcesRefresh = () => { - if (!isCatalogCacheFetching(dataSourcesLoadStatus) && clusterList.length > 0) { - startLoadingDataSources(clusterList.map((cluster) => cluster.id)); + if (!isCatalogCacheFetching(dataSourcesLoadStatus) && dataSources.length > 0) { + startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); } }; - const getInitialQuery = (dataSet: SimpleDataSet) => { - const language = uiService.Settings.getUserQueryLanguage(); - const input = uiService.Settings.getQueryEnhancements(language)?.searchBar?.queryStringInput - ?.initialValue; - - if (!dataSet || !input) - return { - query: '', - language, - }; - - return { - query: input.replace('', dataSet.title), - language, - }; - }; - useEffect(() => { - setLoading(true); + setIsLoading(true); Promise.all([ fetchIndexPatterns(props.savedObjectsClient!, ''), @@ -149,13 +120,13 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { } }) .finally(() => { - setLoading(false); + setIsLoading(false); }); }, [indexPatternsService, props.dataSetId, props.savedObjectsClient, selectedDataSet]); useEffect(() => { if (selectedDataSource) { - setLoading(true); + setIsLoading(true); fetchIndices(searchService, selectedDataSource.id).then((indices) => { const objects = indices.map(({ indexName }: { indexName: string }) => ({ id: indexName, @@ -167,7 +138,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { }, })); setSelectedDataSourceObjects(objects); - setLoading(false); + setIsLoading(false); }); } }, [searchService, selectedDataSource]); @@ -215,7 +186,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { if (selectedExternalDataSource) { const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( selectedExternalDataSource.name, - selectedExternalDataSource.id + selectedExternalDataSource.dataSourceRef ); if ( (dataSourceCache.status === CachedDataSourceStatus.Empty || @@ -224,7 +195,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { ) { startLoadingDatabases({ dataSourceName: selectedExternalDataSource.name, - dataSourceMDSId: selectedExternalDataSource.id, + dataSourceMDSId: selectedExternalDataSource.dataSourceRef, }); } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { setCachedDatabases(dataSourceCache.databases); @@ -238,7 +209,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { if (selectedExternalDataSource) { const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( selectedExternalDataSource?.name, - selectedExternalDataSource?.id + selectedExternalDataSource?.dataSourceRef ); if (status === DirectQueryLoadingStatus.SUCCESS) { setCachedDatabases(dataSourceCache.databases); @@ -259,7 +230,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { databaseCache = CatalogCacheManager.getDatabase( selectedExternalDataSource.name, selectedDatabase, - selectedExternalDataSource.id + selectedExternalDataSource.dataSourceRef ); } catch (error) { return; @@ -272,7 +243,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { startLoadingTables({ dataSourceName: selectedExternalDataSource.name, databaseName: selectedDatabase, - dataSourceMDSId: selectedExternalDataSource.id, + dataSourceMDSId: selectedExternalDataSource.dataSourceRef, }); } else if (databaseCache.status === CachedDataSourceStatus.Updated) { setCachedTables(databaseCache.tables); @@ -289,7 +260,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { databaseCache = CatalogCacheManager.getDatabase( selectedExternalDataSource.name, selectedDatabase, - selectedExternalDataSource.id + selectedExternalDataSource.dataSourceRef ); } catch (error) { return; @@ -305,6 +276,62 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { } }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus]); + useEffect(() => { + const createTemporaryIndexPattern = async (dataSet: SimpleDataSet) => { + const fieldsMap = dataSet.fields?.reduce((acc: any, field: any) => { + acc[field.name] = field; + return acc; + }); + const temporaryIndexPattern = await indexPatternsService.create( + { + id: dataSet.id, + title: dataSet.title, + fields: fieldsMap, + dataSourceRef: { + id: dataSet.dataSourceRef?.id!, + name: dataSet.dataSourceRef?.name!, + type: dataSet.dataSourceRef?.type!, + }, + timeFieldName: dataSet.timeFieldName, + }, + true + ); + indexPatternsService.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); + }; + + const getInitialQuery = (dataSet: SimpleDataSet) => { + const language = uiService.Settings.getUserQueryLanguage(); + const input = uiService.Settings.getQueryEnhancements(language)?.searchBar?.queryStringInput + ?.initialValue; + + if (!dataSet || !input) + return { + query: '', + language, + }; + + return { + query: input.replace('', dataSet.title), + language, + }; + }; + + const onDataSetSelected = async (dataSet: SimpleDataSet) => { + if (dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY) { + await createTemporaryIndexPattern(dataSet); + } + // settings.setSelectedDataSet(dataSet); + // CatalogCacheManager.addRecentDataSet(dataSet); + props.onSelectDataSet(dataSet); + queryService.queryString.setQuery(getInitialQuery(dataSet)); + closePopover(); + }; + + if (selectedDataSet) { + onDataSetSelected(selectedDataSet); + } + }, [indexPatternsService, props, queryService.queryString, selectedDataSet, uiService.Settings]); + const RefreshButton = ( { ? [ { name: 'Recently Used', - panel: 7, + panel: 8, }, ] : []), @@ -407,7 +434,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { }, ...(indexNum < arr.length - 1 ? [{ isSeparator: true }] : []), ]) as EuiContextMenuPanelItemDescriptor[], - content:
{loading && LoadingSpinner}
, + content:
{isLoading && LoadingSpinner}
, }, { id: 2, @@ -418,7 +445,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { onClick: () => setSelectedDataSource(dataSource), })), ], - content:
{loading && LoadingSpinner}
, + content:
{isLoading && LoadingSpinner}
, }, { id: 3, @@ -431,7 +458,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { type: SIMPLE_DATA_SET_TYPES.TEMPORARY, }), })), - content:
{loading && LoadingSpinner}
, + content:
{isLoading && LoadingSpinner}
, }, { id: 4, @@ -474,10 +501,10 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { content:
{isCatalogCacheFetching(tablesLoadStatus) && LoadingSpinner}
, }, { - id: 8, + id: 7, title: selectedObject?.title, content: - loading && !selectedObject ? ( + isLoading && !selectedObject ? (
{LoadingSpinner}
) : ( @@ -524,7 +551,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { ), }, { - id: 9, + id: 8, title: 'Recently Used', items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ name: ds.name, diff --git a/src/plugins/data/public/ui/types.ts b/src/plugins/data/public/ui/types.ts index a5938e6aa129..9c092e9d9510 100644 --- a/src/plugins/data/public/ui/types.ts +++ b/src/plugins/data/public/ui/types.ts @@ -65,12 +65,12 @@ export interface IUiSetup { */ export interface IUiStart { IndexPatternSelect: React.ComponentType; + DataSetNavigator: React.ComponentType; SearchBar: React.ComponentType; SuggestionsComponent: React.ComponentType; /** * @experimental - Subject to change */ Settings: Settings; - DataSetNavigator: (onDataSetSelected: any) => React.ComponentType; container$: Observable; } diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index d4020210eeca..6a6142184888 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -82,15 +82,7 @@ export class UiService implements Plugin { return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), - DataSetNavigator: (onSelectedDataSet) => createDataSetNavigator( - Settings, - core.savedObjects.client, - dataServices.indexPatterns, - dataServices.search, - onSelectedDataSet, - core.http, - core.notifications, - ), + DataSetNavigator: createDataSetNavigator(core.savedObjects.client, core.http), SearchBar, SuggestionsComponent, Settings, From 2cee188d611bc731b79adbefc0e0d4c878abba8d Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 21:59:40 +0000 Subject: [PATCH 27/59] no linter complaints Signed-off-by: Kawika Avilla --- src/plugins/data/common/data_sets/types.ts | 1 + .../dataset_navigator/dataset_navigator.tsx | 24 +++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/plugins/data/common/data_sets/types.ts b/src/plugins/data/common/data_sets/types.ts index 7db48f4f718c..cca70bf70cf9 100644 --- a/src/plugins/data/common/data_sets/types.ts +++ b/src/plugins/data/common/data_sets/types.ts @@ -13,6 +13,7 @@ export enum SIMPLE_DATA_SOURCE_TYPES { export enum SIMPLE_DATA_SET_TYPES { INDEX_PATTERN = 'index-pattern', TEMPORARY = 'temporary', + TEMPORARY_ASYNC = 'temporary-async', } export interface SimpleObject { diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index cc6755538c64..1a3cb5b29e1f 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -19,6 +19,7 @@ import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/pub import _ from 'lodash'; import { SIMPLE_DATA_SET_TYPES, + SIMPLE_DATA_SOURCE_TYPES, SimpleDataSet, SimpleDataSource, SimpleObject, @@ -69,7 +70,6 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const [indexPatterns, setIndexPatterns] = useState([]); const [selectedDatabase, setSelectedDatabase] = useState(); - const [cacheRefreshable, setCacheRefreshable] = useState(false); const [cachedDatabases, setCachedDatabases] = useState([]); const [cachedTables, setCachedTables] = useState([]); const [failed, setFailed] = useState(false); @@ -320,8 +320,12 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { if (dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY) { await createTemporaryIndexPattern(dataSet); } - // settings.setSelectedDataSet(dataSet); - // CatalogCacheManager.addRecentDataSet(dataSet); + + CatalogCacheManager.addRecentDataSet({ + id: dataSet.id, + name: dataSet.title, + dataSourceRef: dataSet.dataSourceRef?.id, + }); props.onSelectDataSet(dataSet); queryService.queryString.setQuery(getInitialQuery(dataSet)); closePopover(); @@ -541,7 +545,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { { + onClick={() => { setSelectedDataSet(selectedObject); }} > @@ -555,7 +559,17 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { title: 'Recently Used', items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ name: ds.name, - onClick: () => onDataSetClick(ds), + onClick: () => + setSelectedDataSet({ + id: ds.id, + title: ds.name, + dataSourceRef: { + id: ds.dataSourceRef!, + name: ds.dataSourceRef!, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + }, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC, + }), })), }, ]} From 6376950510f6fbe335c326a4b358ef5d761b4713 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 22:32:28 +0000 Subject: [PATCH 28/59] update to sidebar Signed-off-by: Kawika Avilla --- .../search/search_source/search_source.ts | 7 +- src/plugins/data/public/ui/_index.scss | 1 + .../_dataset_navigator.scss | 0 .../public/ui/dataset_navigator/_index.scss | 1 + .../data/public/ui/query_editor/_index.scss | 1 - .../public/ui/query_editor/query_editor.tsx | 25 +----- .../ui/query_editor/query_editor_top_row.tsx | 7 +- .../ui/search_bar/create_search_bar.tsx | 19 ++--- .../data/public/ui/search_bar/search_bar.tsx | 6 +- src/plugins/data/public/ui/types.ts | 2 +- src/plugins/data/public/ui/ui_service.ts | 16 ++-- .../public/components/sidebar/index.tsx | 78 +++++++------------ .../utils/state_management/metadata_slice.ts | 8 +- 13 files changed, 58 insertions(+), 113 deletions(-) rename src/plugins/data/public/ui/{query_editor => dataset_navigator}/_dataset_navigator.scss (100%) create mode 100644 src/plugins/data/public/ui/dataset_navigator/_index.scss diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index d9518e6a6cab..2c553fdbed97 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -324,7 +324,12 @@ export class SearchSource { const dataFrame = createDataFrame({ name: searchRequest.index.title || searchRequest.index, fields: [], - ...(rawQueryString && { meta: { queryConfig: parseRawQueryString(rawQueryString) } }), + ...(rawQueryString && { + meta: { + queryConfig: parseRawQueryString(rawQueryString), + ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }), + }, + }), }); await this.setDataFrame(dataFrame); return this.getDataFrame(); diff --git a/src/plugins/data/public/ui/_index.scss b/src/plugins/data/public/ui/_index.scss index f7c738b8d09f..4aa425041f58 100644 --- a/src/plugins/data/public/ui/_index.scss +++ b/src/plugins/data/public/ui/_index.scss @@ -2,5 +2,6 @@ @import "./typeahead/index"; @import "./saved_query_management/index"; @import "./query_string_input/index"; +@import "./dataset_navigator/index"; @import "./query_editor/index"; @import "./shard_failure_modal/shard_failure_modal"; diff --git a/src/plugins/data/public/ui/query_editor/_dataset_navigator.scss b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss similarity index 100% rename from src/plugins/data/public/ui/query_editor/_dataset_navigator.scss rename to src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss diff --git a/src/plugins/data/public/ui/dataset_navigator/_index.scss b/src/plugins/data/public/ui/dataset_navigator/_index.scss new file mode 100644 index 000000000000..53acdffad43d --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/_index.scss @@ -0,0 +1 @@ +@import "./dataset_navigator"; diff --git a/src/plugins/data/public/ui/query_editor/_index.scss b/src/plugins/data/public/ui/query_editor/_index.scss index f51d45b8245c..64fb0056cb71 100644 --- a/src/plugins/data/public/ui/query_editor/_index.scss +++ b/src/plugins/data/public/ui/query_editor/_index.scss @@ -1,3 +1,2 @@ @import "./language_selector"; -@import "./dataset_navigator"; @import "./query_editor"; diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 5ed400377edc..2889daca4d18 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -41,10 +41,7 @@ export interface QueryEditorProps { indexPatterns: Array; dataSource?: DataSource; query: Query; - container?: HTMLDivElement; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; - languageSelectorContainerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; settings: Settings; disableAutoFocus?: boolean; screenTitle?: string; @@ -74,8 +71,6 @@ interface Props extends QueryEditorProps { } interface State { - isDataSourcesVisible: boolean; - isDataSetsVisible: boolean; isSuggestionsVisible: boolean; index: number | null; suggestions: QuerySuggestion[]; @@ -102,8 +97,6 @@ const KEY_CODES = { // eslint-disable-next-line import/no-default-export export default class QueryEditorUI extends Component { public state: State = { - isDataSourcesVisible: false, - isDataSetsVisible: true, isSuggestionsVisible: false, index: null, suggestions: [], @@ -118,7 +111,6 @@ export default class QueryEditorUI extends Component { private persistedLog: PersistedLog | undefined; private abortController?: AbortController; private services = this.props.opensearchDashboards.services; - private componentIsUnmounting = false; private headerRef: RefObject = createRef(); private bannerRef: RefObject = createRef(); private extensionMap = this.props.settings?.getQueryEditorExtensionMap(); @@ -282,20 +274,6 @@ export default class QueryEditorUI extends Component { : getQueryLog(uiSettings, storage, appName, this.props.query.language); }; - private initDataSourcesVisibility = () => { - if (this.componentIsUnmounting) return; - - return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar - ?.showDataSourcesSelector; - }; - - private initDataSetsVisibility = () => { - if (this.componentIsUnmounting) return; - - return this.props.settings.getQueryEnhancements(this.props.query.language)?.searchBar - ?.showDataSetsSelector; - }; - public onMouseEnterSuggestion = (index: number) => { this.setState({ index }); }; @@ -323,7 +301,6 @@ export default class QueryEditorUI extends Component { public componentWillUnmount() { if (this.abortController) this.abortController.abort(); - this.componentIsUnmounting = true; } handleOnFocus = () => { diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index f59155ffd24f..c162f50b5ff7 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -38,8 +38,7 @@ const QueryEditor = withOpenSearchDashboards(QueryEditorUI); // @internal export interface QueryEditorTopRowProps { query?: Query; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; settings?: Settings; onSubmit: (payload: { dateRange: TimeRange; query?: Query }) => void; onChange: (payload: { dateRange: TimeRange; query?: Query }) => void; @@ -206,7 +205,6 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ) return ''; - console.log('settings ds:', settings?.getSelectedDataSet()); const defaultDataSet = settings?.getSelectedDataSet() ?? indexPatterns[0]; const dataSet = typeof defaultDataSet === 'string' ? defaultDataSet : defaultDataSet.title; @@ -223,8 +221,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { dataSource={props.dataSource} prepend={props.prepend} query={parsedQuery} - dataSourceContainerRef={props.dataSourceContainerRef} - containerRef={props.containerRef} + dataSetContainerRef={props.dataSetContainerRef} settings={props.settings!} screenTitle={props.screenTitle} onChange={onQueryChange} diff --git a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx index 244f4296216c..7fe38f548048 100644 --- a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx @@ -48,8 +48,7 @@ interface StatefulSearchBarDeps { data: Omit; storage: IStorageWrapper; settings: Settings; - setDataSourceContainerRef: (ref: HTMLDivElement | null) => void; - setContainerRef: (ref: HTMLDivElement | null) => void; + setDataSetContainerRef: (ref: HTMLDivElement | null) => void; } export type StatefulSearchBarProps = SearchBarOwnProps & { @@ -139,8 +138,7 @@ export function createSearchBar({ storage, data, settings, - setDataSourceContainerRef, - setContainerRef, + setDataSetContainerRef, }: StatefulSearchBarDeps) { // App name should come from the core application service. // Until it's available, we'll ask the user to provide it for the pre-wired component. @@ -176,15 +174,9 @@ export function createSearchBar({ notifications: core.notifications, }); - const dataSourceContainerRef = useCallback((node) => { + const dataSetContainerRef = useCallback((node) => { if (node) { - setDataSourceContainerRef(node); - } - }, []); - - const containerRef = useCallback((node) => { - if (node) { - setContainerRef(node); + setDataSetContainerRef(node); } }, []); @@ -228,8 +220,7 @@ export function createSearchBar({ filters={filters} query={query} settings={settings} - dataSourceContainerRef={dataSourceContainerRef} - containerRef={containerRef} + dataSetContainerRef={dataSetContainerRef} onFiltersUpdated={defaultFiltersUpdated(data.query)} onRefreshChange={defaultOnRefreshChange(data.query)} savedQuery={savedQuery} diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index 11914f134443..0855a0fc6a4f 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -80,8 +80,7 @@ export interface SearchBarOwnProps { // Query bar - should be in SearchBarInjectedDeps query?: Query; settings?: Settings; - dataSourceContainerRef?: React.RefCallback; - containerRef?: React.RefCallback; + dataSetContainerRef?: React.RefCallback; // Show when user has privileges to save showSaveQuery?: boolean; savedQuery?: SavedQuery; @@ -491,8 +490,7 @@ class SearchBarUI extends Component { queryEditor = ( ; + dataSetContainer$: Observable; } diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 6a6142184888..b882f3eb7258 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -30,8 +30,7 @@ export class UiService implements Plugin { enhancementsConfig: ConfigSchema['enhancements']; private queryEnhancements: Map = new Map(); private queryEditorExtensionMap: Record = {}; - private dataSourceContainer$ = new BehaviorSubject(null); - private container$ = new BehaviorSubject(null); + private dataSetContainer$ = new BehaviorSubject(null); constructor(initializerContext: PluginInitializerContext) { const { enhancements } = initializerContext.config.get(); @@ -63,12 +62,8 @@ export class UiService implements Plugin { queryEditorExtensionMap: this.queryEditorExtensionMap, }); - const setDataSourceContainerRef = (ref: HTMLDivElement | null) => { - this.dataSourceContainer$.next(ref); - }; - - const setContainerRef = (ref: HTMLDivElement | null) => { - this.container$.next(ref); + const setDataSetContainerRef = (ref: HTMLDivElement | null) => { + this.dataSetContainer$.next(ref); }; const SearchBar = createSearchBar({ @@ -76,8 +71,7 @@ export class UiService implements Plugin { data: dataServices, storage, settings: Settings, - setDataSourceContainerRef, - setContainerRef, + setDataSetContainerRef, }); return { @@ -86,7 +80,7 @@ export class UiService implements Plugin { SearchBar, SuggestionsComponent, Settings, - container$: this.container$, + dataSetContainer$: this.dataSetContainer$, }; } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 413293ca6e41..eb0850f8f56d 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -3,22 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { ComponentType, FC, useCallback, useEffect, useRef, useState } from 'react'; +import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; +import { batch } from 'react-redux'; import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; import { - setIndexPattern, setDataSet, + setIndexPattern, useTypedDispatch, useTypedSelector, } from '../../utils/state_management'; import './index.scss'; -import { DataSetNavigatorProps } from '../../../../data/public/ui/dataset_navigator'; -import { batch } from 'react-redux'; export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); @@ -27,8 +26,6 @@ export const Sidebar: FC = ({ children }) => { const [dataSourceOptionList, setDataSourceOptionList] = useState([]); const [activeDataSources, setActiveDataSources] = useState([]); const [isEnhancementsEnabled, setIsEnhancementsEnabled] = useState(false); - const [DataSetNavigator, setDataSetNavigator] = useState(); - const [selectedDataSet, setSelectedDataSet] = useState(); const containerRef = useRef(null); const { @@ -39,40 +36,19 @@ export const Sidebar: FC = ({ children }) => { }, } = useOpenSearchDashboards(); - const handleDataSetSelection = useCallback( - (selectedDataSet: any) => { - setSelectedDataSet(selectedDataSet); - setSelectedSources([ - { - key: selectedDataSet.id, - name: selectedDataSet.name, - label: selectedDataSet.name, - value: selectedDataSet.id, - type: 'OpenSearch Default', - ds: activeDataSources[0], - }, - ]); - batch(() => { - dispatch(setIndexPattern(selectedDataSet.id)); - dispatch(setDataSet(selectedDataSet)); - }); - }, - [dispatch] - ); + const { DataSetNavigator } = ui; useEffect(() => { const subscriptions = ui.Settings.getEnabledQueryEnhancementsUpdated$().subscribe( (enabledQueryEnhancements) => { setIsEnhancementsEnabled(enabledQueryEnhancements); - if (enabledQueryEnhancements) - setDataSetNavigator(ui.DataSetNavigator(handleDataSetSelection)); } ); return () => { subscriptions.unsubscribe(); }; - }, [ui.Settings, handleDataSetSelection]); + }, [ui.Settings]); const setContainerRef = useCallback((uiContainerRef) => { uiContainerRef.appendChild(containerRef.current); @@ -80,17 +56,17 @@ export const Sidebar: FC = ({ children }) => { useEffect(() => { if (!isEnhancementsEnabled) return; - const subscriptions = ui.container$.subscribe((container) => { - if (container === null) return; + const subscriptions = ui.dataSetContainer$.subscribe((dataSetContainer) => { + if (dataSetContainer === null) return; if (containerRef.current) { - setContainerRef(container); + setContainerRef(dataSetContainer); } }); return () => { subscriptions.unsubscribe(); }; - }, [ui.container$, containerRef, setContainerRef, isEnhancementsEnabled]); + }, [ui.dataSetContainer$, containerRef, setContainerRef, isEnhancementsEnabled]); useEffect(() => { let isMounted = true; @@ -162,23 +138,20 @@ export const Sidebar: FC = ({ children }) => { [toasts] ); + const handleDataSetSelection = useCallback( + (dataSet: any) => { + batch(() => { + dispatch(setIndexPattern(dataSet!.id)); + dispatch(setDataSet(dataSet)); + }); + }, + [dispatch] + ); + const memorizedReload = useCallback(() => { dataSources.dataSourceService.reload(); }, [dataSources.dataSourceService]); - const dataSourceSelector = ( - - ); - return ( { containerRef.current = node; }} > - {DataSetNavigator} + )} {!isEnhancementsEnabled && ( @@ -203,7 +176,16 @@ export const Sidebar: FC = ({ children }) => { color="transparent" className="deSidebar_dataSource" > - {dataSourceSelector} + )} diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index a2842916a78b..a8bc30e89b0d 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -18,14 +18,14 @@ export interface DataSet { timestampField: string; mapping?: any; }; - type?: 'dataset' | 'temporary'; + type?: 'dataSet' | 'temporary'; } export interface MetadataState { indexPattern?: string; originatingApp?: string; view?: string; - dataset?: DataSet; + dataSet?: DataSet; } const initialState: MetadataState = {}; @@ -44,7 +44,7 @@ export const getPreloadedState = async ({ ...initialState, originatingApp, indexPattern: defaultIndexPattern?.id, - dataset: { + dataSet: { id: defaultIndexPattern?.id, }, }; @@ -60,7 +60,7 @@ export const slice = createSlice({ state.indexPattern = action.payload; }, setDataSet: (state, action: PayloadAction) => { - state.dataset = action.payload; + state.dataSet = action.payload; }, setOriginatingApp: (state, action: PayloadAction) => { state.originatingApp = action.payload; From ddb433017d429f43083eade846708ea3aabafe5e Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 22:45:18 +0000 Subject: [PATCH 29/59] handle import errors Signed-off-by: Kawika Avilla --- package.json | 3 +-- .../public/ui/dataset_navigator/dataset_navigator.tsx | 4 ++-- .../dataset_navigator/lib/catalog_cache/cache_loader.tsx | 5 ++--- .../ui/dataset_navigator/lib/catalog_cache/index.tsx | 8 ++++++++ .../data/public/ui/dataset_navigator/lib/hooks/index.tsx | 6 ++++++ .../data/public/ui/dataset_navigator/lib/index.tsx | 3 +++ .../public/ui/dataset_navigator/lib/requests/index.tsx | 6 ++++++ .../ui/dataset_navigator/lib/utils/fetch_data_sources.ts | 2 +- 8 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx create mode 100644 src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx diff --git a/package.json b/package.json index 39d2f48be83f..ee1ea34f2e97 100644 --- a/package.json +++ b/package.json @@ -65,8 +65,7 @@ "start": "scripts/use_node scripts/opensearch_dashboards --dev", "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST", "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security", - "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true", - "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", + "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true", "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev", "lint": "yarn run lint:es && yarn run lint:style", "lint:es": "scripts/use_node scripts/eslint", diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 1a3cb5b29e1f..792d09e31c9c 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -17,14 +17,14 @@ import { } from '@elastic/eui'; import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; import _ from 'lodash'; +import { i18n } from '@osd/i18n'; import { SIMPLE_DATA_SET_TYPES, SIMPLE_DATA_SOURCE_TYPES, SimpleDataSet, SimpleDataSource, SimpleObject, -} from 'src/plugins/data/common'; -import { i18n } from '@osd/i18n'; +} from '../../../common'; import { useLoadDatabasesToCache, useLoadExternalDataSourcesToCache, diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx index f1e28a2b7fcb..16ce324ad64f 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx @@ -8,7 +8,6 @@ import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants'; import { AsyncPollingResult, - CachedAccelerations, CachedColumn, CachedDataSourceStatus, CachedTable, @@ -27,7 +26,7 @@ import { import { usePolling } from '../utils/use_polling'; import { SQLService } from '../requests/sql'; import { CatalogCacheManager } from './cache_manager'; -import { fetchExternalDataSources } from '../../utils/fetch_external_data_sources'; +import { fetchExternalDataSources } from '../utils'; export const updateDatabasesToCache = ( dataSourceName: string, @@ -146,7 +145,7 @@ export const updateAccelerationsToCache = ( const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - const newAccelerations: CachedAccelerations[] = combinedData.map((row: any) => ({ + const newAccelerations: any[] = combinedData.map((row: any) => ({ flintIndexName: row.flint_index_name, type: row.kind === 'mv' ? 'materialized' : row.kind, database: row.database, diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx new file mode 100644 index 000000000000..5449277b2bd8 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './cache_intercept'; +export * from './cache_loader'; +export * from './cache_manager'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx new file mode 100644 index 000000000000..88974a7c9420 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/hooks/index.tsx @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './direct_query_hook'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx index 079132ce99d2..771fbd6eef3a 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx @@ -3,4 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +export * from './catalog_cache'; +export * from './hooks'; +export * from './requests'; export * from './utils'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx new file mode 100644 index 000000000000..3918a896bd0b --- /dev/null +++ b/src/plugins/data/public/ui/dataset_navigator/lib/requests/index.tsx @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './sql'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts index 9d0a47f6ee26..7a10d7badb58 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts @@ -4,7 +4,7 @@ */ import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { SimpleDataSource } from 'src/plugins/data/common'; +import { SimpleDataSource } from '../../../../../common'; export const fetchDataSources = async (client: SavedObjectsClientContract) => { const resp = await client.find({ From 1ccf16f3f9dae5f4070e0be9cb5102597497b6ba Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 22:57:41 +0000 Subject: [PATCH 30/59] rebased Signed-off-by: Kawika Avilla --- .../public/ui/query_editor/query_editor.tsx | 41 +++++++------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 2889daca4d18..389a0104644d 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -369,6 +369,15 @@ export default class QueryEditorUI extends Component { const useQueryEditor = this.props.query.language !== 'kuery' && this.props.query.language !== 'lucene'; + const languageSelector = ( + + ); + return (
@@ -381,11 +390,9 @@ export default class QueryEditorUI extends Component { isCollapsed={!this.state.isCollapsed} /> - {this.state.isDataSetsVisible && ( - -
- - )} + +
+ {(this.state.isCollapsed || !useQueryEditor) && ( @@ -424,14 +431,7 @@ export default class QueryEditorUI extends Component { )} {!useQueryEditor && ( -
- -
+
{languageSelector}
 )} @@ -444,11 +444,6 @@ export default class QueryEditorUI extends Component { > {this.props.prepend} - {this.state.isDataSetsVisible && ( - -
- - )} @@ -486,15 +481,7 @@ export default class QueryEditorUI extends Component { } > - - - + {languageSelector} {this.state.lineCount} {this.state.lineCount === 1 ? 'line' : 'lines'} From 7de0fe25da462034291e50ac9e5779cb20ac5c5d Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 23:07:33 +0000 Subject: [PATCH 31/59] fix a messy merge for package.json Signed-off-by: Kawika Avilla --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index ee1ea34f2e97..9a4229d43671 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,8 @@ "start": "scripts/use_node scripts/opensearch_dashboards --dev", "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST", "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security", - "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true", "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", + "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true", + "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev", "lint": "yarn run lint:es && yarn run lint:style", "lint:es": "scripts/use_node scripts/eslint", From 204cde0e56ee1c604768658fcc43ce853bb709f6 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Mon, 22 Jul 2024 23:33:36 +0000 Subject: [PATCH 32/59] so lnse for catching this Signed-off-by: Kawika Avilla --- .../dataset_navigator/dataset_navigator.tsx | 2 +- .../public/ui/query_editor/query_editor.tsx | 35 +------------------ .../ui/query_editor/query_editor_top_row.tsx | 2 +- .../data/public/ui/settings/settings.ts | 22 ------------ .../utils/use_index_pattern.ts | 3 +- 5 files changed, 4 insertions(+), 60 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 792d09e31c9c..d84b65f767d9 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -508,7 +508,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { id: 7, title: selectedObject?.title, content: - isLoading && !selectedObject ? ( + isLoading || !selectedObject ? (
{LoadingSpinner}
) : ( diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index 389a0104644d..fce4e2f438e9 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -3,14 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { - EuiFlexGroup, - EuiFlexItem, - EuiForm, - EuiFormRow, - htmlIdGenerator, - PopoverAnchorPosition, -} from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, htmlIdGenerator, PopoverAnchorPosition } from '@elastic/eui'; import classNames from 'classnames'; import { isEqual } from 'lodash'; import React, { Component, createRef, RefObject } from 'react'; @@ -207,32 +200,6 @@ export default class QueryEditorUI extends Component { } }; - // private onSelectDataSet = (dataSetName: string) => { - // const newQuery = { - // query: this.props.getQueryStringInitialValue?.(this.props.query.language, dataSetName) ?? '', - // language: this.props.query.language, - // }; - - // const enhancement = this.props.settings.getQueryEnhancements(newQuery.language); - // const fields = enhancement?.fields; - // const newSettings: DataSettings = { - // userQueryLanguage: newQuery.language, - // userQueryString: newQuery.query, - // ...(fields && { uiOverrides: { fields } }), - // }; - // this.props.settings?.updateSettings(newSettings); - - // const dateRangeEnhancement = enhancement?.searchBar?.dateRange; - // const dateRange = dateRangeEnhancement - // ? { - // from: dateRangeEnhancement.initialFrom!, - // to: dateRangeEnhancement.initialTo!, - // } - // : undefined; - // this.onChange(newQuery, dateRange); - // this.onSubmit(newQuery, dateRange); - // }; - // TODO: MQL consider moving language select language of setting search source here private onSelectLanguage = (language: string) => { // Send telemetry info every time the user opts in or out of kuery diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index c162f50b5ff7..97695d26d2c1 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -205,7 +205,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ) return ''; - const defaultDataSet = settings?.getSelectedDataSet() ?? indexPatterns[0]; + const defaultDataSet = indexPatterns[0]; const dataSet = typeof defaultDataSet === 'string' ? defaultDataSet : defaultDataSet.title; return input.replace('', dataSet); diff --git a/src/plugins/data/public/ui/settings/settings.ts b/src/plugins/data/public/ui/settings/settings.ts index ebdde370db26..96c806ad0bc3 100644 --- a/src/plugins/data/public/ui/settings/settings.ts +++ b/src/plugins/data/public/ui/settings/settings.ts @@ -27,7 +27,6 @@ export class Settings { private isEnabled = false; private enabledQueryEnhancementsUpdated$ = new BehaviorSubject(this.isEnabled); private enhancedAppNames: string[] = []; - private selectedDataSet$ = new BehaviorSubject(null); constructor( private readonly config: ConfigSchema['enhancements'], @@ -39,29 +38,8 @@ export class Settings { this.isEnabled = true; this.setUserQueryEnhancementsEnabled(this.isEnabled); this.enhancedAppNames = this.isEnabled ? this.config.supportedAppNames : []; - this.setSelectedDataSet(this.getSelectedDataSet()); } - /** - * @experimental - Sets the dataset BehaviorSubject - */ - setSelectedDataSet = (dataSet: any) => { - console.log('dataSet in settings:', dataSet); - this.storage.set('opensearchDashboards.userQueryDataSet', dataSet); - this.selectedDataSet$.next(dataSet); - }; - - /** - * @experimental - Gets the dataset Observable - */ - getSelectedDataSet$ = () => { - return this.selectedDataSet$.asObservable(); - }; - - getSelectedDataSet = () => { - return this.storage.get('opensearchDashboards.userQueryDataSet'); - }; - supportsEnhancementsEnabled(appName: string) { return this.enhancedAppNames.includes(appName); } diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index 05b7cc648ecd..e8a81234278e 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -5,9 +5,8 @@ import { useEffect, useState } from 'react'; import { i18n } from '@osd/i18n'; -import { batch } from 'react-redux'; import { IndexPattern } from '../../../../../data/public'; -import { updateDataSet, useSelector, updateIndexPattern } from '../../utils/state_management'; +import { useSelector, updateIndexPattern } from '../../utils/state_management'; import { DiscoverViewServices } from '../../../build_services'; import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; From 238efad83e20a4a1bb11b4c8997d3085b37df3d7 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:04:20 +0000 Subject: [PATCH 33/59] fix nav Signed-off-by: Kawika Avilla --- .../dataset_navigator/dataset_navigator.tsx | 183 ++++++++++-------- 1 file changed, 97 insertions(+), 86 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index d84b65f767d9..228a9b9bc09c 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { EuiButton, EuiButtonEmpty, @@ -48,7 +48,12 @@ export interface DataSetNavigatorProps { onSelectDataSet: (dataSet: SimpleDataSet) => void; } -export const DataSetNavigator = (props: DataSetNavigatorProps) => { +export const DataSetNavigator = ({ + dataSetId, + savedObjectsClient, + http, + onSelectDataSet, +}: DataSetNavigatorProps) => { const searchService = getSearchService(); const queryService = getQueryService(); const uiService = getUiService(); @@ -57,6 +62,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const [isOpen, setIsOpen] = useState(false); const [isLoading, setIsLoading] = useState(false); + const [isMounted, setIsMounted] = useState(false); const [selectedDataSet, setSelectedDataSet] = useState(); const [selectedObject, setSelectedObject] = useState(); const [selectedDataSource, setSelectedDataSource] = useState(); @@ -77,13 +83,13 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const { loadStatus: dataSourcesLoadStatus, loadExternalDataSources: startLoadingDataSources, - } = useLoadExternalDataSourcesToCache(props.http!, notifications); + } = useLoadExternalDataSourcesToCache(http!, notifications); const { loadStatus: databasesLoadStatus, startLoading: startLoadingDatabases, - } = useLoadDatabasesToCache(props.http!, notifications); + } = useLoadDatabasesToCache(http!, notifications); const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache( - props.http!, + http!, notifications ); @@ -96,33 +102,34 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { }; useEffect(() => { - setIsLoading(true); - - Promise.all([ - fetchIndexPatterns(props.savedObjectsClient!, ''), - fetchDataSources(props.savedObjectsClient!), - ]) - .then(([defaultIndexPatterns, defaultDataSources]) => { - setIndexPatterns(defaultIndexPatterns); - setDataSources(defaultDataSources); - - if (!selectedDataSet && props.dataSetId) { - const selectedPattern = defaultIndexPatterns.find( - (pattern) => pattern.id === props.dataSetId - ); - if (selectedPattern) { - setSelectedDataSet({ - id: selectedPattern.id ?? selectedPattern.title, - title: selectedPattern.title, - type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, - }); + if (!isMounted) { + setIsLoading(true); + Promise.all([ + fetchIndexPatterns(savedObjectsClient!, ''), + fetchDataSources(savedObjectsClient!), + ]) + .then(([defaultIndexPatterns, defaultDataSources]) => { + setIndexPatterns(defaultIndexPatterns); + setDataSources(defaultDataSources); + if (!selectedDataSet && dataSetId) { + const selectedPattern = defaultIndexPatterns.find( + (pattern) => pattern.id === dataSetId + ); + if (selectedPattern) { + setSelectedDataSet({ + id: selectedPattern.id ?? selectedPattern.title, + title: selectedPattern.title, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + }); + } } - } - }) - .finally(() => { - setIsLoading(false); - }); - }, [indexPatternsService, props.dataSetId, props.savedObjectsClient, selectedDataSet]); + }) + .finally(() => { + setIsMounted(true); + setIsLoading(false); + }); + } + }, [indexPatternsService, dataSetId, savedObjectsClient, selectedDataSet, isMounted]); useEffect(() => { if (selectedDataSource) { @@ -276,65 +283,69 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { } }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus]); - useEffect(() => { - const createTemporaryIndexPattern = async (dataSet: SimpleDataSet) => { - const fieldsMap = dataSet.fields?.reduce((acc: any, field: any) => { - acc[field.name] = field; - return acc; - }); - const temporaryIndexPattern = await indexPatternsService.create( - { - id: dataSet.id, - title: dataSet.title, - fields: fieldsMap, - dataSourceRef: { - id: dataSet.dataSourceRef?.id!, - name: dataSet.dataSourceRef?.name!, - type: dataSet.dataSourceRef?.type!, + const handleSelectedDataSet = useCallback( + async (ds: any) => { + const createTemporaryIndexPattern = async (dataSet: SimpleDataSet) => { + const fieldsMap = dataSet.fields?.reduce((acc: any, field: any) => { + acc[field.name] = field; + return acc; + }); + const temporaryIndexPattern = await indexPatternsService.create( + { + id: dataSet.id, + title: dataSet.title, + fields: fieldsMap, + dataSourceRef: { + id: dataSet.dataSourceRef?.id!, + name: dataSet.dataSourceRef?.name!, + type: dataSet.dataSourceRef?.type!, + }, + timeFieldName: dataSet.timeFieldName, }, - timeFieldName: dataSet.timeFieldName, - }, - true - ); - indexPatternsService.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); - }; + true + ); + indexPatternsService.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); + }; + + const getInitialQuery = (dataSet: SimpleDataSet) => { + const language = uiService.Settings.getUserQueryLanguage(); + const input = uiService.Settings.getQueryEnhancements(language)?.searchBar?.queryStringInput + ?.initialValue; - const getInitialQuery = (dataSet: SimpleDataSet) => { - const language = uiService.Settings.getUserQueryLanguage(); - const input = uiService.Settings.getQueryEnhancements(language)?.searchBar?.queryStringInput - ?.initialValue; + if (!dataSet || !input) + return { + query: '', + language, + }; - if (!dataSet || !input) return { - query: '', + query: input.replace('', dataSet.title), language, }; - - return { - query: input.replace('', dataSet.title), - language, }; - }; - const onDataSetSelected = async (dataSet: SimpleDataSet) => { - if (dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY) { - await createTemporaryIndexPattern(dataSet); - } + const onDataSetSelected = async (dataSet: SimpleDataSet) => { + if (dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY) { + await createTemporaryIndexPattern(dataSet); + } - CatalogCacheManager.addRecentDataSet({ - id: dataSet.id, - name: dataSet.title, - dataSourceRef: dataSet.dataSourceRef?.id, - }); - props.onSelectDataSet(dataSet); - queryService.queryString.setQuery(getInitialQuery(dataSet)); - closePopover(); - }; + CatalogCacheManager.addRecentDataSet({ + id: dataSet.id, + name: dataSet.title, + dataSourceRef: dataSet.dataSourceRef?.id, + }); + onSelectDataSet(dataSet); + queryService.queryString.setQuery(getInitialQuery(dataSet)); + closePopover(); + }; - if (selectedDataSet) { - onDataSetSelected(selectedDataSet); - } - }, [indexPatternsService, props, queryService.queryString, selectedDataSet, uiService.Settings]); + if (ds) { + onDataSetSelected(ds); + setSelectedDataSet(ds); + } + }, + [indexPatternsService, onSelectDataSet, queryService.queryString, uiService.Settings] + ); const RefreshButton = ( { panels={[ { id: 0, - title: 'Data', items: [ ...(CatalogCacheManager.getRecentDataSets().length > 0 ? [ @@ -399,7 +409,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { }, { name: indicesLabel, - panel: 3, + panel: 2, }, { name: 'Connected data sources', @@ -442,6 +452,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { }, { id: 2, + title: 'Clusters', items: [ ...dataSources.map((dataSource) => ({ name: dataSource.name, @@ -545,8 +556,8 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { { - setSelectedDataSet(selectedObject); + onClick={async () => { + await handleSelectedDataSet(selectedObject); }} > Select @@ -559,8 +570,8 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { title: 'Recently Used', items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ name: ds.name, - onClick: () => - setSelectedDataSet({ + onClick: async () => + await handleSelectedDataSet({ id: ds.id, title: ds.name, dataSourceRef: { From f8aee58a270eab0c4abf926bb0f84f721c63e50d Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:19:30 +0000 Subject: [PATCH 34/59] no recalling Signed-off-by: Kawika Avilla --- .../dataset_navigator/dataset_navigator.tsx | 101 +++++++++--------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 228a9b9bc09c..67d7c7383220 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -131,50 +131,6 @@ export const DataSetNavigator = ({ } }, [indexPatternsService, dataSetId, savedObjectsClient, selectedDataSet, isMounted]); - useEffect(() => { - if (selectedDataSource) { - setIsLoading(true); - fetchIndices(searchService, selectedDataSource.id).then((indices) => { - const objects = indices.map(({ indexName }: { indexName: string }) => ({ - id: indexName, - title: indexName, - dataSourceRef: { - id: selectedDataSource.id, - name: selectedDataSource.name, - type: selectedDataSource.type, - }, - })); - setSelectedDataSourceObjects(objects); - setIsLoading(false); - }); - } - }, [searchService, selectedDataSource]); - - useEffect(() => { - const getFieldsForWildcard = async (object: SimpleObject | SimpleDataSet) => { - const fields = await indexPatternsService.getFieldsForWildcard({ - pattern: object.title, - dataSourceId: object.dataSourceRef?.id, - }); - - const timeFields = fields.filter((field: any) => field.type === 'date'); - - setSelectedObject({ - id: object.id, - title: object.title, - fields, - timeFields, - ...(timeFields[0]?.name ? { timeFieldName: timeFields[0].name } : {}), - dataSourceRef: object.dataSourceRef, - type: SIMPLE_DATA_SET_TYPES.TEMPORARY, - }); - }; - - if (selectedObject) { - getFieldsForWildcard(selectedObject); - } - }, [indexPatternsService, searchService, selectedObject]); - useEffect(() => { const status = dataSourcesLoadStatus.toLowerCase(); const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); @@ -283,6 +239,54 @@ export const DataSetNavigator = ({ } }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus]); + const handleSelectedDataSource = useCallback( + async (source) => { + if (source) { + setIsLoading(true); + await fetchIndices(searchService, source.id).then((indices) => { + const objects = indices.map((indexName: string) => ({ + id: indexName, + title: indexName, + dataSourceRef: { + id: source.id, + name: source.name, + type: source.type, + }, + })); + setSelectedDataSourceObjects(objects); + setIsLoading(false); + }); + } + }, + [searchService] + ); + + const handleSelectedObject = useCallback( + async (object) => { + setIsLoading(true); + if (object) { + const fields = await indexPatternsService.getFieldsForWildcard({ + pattern: object.title, + dataSourceId: object.dataSourceRef?.id, + }); + + const timeFields = fields.filter((field: any) => field.type === 'date'); + + setSelectedObject({ + id: object.id, + title: object.title, + fields, + timeFields, + ...(timeFields[0]?.name ? { timeFieldName: timeFields[0].name } : {}), + dataSourceRef: object.dataSourceRef, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY, + }); + setIsLoading(false); + } + }, + [indexPatternsService] + ); + const handleSelectedDataSet = useCallback( async (ds: any) => { const createTemporaryIndexPattern = async (dataSet: SimpleDataSet) => { @@ -457,7 +461,7 @@ export const DataSetNavigator = ({ ...dataSources.map((dataSource) => ({ name: dataSource.name, panel: 3, - onClick: () => setSelectedDataSource(dataSource), + onClick: async () => await handleSelectedDataSource(dataSource), })), ], content:
{isLoading && LoadingSpinner}
, @@ -467,11 +471,8 @@ export const DataSetNavigator = ({ title: selectedDataSource?.name ?? indicesLabel, items: selectedDataSourceObjects.map((object) => ({ name: object.title, - onClick: () => - setSelectedObject({ - ...object, - type: SIMPLE_DATA_SET_TYPES.TEMPORARY, - }), + onClick: async () => + await handleSelectedObject({ ...object, type: SIMPLE_DATA_SET_TYPES.TEMPORARY }), })), content:
{isLoading && LoadingSpinner}
, }, From a687417ff75856524cac1958847044da565005a2 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:23:23 +0000 Subject: [PATCH 35/59] number panels Signed-off-by: Kawika Avilla --- .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 67d7c7383220..d0321ace257d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -471,6 +471,7 @@ export const DataSetNavigator = ({ title: selectedDataSource?.name ?? indicesLabel, items: selectedDataSourceObjects.map((object) => ({ name: object.title, + panel: 7, onClick: async () => await handleSelectedObject({ ...object, type: SIMPLE_DATA_SET_TYPES.TEMPORARY }), })), From 4ab55c967169711371901acdb7868de9d2eff98e Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:25:22 +0000 Subject: [PATCH 36/59] add wait Signed-off-by: Kawika Avilla --- .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index d0321ace257d..6da6bb20c83d 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -344,7 +344,7 @@ export const DataSetNavigator = ({ }; if (ds) { - onDataSetSelected(ds); + await onDataSetSelected(ds); setSelectedDataSet(ds); } }, From bbdbdd16f4dd5b5673e926992f6b24f7b978e6b8 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:26:42 +0000 Subject: [PATCH 37/59] index patterns worknig agan Signed-off-by: Kawika Avilla --- .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 6da6bb20c83d..b44576290aef 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -440,8 +440,8 @@ export const DataSetNavigator = ({ items: indexPatterns.flatMap((indexPattern, indexNum, arr) => [ { name: indexPattern.title, - onClick: () => { - setSelectedDataSet({ + onClick: async () => { + await handleSelectedDataSet({ id: indexPattern.id ?? indexPattern.title, title: indexPattern.title, fields: indexPattern.fields, From b651b941d2da56f61da85aa433ae968bf85a4687 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:38:17 +0000 Subject: [PATCH 38/59] fix and clearing it df out Signed-off-by: Kawika Avilla --- .../public/ui/dataset_navigator/dataset_navigator.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index b44576290aef..17f441646717 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -338,6 +338,7 @@ export const DataSetNavigator = ({ name: dataSet.title, dataSourceRef: dataSet.dataSourceRef?.id, }); + searchService.df.clear(); onSelectDataSet(dataSet); queryService.queryString.setQuery(getInitialQuery(dataSet)); closePopover(); @@ -348,7 +349,13 @@ export const DataSetNavigator = ({ setSelectedDataSet(ds); } }, - [indexPatternsService, onSelectDataSet, queryService.queryString, uiService.Settings] + [ + indexPatternsService, + onSelectDataSet, + queryService.queryString, + searchService.df, + uiService.Settings, + ] ); const RefreshButton = ( From 7a0a81dc6069948f2f1ca266a081e8007c33e557 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:43:36 +0000 Subject: [PATCH 39/59] disable hooks dependency Signed-off-by: Kawika Avilla --- .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 17f441646717..cadcd1d4fbac 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -164,7 +164,8 @@ export const DataSetNavigator = ({ setCachedDatabases(dataSourceCache.databases); } } - }, [databasesLoadStatus, selectedExternalDataSource, startLoadingDatabases]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedExternalDataSource]); // Retrieve databases from cache upon success useEffect(() => { @@ -212,7 +213,8 @@ export const DataSetNavigator = ({ setCachedTables(databaseCache.tables); } } - }, [selectedExternalDataSource, selectedDatabase, tablesLoadStatus, startLoadingTables]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedExternalDataSource, selectedDatabase]); // Retrieve tables from cache upon success useEffect(() => { From db519def79c1ec661b0d427a1ff46b19b4d832de Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:46:42 +0000 Subject: [PATCH 40/59] fix the issue with import Signed-off-by: Kawika Avilla --- .../public/search/ppl_search_interceptor.ts | 4 ++-- .../public/search/sql_async_search_interceptor.ts | 2 +- .../public/search/sql_search_interceptor.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts index bca9961fea3b..0d1e5292b5df 100644 --- a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts @@ -34,7 +34,7 @@ import { fetchDataFrame, } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; +import { ConnectionsService } from '../services'; export class PPLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; @@ -166,7 +166,7 @@ export class PPLSearchInterceptor extends SearchInterceptor { queryConfig: { ...dataFrame.meta.queryConfig, ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.id, + dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, }), }, }; diff --git a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts index 9232ef146cdb..4dafdab49628 100644 --- a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts @@ -26,7 +26,7 @@ import { fetchDataFramePolling, } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; +import { ConnectionsService } from '../services'; export class SQLAsyncSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index 5a3b8278c65a..b41f97293583 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -18,7 +18,7 @@ import { } from '../../../data/public'; import { API, FetchDataFrameContext, SEARCH_STRATEGY, fetchDataFrame } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../data_source_connection'; +import { ConnectionsService } from '../services'; export class SQLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; @@ -60,7 +60,7 @@ export class SQLSearchInterceptor extends SearchInterceptor { queryConfig: { ...dataFrame.meta.queryConfig, ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.id, + dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, }), }, }; From 2cd93b88e58287d9553eb2b692adabf9416af934 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:49:45 +0000 Subject: [PATCH 41/59] fixed passing the data source id Signed-off-by: Kawika Avilla --- .../public/search/ppl_search_interceptor.ts | 3 --- .../public/search/sql_async_search_interceptor.ts | 4 ---- .../public/search/sql_search_interceptor.ts | 3 --- 3 files changed, 10 deletions(-) diff --git a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts index 0d1e5292b5df..ffaea6718b99 100644 --- a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts @@ -165,9 +165,6 @@ export class PPLSearchInterceptor extends SearchInterceptor { ...dataFrame.meta, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, - }), }, }; const aggConfig = getAggConfig( diff --git a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts index 4dafdab49628..b8410333f63f 100644 --- a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts @@ -71,10 +71,6 @@ export class SQLAsyncSearchInterceptor extends SearchInterceptor { ...dataFrame.meta, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && - this.connectionsService.getSelectedConnection()?.dataSource && { - dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, - }), }, }; diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index b41f97293583..9206b4300b83 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -59,9 +59,6 @@ export class SQLSearchInterceptor extends SearchInterceptor { ...dataFrame.meta, queryConfig: { ...dataFrame.meta.queryConfig, - ...(this.connectionsService.getSelectedConnection() && { - dataSourceId: this.connectionsService.getSelectedConnection()?.dataSource.id, - }), }, }; From f017dc54ce0d5f6d51746e2a8c5ec84106e35510 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 00:58:55 +0000 Subject: [PATCH 42/59] loading spinner Signed-off-by: Kawika Avilla --- .../ui/dataset_navigator/_dataset_navigator.scss | 4 ++++ .../public/ui/dataset_navigator/dataset_navigator.tsx | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss index 568d5532bc4a..358916830f4b 100644 --- a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss +++ b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss @@ -6,3 +6,7 @@ min-width: 350px; border-bottom: $euiBorderThin !important; } + +.dataSetNavigator__loading { + padding: $euiSizeS; +} diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index cadcd1d4fbac..3b487346d1f6 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -12,6 +12,7 @@ import { EuiForm, EuiFormRow, EuiLoadingSpinner, + EuiPanel, EuiPopover, EuiSelect, } from '@elastic/eui'; @@ -368,7 +369,11 @@ export const DataSetNavigator = ({ /> ); - const LoadingSpinner = ; + const LoadingSpinner = ( + + + + ); const indexPatternsLabel = i18n.translate('data.query.dataSetNavigator.indexPatternsName', { defaultMessage: 'Index patterns', @@ -552,8 +557,8 @@ export const DataSetNavigator = ({ : []), { value: 'no-time-filter', text: "I don't want to use a time filter" }, ]} - onChange={(event) => { - setSelectedObject({ + onChange={async (event) => { + await handleSelectedObject({ ...selectedObject, timeFieldName: event.target.value !== 'no-time-filter' From 10f2e1ebbdbe3da2131c5fcaf991aec3c7a53089 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 01:03:05 +0000 Subject: [PATCH 43/59] pading Signed-off-by: Kawika Avilla --- .../data/public/ui/dataset_navigator/_dataset_navigator.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss index 358916830f4b..73a8c8719500 100644 --- a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss +++ b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss @@ -7,6 +7,10 @@ border-bottom: $euiBorderThin !important; } +.dataSetNavigatorFormWrapper { + padding: $euiSizeS; +} + .dataSetNavigator__loading { padding: $euiSizeS; } From 38d93fcb9e9b54507e391918715f5202114d478e Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 02:02:23 +0000 Subject: [PATCH 44/59] conditionally show s3 Signed-off-by: Kawika Avilla --- .../ui/dataset_navigator/dataset_navigator.tsx | 17 +++++++++++++---- .../lib/utils/fetch_external_data_sources.ts | 9 +++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 3b487346d1f6..c7bc2fbd72ca 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -40,7 +40,13 @@ import { getSearchService, getUiService, } from '../../services'; -import { fetchDataSources, fetchIndexPatterns, fetchIndices, isCatalogCacheFetching } from './lib'; +import { + fetchDataSources, + fetchIndexPatterns, + fetchIndices, + isCatalogCacheFetching, + fetchIfExternalDataSourcesEnabled, +} from './lib'; export interface DataSetNavigatorProps { dataSetId: string | undefined; @@ -64,6 +70,7 @@ export const DataSetNavigator = ({ const [isOpen, setIsOpen] = useState(false); const [isLoading, setIsLoading] = useState(false); const [isMounted, setIsMounted] = useState(false); + const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false); const [selectedDataSet, setSelectedDataSet] = useState(); const [selectedObject, setSelectedObject] = useState(); const [selectedDataSource, setSelectedDataSource] = useState(); @@ -108,8 +115,10 @@ export const DataSetNavigator = ({ Promise.all([ fetchIndexPatterns(savedObjectsClient!, ''), fetchDataSources(savedObjectsClient!), + fetchIfExternalDataSourcesEnabled(http!), ]) - .then(([defaultIndexPatterns, defaultDataSources]) => { + .then(([defaultIndexPatterns, defaultDataSources, isExternalDSEnabled]) => { + setIsExternalDataSourcesEnabled(isExternalDSEnabled); setIndexPatterns(defaultIndexPatterns); setDataSources(defaultDataSources); if (!selectedDataSet && dataSetId) { @@ -130,7 +139,7 @@ export const DataSetNavigator = ({ setIsLoading(false); }); } - }, [indexPatternsService, dataSetId, savedObjectsClient, selectedDataSet, isMounted]); + }, [indexPatternsService, dataSetId, savedObjectsClient, selectedDataSet, isMounted, http]); useEffect(() => { const status = dataSourcesLoadStatus.toLowerCase(); @@ -495,7 +504,7 @@ export const DataSetNavigator = ({ id: 4, title: (
- Connected data sources + S3 {CatalogCacheManager.getExternalDataSourcesCache().status === CachedDataSourceStatus.Updated && RefreshButton}
diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts index 8f104c6288d9..81c6a3c4a2bb 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts @@ -5,6 +5,15 @@ import { HttpStart } from 'opensearch-dashboards/public'; +export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => { + try { + const resp = await http.head('/api/dataconnections'); + return resp.status === 200; + } catch (e) { + return false; + } +}; + export const fetchExternalDataSources = async (http: HttpStart, connectedClusters: string[]) => { const results = await Promise.all( connectedClusters.map(async (cluster) => { From 56b2510f7883802662f52cda7f2a9f8e237e87e1 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 02:07:39 +0000 Subject: [PATCH 45/59] remove sql Signed-off-by: Kawika Avilla --- .../dataset_navigator/dataset_navigator.tsx | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index c7bc2fbd72ca..97762f9ff57e 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -438,23 +438,29 @@ export const DataSetNavigator = ({ name: indicesLabel, panel: 2, }, - { - name: 'Connected data sources', - panel: 4, - onClick: () => { - const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); - if ( - (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || - externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && - !isCatalogCacheFetching(dataSourcesLoadStatus) && - dataSources.length > 0 - ) { - startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); - } else if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) { - setExternalDataSources(externalDataSourcesCache.externalDataSources); - } - }, - }, + ...(isExternalDataSourcesEnabled + ? [ + { + name: 'Connected data sources', + panel: 4, + onClick: () => { + const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); + if ( + (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || + externalDataSourcesCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(dataSourcesLoadStatus) && + dataSources.length > 0 + ) { + startLoadingDataSources(dataSources.map((dataSource) => dataSource.id)); + } else if ( + externalDataSourcesCache.status === CachedDataSourceStatus.Updated + ) { + setExternalDataSources(externalDataSourcesCache.externalDataSources); + } + }, + }, + ] + : []), ], }, { From 7b19eb97410c5a65a90f034dd5b16a70f2595b25 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 02:18:56 +0000 Subject: [PATCH 46/59] let it fail Signed-off-by: Kawika Avilla --- .../common/search/opensearch_search/types.ts | 4 + .../lib/utils/fetch_external_data_sources.ts | 4 +- .../query_enhancements/public/plugin.tsx | 52 ++----- .../query_enhancements/public/search/index.ts | 1 - .../search/sql_async_search_interceptor.ts | 133 ------------------ .../public/search/sql_search_interceptor.ts | 106 ++++++++++++-- 6 files changed, 111 insertions(+), 189 deletions(-) delete mode 100644 src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index f90a3f1de245..6d24e8c36dd3 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -48,6 +48,10 @@ export interface ISearchOptions { * Use this option to enable support for long numerals. */ withLongNumeralsSupport?: boolean; + /** + * Use this option to enable support for async. + */ + isAsync?: boolean; } export type ISearchRequestParams> = { diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts index 81c6a3c4a2bb..a9272155e602 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts +++ b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts @@ -7,8 +7,8 @@ import { HttpStart } from 'opensearch-dashboards/public'; export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => { try { - const resp = await http.head('/api/dataconnections'); - return resp.status === 200; + await http.get('/api/dataconnections'); + return true; } catch (e) { return false; } diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index 9baec694f1a9..f760e44dc53b 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -9,7 +9,7 @@ import { IStorageWrapper, Storage } from '../../opensearch_dashboards_utils/publ import { ConfigSchema } from '../common/config'; import { ConnectionsService, setData, setStorage } from './services'; import { createQueryAssistExtension } from './query_assist'; -import { PPLSearchInterceptor, SQLAsyncSearchInterceptor, SQLSearchInterceptor } from './search'; +import { PPLSearchInterceptor, SQLSearchInterceptor } from './search'; import { QueryEnhancementsPluginSetup, QueryEnhancementsPluginSetupDependencies, @@ -54,27 +54,13 @@ export class QueryEnhancementsPlugin this.connectionsService ); - const sqlSearchInterceptor = new SQLSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); - - const sqlAsyncSearchInterceptor = new SQLAsyncSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); + const sqlSearchInterceptor = new SQLSearchInterceptor({ + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }); data.__enhance({ ui: { @@ -122,28 +108,6 @@ export class QueryEnhancementsPlugin }, }); - data.__enhance({ - ui: { - query: { - language: 'SQLAsync', - search: sqlAsyncSearchInterceptor, - searchBar: { - showDatePicker: false, - showFilterBar: false, - showDataSetsSelector: false, - showDataSourcesSelector: true, - queryStringInput: { initialValue: 'SHOW DATABASES IN ::mys3::' }, - }, - fields: { - filterable: false, - visualizable: false, - }, - showDocLinks: false, - supportedAppNames: ['discover'], - }, - }, - }); - data.__enhance({ ui: { queryEditorExtension: createQueryAssistExtension( diff --git a/src/plugins/query_enhancements/public/search/index.ts b/src/plugins/query_enhancements/public/search/index.ts index 9835c1345f02..624e7cf6e7b5 100644 --- a/src/plugins/query_enhancements/public/search/index.ts +++ b/src/plugins/query_enhancements/public/search/index.ts @@ -5,4 +5,3 @@ export { PPLSearchInterceptor } from './ppl_search_interceptor'; export { SQLSearchInterceptor } from './sql_search_interceptor'; -export { SQLAsyncSearchInterceptor } from './sql_async_search_interceptor'; diff --git a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts deleted file mode 100644 index b8410333f63f..000000000000 --- a/src/plugins/query_enhancements/public/search/sql_async_search_interceptor.ts +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { trimEnd } from 'lodash'; -import { BehaviorSubject, Observable, throwError } from 'rxjs'; -import { i18n } from '@osd/i18n'; -import { concatMap, map } from 'rxjs/operators'; -import { - DATA_FRAME_TYPES, - DataPublicPluginStart, - IOpenSearchDashboardsSearchRequest, - IOpenSearchDashboardsSearchResponse, - ISearchOptions, - SearchInterceptor, - SearchInterceptorDeps, -} from '../../../data/public'; -import { getRawDataFrame, getRawQueryString, IDataFrameResponse } from '../../../data/common'; -import { - API, - DataFramePolling, - FetchDataFrameContext, - SEARCH_STRATEGY, - fetchDataFrame, - fetchDataFramePolling, -} from '../../common'; -import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../services'; - -export class SQLAsyncSearchInterceptor extends SearchInterceptor { - protected queryService!: DataPublicPluginStart['query']; - protected aggsService!: DataPublicPluginStart['search']['aggs']; - protected indexPatterns!: DataPublicPluginStart['indexPatterns']; - protected dataFrame$ = new BehaviorSubject(undefined); - - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { - super(deps); - - deps.startServices.then(([coreStart, depsStart]) => { - this.queryService = (depsStart as QueryEnhancementsPluginStartDependencies).data.query; - this.aggsService = (depsStart as QueryEnhancementsPluginStartDependencies).data.search.aggs; - }); - } - - protected runSearch( - request: IOpenSearchDashboardsSearchRequest, - signal?: AbortSignal, - strategy?: string - ): Observable { - const { id, ...searchRequest } = request; - const path = trimEnd(API.SQL_ASYNC_SEARCH); - const dfContext: FetchDataFrameContext = { - http: this.deps.http, - path, - signal, - }; - - const dataFrame = getRawDataFrame(searchRequest); - if (!dataFrame) { - return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); - } - - const queryString = - dataFrame.meta?.queryConfig?.formattedQs() ?? getRawQueryString(searchRequest) ?? ''; - - dataFrame.meta = { - ...dataFrame.meta, - queryConfig: { - ...dataFrame.meta.queryConfig, - }, - }; - - const onPollingSuccess = (pollingResult: any) => { - if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { - return false; - } - if (pollingResult && pollingResult.body.meta.status === 'FAILED') { - const jsError = new Error(pollingResult.data.error.response); - this.deps.toasts.addError(jsError, { - title: i18n.translate('queryEnhancements.sqlQueryError', { - defaultMessage: 'Could not complete the SQL async query', - }), - toastMessage: pollingResult.data.error.response, - }); - return false; - } - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryPolling', { - defaultMessage: 'Polling query job results...', - }), - }); - - return true; - }; - - const onPollingError = (error: Error) => { - throw new Error(error.message); - }; - - this.deps.toasts.addInfo({ - title: i18n.translate('queryEnhancements.sqlQueryInfo', { - defaultMessage: 'Starting query job...', - }), - }); - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( - concatMap((jobResponse) => { - const df = jobResponse.body; - const dataFramePolling = new DataFramePolling( - () => fetchDataFramePolling(dfContext, df), - 5000, - onPollingSuccess, - onPollingError - ); - return dataFramePolling.fetch().pipe( - map(() => { - const dfPolling = dataFramePolling.data; - dfPolling.type = DATA_FRAME_TYPES.DEFAULT; - return dfPolling; - }) - ); - }) - ); - } - - public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); - } -} diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index 9206b4300b83..433fd98ef888 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -6,8 +6,8 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; import { i18n } from '@osd/i18n'; -import { concatMap } from 'rxjs/operators'; -import { getRawDataFrame, getRawQueryString } from '../../../data/common'; +import { concatMap, map } from 'rxjs/operators'; +import { DATA_FRAME_TYPES, getRawDataFrame, getRawQueryString } from '../../../data/common'; import { DataPublicPluginStart, IOpenSearchDashboardsSearchRequest, @@ -16,18 +16,21 @@ import { SearchInterceptor, SearchInterceptorDeps, } from '../../../data/public'; -import { API, FetchDataFrameContext, SEARCH_STRATEGY, fetchDataFrame } from '../../common'; +import { + API, + DataFramePolling, + FetchDataFrameContext, + SEARCH_STRATEGY, + fetchDataFrame, + fetchDataFramePolling, +} from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../services'; export class SQLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { + constructor(deps: SearchInterceptorDeps) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { @@ -78,7 +81,92 @@ export class SQLSearchInterceptor extends SearchInterceptor { return fetchDataFrame(dfContext, queryString, dataFrame); } + protected runSearchAsync( + request: IOpenSearchDashboardsSearchRequest, + signal?: AbortSignal, + strategy?: string + ): Observable { + const { id, ...searchRequest } = request; + const path = trimEnd(API.SQL_ASYNC_SEARCH); + const dfContext: FetchDataFrameContext = { + http: this.deps.http, + path, + signal, + }; + + const dataFrame = getRawDataFrame(searchRequest); + if (!dataFrame) { + return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); + } + + const queryString = + dataFrame.meta?.queryConfig?.formattedQs() ?? getRawQueryString(searchRequest) ?? ''; + + dataFrame.meta = { + ...dataFrame.meta, + queryConfig: { + ...dataFrame.meta.queryConfig, + }, + }; + + const onPollingSuccess = (pollingResult: any) => { + if (pollingResult && pollingResult.body.meta.status === 'SUCCESS') { + return false; + } + if (pollingResult && pollingResult.body.meta.status === 'FAILED') { + const jsError = new Error(pollingResult.data.error.response); + this.deps.toasts.addError(jsError, { + title: i18n.translate('queryEnhancements.sqlQueryError', { + defaultMessage: 'Could not complete the SQL async query', + }), + toastMessage: pollingResult.data.error.response, + }); + return false; + } + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryPolling', { + defaultMessage: 'Polling query job results...', + }), + }); + + return true; + }; + + const onPollingError = (error: Error) => { + throw new Error(error.message); + }; + + this.deps.toasts.addInfo({ + title: i18n.translate('queryEnhancements.sqlQueryInfo', { + defaultMessage: 'Starting query job...', + }), + }); + return fetchDataFrame(dfContext, queryString, dataFrame).pipe( + concatMap((jobResponse) => { + const df = jobResponse.body; + const dataFramePolling = new DataFramePolling( + () => fetchDataFramePolling(dfContext, df), + 5000, + onPollingSuccess, + onPollingError + ); + return dataFramePolling.fetch().pipe( + map(() => { + const dfPolling = dataFramePolling.data; + dfPolling.type = DATA_FRAME_TYPES.DEFAULT; + return dfPolling; + }) + ); + }) + ); + } + public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL); + return this.runSearch( + request, + options.abortSignal, + !!options.isAsync ? SEARCH_STRATEGY.SQL_ASYNC : SEARCH_STRATEGY.SQL + ); } } From 6b5e2db6d26c47eca0d25230aa6ec245487c23f5 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Mon, 22 Jul 2024 18:55:42 -0700 Subject: [PATCH 47/59] s3 datasources Signed-off-by: Kawika Avilla --- .../dataset_navigator/dataset_navigator.tsx | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 97762f9ff57e..c8c1553c74de 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -75,11 +75,9 @@ export const DataSetNavigator = ({ const [selectedObject, setSelectedObject] = useState(); const [selectedDataSource, setSelectedDataSource] = useState(); const [selectedDataSourceObjects, setSelectedDataSourceObjects] = useState([]); - const [selectedExternalDataSource, setSelectedExternalDataSource] = useState< - ExternalDataSource - >(); + const [selectedExternalDataSource, setSelectedExternalDataSource] = useState(); const [dataSources, setDataSources] = useState([]); - const [externalDataSources, setExternalDataSources] = useState([]); + const [externalDataSources, setExternalDataSources] = useState([]); // TODO iindexpattern const [indexPatterns, setIndexPatterns] = useState([]); @@ -145,7 +143,13 @@ export const DataSetNavigator = ({ const status = dataSourcesLoadStatus.toLowerCase(); const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); if (status === DirectQueryLoadingStatus.SUCCESS) { - setExternalDataSources(externalDataSourcesCache.externalDataSources); + setExternalDataSources( + externalDataSourcesCache.externalDataSources.map((ds) => ({ + id: ds.dataSourceRef, + name: ds.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + })) + ); } else if ( status === DirectQueryLoadingStatus.CANCELED || status === DirectQueryLoadingStatus.FAILED @@ -159,7 +163,7 @@ export const DataSetNavigator = ({ if (selectedExternalDataSource) { const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( selectedExternalDataSource.name, - selectedExternalDataSource.dataSourceRef + selectedExternalDataSource.id ); if ( (dataSourceCache.status === CachedDataSourceStatus.Empty || @@ -168,7 +172,7 @@ export const DataSetNavigator = ({ ) { startLoadingDatabases({ dataSourceName: selectedExternalDataSource.name, - dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + dataSourceMDSId: selectedExternalDataSource.id, }); } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { setCachedDatabases(dataSourceCache.databases); @@ -183,7 +187,7 @@ export const DataSetNavigator = ({ if (selectedExternalDataSource) { const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( selectedExternalDataSource?.name, - selectedExternalDataSource?.dataSourceRef + selectedExternalDataSource?.id ); if (status === DirectQueryLoadingStatus.SUCCESS) { setCachedDatabases(dataSourceCache.databases); @@ -204,7 +208,7 @@ export const DataSetNavigator = ({ databaseCache = CatalogCacheManager.getDatabase( selectedExternalDataSource.name, selectedDatabase, - selectedExternalDataSource.dataSourceRef + selectedExternalDataSource.id ); } catch (error) { return; @@ -217,7 +221,7 @@ export const DataSetNavigator = ({ startLoadingTables({ dataSourceName: selectedExternalDataSource.name, databaseName: selectedDatabase, - dataSourceMDSId: selectedExternalDataSource.dataSourceRef, + dataSourceMDSId: selectedExternalDataSource.id, }); } else if (databaseCache.status === CachedDataSourceStatus.Updated) { setCachedTables(databaseCache.tables); @@ -235,7 +239,7 @@ export const DataSetNavigator = ({ databaseCache = CatalogCacheManager.getDatabase( selectedExternalDataSource.name, selectedDatabase, - selectedExternalDataSource.dataSourceRef + selectedExternalDataSource.id ); } catch (error) { return; @@ -390,6 +394,9 @@ export const DataSetNavigator = ({ const indicesLabel = i18n.translate('data.query.dataSetNavigator.indicesName', { defaultMessage: 'Indexes', }); + const S3DataSourcesLabel = i18n.translate('data.query.dataSetNavigator.S3DataSourcesLabel', { + defaultMessage: 'S3', + }); return ( { const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); @@ -455,7 +462,13 @@ export const DataSetNavigator = ({ } else if ( externalDataSourcesCache.status === CachedDataSourceStatus.Updated ) { - setExternalDataSources(externalDataSourcesCache.externalDataSources); + setExternalDataSources( + externalDataSourcesCache.externalDataSources.map((ds) => ({ + id: ds.dataSourceRef, + name: ds.name, + type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, + })) + ); } }, }, @@ -510,7 +523,7 @@ export const DataSetNavigator = ({ id: 4, title: (
- S3 + {S3DataSourcesLabel} {CatalogCacheManager.getExternalDataSourcesCache().status === CachedDataSourceStatus.Updated && RefreshButton}
From 43a83e08415c7e8b5fdf70a5a6290590e01690f2 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 02:35:00 +0000 Subject: [PATCH 48/59] set data set for table Signed-off-by: Kawika Avilla --- .../dataset_navigator/dataset_navigator.tsx | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index c8c1553c74de..8db143263df6 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -32,7 +32,7 @@ import { useLoadTablesToCache, } from './lib/catalog_cache/cache_loader'; import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; -import { CachedDataSourceStatus, DirectQueryLoadingStatus, ExternalDataSource } from './lib/types'; +import { CachedDataSourceStatus, DirectQueryLoadingStatus } from './lib/types'; import { getIndexPatterns, getNotifications, @@ -315,6 +315,7 @@ export const DataSetNavigator = ({ id: dataSet.id, title: dataSet.title, fields: fieldsMap, + type: dataSet.type, dataSourceRef: { id: dataSet.dataSourceRef?.id!, name: dataSet.dataSourceRef?.name!, @@ -345,7 +346,10 @@ export const DataSetNavigator = ({ }; const onDataSetSelected = async (dataSet: SimpleDataSet) => { - if (dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY) { + if ( + dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY || + dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC + ) { await createTemporaryIndexPattern(dataSet); } @@ -555,6 +559,18 @@ export const DataSetNavigator = ({ items: [ ...cachedTables.map((table) => ({ name: table.name, + onClick: async () => { + await handleSelectedDataSet({ + id: table.name, + title: `${selectedExternalDataSource!.name}.${selectedDatabase}.${table.name}`, + dataSourceRef: { + id: selectedExternalDataSource!.id, + name: selectedExternalDataSource!.name, + type: selectedExternalDataSource!.type, + }, + type: SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC, + }); + }, })), ], content:
{isCatalogCacheFetching(tablesLoadStatus) && LoadingSpinner}
, From e303635c8c68850b3b53d6e9c2d2e35450b745ed Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 02:40:21 +0000 Subject: [PATCH 49/59] pass it in the options Signed-off-by: Kawika Avilla --- src/plugins/data/common/search/search_source/search_source.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 2c553fdbed97..6f1ac7a3bacc 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -358,6 +358,7 @@ export class SearchSource { if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) { response = await this.legacyFetch(searchRequest, options); } else if (this.isUnsupportedRequest(searchRequest)) { + options = { ...options, isAsync: this.getField('type')?.includes('ASYNC') }; response = await this.fetchExternalSearch(searchRequest, options); } else { const indexPattern = this.getField('index'); From 41ae189c542015b13ed30f667f1b452921914fa7 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 02:43:15 +0000 Subject: [PATCH 50/59] run query different Signed-off-by: Kawika Avilla --- .../public/search/sql_search_interceptor.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index 433fd98ef888..eef68910fe2d 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -163,10 +163,9 @@ export class SQLSearchInterceptor extends SearchInterceptor { } public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - return this.runSearch( - request, - options.abortSignal, - !!options.isAsync ? SEARCH_STRATEGY.SQL_ASYNC : SEARCH_STRATEGY.SQL - ); + if (options.isAsync) { + return this.runSearchAsync(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); + } + return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL); } } From 2540a92166b1f8cc7d6720ab4d443794662eec81 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 03:14:00 +0000 Subject: [PATCH 51/59] dont parse query string Signed-off-by: Kawika Avilla --- .../query_enhancements/public/search/sql_search_interceptor.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index eef68910fe2d..429c7f4dc543 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -99,8 +99,7 @@ export class SQLSearchInterceptor extends SearchInterceptor { return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); } - const queryString = - dataFrame.meta?.queryConfig?.formattedQs() ?? getRawQueryString(searchRequest) ?? ''; + const queryString = getRawQueryString(searchRequest) ?? ''; dataFrame.meta = { ...dataFrame.meta, From 52b7bd1e90cfeef232fe09ee7fef97e7a688acb8 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 03:23:42 +0000 Subject: [PATCH 52/59] not capitalization Signed-off-by: Kawika Avilla --- src/plugins/data/common/search/search_source/search_source.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 6f1ac7a3bacc..90d7b22e6c96 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -358,7 +358,7 @@ export class SearchSource { if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) { response = await this.legacyFetch(searchRequest, options); } else if (this.isUnsupportedRequest(searchRequest)) { - options = { ...options, isAsync: this.getField('type')?.includes('ASYNC') }; + options = { ...options, isAsync: this.getField('type')?.includes('async') }; response = await this.fetchExternalSearch(searchRequest, options); } else { const indexPattern = this.getField('index'); From 046526b7ce599ae91a00579b96b72be4197230bd Mon Sep 17 00:00:00 2001 From: "opensearch-changeset-bot[bot]" <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 03:57:39 +0000 Subject: [PATCH 53/59] Changeset file for PR #7368 created/updated --- changelogs/fragments/7368.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/7368.yml diff --git a/changelogs/fragments/7368.yml b/changelogs/fragments/7368.yml new file mode 100644 index 000000000000..c8316dc939f0 --- /dev/null +++ b/changelogs/fragments/7368.yml @@ -0,0 +1,2 @@ +feat: +- [Discover] Adds a dataset selector for Discover ([#7368](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7368)) \ No newline at end of file From 8a57b08021a9c003f3c60acb8475f1c5e30e67e9 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 05:22:36 +0000 Subject: [PATCH 54/59] connections imported right Signed-off-by: Kawika Avilla --- src/plugins/data/public/antlr/shared/utils.ts | 24 ++-- .../dataset_navigator/dataset_navigator.tsx | 103 ++++++++++-------- .../lib/catalog_cache/cache_loader.tsx | 4 +- .../query_enhancements/public/plugin.tsx | 17 ++- .../components/query_assist_bar.tsx | 4 +- .../query_assist/utils/create_extension.tsx | 4 +- .../public/search/ppl_search_interceptor.ts | 7 +- 7 files changed, 84 insertions(+), 79 deletions(-) diff --git a/src/plugins/data/public/antlr/shared/utils.ts b/src/plugins/data/public/antlr/shared/utils.ts index b2658b304e0f..48533048d70b 100644 --- a/src/plugins/data/public/antlr/shared/utils.ts +++ b/src/plugins/data/public/antlr/shared/utils.ts @@ -12,7 +12,7 @@ export interface IDataSourceRequestHandlerParams { } export const getRawSuggestionData$ = ( - connectionsService, + connectionsService: { getSelectedConnection$: any }, dataSourceReuqstHandler: ({ dataSourceId, title, @@ -21,11 +21,11 @@ export const getRawSuggestionData$ = ( ) => connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), - switchMap((connection) => { + switchMap((connection: any) => { if (connection === undefined) { return from(defaultReuqstHandler()); } - const dataSourceId = connection?.id; + const dataSourceId = connection?.dataSource.id; const title = connection?.attributes?.title; return from(dataSourceReuqstHandler({ dataSourceId, title })); }) @@ -34,8 +34,8 @@ export const getRawSuggestionData$ = ( export const fetchData = ( tables: string[], queryFormatter: (table: string, dataSourceId?: string, title?: string) => any, - api, - connectionService + api: any, + connectionService: any ): Promise => { return new Promise((resolve, reject) => { getRawSuggestionData$( @@ -65,8 +65,8 @@ export const fetchData = ( ); } ).subscribe({ - next: (dataFrames) => resolve(dataFrames), - error: (err) => { + next: (dataFrames: any) => resolve(dataFrames), + error: (err: any) => { // TODO: pipe error to UI reject(err); }, @@ -74,7 +74,11 @@ export const fetchData = ( }); }; -export const fetchTableSchemas = (tables: string[], api, connectionService): Promise => { +export const fetchTableSchemas = ( + tables: string[], + api: any, + connectionService: any +): Promise => { return fetchData( tables, (table, dataSourceId, title) => ({ @@ -96,8 +100,8 @@ export const fetchTableSchemas = (tables: string[], api, connectionService): Pro export const fetchColumnValues = ( tables: string[], column: string, - api, - connectionService + api: any, + connectionService: any ): Promise => { return fetchData( tables, diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index 8db143263df6..a0f134a47bd2 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -158,29 +158,6 @@ export const DataSetNavigator = ({ } }, [dataSourcesLoadStatus]); - // Start loading databases for datasource - useEffect(() => { - if (selectedExternalDataSource) { - const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( - selectedExternalDataSource.name, - selectedExternalDataSource.id - ); - if ( - (dataSourceCache.status === CachedDataSourceStatus.Empty || - dataSourceCache.status === CachedDataSourceStatus.Failed) && - !isCatalogCacheFetching(databasesLoadStatus) - ) { - startLoadingDatabases({ - dataSourceName: selectedExternalDataSource.name, - dataSourceMDSId: selectedExternalDataSource.id, - }); - } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { - setCachedDatabases(dataSourceCache.databases); - } - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [selectedExternalDataSource]); - // Retrieve databases from cache upon success useEffect(() => { const status = databasesLoadStatus.toLowerCase(); @@ -200,35 +177,65 @@ export const DataSetNavigator = ({ } }, [selectedExternalDataSource, databasesLoadStatus]); - // Start loading tables for selected database - useEffect(() => { - if (selectedExternalDataSource && selectedDatabase) { - let databaseCache; - try { - databaseCache = CatalogCacheManager.getDatabase( + // Start loading databases for datasource + const handleSelectExternalDataSource = useCallback( + async (externalDataSource) => { + if (selectedExternalDataSource) { + const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( selectedExternalDataSource.name, - selectedDatabase, selectedExternalDataSource.id ); - } catch (error) { - return; + if ( + (dataSourceCache.status === CachedDataSourceStatus.Empty || + dataSourceCache.status === CachedDataSourceStatus.Failed) && + !isCatalogCacheFetching(databasesLoadStatus) + ) { + startLoadingDatabases({ + dataSourceName: selectedExternalDataSource.name, + dataSourceMDSId: selectedExternalDataSource.id, + }); + } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { + setCachedDatabases(dataSourceCache.databases); + } } - if ( - databaseCache.status === CachedDataSourceStatus.Empty || - (databaseCache.status === CachedDataSourceStatus.Failed && - !isCatalogCacheFetching(tablesLoadStatus)) - ) { - startLoadingTables({ - dataSourceName: selectedExternalDataSource.name, - databaseName: selectedDatabase, - dataSourceMDSId: selectedExternalDataSource.id, - }); - } else if (databaseCache.status === CachedDataSourceStatus.Updated) { - setCachedTables(databaseCache.tables); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [selectedExternalDataSource] + ); + + // Start loading tables for selected database + const handleSelectExternalDatabase = useCallback( + async (externalDatabase) => { + if (selectedExternalDataSource && externalDatabase) { + let databaseCache; + try { + databaseCache = CatalogCacheManager.getDatabase( + selectedExternalDataSource.name, + selectedDatabase, + selectedExternalDataSource.id + ); + } catch (error) { + return; + } + if ( + databaseCache.status === CachedDataSourceStatus.Empty || + (databaseCache.status === CachedDataSourceStatus.Failed && + !isCatalogCacheFetching(tablesLoadStatus)) + ) { + await startLoadingTables({ + dataSourceName: selectedExternalDataSource.name, + databaseName: externalDatabase, + dataSourceMDSId: selectedExternalDataSource.id, + }); + setSelectedDatabase(externalDatabase); + } else if (databaseCache.status === CachedDataSourceStatus.Updated) { + setCachedTables(databaseCache.tables); + } } - } + }, // eslint-disable-next-line react-hooks/exhaustive-deps - }, [selectedExternalDataSource, selectedDatabase]); + [selectedExternalDataSource, selectedDatabase] + ); // Retrieve tables from cache upon success useEffect(() => { @@ -547,7 +554,9 @@ export const DataSetNavigator = ({ items: [ ...cachedDatabases.map((db) => ({ name: db.name, - onClick: () => setSelectedDatabase(db.name), + onClick: async () => { + await handleSelectExternalDatabase(db.name); + }, panel: 6, })), ], diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx index 16ce324ad64f..9a42bfc74aa8 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx @@ -318,7 +318,7 @@ export const useLoadToCache = ( ); }; - const startLoading = ({ + const startLoading = async ({ dataSourceName, dataSourceMDSId, databaseName, @@ -340,7 +340,7 @@ export const useLoadToCache = ( if (sessionId) { requestPayload = { ...requestPayload, sessionId }; } - sqlService + await sqlService .fetch(requestPayload, dataSourceMDSId) .then((result) => { setAsyncSessionId(dataSourceName, getObjValue(result, 'sessionId', null)); diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index 3f1d062e134d..b74c00ced7e0 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -43,16 +43,13 @@ export class QueryEnhancementsPlugin http: core.http, }); - const pplSearchInterceptor = new PPLSearchInterceptor( - { - toasts: core.notifications.toasts, - http: core.http, - uiSettings: core.uiSettings, - startServices: core.getStartServices(), - usageCollector: data.search.usageCollector, - }, - this.connectionsService - ); + const pplSearchInterceptor = new PPLSearchInterceptor({ + toasts: core.notifications.toasts, + http: core.http, + uiSettings: core.uiSettings, + startServices: core.getStartServices(), + usageCollector: data.search.usageCollector, + }); const sqlSearchInterceptor = new SQLSearchInterceptor({ toasts: core.notifications.toasts, diff --git a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx index 84ac854c980f..92393ef1a6c0 100644 --- a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx +++ b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx @@ -12,7 +12,7 @@ import { } from '../../../../data/public'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { QueryAssistParameters } from '../../../common/query_assist'; -import { ConnectionsService } from '../../data_source_connection'; +import { ConnectionsService } from '../../services'; import { getStorage } from '../../services'; import { useGenerateQuery } from '../hooks'; import { getPersistedLog, ProhibitedQueryError } from '../utils'; @@ -45,7 +45,7 @@ export const QueryAssistBar: React.FC = (props) => { const subscription = props.connectionsService .getSelectedConnection$() .subscribe((connection) => { - dataSourceIdRef.current = connection?.id; + dataSourceIdRef.current = connection?.dataSource.id; }); return () => subscription.unsubscribe(); }, [props.connectionsService]); diff --git a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx index bd990b57f5a7..c76cd13bbf5e 100644 --- a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx @@ -13,7 +13,7 @@ import { } from '../../../../data/public'; import { API } from '../../../common'; import { ConfigSchema } from '../../../common/config'; -import { ConnectionsService } from '../../data_source_connection'; +import { ConnectionsService } from '../../services'; import { QueryAssistBar, QueryAssistBanner } from '../components'; /** @@ -28,7 +28,7 @@ const getAvailableLanguages$ = ( connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), switchMap(async (connection) => { - const dataSourceId = connection?.id; + const dataSourceId = connection?.dataSource.id; const cached = availableLanguagesByDataSource.get(dataSourceId); if (cached !== undefined) return cached; const languages = await http diff --git a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts index ffaea6718b99..cc218fb6990f 100644 --- a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts @@ -5,7 +5,6 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; -import { i18n } from '@osd/i18n'; import { concatMap } from 'rxjs/operators'; import { DataFrameAggConfig, @@ -34,16 +33,12 @@ import { fetchDataFrame, } from '../../common'; import { QueryEnhancementsPluginStartDependencies } from '../types'; -import { ConnectionsService } from '../services'; export class PPLSearchInterceptor extends SearchInterceptor { protected queryService!: DataPublicPluginStart['query']; protected aggsService!: DataPublicPluginStart['search']['aggs']; - constructor( - deps: SearchInterceptorDeps, - private readonly connectionsService: ConnectionsService - ) { + constructor(deps: SearchInterceptorDeps) { super(deps); deps.startServices.then(([coreStart, depsStart]) => { From f2ca5b001b6ad52728005be55d243c29062f6375 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 06:15:30 +0000 Subject: [PATCH 55/59] handle select external Signed-off-by: Kawika Avilla --- .../data/public/ui/dataset_navigator/dataset_navigator.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index a0f134a47bd2..a08fb627bf32 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -461,7 +461,7 @@ export const DataSetNavigator = ({ { name: S3DataSourcesLabel, panel: 4, - onClick: () => { + onClick: async () => { const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); if ( (externalDataSourcesCache.status === CachedDataSourceStatus.Empty || @@ -473,7 +473,7 @@ export const DataSetNavigator = ({ } else if ( externalDataSourcesCache.status === CachedDataSourceStatus.Updated ) { - setExternalDataSources( + await handleSelectExternalDataSource( externalDataSourcesCache.externalDataSources.map((ds) => ({ id: ds.dataSourceRef, name: ds.name, From 7f1a07d3162bcaff56c41ad59fba89632b760f76 Mon Sep 17 00:00:00 2001 From: Sean Li Date: Mon, 22 Jul 2024 23:28:48 -0700 Subject: [PATCH 56/59] initial implementation for url state Signed-off-by: Sean Li --- .../search/search_source/search_source.ts | 3 ++ .../dataset_navigator/dataset_navigator.tsx | 40 ++++++++++++------- .../public/ui/dataset_navigator/lib/types.tsx | 18 +++++++++ .../public/components/sidebar/index.tsx | 15 +++++-- .../utils/state_management/metadata_slice.ts | 39 ++++++++++-------- .../utils/update_search_source.ts | 24 +++++------ .../utils/use_index_pattern.ts | 24 ++++++++++- 7 files changed, 114 insertions(+), 49 deletions(-) diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 90d7b22e6c96..3236da2e572c 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -358,6 +358,9 @@ export class SearchSource { if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) { response = await this.legacyFetch(searchRequest, options); } else if (this.isUnsupportedRequest(searchRequest)) { + const indexPattern = this.getField('index'); + searchRequest.dataSourceId = indexPattern?.dataSourceRef?.id; + options = { ...options, isAsync: this.getField('type')?.includes('async') }; response = await this.fetchExternalSearch(searchRequest, options); } else { diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index a08fb627bf32..c94e956710b9 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -32,7 +32,7 @@ import { useLoadTablesToCache, } from './lib/catalog_cache/cache_loader'; import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; -import { CachedDataSourceStatus, DirectQueryLoadingStatus } from './lib/types'; +import { CachedDataSourceStatus, DataSet, DirectQueryLoadingStatus } from './lib/types'; import { getIndexPatterns, getNotifications, @@ -49,14 +49,16 @@ import { } from './lib'; export interface DataSetNavigatorProps { - dataSetId: string | undefined; + dataSet: Omit | undefined; + indexPatternId: string | undefined; savedObjectsClient?: SavedObjectsClientContract; http?: HttpStart; onSelectDataSet: (dataSet: SimpleDataSet) => void; } export const DataSetNavigator = ({ - dataSetId, + dataSet, + indexPatternId, savedObjectsClient, http, onSelectDataSet, @@ -71,7 +73,7 @@ export const DataSetNavigator = ({ const [isLoading, setIsLoading] = useState(false); const [isMounted, setIsMounted] = useState(false); const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false); - const [selectedDataSet, setSelectedDataSet] = useState(); + // const [selectedDataSet, setSelectedDataSet] = useState({ id: dataSet?.id, title: dataSet?.datasource?.name, }); const [selectedObject, setSelectedObject] = useState(); const [selectedDataSource, setSelectedDataSource] = useState(); const [selectedDataSourceObjects, setSelectedDataSourceObjects] = useState([]); @@ -119,12 +121,18 @@ export const DataSetNavigator = ({ setIsExternalDataSourcesEnabled(isExternalDSEnabled); setIndexPatterns(defaultIndexPatterns); setDataSources(defaultDataSources); - if (!selectedDataSet && dataSetId) { + // If there's no dataset, then should be an index pattern + if (!dataSet && indexPatternId) { const selectedPattern = defaultIndexPatterns.find( - (pattern) => pattern.id === dataSetId + (pattern) => pattern.id === indexPatternId ); if (selectedPattern) { - setSelectedDataSet({ + // setSelectedDataSet({ + // id: selectedPattern.id ?? selectedPattern.title, + // title: selectedPattern.title, + // type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + // }); + onSelectDataSet({ id: selectedPattern.id ?? selectedPattern.title, title: selectedPattern.title, type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, @@ -137,7 +145,7 @@ export const DataSetNavigator = ({ setIsLoading(false); }); } - }, [indexPatternsService, dataSetId, savedObjectsClient, selectedDataSet, isMounted, http]); + }, [indexPatternsService, savedObjectsClient, isMounted, http]); useEffect(() => { const status = dataSourcesLoadStatus.toLowerCase(); @@ -366,14 +374,19 @@ export const DataSetNavigator = ({ dataSourceRef: dataSet.dataSourceRef?.id, }); searchService.df.clear(); - onSelectDataSet(dataSet); + onSelectDataSet({ + id: dataSet.id, + title: dataSet.title, + dataSourceRef: dataSet.dataSourceRef, + type: dataSet.type, + }); queryService.queryString.setQuery(getInitialQuery(dataSet)); closePopover(); }; if (ds) { await onDataSetSelected(ds); - setSelectedDataSet(ds); + // setSelectedDataSet(ds); } }, [ @@ -419,11 +432,8 @@ export const DataSetNavigator = ({ iconSide="right" onClick={() => setIsOpen(!isOpen)} > - {`${selectedDataSet?.dataSourceRef ? `${selectedDataSet.dataSourceRef.name}::` : ''}${ - selectedDataSet?.title ?? - i18n.translate('data.query.dataSetNavigator.selectDataSet', { - defaultMessage: 'Select data set', - }) + {`${dataSet?.dataSourceRef?.name ? `${dataSet.dataSourceRef?.name}::` : ''}${ + dataSet?.title }`} } diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx index a51870b4e74f..8d90e84cb716 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx @@ -314,3 +314,21 @@ export interface ExternalDataSourcesCacheData { lastUpdated: string; status: CachedDataSourceStatus; } + +interface DataSourceMeta { + // ref: string; // MDS ID + // dsName?: string; // flint datasource + id: string; + name: string; + type?: string; +} + +export interface DataSet { + id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table + datasource?: DataSourceMeta; + meta?: { + timestampField: string; + mapping?: any; + }; + type?: 'dataSet' | 'temporary'; +} diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index eb0850f8f56d..16bd8f72bcb7 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -20,7 +20,9 @@ import { import './index.scss'; export const Sidebar: FC = ({ children }) => { - const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); + const { indexPattern: indexPatternId, dataSet: dataSet } = useTypedSelector( + (state) => state.metadata + ); const dispatch = useTypedDispatch(); const [selectedSources, setSelectedSources] = useState([]); const [dataSourceOptionList, setDataSourceOptionList] = useState([]); @@ -141,8 +143,9 @@ export const Sidebar: FC = ({ children }) => { const handleDataSetSelection = useCallback( (dataSet: any) => { batch(() => { - dispatch(setIndexPattern(dataSet!.id)); - dispatch(setDataSet(dataSet)); + const { id, ...ds } = dataSet; + dispatch(setIndexPattern(id)); + dispatch(setDataSet(ds)); }); }, [dispatch] @@ -166,7 +169,11 @@ export const Sidebar: FC = ({ children }) => { containerRef.current = node; }} > - + )} {!isEnhancementsEnabled && ( diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index a8bc30e89b0d..c9700a086398 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -5,27 +5,32 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import { DataExplorerServices } from '../../types'; +import { SimpleDataSet } from '../../../../data/common'; -interface DataSourceMeta { - ref: string; // MDS ID - dsName?: string; // flint datasource -} +// interface DataSourceMeta { +// // ref: string; // MDS ID +// // dsName?: string; // flint datasource +// id: string; +// name: string; +// type?: string; +// } -export interface DataSet { - id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table - datasource?: DataSourceMeta; - meta?: { - timestampField: string; - mapping?: any; - }; - type?: 'dataSet' | 'temporary'; -} +// export interface DataSet { +// // id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table +// // TODO: treating index pattern ID as DataSet ID +// datasource?: DataSourceMeta; +// meta?: { +// timestampField: string; +// mapping?: any; +// }; +// type?: 'dataSet' | 'temporary'; +// } export interface MetadataState { indexPattern?: string; originatingApp?: string; view?: string; - dataSet?: DataSet; + dataSet?: Omit; } const initialState: MetadataState = {}; @@ -44,9 +49,9 @@ export const getPreloadedState = async ({ ...initialState, originatingApp, indexPattern: defaultIndexPattern?.id, - dataSet: { - id: defaultIndexPattern?.id, - }, + // dataSet: { + // id: defaultIndexPattern?.id, + // }, }; return preloadedState; diff --git a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts index a8480fdad18a..aa35dd37c229 100644 --- a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts +++ b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts @@ -30,18 +30,18 @@ export const updateSearchSource = async ({ histogramConfigs, }: Props) => { const { uiSettings, data } = services; - let dataSet = indexPattern; - const dataFrame = searchSource?.getDataFrame(); - if ( - searchSource && - dataFrame && - dataFrame.name && - dataFrame.name !== '' && - dataSet.title !== dataFrame.name - ) { - dataSet = data.indexPatterns.getByTitle(dataFrame.name, true) ?? dataSet; - searchSource.setField('index', dataSet); - } + const dataSet = indexPattern; + // const dataFrame = searchSource?.getDataFrame(); + // if ( + // searchSource && + // dataFrame && + // dataFrame.name && + // dataFrame.name !== '' && + // dataSet.title !== dataFrame.name + // ) { + // dataSet = data.indexPatterns.getByTitle(dataFrame.name, true) ?? dataSet; + // searchSource.setField('index', dataSet); + // } const sortForSearchSource = getSortForSearchSource( sort, diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index e8a81234278e..58c83969bc7a 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -26,10 +26,32 @@ import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; */ export const useIndexPattern = (services: DiscoverViewServices) => { const indexPatternIdFromState = useSelector((state) => state.metadata.indexPattern); + const dataSetFromState = useSelector((state) => state.metadata.dataSet); const [indexPattern, setIndexPattern] = useState(undefined); const { data, toastNotifications, uiSettings: config, store } = services; useEffect(() => { + const checkDataSet = async (dataset: any, indexPatternIdFromState: string) => { + if (dataset) { + const temporaryIndexPattern = await data.indexPatterns.create( + { + id: dataset.id, + title: dataset.id, + type: dataset.type, + dataSourceRef: { + id: dataset.datasource?.ref!, + name: dataset.datasource?.dsName!, + type: dataset.type!, + }, + timeFieldName: dataset.meta?.timestampField, + }, + true + ); + data.indexPatterns.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); + } + fetchIndexPatternDetails(indexPatternIdFromState); + }; + let isMounted = true; const fetchIndexPatternDetails = (id: string) => { @@ -65,7 +87,7 @@ export const useIndexPattern = (services: DiscoverViewServices) => { fetchIndexPatternDetails(newId); }); } else { - fetchIndexPatternDetails(indexPatternIdFromState); + checkDataSet(dataSetFromState, indexPatternIdFromState); } return () => { From 6a1f4d13c57ca356f2253f1a6caf54e8e86667e4 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 17:57:59 +0000 Subject: [PATCH 57/59] restore the dataset metadata stuff Signed-off-by: Kawika Avilla --- .../public/components/sidebar/index.tsx | 4 +- src/plugins/data_explorer/public/index.ts | 2 +- .../utils/state_management/metadata_slice.ts | 24 +----------- .../utils/state_management/index.ts | 2 +- .../utils/use_index_pattern.ts | 39 ++++++++++++------- 5 files changed, 30 insertions(+), 41 deletions(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 16bd8f72bcb7..4908e678be7b 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -141,9 +141,9 @@ export const Sidebar: FC = ({ children }) => { ); const handleDataSetSelection = useCallback( - (dataSet: any) => { + (selectedDataSet: any) => { batch(() => { - const { id, ...ds } = dataSet; + const { id, ...ds } = selectedDataSet; dispatch(setIndexPattern(id)); dispatch(setDataSet(ds)); }); diff --git a/src/plugins/data_explorer/public/index.ts b/src/plugins/data_explorer/public/index.ts index 6825e1d82e53..6b0561261c16 100644 --- a/src/plugins/data_explorer/public/index.ts +++ b/src/plugins/data_explorer/public/index.ts @@ -18,5 +18,5 @@ export { useTypedSelector, useTypedDispatch, setIndexPattern, - setDataset, + setDataSet, } from './utils/state_management'; diff --git a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts index c9700a086398..fa41a29259e3 100644 --- a/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts +++ b/src/plugins/data_explorer/public/utils/state_management/metadata_slice.ts @@ -7,25 +7,6 @@ import { createSlice, PayloadAction } from '@reduxjs/toolkit'; import { DataExplorerServices } from '../../types'; import { SimpleDataSet } from '../../../../data/common'; -// interface DataSourceMeta { -// // ref: string; // MDS ID -// // dsName?: string; // flint datasource -// id: string; -// name: string; -// type?: string; -// } - -// export interface DataSet { -// // id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table -// // TODO: treating index pattern ID as DataSet ID -// datasource?: DataSourceMeta; -// meta?: { -// timestampField: string; -// mapping?: any; -// }; -// type?: 'dataSet' | 'temporary'; -// } - export interface MetadataState { indexPattern?: string; originatingApp?: string; @@ -49,9 +30,6 @@ export const getPreloadedState = async ({ ...initialState, originatingApp, indexPattern: defaultIndexPattern?.id, - // dataSet: { - // id: defaultIndexPattern?.id, - // }, }; return preloadedState; @@ -64,7 +42,7 @@ export const slice = createSlice({ setIndexPattern: (state, action: PayloadAction) => { state.indexPattern = action.payload; }, - setDataSet: (state, action: PayloadAction) => { + setDataSet: (state, action: PayloadAction>) => { state.dataSet = action.payload; }, setOriginatingApp: (state, action: PayloadAction) => { diff --git a/src/plugins/discover/public/application/utils/state_management/index.ts b/src/plugins/discover/public/application/utils/state_management/index.ts index a8b732b1c583..e6df7e4774b8 100644 --- a/src/plugins/discover/public/application/utils/state_management/index.ts +++ b/src/plugins/discover/public/application/utils/state_management/index.ts @@ -7,7 +7,7 @@ import { TypedUseSelectorHook } from 'react-redux'; import { RootState, setIndexPattern as updateIndexPattern, - setDataset as updateDataSet, + setDataSet as updateDataSet, useTypedDispatch, useTypedSelector, } from '../../../../../data_explorer/public'; diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index 58c83969bc7a..28b239064996 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -6,6 +6,7 @@ import { useEffect, useState } from 'react'; import { i18n } from '@osd/i18n'; import { IndexPattern } from '../../../../../data/public'; +import { SimpleDataSet } from '../../../../../data/common'; import { useSelector, updateIndexPattern } from '../../utils/state_management'; import { DiscoverViewServices } from '../../../build_services'; import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; @@ -31,25 +32,28 @@ export const useIndexPattern = (services: DiscoverViewServices) => { const { data, toastNotifications, uiSettings: config, store } = services; useEffect(() => { - const checkDataSet = async (dataset: any, indexPatternIdFromState: string) => { - if (dataset) { + const checkDataSet = async (idFromState: string, dataSet?: Omit) => { + if (dataSet) { const temporaryIndexPattern = await data.indexPatterns.create( { - id: dataset.id, - title: dataset.id, - type: dataset.type, - dataSourceRef: { - id: dataset.datasource?.ref!, - name: dataset.datasource?.dsName!, - type: dataset.type!, - }, - timeFieldName: dataset.meta?.timestampField, + id: idFromState, + title: dataSet.title, + ...(dataSet.dataSourceRef + ? { + dataSourceRef: { + id: dataSet.dataSourceRef.id ?? dataSet.dataSourceRef.name, + name: dataSet.dataSourceRef.name, + type: dataSet.type!, + }, + } + : {}), + timeFieldName: dataSet.timeFieldName, }, true ); data.indexPatterns.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); } - fetchIndexPatternDetails(indexPatternIdFromState); + fetchIndexPatternDetails(idFromState); }; let isMounted = true; @@ -87,13 +91,20 @@ export const useIndexPattern = (services: DiscoverViewServices) => { fetchIndexPatternDetails(newId); }); } else { - checkDataSet(dataSetFromState, indexPatternIdFromState); + checkDataSet(indexPatternIdFromState, dataSetFromState); } return () => { isMounted = false; }; - }, [indexPatternIdFromState, data.indexPatterns, toastNotifications, config, store]); + }, [ + indexPatternIdFromState, + data.indexPatterns, + toastNotifications, + config, + store, + dataSetFromState, + ]); return indexPattern; }; From 7b393782f2818ab6656f7cf3c67c502719281555 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 18:20:38 +0000 Subject: [PATCH 58/59] build dataset in state Signed-off-by: Kawika Avilla --- src/plugins/data/common/data_sets/types.ts | 6 +- .../dataset_navigator/dataset_navigator.tsx | 108 ++++++++---------- .../public/ui/dataset_navigator/lib/types.tsx | 2 +- .../public/components/sidebar/index.tsx | 6 +- 4 files changed, 55 insertions(+), 67 deletions(-) diff --git a/src/plugins/data/common/data_sets/types.ts b/src/plugins/data/common/data_sets/types.ts index cca70bf70cf9..4f4236d20275 100644 --- a/src/plugins/data/common/data_sets/types.ts +++ b/src/plugins/data/common/data_sets/types.ts @@ -17,8 +17,8 @@ export enum SIMPLE_DATA_SET_TYPES { } export interface SimpleObject { - id: string; - title: string; + id?: string; + title?: string; dataSourceRef?: SimpleDataSource; } @@ -33,5 +33,5 @@ export interface SimpleDataSet extends SimpleObject { fields?: any[]; timeFieldName?: string; timeFields?: any[]; - type: SIMPLE_DATA_SET_TYPES; + type?: SIMPLE_DATA_SET_TYPES; } diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index c94e956710b9..f104a9dea9f9 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -49,20 +49,14 @@ import { } from './lib'; export interface DataSetNavigatorProps { - dataSet: Omit | undefined; - indexPatternId: string | undefined; + dataSet: SimpleDataSet | undefined; savedObjectsClient?: SavedObjectsClientContract; http?: HttpStart; onSelectDataSet: (dataSet: SimpleDataSet) => void; } -export const DataSetNavigator = ({ - dataSet, - indexPatternId, - savedObjectsClient, - http, - onSelectDataSet, -}: DataSetNavigatorProps) => { +export const DataSetNavigator = (props: DataSetNavigatorProps) => { + const { savedObjectsClient, http, onSelectDataSet } = props; const searchService = getSearchService(); const queryService = getQueryService(); const uiService = getUiService(); @@ -73,14 +67,12 @@ export const DataSetNavigator = ({ const [isLoading, setIsLoading] = useState(false); const [isMounted, setIsMounted] = useState(false); const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false); - // const [selectedDataSet, setSelectedDataSet] = useState({ id: dataSet?.id, title: dataSet?.datasource?.name, }); const [selectedObject, setSelectedObject] = useState(); const [selectedDataSource, setSelectedDataSource] = useState(); const [selectedDataSourceObjects, setSelectedDataSourceObjects] = useState([]); const [selectedExternalDataSource, setSelectedExternalDataSource] = useState(); const [dataSources, setDataSources] = useState([]); const [externalDataSources, setExternalDataSources] = useState([]); - // TODO iindexpattern const [indexPatterns, setIndexPatterns] = useState([]); const [selectedDatabase, setSelectedDatabase] = useState(); @@ -121,23 +113,15 @@ export const DataSetNavigator = ({ setIsExternalDataSourcesEnabled(isExternalDSEnabled); setIndexPatterns(defaultIndexPatterns); setDataSources(defaultDataSources); - // If there's no dataset, then should be an index pattern - if (!dataSet && indexPatternId) { - const selectedPattern = defaultIndexPatterns.find( - (pattern) => pattern.id === indexPatternId - ); - if (selectedPattern) { - // setSelectedDataSet({ - // id: selectedPattern.id ?? selectedPattern.title, - // title: selectedPattern.title, - // type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, - // }); - onSelectDataSet({ - id: selectedPattern.id ?? selectedPattern.title, - title: selectedPattern.title, - type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, - }); - } + const selectedPattern = defaultIndexPatterns.find( + (pattern) => pattern.id === props.dataSet?.id + ); + if (selectedPattern) { + onSelectDataSet({ + id: selectedPattern.id ?? selectedPattern.title, + title: selectedPattern.title, + type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, + }); } }) .finally(() => { @@ -145,7 +129,14 @@ export const DataSetNavigator = ({ setIsLoading(false); }); } - }, [indexPatternsService, savedObjectsClient, isMounted, http]); + }, [ + indexPatternsService, + savedObjectsClient, + isMounted, + http, + onSelectDataSet, + props.dataSet?.id, + ]); useEffect(() => { const status = dataSourcesLoadStatus.toLowerCase(); @@ -319,30 +310,7 @@ export const DataSetNavigator = ({ ); const handleSelectedDataSet = useCallback( - async (ds: any) => { - const createTemporaryIndexPattern = async (dataSet: SimpleDataSet) => { - const fieldsMap = dataSet.fields?.reduce((acc: any, field: any) => { - acc[field.name] = field; - return acc; - }); - const temporaryIndexPattern = await indexPatternsService.create( - { - id: dataSet.id, - title: dataSet.title, - fields: fieldsMap, - type: dataSet.type, - dataSourceRef: { - id: dataSet.dataSourceRef?.id!, - name: dataSet.dataSourceRef?.name!, - type: dataSet.dataSourceRef?.type!, - }, - timeFieldName: dataSet.timeFieldName, - }, - true - ); - indexPatternsService.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); - }; - + async (selectedDataSet: SimpleDataSet) => { const getInitialQuery = (dataSet: SimpleDataSet) => { const language = uiService.Settings.getUserQueryLanguage(); const input = uiService.Settings.getQueryEnhancements(language)?.searchBar?.queryStringInput @@ -355,7 +323,7 @@ export const DataSetNavigator = ({ }; return { - query: input.replace('', dataSet.title), + query: input.replace('', dataSet.title!), language, }; }; @@ -365,12 +333,31 @@ export const DataSetNavigator = ({ dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY || dataSet.type === SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC ) { - await createTemporaryIndexPattern(dataSet); + const fieldsMap = dataSet.fields?.reduce((acc: any, field: any) => { + acc[field.name] = field; + return acc; + }); + const temporaryIndexPattern = await indexPatternsService.create( + { + id: dataSet.id, + title: dataSet.title, + fields: fieldsMap, + type: dataSet.type, + dataSourceRef: { + id: dataSet.dataSourceRef?.id!, + name: dataSet.dataSourceRef?.name!, + type: dataSet.dataSourceRef?.type!, + }, + timeFieldName: dataSet.timeFieldName, + }, + true + ); + indexPatternsService.saveToCache(temporaryIndexPattern.title, temporaryIndexPattern); } CatalogCacheManager.addRecentDataSet({ id: dataSet.id, - name: dataSet.title, + name: dataSet.title ?? dataSet.id!, dataSourceRef: dataSet.dataSourceRef?.id, }); searchService.df.clear(); @@ -384,9 +371,8 @@ export const DataSetNavigator = ({ closePopover(); }; - if (ds) { - await onDataSetSelected(ds); - // setSelectedDataSet(ds); + if (selectedDataSet) { + await onDataSetSelected(selectedDataSet); } }, [ @@ -432,8 +418,8 @@ export const DataSetNavigator = ({ iconSide="right" onClick={() => setIsOpen(!isOpen)} > - {`${dataSet?.dataSourceRef?.name ? `${dataSet.dataSourceRef?.name}::` : ''}${ - dataSet?.title + {`${props.dataSet?.dataSourceRef?.name ? `${props.dataSet.dataSourceRef?.name}::` : ''}${ + props.dataSet?.title }`} } diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx index 8d90e84cb716..1718323ca531 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx @@ -292,7 +292,7 @@ export interface RenderAccelerationDetailsFlyoutParams { } export interface DataSetOption { - id: string; + id?: string; name: string; dataSourceRef?: string; } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 4908e678be7b..a6bd995c1f58 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -170,8 +170,10 @@ export const Sidebar: FC = ({ children }) => { }} > From d6f84850e44cb78658c41784c976e1f000be8bbb Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 23 Jul 2024 21:19:33 +0000 Subject: [PATCH 59/59] null pointer Signed-off-by: Kawika Avilla --- src/plugins/data/public/antlr/shared/utils.ts | 2 +- .../dataset_navigator/dataset_navigator.tsx | 30 ++++++++++--------- .../components/query_assist_bar.tsx | 2 +- .../query_assist/utils/create_extension.tsx | 2 +- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/plugins/data/public/antlr/shared/utils.ts b/src/plugins/data/public/antlr/shared/utils.ts index 48533048d70b..94236aa81387 100644 --- a/src/plugins/data/public/antlr/shared/utils.ts +++ b/src/plugins/data/public/antlr/shared/utils.ts @@ -25,7 +25,7 @@ export const getRawSuggestionData$ = ( if (connection === undefined) { return from(defaultReuqstHandler()); } - const dataSourceId = connection?.dataSource.id; + const dataSourceId = connection?.dataSource?.id; const title = connection?.attributes?.title; return from(dataSourceReuqstHandler({ dataSourceId, title })); }) diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx index f104a9dea9f9..fefe37b36a00 100644 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx @@ -67,6 +67,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const [isLoading, setIsLoading] = useState(false); const [isMounted, setIsMounted] = useState(false); const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false); + const [selectedTimeFieldName, setSelectedTimeFieldName] = useState(); const [selectedObject, setSelectedObject] = useState(); const [selectedDataSource, setSelectedDataSource] = useState(); const [selectedDataSourceObjects, setSelectedDataSourceObjects] = useState([]); @@ -293,20 +294,21 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { }); const timeFields = fields.filter((field: any) => field.type === 'date'); - + const timeFieldName = timeFields?.length > 0 ? timeFields[0].name : undefined; + setSelectedTimeFieldName(timeFieldName); setSelectedObject({ id: object.id, title: object.title, fields, timeFields, - ...(timeFields[0]?.name ? { timeFieldName: timeFields[0].name } : {}), + timeFieldName: selectedTimeFieldName, dataSourceRef: object.dataSourceRef, type: SIMPLE_DATA_SET_TYPES.TEMPORARY, }); setIsLoading(false); } }, - [indexPatternsService] + [indexPatternsService, selectedTimeFieldName] ); const handleSelectedDataSet = useCallback( @@ -336,13 +338,13 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { const fieldsMap = dataSet.fields?.reduce((acc: any, field: any) => { acc[field.name] = field; return acc; - }); + }, {}); const temporaryIndexPattern = await indexPatternsService.create( { id: dataSet.id, title: dataSet.title, + // type: dataSet.type, fields: fieldsMap, - type: dataSet.type, dataSourceRef: { id: dataSet.dataSourceRef?.id!, name: dataSet.dataSourceRef?.name!, @@ -365,6 +367,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { id: dataSet.id, title: dataSet.title, dataSourceRef: dataSet.dataSourceRef, + timeFieldName: dataSet.timeFieldName, type: dataSet.type, }); queryService.queryString.setQuery(getInitialQuery(dataSet)); @@ -606,14 +609,10 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { : []), { value: 'no-time-filter', text: "I don't want to use a time filter" }, ]} - onChange={async (event) => { - await handleSelectedObject({ - ...selectedObject, - timeFieldName: - event.target.value !== 'no-time-filter' - ? event.target.value - : undefined, - } as SimpleDataSet); + onChange={(event) => { + setSelectedTimeFieldName( + event.target.value !== 'no-time-filter' ? event.target.value : undefined + ); }} aria-label="Select a date field" /> @@ -622,7 +621,10 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => { size="s" fullWidth onClick={async () => { - await handleSelectedDataSet(selectedObject); + await handleSelectedDataSet({ + ...selectedObject, + timeFieldName: selectedTimeFieldName, + } as SimpleDataSet); }} > Select diff --git a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx index 92393ef1a6c0..c28c5cb8b0be 100644 --- a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx +++ b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx @@ -45,7 +45,7 @@ export const QueryAssistBar: React.FC = (props) => { const subscription = props.connectionsService .getSelectedConnection$() .subscribe((connection) => { - dataSourceIdRef.current = connection?.dataSource.id; + dataSourceIdRef.current = connection?.dataSource?.id; }); return () => subscription.unsubscribe(); }, [props.connectionsService]); diff --git a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx index c76cd13bbf5e..23611e39501e 100644 --- a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx @@ -28,7 +28,7 @@ const getAvailableLanguages$ = ( connectionsService.getSelectedConnection$().pipe( distinctUntilChanged(), switchMap(async (connection) => { - const dataSourceId = connection?.dataSource.id; + const dataSourceId = connection?.dataSource?.id; const cached = availableLanguagesByDataSource.get(dataSourceId); if (cached !== undefined) return cached; const languages = await http