diff --git a/query/internal/plugins/builtin/observability/api/api.go b/query/internal/plugins/builtin/observability/api/api.go index 46d0823ce..f72373184 100644 --- a/query/internal/plugins/builtin/observability/api/api.go +++ b/query/internal/plugins/builtin/observability/api/api.go @@ -1,5 +1,18 @@ package api +import ( + ch "github.com/ClickHouse/clickhouse-go/v2" + "github.com/DataObserve/datav/query/pkg/models" + "github.com/gin-gonic/gin" +) + const ( - TestDatasourceAPI = "testDatasource" + TestDatasourceAPI = "testDatasource" + GetServiceInfoListAPI = "getServiceInfoList" + GetServiceNamesAPI = "getServiceNames" ) + +var APIRoutes = map[string]func(c *gin.Context, ds *models.Datasource, conn ch.Conn) models.PluginResult{ + GetServiceInfoListAPI: GetServiceInfoList, + GetServiceNamesAPI: GetServiceNames, +} diff --git a/query/internal/plugins/builtin/observability/api/service.go b/query/internal/plugins/builtin/observability/api/service.go new file mode 100644 index 000000000..7cd34c974 --- /dev/null +++ b/query/internal/plugins/builtin/observability/api/service.go @@ -0,0 +1,84 @@ +package api + +import ( + "fmt" + + ch "github.com/ClickHouse/clickhouse-go/v2" + "github.com/DataObserve/datav/query/pkg/colorlog" + "github.com/DataObserve/datav/query/pkg/models" + "github.com/gin-gonic/gin" +) + +var logger = colorlog.RootLogger.New("logger", "observability") + +type ServiceNameRes struct { + ServiceName string `ch:"serviceName"` +} + +func GetServiceNames(c *gin.Context, ds *models.Datasource, conn ch.Conn) models.PluginResult { + start := c.Query("start") + end := c.Query("end") + query := fmt.Sprintf("SELECT serviceName FROM signoz_traces.distributed_signoz_index_v2 WHERE timestamp >= %s AND timestamp <= %s GROUP BY serviceName", start, end) + + var res []ServiceNameRes + err := conn.Select(c.Request.Context(), &res, query) + if err != nil { + return models.GenPluginResult(models.PluginStatusError, err.Error(), nil) + } + + logger.Info("Query service names", "query", query) + + columns := []string{"service"} + data := make([][]interface{}, 0) + for _, v := range res { + data = append(data, []interface{}{v.ServiceName}) + } + + return models.GenPluginResult(models.PluginStatusSuccess, "", models.PluginResultData{ + Columns: columns, + Data: data, + }) +} + +func GetServiceInfoList(c *gin.Context, ds *models.Datasource, conn ch.Conn) models.PluginResult { + fmt.Println("here33333") + // rows, err := conn.Query(c.Request.Context(), query) + // if err != nil { + // colorlog.RootLogger.Info("Error query clickhouse :", "error", err, "ds_id", ds.Id, "query:", query) + // return models.GenPluginResult(models.PluginStatusError, err.Error(), nil) + // } + // defer rows.Close() + + // columns := rows.Columns() + // columnTypes := rows.ColumnTypes() + // types := make(map[string]string) + // data := make([][]interface{}, 0) + // for rows.Next() { + // v := make([]interface{}, len(columns)) + // for i := range v { + // t := columnTypes[i].ScanType() + // v[i] = reflect.New(t).Interface() + + // tp := t.String() + // if tp == "time.Time" { + // types[columns[i]] = "time" + // } + // } + + // err = rows.Scan(v...) + // if err != nil { + // colorlog.RootLogger.Info("Error scan clickhouse :", "error", err, "ds_id", ds.Id) + // continue + // } + + // for i, v0 := range v { + // v1, ok := v0.(*time.Time) + // if ok { + // v[i] = v1.Unix() + // } + // } + + // data = append(data, v) + // } + return models.GenPluginResult(models.PluginStatusSuccess, "", nil) +} diff --git a/query/internal/plugins/builtin/observability/api/testDatasource.go b/query/internal/plugins/builtin/observability/api/testDatasource.go deleted file mode 100644 index bf4901ab1..000000000 --- a/query/internal/plugins/builtin/observability/api/testDatasource.go +++ /dev/null @@ -1,10 +0,0 @@ -package api - -import ( - "github.com/DataObserve/datav/query/pkg/models" - "github.com/gin-gonic/gin" -) - -func TestDatasource(c *gin.Context, ds *models.Datasource) models.PluginResult { - return models.GenPluginResult(models.PluginStatusSuccess, "", nil) -} diff --git a/query/internal/plugins/builtin/observability/observability.go b/query/internal/plugins/builtin/observability/observability.go index df67fa7fe..4f468d112 100644 --- a/query/internal/plugins/builtin/observability/observability.go +++ b/query/internal/plugins/builtin/observability/observability.go @@ -1,9 +1,7 @@ package clickhouse import ( - "reflect" "sync" - "time" ch "github.com/ClickHouse/clickhouse-go/v2" "github.com/DataObserve/datav/query/internal/plugins/builtin/observability/api" @@ -41,54 +39,17 @@ func (p *ObservabilityPlugin) Query(c *gin.Context, ds *models.Datasource) model conns[ds.Id] = conn connsLock.Unlock() } - - rows, err := conn.Query(c.Request.Context(), query) - if err != nil { - colorlog.RootLogger.Info("Error query clickhouse :", "error", err, "ds_id", ds.Id, "query:", query) - return models.GenPluginResult(models.PluginStatusError, err.Error(), nil) - } - defer rows.Close() - - columns := rows.Columns() - columnTypes := rows.ColumnTypes() - types := make(map[string]string) - data := make([][]interface{}, 0) - for rows.Next() { - v := make([]interface{}, len(columns)) - for i := range v { - t := columnTypes[i].ScanType() - v[i] = reflect.New(t).Interface() - - tp := t.String() - if tp == "time.Time" { - types[columns[i]] = "time" - } + route, ok := api.APIRoutes[query] + if ok { + res := route(c, ds, conn) + return models.PluginResult{ + Status: models.PluginStatusSuccess, + Error: "", + Data: res, } - - err = rows.Scan(v...) - if err != nil { - colorlog.RootLogger.Info("Error scan clickhouse :", "error", err, "ds_id", ds.Id) - continue - } - - for i, v0 := range v { - v1, ok := v0.(*time.Time) - if ok { - v[i] = v1.Unix() - } - } - - data = append(data, v) + } else { + return models.GenPluginResult(models.PluginStatusError, "api not found", nil) } - - return models.PluginResult{ - Status: models.PluginStatusSuccess, - Error: "", - Data: map[string]interface{}{ - "columns": columns, - "data": data, - "types": types, - }} } func init() { diff --git a/query/internal/plugins/external/clickhouse/clickhouse.go b/query/internal/plugins/external/clickhouse/clickhouse.go index b57a821c4..a970e44cc 100644 --- a/query/internal/plugins/external/clickhouse/clickhouse.go +++ b/query/internal/plugins/external/clickhouse/clickhouse.go @@ -82,10 +82,10 @@ func (p *ClickHousePlugin) Query(c *gin.Context, ds *models.Datasource) models.P return models.PluginResult{ Status: models.PluginStatusSuccess, Error: "", - Data: map[string]interface{}{ - "columns": columns, - "data": data, - "types": types, + Data: models.PluginResultData{ + Columns: columns, + Data: data, + ColumnTypes: types, }} } diff --git a/query/pkg/models/plugin.go b/query/pkg/models/plugin.go index ff3da5c80..d255a86f6 100644 --- a/query/pkg/models/plugin.go +++ b/query/pkg/models/plugin.go @@ -13,6 +13,19 @@ type PluginResult struct { Data interface{} `json:"data,omitempty"` } +const ( + PluginResultFormatTable = "table" + PluginResultFormatMetrics = "metrics" + PluginResultFormatTrace = "traces" + PluginResultFormatLog = "logs" +) + +type PluginResultData struct { + Columns []string `json:"columns"` + Data [][]interface{} `json:"data"` + ColumnTypes map[string]string `json:"types,omitempty"` +} + type Plugin interface { Query(c *gin.Context, ds *Datasource) PluginResult } diff --git a/ui/src/types/plugin.ts b/ui/src/types/plugin.ts index e4fd925d8..d94b6df17 100644 --- a/ui/src/types/plugin.ts +++ b/ui/src/types/plugin.ts @@ -40,5 +40,11 @@ export interface DatasourcePluginComponents { export interface QueryPluginResult { status: "success" | "error" error: string - data: any + data: QueryPluginData +} + +export interface QueryPluginData { + columns: string[] + data: any[][] + types: Record } \ No newline at end of file diff --git a/ui/src/utils/plugins.ts b/ui/src/utils/plugins.ts index 7ed74bab1..ab60fdc2a 100644 --- a/ui/src/utils/plugins.ts +++ b/ui/src/utils/plugins.ts @@ -11,8 +11,138 @@ // See the License for the specific language governing permissions and // limitations under the License. +import { PanelQuery } from "types/dashboard" +import { QueryPluginData } from "types/plugin" +import { FieldType, SeriesData } from "types/seriesData" +import { jsonToEqualPairs1, parseLegendFormat } from "./format" +import { alignTimeSeriesData } from "./seriesData" +import { isEmpty } from "./validate" +import { replaceWithVariables } from "./variable" + export const isPluginDisabled = (p) => { if (p && p.settings?.disabled) { return p.settings.disabled() } -} \ No newline at end of file +} + + + +export const queryPluginDataToTimeSeries = (data: QueryPluginData, query: PanelQuery) => { + const seriesMap: Record = {} + const formats = parseLegendFormat(query.legend) + + for (var i=0;i { + const labelName = data.columns[i] + const valueType = data.types[labelName] ?? typeof v as any + if (valueType == FieldType.Time) { + if (!timeValue) { + timeValue = v + timeFieldName = labelName + } + } else if (valueType == FieldType.Number) { + if (!value) { + value = v + valueFieldName = labelName + } + } else { + labels[labelName] = v + } + }) + + if (!timeFieldName) { + return [] + } + + let seriesName; + if (isEmpty(labels)) { + seriesName = query.id + } else { + seriesName = jsonToEqualPairs1(labels) + } + + const series = seriesMap[seriesName] + if (!series) { + seriesMap[seriesName] = { + queryId: query.id, + name: seriesName, + labels: labels, + fields: [ + { + name: timeFieldName, + type: FieldType.Time, + values: [timeValue] + }, + { + name: valueFieldName, + type: FieldType.Number, + values: [value] + }, + ] + } + } else { + series.fields[0].values.push(timeValue) + series.fields[1].values.push(value) + } + } + + + const res = Object.values(seriesMap) + for (const s of res) { + if (!isEmpty(query.legend)) { + s.name = query.legend + if (!isEmpty(formats)) { + for (const format of formats) { + const l = s.labels[format] + if (l) { + s.name= s.name.replaceAll(`{{${format}}}`, l) + } + } + } + // replace ${xxx} format with corresponding variables + s.name= replaceWithVariables(s.name) + } + + } + + + const seriesList = Object.values(seriesMap) + alignTimeSeriesData(seriesList) + + return seriesList +} + + +export const queryPluginDataToTable= (data: QueryPluginData, query: PanelQuery) => { + const series: SeriesData = { + queryId: query.id, + name: isEmpty(query.legend) ? query.id.toString() : query.legend, + fields: [] + } + + data.columns.forEach((c,i) => { + series.fields.push({ + name: c, + values: [] + }) + }) + + data.data.forEach((row,i) => { + row.forEach((v,i) => { + const f = series.fields[i] + if (!f.type && data.types) { + f.type = data.types[f.name] ?? typeof v as any + } + f.values.push(v) + }) + }) + + return [series] +} + diff --git a/ui/src/views/dashboard/plugins/built-in/datasource/observability/QueryEditor.tsx b/ui/src/views/dashboard/plugins/built-in/datasource/observability/QueryEditor.tsx index 84b35854e..1f6d6c018 100644 --- a/ui/src/views/dashboard/plugins/built-in/datasource/observability/QueryEditor.tsx +++ b/ui/src/views/dashboard/plugins/built-in/datasource/observability/QueryEditor.tsx @@ -23,6 +23,8 @@ import { locale } from "src/i18n/i18n" import InputSelect from "components/select/InputSelect" import { MobileVerticalBreakpoint } from "src/data/constants" import CodeEditor from "components/CodeEditor/CodeEditor" +import SelectDataFormat from "../../../components/query-edtitor/SelectDataFormat" +import { DataFormat } from "types/format" const HttpQueryEditor = ({ panel, datasource, query, onChange }: DatasourceEditorProps) => { const code = useStore(locale) @@ -39,6 +41,14 @@ const HttpQueryEditor = ({ panel, datasource, query, onChange }: DatasourceEdito onChange(q) } } + + if (!tempQuery.data['format']) { + tempQuery.data['format'] = api?.format ?? DataFormat.Table + const q = cloneDeep(tempQuery) + setTempQuery(q) + onChange(q) + } + const [isMobileScreen] = useMediaQuery(MobileVerticalBreakpoint) return (<>
@@ -92,6 +102,8 @@ const HttpQueryEditor = ({ panel, datasource, query, onChange }: DatasourceEdito /> } + + ) } @@ -114,6 +126,7 @@ const apiList = [{ params: `{ "env": "test" }`, - paramsDesc: [["env", "environment name, such as dev, test, prod etc"]] + paramsDesc: [["env", "environment name, such as dev, test, prod etc"]], + format: DataFormat.Table } ] \ No newline at end of file diff --git a/ui/src/views/dashboard/plugins/built-in/datasource/observability/query_runner.ts b/ui/src/views/dashboard/plugins/built-in/datasource/observability/query_runner.ts index 9ab92a3ca..e34b28e89 100644 --- a/ui/src/views/dashboard/plugins/built-in/datasource/observability/query_runner.ts +++ b/ui/src/views/dashboard/plugins/built-in/datasource/observability/query_runner.ts @@ -24,7 +24,9 @@ import { requestApi } from "utils/axios/request" import { isEmpty } from "utils/validate" import { roundDsTime } from "utils/datasource" import { $variables } from "src/views/variables/store" -import { QueryPluginResult } from "types/plugin" +import { QueryPluginData, QueryPluginResult } from "types/plugin" +import { queryPluginDataToTable, queryPluginDataToTimeSeries } from "utils/plugins" +import { DataFormat } from "types/format" export const runQuery = async (panel: Panel, q: PanelQuery, range: TimeRange, ds: Datasource) => { if (isEmpty(q.metrics)) { @@ -39,7 +41,11 @@ export const runQuery = async (panel: Panel, q: PanelQuery, range: TimeRange, ds - const res: QueryPluginResult = await requestApi.get(`/proxy/${ds.id}?query=${replaceWithVariables(q.metrics)}¶ms=${q.data.params}&start=${start}&end=${end}&step=${q.interval}`) + const res: { + status: string, + error: string, + data: QueryPluginResult + } = await requestApi.get(`/proxy/${ds.id}?query=${replaceWithVariables(q.metrics)}¶ms=${q.data.params}&start=${start}&end=${end}&step=${q.interval}`) if (res.status !== "success") { console.log("Failed to fetch data from target datasource", res) @@ -48,10 +54,21 @@ export const runQuery = async (panel: Panel, q: PanelQuery, range: TimeRange, ds data: [] } } - + let data; + switch (q.data["format"]) { + case DataFormat.TimeSeries: + data = queryPluginDataToTimeSeries(res.data.data, q) + break + case DataFormat.Table: + data = queryPluginDataToTable(res.data.data, q) + break + default: + data = queryPluginDataToTable(res.data.data, q) + } + return { error: null, - data: res.data, + data } } diff --git a/ui/src/views/dashboard/plugins/components/query-edtitor/SelectDataFormat.tsx b/ui/src/views/dashboard/plugins/components/query-edtitor/SelectDataFormat.tsx new file mode 100644 index 000000000..a40cd7912 --- /dev/null +++ b/ui/src/views/dashboard/plugins/components/query-edtitor/SelectDataFormat.tsx @@ -0,0 +1,37 @@ +// Copyright 2023 Datav.io Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { Select } from "@chakra-ui/react" +import { useStore } from "@nanostores/react" +import FormItem from "components/form/Item" +import { cloneDeep } from "lodash" +import React from "react" +import { locale } from "src/i18n/i18n" +import { DataFormat } from "types/format" + +const SelectDataFormat = ({tempQuery,setTempQuery,onChange, labelWidth="150px"}) => { + let lang = useStore(locale) + return + + +} + +export default SelectDataFormat \ No newline at end of file diff --git a/ui/src/views/dashboard/plugins/external/datasource/clickhouse/QueryEditor.tsx b/ui/src/views/dashboard/plugins/external/datasource/clickhouse/QueryEditor.tsx index a86212d2f..fb5de669f 100644 --- a/ui/src/views/dashboard/plugins/external/datasource/clickhouse/QueryEditor.tsx +++ b/ui/src/views/dashboard/plugins/external/datasource/clickhouse/QueryEditor.tsx @@ -26,6 +26,7 @@ import { $datasources } from "src/views/datasource/store"; import { DataFormat } from "types/format"; import { locale } from "src/i18n/i18n"; import ExpandTimeline from "../../../components/query-edtitor/ExpandTimeline"; +import SelectDataFormat from "../../../components/query-edtitor/SelectDataFormat"; @@ -77,17 +78,7 @@ const QueryEditor = ({ datasource, query, onChange }: DatasourceEditorProps) => size="sm" /> - - - + {/* */} diff --git a/ui/src/views/dashboard/plugins/external/datasource/clickhouse/types.ts b/ui/src/views/dashboard/plugins/external/datasource/clickhouse/types.ts index cda5d6678..abfe74421 100644 --- a/ui/src/views/dashboard/plugins/external/datasource/clickhouse/types.ts +++ b/ui/src/views/dashboard/plugins/external/datasource/clickhouse/types.ts @@ -1,7 +1,2 @@ export const DatasourceTypeVM = "clickhouse" -export interface ChPluginData { - columns: string[] - data: any[][] - types: Record -} \ No newline at end of file diff --git a/ui/src/views/dashboard/plugins/external/datasource/clickhouse/utils.ts b/ui/src/views/dashboard/plugins/external/datasource/clickhouse/utils.ts index cac22dc4f..c1ee0a733 100644 --- a/ui/src/views/dashboard/plugins/external/datasource/clickhouse/utils.ts +++ b/ui/src/views/dashboard/plugins/external/datasource/clickhouse/utils.ts @@ -12,20 +12,17 @@ // limitations under the License. import { Panel, PanelQuery } from "types/dashboard"; -import { FieldType, SeriesData } from "types/seriesData"; import { TimeRange } from "types/time"; -import { jsonToEqualPairs1, parseLegendFormat } from "utils/format"; import { isEmpty } from "utils/validate"; -import { replaceWithVariables } from "utils/variable"; import { PanelTypeGraph } from "../../../built-in/panel/graph/types"; import { PanelTypeBar } from "../../../built-in/panel/bar/types"; import { PanelTypeStat } from "../../../built-in/panel/stat/types"; -import { ChPluginData } from "./types"; import { DataFormat } from "types/format"; -import { alignTimeSeriesData } from "utils/seriesData"; +import { queryPluginDataToTable, queryPluginDataToTimeSeries } from "utils/plugins"; +import { QueryPluginData } from "types/plugin"; -export const clickhouseToPanelData = (data: ChPluginData, panel: Panel, query: PanelQuery, range: TimeRange) => { +export const clickhouseToPanelData = (data: QueryPluginData, panel: Panel, query: PanelQuery, range: TimeRange) => { if (isEmpty(data) || data.columns.length == 0 || data.data.length == 0) { return null } @@ -41,131 +38,11 @@ export const clickhouseToPanelData = (data: ChPluginData, panel: Panel, query: P switch (query.data["format"]) { case DataFormat.TimeSeries: - return toTimeSeries(data, query) + return queryPluginDataToTimeSeries(data, query) case DataFormat.Table: - return toTable(data, query) + return queryPluginDataToTable(data, query) default: - return toTimeSeries(data, query) + return queryPluginDataToTimeSeries(data, query) } } - -const toTimeSeries = (data: ChPluginData, query: PanelQuery) => { - const seriesMap: Record = {} - const formats = parseLegendFormat(query.legend) - - for (var i=0;i { - const labelName = data.columns[i] - const valueType = data.types[labelName] ?? typeof v as any - if (valueType == FieldType.Time) { - if (!timeValue) { - timeValue = v - timeFieldName = labelName - } - } else if (valueType == FieldType.Number) { - if (!value) { - value = v - valueFieldName = labelName - } - } else { - labels[labelName] = v - } - }) - - if (!timeFieldName) { - return [] - } - - let seriesName; - if (isEmpty(labels)) { - seriesName = query.id - } else { - seriesName = jsonToEqualPairs1(labels) - } - - const series = seriesMap[seriesName] - if (!series) { - seriesMap[seriesName] = { - queryId: query.id, - name: seriesName, - labels: labels, - fields: [ - { - name: timeFieldName, - type: FieldType.Time, - values: [timeValue] - }, - { - name: valueFieldName, - type: FieldType.Number, - values: [value] - }, - ] - } - } else { - series.fields[0].values.push(timeValue) - series.fields[1].values.push(value) - } - } - - - const res = Object.values(seriesMap) - for (const s of res) { - if (!isEmpty(query.legend)) { - s.name = query.legend - if (!isEmpty(formats)) { - for (const format of formats) { - const l = s.labels[format] - if (l) { - s.name= s.name.replaceAll(`{{${format}}}`, l) - } - } - } - // replace ${xxx} format with corresponding variables - s.name= replaceWithVariables(s.name) - } - - } - - - const seriesList = Object.values(seriesMap) - alignTimeSeriesData(seriesList) - - return seriesList -} - - -const toTable= (data: ChPluginData, query: PanelQuery) => { - const series: SeriesData = { - queryId: query.id, - name: isEmpty(query.legend) ? query.id.toString() : query.legend, - fields: [] - } - - data.columns.forEach((c,i) => { - series.fields.push({ - name: c, - values: [] - }) - }) - - data.data.forEach((row,i) => { - row.forEach((v,i) => { - const f = series.fields[i] - if (!f.type) { - f.type = data.types[f.name] ?? typeof v as any - } - f.values.push(v) - }) - }) - - return [series] -} -