From 3abd3d2fb2c9849a501db621306570ea9de63ae9 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Sun, 29 Dec 2024 15:11:53 +0100 Subject: [PATCH] fix: update get query parameters --- services/inventory/client/inventory.go | 2 +- services/inventory/http_routes.go | 4 +- services/metadata/api/query_parameter.go | 15 +-- services/metadata/http_handler.go | 8 ++ services/metadata/http_routes.go | 125 ++++++++++++++++-- .../internal/database/query_parameter.go | 9 ++ 6 files changed, 140 insertions(+), 23 deletions(-) diff --git a/services/inventory/client/inventory.go b/services/inventory/client/inventory.go index 157c78a69..1dc2eaf71 100644 --- a/services/inventory/client/inventory.go +++ b/services/inventory/client/inventory.go @@ -70,7 +70,7 @@ func (s *inventoryClient) CountResources(ctx *httpclient.Context) (int64, error) } func (s *inventoryClient) GetQuery(ctx *httpclient.Context, id string) (*api.NamedQueryItemV2, error) { - url := fmt.Sprintf("%s/api/v3/query/%s", s.baseURL, id) + url := fmt.Sprintf("%s/api/v3/queries/%s", s.baseURL, id) var namedQuery api.NamedQueryItemV2 if statusCode, err := httpclient.DoRequest(ctx.Ctx, http.MethodGet, url, ctx.ToHeaders(), nil, &namedQuery); err != nil { diff --git a/services/inventory/http_routes.go b/services/inventory/http_routes.go index 41de76cb5..61e9f63cf 100644 --- a/services/inventory/http_routes.go +++ b/services/inventory/http_routes.go @@ -77,7 +77,7 @@ func (h *HttpHandler) Register(e *echo.Echo) { v3 := e.Group("/api/v3") v3.POST("/queries", httpserver.AuthorizeHandler(h.ListQueriesV2, api.ViewerRole)) v3.GET("/queries/filters", httpserver.AuthorizeHandler(h.ListQueriesFilters, api.ViewerRole)) - v3.GET("/query/:query_id", httpserver.AuthorizeHandler(h.GetQuery, api.ViewerRole)) + v3.GET("/queries/:query_id", httpserver.AuthorizeHandler(h.GetQuery, api.ViewerRole)) v3.GET("/queries/tags", httpserver.AuthorizeHandler(h.ListQueriesTags, api.ViewerRole)) v3.POST("/query/run", httpserver.AuthorizeHandler(h.RunQueryByID, api.ViewerRole)) v3.GET("/query/async/run/:run_id/result", httpserver.AuthorizeHandler(h.GetAsyncQueryRunResult, api.ViewerRole)) @@ -385,7 +385,7 @@ func (h *HttpHandler) ListQueriesV2(ctx echo.Context) error { // @Produce json // @Param query_id path string true "QueryID" // @Success 200 {object} inventoryApi.NamedQueryItemV2 -// @Router /inventory/api/v3/query/{query_id} [get] +// @Router /inventory/api/v3/queries/{query_id} [get] func (h *HttpHandler) GetQuery(ctx echo.Context) error { queryID := ctx.Param("query_id") diff --git a/services/metadata/api/query_parameter.go b/services/metadata/api/query_parameter.go index e8b4197e6..17111b8f5 100644 --- a/services/metadata/api/query_parameter.go +++ b/services/metadata/api/query_parameter.go @@ -1,18 +1,17 @@ package api type QueryParameter struct { - Key string `json:"key"` - Value string `json:"value"` + Key string `json:"key"` + Value string `json:"value"` + ControlsCount int `json:"controls_count"` + QueriesCount int `json:"queries_count"` } type SetQueryParameterRequest struct { - QueryParameters []QueryParameter `json:"queryParameters"` -} - -type GetQueryParameterResponse struct { - QueryParameter QueryParameter `json:"queryParameter"` + QueryParameters []QueryParameter `json:"query_parameters"` } type ListQueryParametersResponse struct { - QueryParameters []QueryParameter `json:"queryParameters"` + Items []QueryParameter `json:"items"` + TotalCount int `json:"total_count"` } diff --git a/services/metadata/http_handler.go b/services/metadata/http_handler.go index b5771498c..82673eef6 100644 --- a/services/metadata/http_handler.go +++ b/services/metadata/http_handler.go @@ -4,6 +4,8 @@ import ( "context" "crypto/rand" "fmt" + complianceClient "github.com/opengovern/opencomply/services/compliance/client" + inventoryClient "github.com/opengovern/opencomply/services/inventory/client" "strings" "time" @@ -37,6 +39,9 @@ type HttpHandler struct { dexClient dexApi.DexClient logger *zap.Logger + complianceClient complianceClient.ComplianceServiceClient + inventoryClient inventoryClient.InventoryServiceClient + viewCheckpoint time.Time } @@ -121,6 +126,9 @@ func InitializeHttpHandler( viewCheckpoint: time.Now().Add(-time.Hour * 2), } + h.complianceClient = complianceClient.NewComplianceClient(cfg.Compliance.BaseURL) + h.inventoryClient = inventoryClient.NewInventoryServiceClient(cfg.Inventory.BaseURL) + switch cfg.Vault.Provider { case vault.AwsKMS: h.vault, err = vault.NewKMSVaultSourceConfig(ctx, cfg.Vault.Aws, cfg.Vault.KeyId) diff --git a/services/metadata/http_routes.go b/services/metadata/http_routes.go index 1bccde1e4..d98f79ce6 100644 --- a/services/metadata/http_routes.go +++ b/services/metadata/http_routes.go @@ -272,30 +272,131 @@ func (h HttpHandler) SetQueryParameter(ctx echo.Context) error { // @Security BearerToken // @Tags metadata // @Produce json +// @Param query_id query string false "Query ID to filter with" +// @Param control_id query string false "Control ID to filter with" +// @Param cursor query int false "Cursor" +// @Param per_page query int false "Per Page" // @Success 200 {object} api.ListQueryParametersResponse // @Router /metadata/api/v1/query_parameter [get] func (h HttpHandler) ListQueryParameters(ctx echo.Context) error { - _, span := tracer.Start(ctx.Request().Context(), "new_ListQueryParameters", trace.WithSpanKind(trace.SpanKindServer)) - span.SetName("new_ListQueryParameters") + clientCtx := &httpclient.Context{UserRole: api3.AdminRole} + + var cursor, perPage int64 + var err error + + cursorStr := ctx.QueryParam("cursor") + if cursorStr != "" { + cursor, err = strconv.ParseInt(cursorStr, 10, 64) + if err != nil { + return err + } + } + perPageStr := ctx.QueryParam("per_page") + if perPageStr != "" { + perPage, err = strconv.ParseInt(perPageStr, 10, 64) + if err != nil { + return err + } + } - queryParams, err := h.db.GetQueryParameters() + queryID := ctx.QueryParam("query_id") + controlID := ctx.QueryParam("control_id") + + controls, err := h.complianceClient.ListControl(clientCtx, nil, nil) if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - h.logger.Error("error getting query parameters", zap.Error(err)) - return err + h.logger.Error("error listing controls", zap.Error(err)) + return echo.NewHTTPError(http.StatusInternalServerError, "error listing controls") + } + namedQueries, err := h.inventoryClient.ListQueriesV2(clientCtx) + if err != nil { + h.logger.Error("error listing queries", zap.Error(err)) + return echo.NewHTTPError(http.StatusInternalServerError, "error listing queries") + } + + var filteredQueryParams []string + if controlID != "" { + control, err := h.complianceClient.GetControl(clientCtx, controlID) + if err != nil { + h.logger.Error("error getting control", zap.Error(err)) + return echo.NewHTTPError(http.StatusInternalServerError, "error getting control") + } + if control == nil { + return echo.NewHTTPError(http.StatusNotFound, "control not found") + } + for _, param := range control.Query.Parameters { + filteredQueryParams = append(filteredQueryParams, param.Key) + } + } else if queryID != "" { + query, err := h.inventoryClient.GetQuery(clientCtx, queryID) + if err != nil { + h.logger.Error("error getting query", zap.Error(err)) + return echo.NewHTTPError(http.StatusInternalServerError, "error getting query") + } + if query == nil { + return echo.NewHTTPError(http.StatusNotFound, "query not found") + } + for _, param := range query.Query.Parameters { + filteredQueryParams = append(filteredQueryParams, param.Key) + } } - span.End() - result := api.ListQueryParametersResponse{ - QueryParameters: make([]api.QueryParameter, 0, len(queryParams)), + var queryParams []models.QueryParameterValues + if len(filteredQueryParams) > 0 { + queryParams, err = h.db.GetQueryParametersByIds(filteredQueryParams) + if err != nil { + h.logger.Error("error getting query parameters", zap.Error(err)) + return err + } + } else { + queryParams, err = h.db.GetQueryParameters() + if err != nil { + h.logger.Error("error getting query parameters", zap.Error(err)) + return err + } } + + parametersMap := make(map[string]*api.QueryParameter) for _, dbParam := range queryParams { apiParam := dbParam.ToAPI() - result.QueryParameters = append(result.QueryParameters, apiParam) + parametersMap[apiParam.Key] = &apiParam + } + + for _, c := range controls { + for _, p := range c.Query.Parameters { + if _, ok := parametersMap[p.Key]; !ok { + parametersMap[p.Key].ControlsCount += 1 + } + } + } + for _, q := range namedQueries.Items { + for _, p := range q.Query.Parameters { + if _, ok := parametersMap[p.Key]; !ok { + parametersMap[p.Key].QueriesCount += 1 + } + } } - return ctx.JSON(http.StatusOK, result) + var items []api.QueryParameter + for _, i := range parametersMap { + items = append(items, *i) + } + + totalCount := len(items) + sort.Slice(items, func(i, j int) bool { + return items[i].Key < items[j].Key + }) + if perPage != 0 { + if cursor == 0 { + items = utils.Paginate(1, perPage, items) + } else { + items = utils.Paginate(cursor, perPage, items) + } + } + + return ctx.JSON(http.StatusOK, api.ListQueryParametersResponse{ + TotalCount: totalCount, + Items: items, + }) } // PurgeSampleData godoc diff --git a/services/metadata/internal/database/query_parameter.go b/services/metadata/internal/database/query_parameter.go index f9523a8a0..492dd12f3 100644 --- a/services/metadata/internal/database/query_parameter.go +++ b/services/metadata/internal/database/query_parameter.go @@ -54,6 +54,15 @@ func (db Database) GetQueryParameters() ([]models.QueryParameterValues, error) { return queryParams, nil } +func (db Database) GetQueryParametersByIds(ids []string) ([]models.QueryParameterValues, error) { + var queryParams []models.QueryParameterValues + err := db.orm.Where("key IN ?", ids).Find(&queryParams).Error + if err != nil { + return nil, err + } + return queryParams, nil +} + func (db Database) DeleteQueryParameter(key string) error { return db.orm.Unscoped().Delete(&models.QueryParameterValues{}, "key = ?", key).Error }