From 3b3425d2770fb3b2b5763ebbbc6016aa37063b95 Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Wed, 27 Nov 2024 17:59:54 +0000 Subject: [PATCH] Integration Discovery Rules: endpoint that returns all the rules This PR adds a new endpoint that returns all the discovery rules that exist for a given Integration. It will include what was the last time the rule was used to fetch resources, so that users can see stale rules and act upon it. The endpoint supports filtering by the resource type to ensure the UI only requests a subset of the rules, depending on the screen the user is visiting (eg, EC2, RDS or EKS). --- lib/web/apiserver.go | 1 + lib/web/integrations.go | 104 +++++++++++++++++++++++++++ lib/web/intgrations_test.go | 139 ++++++++++++++++++++++++++++++++++++ lib/web/ui/integration.go | 25 +++++++ 4 files changed, 269 insertions(+) diff --git a/lib/web/apiserver.go b/lib/web/apiserver.go index fb8794c1272e6..4b0b861d33a3d 100644 --- a/lib/web/apiserver.go +++ b/lib/web/apiserver.go @@ -977,6 +977,7 @@ func (h *Handler) bindDefaultEndpoints() { h.GET("/webapi/sites/:site/integrations/:name", h.WithClusterAuth(h.integrationsGet)) h.PUT("/webapi/sites/:site/integrations/:name", h.WithClusterAuth(h.integrationsUpdate)) h.GET("/webapi/sites/:site/integrations/:name/stats", h.WithClusterAuth(h.integrationStats)) + h.GET("/webapi/sites/:site/integrations/:name/discoveryrules", h.WithClusterAuth(h.integrationDiscoveryRules)) h.DELETE("/webapi/sites/:site/integrations/:name_or_subkind", h.WithClusterAuth(h.integrationsDelete)) // GET the Microsoft Teams plugin app.zip file. diff --git a/lib/web/integrations.go b/lib/web/integrations.go index 4693418dd3b9d..89f52a7ec00d6 100644 --- a/lib/web/integrations.go +++ b/lib/web/integrations.go @@ -36,6 +36,7 @@ import ( "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/httplib" "github.com/gravitational/teleport/lib/reversetunnelclient" + libui "github.com/gravitational/teleport/lib/ui" "github.com/gravitational/teleport/lib/web/ui" ) @@ -319,6 +320,109 @@ func rulesWithIntegration(dc *discoveryconfig.DiscoveryConfig, matcherType strin return ret } +// integrationDiscoveryRules returns the Discovery Rules that are using a given integration. +// A Discovery Rule is just like a DiscoveryConfig Matcher, except that it breaks down by region. +// So, if a Matcher exists for two regions, that will be represented as two Rules. +func (h *Handler) integrationDiscoveryRules(w http.ResponseWriter, r *http.Request, p httprouter.Params, sctx *SessionContext, site reversetunnelclient.RemoteSite) (interface{}, error) { + integrationName := p.ByName("name") + if integrationName == "" { + return nil, trace.BadParameter("an integration name is required") + } + + values := r.URL.Query() + startKey := values.Get("startKey") + resourceType := values.Get("resourceType") + + clt, err := sctx.GetUserClient(r.Context(), site) + if err != nil { + return nil, trace.Wrap(err) + } + + ig, err := clt.GetIntegration(r.Context(), integrationName) + if err != nil { + return nil, trace.Wrap(err) + } + + rules, err := collectAutoDiscoverRules(r.Context(), ig.GetName(), startKey, resourceType, clt.DiscoveryConfigClient()) + if err != nil { + return nil, trace.Wrap(err) + } + + return rules, nil +} + +// collectAutoDiscoverRules will iterate over all DiscoveryConfigs's Matchers and collect the Discover Rules that exist in them for the given integration. +// It can also be filtered by Matcher Type (eg ec2, rds, eks) +// A Discover Rule is a close match to a DiscoveryConfig's Matcher, except that it will count as many rules as regions exist. +// Eg if a DiscoveryConfig's Matcher has two regions, then it will output two (almost equal) Rules, one for each Region. +func collectAutoDiscoverRules( + ctx context.Context, + integrationName string, + nextPage string, + resourceTypeFilter string, + clt interface { + ListDiscoveryConfigs(ctx context.Context, pageSize int, nextToken string) ([]*discoveryconfig.DiscoveryConfig, string, error) + }, +) (ui.IntegrationDiscoveryRules, error) { + const ( + maxPerPage = 100 + ) + var ret ui.IntegrationDiscoveryRules + for { + discoveryConfigs, nextToken, err := clt.ListDiscoveryConfigs(ctx, 0, nextPage) + if err != nil { + return ret, trace.Wrap(err) + } + for _, dc := range discoveryConfigs { + lastSync := &dc.Status.LastSyncTime + if lastSync.IsZero() { + lastSync = nil + } + + for _, matcher := range dc.Spec.AWS { + if matcher.Integration != integrationName { + continue + } + + for _, resourceType := range matcher.Types { + if resourceTypeFilter != "" && resourceType != resourceTypeFilter { + continue + } + + for _, region := range matcher.Regions { + uiLables := make([]libui.Label, 0, len(matcher.Tags)) + for labelKey, labelValues := range matcher.Tags { + for _, labelValue := range labelValues { + uiLables = append(uiLables, libui.Label{ + Name: labelKey, + Value: labelValue, + }) + } + } + ret.Rules = append(ret.Rules, ui.IntegrationDiscoveryRule{ + ResourceType: resourceType, + Region: region, + LabelMatcher: uiLables, + DiscoveryConfig: dc.GetName(), + LastSync: lastSync, + }) + } + } + } + } + + ret.NextKey = nextToken + + if nextToken == "" || len(ret.Rules) > maxPerPage { + break + } + + nextPage = nextToken + } + + return ret, nil +} + // integrationsList returns a page of Integrations func (h *Handler) integrationsList(w http.ResponseWriter, r *http.Request, p httprouter.Params, sctx *SessionContext, site reversetunnelclient.RemoteSite) (interface{}, error) { clt, err := sctx.GetUserClient(r.Context(), site) diff --git a/lib/web/intgrations_test.go b/lib/web/intgrations_test.go index f60eabd623041..25a31490dfcad 100644 --- a/lib/web/intgrations_test.go +++ b/lib/web/intgrations_test.go @@ -24,12 +24,15 @@ import ( "testing" "time" + "github.com/google/uuid" "github.com/stretchr/testify/require" discoveryconfigv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/discoveryconfig/v1" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/api/types/discoveryconfig" + "github.com/gravitational/teleport/api/types/header" "github.com/gravitational/teleport/lib/services" + libui "github.com/gravitational/teleport/lib/ui" "github.com/gravitational/teleport/lib/web/ui" ) @@ -212,6 +215,142 @@ func TestCollectAWSOIDCAutoDiscoverStats(t *testing.T) { }) } +func TestCollectAutoDiscoverRules(t *testing.T) { + ctx := context.Background() + integrationName := "my-integration" + + t.Run("without discovery configs, returns no rules", func(t *testing.T) { + clt := &mockDiscoveryConfigsGetter{ + discoveryConfigs: make([]*discoveryconfig.DiscoveryConfig, 0), + } + + gotRules, err := collectAutoDiscoverRules(ctx, integrationName, "", "", clt) + require.NoError(t, err) + expectedRules := ui.IntegrationDiscoveryRules{} + require.Equal(t, expectedRules, gotRules) + }) + + t.Run("collects multiple discovery configs", func(t *testing.T) { + syncTime := time.Now() + dcForEC2 := &discoveryconfig.DiscoveryConfig{ + ResourceHeader: header.ResourceHeader{Metadata: header.Metadata{ + Name: uuid.NewString(), + }}, + Spec: discoveryconfig.Spec{AWS: []types.AWSMatcher{{ + Integration: integrationName, + Types: []string{"ec2"}, + Regions: []string{"us-east-1"}, + Tags: types.Labels{"*": []string{"*"}}, + }}}, + Status: discoveryconfig.Status{ + LastSyncTime: syncTime, + }, + } + dcForRDS := &discoveryconfig.DiscoveryConfig{ + ResourceHeader: header.ResourceHeader{Metadata: header.Metadata{ + Name: uuid.NewString(), + }}, + Spec: discoveryconfig.Spec{AWS: []types.AWSMatcher{{ + Integration: integrationName, + Types: []string{"rds"}, + Regions: []string{"us-east-1", "us-east-2"}, + Tags: types.Labels{ + "env": []string{"dev", "prod"}, + }, + }}}, + Status: discoveryconfig.Status{ + LastSyncTime: syncTime, + }, + } + dcForEKS := &discoveryconfig.DiscoveryConfig{ + ResourceHeader: header.ResourceHeader{Metadata: header.Metadata{ + Name: uuid.NewString(), + }}, + Spec: discoveryconfig.Spec{AWS: []types.AWSMatcher{{ + Integration: integrationName, + Types: []string{"eks"}, + Regions: []string{"us-east-1"}, + Tags: types.Labels{"*": []string{"*"}}, + }}}, + Status: discoveryconfig.Status{ + LastSyncTime: syncTime, + }, + } + dcForEKSWithoutStatus := &discoveryconfig.DiscoveryConfig{ + ResourceHeader: header.ResourceHeader{Metadata: header.Metadata{ + Name: uuid.NewString(), + }}, + Spec: discoveryconfig.Spec{AWS: []types.AWSMatcher{{ + Integration: integrationName, + Types: []string{"eks"}, + Regions: []string{"eu-west-1"}, + Tags: types.Labels{"*": []string{"*"}}, + }}}, + } + clt := &mockDiscoveryConfigsGetter{ + discoveryConfigs: []*discoveryconfig.DiscoveryConfig{ + dcForEC2, + dcForRDS, + dcForEKS, + dcForEKSWithoutStatus, + }, + } + + got, err := collectAutoDiscoverRules(ctx, integrationName, "", "", clt) + require.NoError(t, err) + expectedRules := []ui.IntegrationDiscoveryRule{ + { + ResourceType: "ec2", + Region: "us-east-1", + LabelMatcher: []libui.Label{ + {Name: "*", Value: "*"}, + }, + DiscoveryConfig: dcForEC2.GetName(), + LastSync: &syncTime, + }, + { + ResourceType: "eks", + Region: "us-east-1", + LabelMatcher: []libui.Label{ + {Name: "*", Value: "*"}, + }, + DiscoveryConfig: dcForEKS.GetName(), + LastSync: &syncTime, + }, + { + ResourceType: "eks", + Region: "eu-west-1", + LabelMatcher: []libui.Label{ + {Name: "*", Value: "*"}, + }, + DiscoveryConfig: dcForEKSWithoutStatus.GetName(), + }, + { + ResourceType: "rds", + Region: "us-east-1", + LabelMatcher: []libui.Label{ + {Name: "env", Value: "dev"}, + {Name: "env", Value: "prod"}, + }, + DiscoveryConfig: dcForRDS.GetName(), + LastSync: &syncTime, + }, + { + ResourceType: "rds", + Region: "us-east-2", + LabelMatcher: []libui.Label{ + {Name: "env", Value: "dev"}, + {Name: "env", Value: "prod"}, + }, + DiscoveryConfig: dcForRDS.GetName(), + LastSync: &syncTime, + }, + } + require.Empty(t, got.NextKey) + require.ElementsMatch(t, expectedRules, got.Rules) + }) +} + type mockDiscoveryConfigsGetter struct { discoveryConfigs []*discoveryconfig.DiscoveryConfig } diff --git a/lib/web/ui/integration.go b/lib/web/ui/integration.go index 3c0aab3c817d8..913f22c647947 100644 --- a/lib/web/ui/integration.go +++ b/lib/web/ui/integration.go @@ -96,6 +96,31 @@ type ResourceTypeSummary struct { ECSDatabaseServiceCount int `json:"ecsDatabaseServiceCount,omitempty"` } +// IntegrationDiscoveryRule describes a discovery rule associated with an integration. +type IntegrationDiscoveryRule struct { + // ResourceType indicates the type of resource that this rule targets. + // This is the same value that is set in DiscoveryConfig.AWS..Types + // Example: ec2, rds, eks + ResourceType string `json:"resourceType,omitempty"` + // Region where this rule applies to. + Region string `json:"region,omitempty"` + // LabelMatcher is the set of labels that are used to filter the resources before trying to auto-enroll them. + LabelMatcher []ui.Label `json:"labelMatcher,omitempty"` + // DiscoveryConfig is the name of the DiscoveryConfig that created this rule. + DiscoveryConfig string `json:"discoveryConfig,omitempty"` + // LastSync contains the time when this rule was used. + // If empty, it indicates that the rule is not being used. + LastSync *time.Time `json:"lastSync,omitempty"` +} + +// IntegrationDiscoveryRules contains the list of discovery rules for a given Integration. +type IntegrationDiscoveryRules struct { + // Rules is the list of integration rules. + Rules []IntegrationDiscoveryRule `json:"rules"` + // NextKey is the position to resume listing rules. + NextKey string `json:"nextKey,omitempty"` +} + // Integration describes Integration fields type Integration struct { // Name is the Integration name.