From 82e120d8d5653e7195697c7074271f753c0688cf Mon Sep 17 00:00:00 2001 From: Marco Dinis Date: Tue, 22 Oct 2024 15:54:36 +0100 Subject: [PATCH 1/2] Add instance name to DiscoverEC2 User Task failed instances list (#47712) * Add instance name to DiscoverEC2 User Task failed instances list * use stdlib slices and maps --- lib/srv/discovery/discovery.go | 2 +- lib/srv/discovery/status.go | 2 +- lib/srv/server/ec2_watcher.go | 4 ++ lib/srv/server/ec2_watcher_test.go | 82 ++++++++++++++++++++++++++++-- lib/srv/server/ssm_install.go | 78 +++++++++++++++++----------- lib/srv/server/ssm_install_test.go | 3 +- 6 files changed, 135 insertions(+), 36 deletions(-) diff --git a/lib/srv/discovery/discovery.go b/lib/srv/discovery/discovery.go index d1b490bbe0830..6e93a8a8eddc6 100644 --- a/lib/srv/discovery/discovery.go +++ b/lib/srv/discovery/discovery.go @@ -990,10 +990,10 @@ func (s *Server) handleEC2RemoteInstallation(instances *server.EC2Instances) err installerScript: req.InstallerScriptName(), }, &usertasksv1.DiscoverEC2Instance{ - // TODO(marco): add instance name DiscoveryConfig: instances.DiscoveryConfig, DiscoveryGroup: s.DiscoveryGroup, InstanceId: instance.InstanceID, + Name: instance.InstanceName, SyncTime: timestamppb.New(s.clock.Now()), }, ) diff --git a/lib/srv/discovery/status.go b/lib/srv/discovery/status.go index 4bb70efaa5b66..7fe0b0f39398d 100644 --- a/lib/srv/discovery/status.go +++ b/lib/srv/discovery/status.go @@ -310,12 +310,12 @@ func (s *Server) ReportEC2SSMInstallationResult(ctx context.Context, result *ser installerScript: result.InstallerScript, }, &usertasksv1.DiscoverEC2Instance{ - // TODO(marco): add instance name InvocationUrl: result.SSMRunEvent.InvocationURL, DiscoveryConfig: result.DiscoveryConfig, DiscoveryGroup: s.DiscoveryGroup, SyncTime: timestamppb.New(result.SSMRunEvent.Time), InstanceId: result.SSMRunEvent.InstanceID, + Name: result.InstanceName, }, ) diff --git a/lib/srv/server/ec2_watcher.go b/lib/srv/server/ec2_watcher.go index f8390b032c4d2..25f12018e0170 100644 --- a/lib/srv/server/ec2_watcher.go +++ b/lib/srv/server/ec2_watcher.go @@ -79,6 +79,7 @@ type EC2Instances struct { // discovered. type EC2Instance struct { InstanceID string + InstanceName string Tags map[string]string OriginalInstance ec2.Instance } @@ -92,6 +93,9 @@ func toEC2Instance(originalInst *ec2.Instance) EC2Instance { for _, tag := range originalInst.Tags { if key := aws.StringValue(tag.Key); key != "" { inst.Tags[key] = aws.StringValue(tag.Value) + if key == "Name" { + inst.InstanceName = aws.StringValue(tag.Value) + } } } return inst diff --git a/lib/srv/server/ec2_watcher_test.go b/lib/srv/server/ec2_watcher_test.go index 2647a7b0f527f..8279cbbd868f1 100644 --- a/lib/srv/server/ec2_watcher_test.go +++ b/lib/srv/server/ec2_watcher_test.go @@ -176,10 +176,16 @@ func TestEC2Watcher(t *testing.T) { present := ec2.Instance{ InstanceId: aws.String("instance-present"), - Tags: []*ec2.Tag{{ - Key: aws.String("teleport"), - Value: aws.String("yes"), - }}, + Tags: []*ec2.Tag{ + { + Key: aws.String("teleport"), + Value: aws.String("yes"), + }, + { + Key: aws.String("Name"), + Value: aws.String("Present"), + }, + }, State: &ec2.InstanceState{ Name: aws.String(ec2.InstanceStateNameRunning), }, @@ -360,3 +366,71 @@ func TestMakeEvents(t *testing.T) { }) } } + +func TestToEC2Instances(t *testing.T) { + sampleInstance := &ec2.Instance{ + InstanceId: aws.String("instance-001"), + Tags: []*ec2.Tag{ + { + Key: aws.String("teleport"), + Value: aws.String("yes"), + }, + { + Key: aws.String("Name"), + Value: aws.String("MyInstanceName"), + }, + }, + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + } + + sampleInstanceWithoutName := &ec2.Instance{ + InstanceId: aws.String("instance-001"), + Tags: []*ec2.Tag{ + { + Key: aws.String("teleport"), + Value: aws.String("yes"), + }, + }, + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + } + + for _, tt := range []struct { + name string + input []*ec2.Instance + expected []EC2Instance + }{ + { + name: "with name", + input: []*ec2.Instance{sampleInstance}, + expected: []EC2Instance{{ + InstanceID: "instance-001", + Tags: map[string]string{ + "Name": "MyInstanceName", + "teleport": "yes", + }, + InstanceName: "MyInstanceName", + OriginalInstance: *sampleInstance, + }}, + }, + { + name: "without name", + input: []*ec2.Instance{sampleInstanceWithoutName}, + expected: []EC2Instance{{ + InstanceID: "instance-001", + Tags: map[string]string{ + "teleport": "yes", + }, + OriginalInstance: *sampleInstanceWithoutName, + }}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got := ToEC2Instances(tt.input) + require.Equal(t, tt.expected, got) + }) + } +} diff --git a/lib/srv/server/ssm_install.go b/lib/srv/server/ssm_install.go index 6c0c2f427708e..3c23f672884a3 100644 --- a/lib/srv/server/ssm_install.go +++ b/lib/srv/server/ssm_install.go @@ -23,6 +23,8 @@ import ( "errors" "fmt" "log/slog" + "maps" + "slices" "strings" "github.com/aws/aws-sdk-go/aws" @@ -67,6 +69,9 @@ type SSMInstallationResult struct { SSMDocumentName string // InstallerScript is the Teleport Installer script name used to install Teleport into the instance. InstallerScript string + // InstanceName is the Instance's name. + // Might be empty. + InstanceName string } // SSMInstaller handles running SSM commands that install Teleport on EC2 instances. @@ -134,9 +139,9 @@ func NewSSMInstaller(cfg SSMInstallerConfig) (*SSMInstaller, error) { // Run executes the SSM document and then blocks until the command has completed. func (si *SSMInstaller) Run(ctx context.Context, req SSMRunRequest) error { - ids := make([]string, 0, len(req.Instances)) + instances := make(map[string]string, len(req.Instances)) for _, inst := range req.Instances { - ids = append(ids, inst.InstanceID) + instances[inst.InstanceID] = inst.InstanceName } params := make(map[string][]*string) @@ -144,8 +149,8 @@ func (si *SSMInstaller) Run(ctx context.Context, req SSMRunRequest) error { params[k] = []*string{aws.String(v)} } - validInstances := ids - instancesState, err := si.describeSSMAgentState(ctx, req, ids) + validInstances := instances + instancesState, err := si.describeSSMAgentState(ctx, req, instances) switch { case trace.IsAccessDenied(err): // describeSSMAgentState uses `ssm:DescribeInstanceInformation` to gather all the instances information. @@ -169,9 +174,10 @@ func (si *SSMInstaller) Run(ctx context.Context, req SSMRunRequest) error { validInstances = instancesState.valid } + validInstanceIDs := instanceIDsFrom(validInstances) output, err := req.SSM.SendCommandWithContext(ctx, &ssm.SendCommandInput{ DocumentName: aws.String(req.DocumentName), - InstanceIds: aws.StringSlice(validInstances), + InstanceIds: aws.StringSlice(validInstanceIDs), Parameters: params, }) if err != nil { @@ -190,7 +196,7 @@ func (si *SSMInstaller) Run(ctx context.Context, req SSMRunRequest) error { delete(params, ParamSSHDConfigPath) output, err = req.SSM.SendCommandWithContext(ctx, &ssm.SendCommandInput{ DocumentName: aws.String(req.DocumentName), - InstanceIds: aws.StringSlice(validInstances), + InstanceIds: aws.StringSlice(validInstanceIDs), Parameters: params, }) if err != nil { @@ -200,16 +206,17 @@ func (si *SSMInstaller) Run(ctx context.Context, req SSMRunRequest) error { g, ctx := errgroup.WithContext(ctx) g.SetLimit(10) - for _, inst := range validInstances { - inst := inst + for instanceID, instanceName := range validInstances { + instanceID := instanceID + instanceName := instanceName g.Go(func() error { - return trace.Wrap(si.checkCommand(ctx, req, output.Command.CommandId, &inst)) + return trace.Wrap(si.checkCommand(ctx, req, output.Command.CommandId, &instanceID, instanceName)) }) } return trace.Wrap(g.Wait()) } -func invalidSSMInstanceInstallationResult(req SSMRunRequest, instanceID, status, issueType string) *SSMInstallationResult { +func invalidSSMInstanceInstallationResult(req SSMRunRequest, instanceID, instanceName, status, issueType string) *SSMInstallationResult { return &SSMInstallationResult{ SSMRunEvent: &apievents.SSMRun{ Metadata: apievents.Metadata{ @@ -228,13 +235,14 @@ func invalidSSMInstanceInstallationResult(req SSMRunRequest, instanceID, status, IssueType: issueType, SSMDocumentName: req.DocumentName, InstallerScript: req.InstallerScriptName(), + InstanceName: instanceName, } } func (si *SSMInstaller) emitInvalidInstanceEvents(ctx context.Context, req SSMRunRequest, instanceIDsState *instanceIDsSSMState) error { var errs []error - for _, instanceID := range instanceIDsState.missing { - installationResult := invalidSSMInstanceInstallationResult(req, instanceID, + for instanceID, instanceName := range instanceIDsState.missing { + installationResult := invalidSSMInstanceInstallationResult(req, instanceID, instanceName, "EC2 Instance is not registered in SSM. Make sure that the instance has AmazonSSMManagedInstanceCore policy assigned.", usertasks.AutoDiscoverEC2IssueSSMInstanceNotRegistered, ) @@ -243,8 +251,8 @@ func (si *SSMInstaller) emitInvalidInstanceEvents(ctx context.Context, req SSMRu } } - for _, instanceID := range instanceIDsState.connectionLost { - installationResult := invalidSSMInstanceInstallationResult(req, instanceID, + for instanceID, instanceName := range instanceIDsState.connectionLost { + installationResult := invalidSSMInstanceInstallationResult(req, instanceID, instanceName, "SSM Agent in EC2 Instance is not connecting to SSM Service. Restart or reinstall the SSM service. See https://docs.aws.amazon.com/systems-manager/latest/userguide/ami-preinstalled-agent.html#verify-ssm-agent-status for more details.", usertasks.AutoDiscoverEC2IssueSSMInstanceConnectionLost, ) @@ -253,8 +261,8 @@ func (si *SSMInstaller) emitInvalidInstanceEvents(ctx context.Context, req SSMRu } } - for _, instanceID := range instanceIDsState.unsupportedOS { - installationResult := invalidSSMInstanceInstallationResult(req, instanceID, + for instanceID, instanceName := range instanceIDsState.unsupportedOS { + installationResult := invalidSSMInstanceInstallationResult(req, instanceID, instanceName, "EC2 instance is running an unsupported Operating System. Only Linux is supported.", usertasks.AutoDiscoverEC2IssueSSMInstanceUnsupportedOS, ) @@ -268,19 +276,29 @@ func (si *SSMInstaller) emitInvalidInstanceEvents(ctx context.Context, req SSMRu // instanceIDsSSMState contains a list of EC2 Instance IDs for a given state. type instanceIDsSSMState struct { - valid []string - missing []string - connectionLost []string - unsupportedOS []string + valid map[string]string + missing map[string]string + connectionLost map[string]string + unsupportedOS map[string]string +} + +func instanceIDsFrom(m map[string]string) []string { + return slices.Collect(maps.Keys(m)) } // describeSSMAgentState returns the instanceIDsSSMState for all the instances. -func (si *SSMInstaller) describeSSMAgentState(ctx context.Context, req SSMRunRequest, allInstanceIDs []string) (*instanceIDsSSMState, error) { - ret := &instanceIDsSSMState{} +func (si *SSMInstaller) describeSSMAgentState(ctx context.Context, req SSMRunRequest, allInstances map[string]string) (*instanceIDsSSMState, error) { + ret := &instanceIDsSSMState{ + valid: make(map[string]string), + missing: make(map[string]string), + connectionLost: make(map[string]string), + unsupportedOS: make(map[string]string), + } + instanceIDs := instanceIDsFrom(allInstances) ssmInstancesInfo, err := req.SSM.DescribeInstanceInformationWithContext(ctx, &ssm.DescribeInstanceInformationInput{ Filters: []*ssm.InstanceInformationStringFilter{ - {Key: aws.String(ssm.InstanceInformationFilterKeyInstanceIds), Values: aws.StringSlice(allInstanceIDs)}, + {Key: aws.String(ssm.InstanceInformationFilterKeyInstanceIds), Values: aws.StringSlice(instanceIDs)}, }, MaxResults: aws.Int64(awsEC2APIChunkSize), }) @@ -294,24 +312,24 @@ func (si *SSMInstaller) describeSSMAgentState(ctx context.Context, req SSMRunReq instanceStateByInstanceID[aws.StringValue(instanceState.InstanceId)] = instanceState } - for _, instanceID := range allInstanceIDs { + for instanceID, instanceName := range allInstances { instanceState, found := instanceStateByInstanceID[instanceID] if !found { - ret.missing = append(ret.missing, instanceID) + ret.missing[instanceID] = instanceName continue } if aws.StringValue(instanceState.PingStatus) == ssm.PingStatusConnectionLost { - ret.connectionLost = append(ret.connectionLost, instanceID) + ret.connectionLost[instanceID] = instanceName continue } if aws.StringValue(instanceState.PlatformType) != ssm.PlatformTypeLinux { - ret.unsupportedOS = append(ret.unsupportedOS, instanceID) + ret.unsupportedOS[instanceID] = instanceName continue } - ret.valid = append(ret.valid, instanceID) + ret.valid[instanceID] = instanceName } return ret, nil @@ -330,7 +348,7 @@ func skipAWSWaitErr(err error) error { return trace.Wrap(err) } -func (si *SSMInstaller) checkCommand(ctx context.Context, req SSMRunRequest, commandID, instanceID *string) error { +func (si *SSMInstaller) checkCommand(ctx context.Context, req SSMRunRequest, commandID, instanceID *string, instanceName string) error { err := req.SSM.WaitUntilCommandExecutedWithContext(ctx, &ssm.GetCommandInvocationInput{ CommandId: commandID, InstanceId: instanceID, @@ -377,6 +395,7 @@ func (si *SSMInstaller) checkCommand(ctx context.Context, req SSMRunRequest, com IssueType: usertasks.AutoDiscoverEC2IssueSSMScriptFailure, SSMDocumentName: req.DocumentName, InstallerScript: req.InstallerScriptName(), + InstanceName: instanceName, })) } @@ -393,6 +412,7 @@ func (si *SSMInstaller) checkCommand(ctx context.Context, req SSMRunRequest, com IssueType: usertasks.AutoDiscoverEC2IssueSSMScriptFailure, SSMDocumentName: req.DocumentName, InstallerScript: req.InstallerScriptName(), + InstanceName: instanceName, })) } } diff --git a/lib/srv/server/ssm_install_test.go b/lib/srv/server/ssm_install_test.go index 2107e91b0e590..c56b286258527 100644 --- a/lib/srv/server/ssm_install_test.go +++ b/lib/srv/server/ssm_install_test.go @@ -102,7 +102,7 @@ func TestSSMInstaller(t *testing.T) { name: "ssm run was successful", req: SSMRunRequest{ Instances: []EC2Instance{ - {InstanceID: "instance-id-1"}, + {InstanceID: "instance-id-1", InstanceName: "my-instance-name"}, }, DocumentName: document, Params: map[string]string{"token": "abcdefg"}, @@ -146,6 +146,7 @@ func TestSSMInstaller(t *testing.T) { }, IssueType: "ec2-ssm-script-failure", SSMDocumentName: "ssmdocument", + InstanceName: "my-instance-name", }}, }, { From 3672bc6da77d14d66582cd7b9bd97400a4d55975 Mon Sep 17 00:00:00 2001 From: Edward Dowling Date: Tue, 22 Oct 2024 16:47:29 +0100 Subject: [PATCH 2/2] Fix pagerduty AMR test to prevent flakiness (#46390) * Fix pagerduty AMR test to prevent flakiness * Update integrations/access/pagerduty/testlib/suite.go Co-authored-by: Zac Bergquist * Swap pagerduty test to use EventuallyWith * Update pagerduty tests to not create several access requests * Make more information available to AMR cache update callback * Update integrations/access/pagerduty/testlib/suite.go Co-authored-by: Tiago Silva * Update integrations/access/pagerduty/testlib/suite.go Co-authored-by: Tiago Silva * Fix formatting * Revert rename of pluginData in pagerduty tests * Remove duplicated ruleHandler init --------- Co-authored-by: Zac Bergquist Co-authored-by: Tiago Silva --- .../access_monitoring_rules.go | 10 +++++++ integrations/access/pagerduty/app.go | 9 ++++--- integrations/access/pagerduty/config.go | 6 +++++ .../access/pagerduty/testlib/suite.go | 27 +++++++++++++++---- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/integrations/access/accessmonitoring/access_monitoring_rules.go b/integrations/access/accessmonitoring/access_monitoring_rules.go index 72eb921f3f4e8..7fb2c045d57ea 100644 --- a/integrations/access/accessmonitoring/access_monitoring_rules.go +++ b/integrations/access/accessmonitoring/access_monitoring_rules.go @@ -48,6 +48,7 @@ type RuleHandler struct { pluginName string fetchRecipientCallback func(ctx context.Context, recipient string) (*common.Recipient, error) + onCacheUpdateCallback func(Operation types.OpType, name string, rule *accessmonitoringrulesv1.AccessMonitoringRule) error } // RuleMap is a concurrent map for access monitoring rules. @@ -65,6 +66,8 @@ type RuleHandlerConfig struct { // FetchRecipientCallback is a callback that maps recipient strings to plugin Recipients. FetchRecipientCallback func(ctx context.Context, recipient string) (*common.Recipient, error) + // OnCacheUpdateCallback is a callback that is called when a rule in the cache is created or updated. + OnCacheUpdateCallback func(Operation types.OpType, name string, rule *accessmonitoringrulesv1.AccessMonitoringRule) error } // NewRuleHandler returns a new RuleHandler. @@ -77,6 +80,7 @@ func NewRuleHandler(conf RuleHandlerConfig) *RuleHandler { pluginType: conf.PluginType, pluginName: conf.PluginName, fetchRecipientCallback: conf.FetchRecipientCallback, + onCacheUpdateCallback: conf.OnCacheUpdateCallback, } } @@ -93,6 +97,9 @@ func (amrh *RuleHandler) InitAccessMonitoringRulesCache(ctx context.Context) err continue } amrh.accessMonitoringRules.rules[amr.GetMetadata().Name] = amr + if amrh.onCacheUpdateCallback != nil { + amrh.onCacheUpdateCallback(types.OpPut, amr.GetMetadata().Name, amr) + } } return nil } @@ -123,6 +130,9 @@ func (amrh *RuleHandler) HandleAccessMonitoringRule(ctx context.Context, event t return nil } amrh.accessMonitoringRules.rules[req.Metadata.Name] = req + if amrh.onCacheUpdateCallback != nil { + amrh.onCacheUpdateCallback(types.OpPut, req.GetMetadata().Name, req) + } return nil case types.OpDelete: delete(amrh.accessMonitoringRules.rules, event.Resource.GetName()) diff --git a/integrations/access/pagerduty/app.go b/integrations/access/pagerduty/app.go index 972ed1bffce11..5eadcc5147cd0 100644 --- a/integrations/access/pagerduty/app.go +++ b/integrations/access/pagerduty/app.go @@ -78,7 +78,6 @@ func NewApp(conf Config) (*App, error) { teleport: conf.Client, statusSink: conf.StatusSink, } - app.mainJob = lib.NewServiceJob(app.run) return app, nil @@ -173,7 +172,7 @@ func (a *App) init(ctx context.Context) error { } } - a.accessMonitoringRules = accessmonitoring.NewRuleHandler(accessmonitoring.RuleHandlerConfig{ + amrhConf := accessmonitoring.RuleHandlerConfig{ Client: a.teleport, PluginType: types.PluginTypePagerDuty, PluginName: pluginName, @@ -184,7 +183,11 @@ func (a *App) init(ctx context.Context) error { Kind: common.RecipientKindSchedule, }, nil }, - }) + } + if a.conf.OnAccessMonitoringRuleCacheUpdateCallback != nil { + amrhConf.OnCacheUpdateCallback = a.conf.OnAccessMonitoringRuleCacheUpdateCallback + } + a.accessMonitoringRules = accessmonitoring.NewRuleHandler(amrhConf) if pong, err = a.checkTeleportVersion(ctx); err != nil { return trace.Wrap(err) diff --git a/integrations/access/pagerduty/config.go b/integrations/access/pagerduty/config.go index f76e9d2f955f2..8bf7060652b01 100644 --- a/integrations/access/pagerduty/config.go +++ b/integrations/access/pagerduty/config.go @@ -24,6 +24,8 @@ import ( "github.com/gravitational/trace" "github.com/pelletier/go-toml" + accessmonitoringrulesv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/accessmonitoringrules/v1" + "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/integrations/access/common" "github.com/gravitational/teleport/integrations/access/common/teleport" "github.com/gravitational/teleport/integrations/lib" @@ -47,6 +49,10 @@ type Config struct { // TeleportUser is the name of the Teleport user that will act // as the access request approver TeleportUser string + + // OnAccessMonitoringRuleCacheUpdateCallback is used for checking when + // the Rule cache is updated in tests + OnAccessMonitoringRuleCacheUpdateCallback func(Operation types.OpType, name string, rule *accessmonitoringrulesv1.AccessMonitoringRule) error } type PagerdutyConfig struct { diff --git a/integrations/access/pagerduty/testlib/suite.go b/integrations/access/pagerduty/testlib/suite.go index c379c85219a5c..b68c4d7e8d706 100644 --- a/integrations/access/pagerduty/testlib/suite.go +++ b/integrations/access/pagerduty/testlib/suite.go @@ -430,6 +430,15 @@ func (s *PagerdutySuiteOSS) TestRecipientsFromAccessMonitoringRule() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) t.Cleanup(cancel) + const ruleName = "test-pagerduty-amr" + var collectedNames []string + var mu sync.Mutex + s.appConfig.OnAccessMonitoringRuleCacheUpdateCallback = func(_ types.OpType, name string, _ *accessmonitoringrulesv1.AccessMonitoringRule) error { + mu.Lock() + collectedNames = append(collectedNames, name) + mu.Unlock() + return nil + } s.startApp() _, err := s.ClientByName(integration.RulerUserName). @@ -438,7 +447,7 @@ func (s *PagerdutySuiteOSS) TestRecipientsFromAccessMonitoringRule() { Kind: types.KindAccessMonitoringRule, Version: types.V1, Metadata: &v1.Metadata{ - Name: "test-pagerduty-amr", + Name: ruleName, }, Spec: &accessmonitoringrulesv1.AccessMonitoringRuleSpec{ Subjects: []string{types.KindAccessRequest}, @@ -453,6 +462,14 @@ func (s *PagerdutySuiteOSS) TestRecipientsFromAccessMonitoringRule() { }) assert.NoError(t, err) + // Incident creation may happen before plugins Access Monitoring Rule cache + // has been updated with new rule. Retry until the new cache picks up the rule. + require.EventuallyWithT(t, func(t *assert.CollectT) { + mu.Lock() + require.Contains(t, collectedNames, ruleName) + mu.Unlock() + }, 3*time.Second, time.Millisecond*100, "new access monitoring rule did not begin applying") + // Test execution: create an access request req := s.CreateAccessRequest(ctx, integration.RequesterOSSUserName, nil) @@ -463,16 +480,16 @@ func (s *PagerdutySuiteOSS) TestRecipientsFromAccessMonitoringRule() { }) incident, err := s.fakePagerduty.CheckNewIncident(ctx) - require.NoError(t, err, "no new incidents stored") - + assert.NoError(t, err, "no new incidents stored") assert.Equal(t, incident.ID, pluginData.IncidentID) - assert.Equal(t, s.pdNotifyService2.ID, pluginData.ServiceID) assert.Equal(t, pagerduty.PdIncidentKeyPrefix+"/"+req.GetName(), incident.IncidentKey) assert.Equal(t, "triggered", incident.Status) + assert.Equal(t, s.pdNotifyService2.ID, pluginData.ServiceID) + assert.NoError(t, s.ClientByName(integration.RulerUserName). - AccessMonitoringRulesClient().DeleteAccessMonitoringRule(ctx, "test-pagerduty-amr")) + AccessMonitoringRulesClient().DeleteAccessMonitoringRule(ctx, ruleName)) } func (s *PagerdutyBaseSuite) assertNewEvent(ctx context.Context, watcher types.Watcher, opType types.OpType, resourceKind, resourceName string) types.Event {