diff --git a/metrics/notifier.go b/metrics/notifier.go index dbb1fe9f3..1c32302d7 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -3,6 +3,7 @@ package metrics // NotifierMetrics is a collection of metrics used in notifier type NotifierMetrics struct { SubsMalformed Meter + SubsBrokenDisabled Meter EventsReceived Meter EventsMalformed Meter EventsProcessingFailed Meter @@ -16,6 +17,7 @@ type NotifierMetrics struct { func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics { return &NotifierMetrics{ SubsMalformed: registry.NewMeter("subs", "malformed"), + SubsBrokenDisabled: registry.NewMeter("subs", "broken_disabled"), EventsReceived: registry.NewMeter("events", "received"), EventsMalformed: registry.NewMeter("events", "malformed"), EventsProcessingFailed: registry.NewMeter("events", "failed"), diff --git a/notifier/notifier.go b/notifier/notifier.go index 6daa4b2e2..0f0b1c666 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -195,6 +195,12 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific switch e := err.(type) { case moira.SenderBrokenContactError: log.Errorf("Cannot send to broken contact: %s", e.Error()) + if count, errorDisabling := disableBrokenContactSubscriptions(&pkg.Contact, notifier.database, log); errorDisabling != nil { + log.Errorf("Failed to disable broken subscriptions: ", errorDisabling) + } else { + notifier.metrics.SubsBrokenDisabled.Mark(int64(count)) + log.Infof("Disabled %d broken subscriptions", count) + } default: log.Errorf("Cannot send notification: %s", err.Error()) @@ -203,3 +209,40 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific } } } + +func disableBrokenContactSubscriptions(brokenContact *moira.ContactData, database moira.Database, log moira.Logger) ( + disabledCount int, e error) { + subsIDs, err := database.GetUserSubscriptionIDs(brokenContact.User) + if err != nil { + return 0, err + } + if len(subsIDs) == 0 { + return 0, nil + } + + subs, e := database.GetSubscriptions(subsIDs) + if e != nil { + return 0, e + } + + disableSubs := []*moira.SubscriptionData{} + for _, s := range subs { + if len(s.Contacts) == 1 { + if s.Contacts[0] == brokenContact.ID { + s.Enabled = false + disableSubs = append(disableSubs, s) + } + } else { + log.Warningf("Skipped disable subscription with broken contact, "+ + "contacts more than 1, contacts = %v", s.Contacts) + } + } + disableCount := len(disableSubs) + if disableCount > 0 { + if err := database.SaveSubscriptions(disableSubs); err != nil { + return 0, err + } + return disableCount, nil + } + return 0, nil +} diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go index d2009a655..6b1f0f71c 100644 --- a/notifier/notifier_test.go +++ b/notifier/notifier_test.go @@ -131,10 +131,12 @@ func TestNoResendForSendToBrokenContact(t *testing.T) { Events: eventsData, Contact: moira.ContactData{ Type: "test", + User: "someuser", }, } sender.EXPECT().SendEvents(eventsData, pkg.Contact, pkg.Trigger, plots, pkg.Throttled). Return(moira.NewSenderBrokenContactError(fmt.Errorf("some sender reason"))) + dataBase.EXPECT().GetUserSubscriptionIDs(pkg.Contact.User).Return([]string{}, nil) var wg sync.WaitGroup notif.Send(&pkg, &wg) @@ -236,7 +238,8 @@ func afterTest() { var subID = "SubscriptionID-000000000000001" var event = moira.NotificationEvent{ - Metric: "generate.event.1", + Metric: "generate.event.1", + State: moira.StateOK, OldState: moira.StateWARN, TriggerID: "triggerID-0000000000001",