Skip to content

Commit

Permalink
Set severity field of the dummy events
Browse files Browse the repository at this point in the history
Otherwise, the notification subject is looking odd, since this field is
used to build the notification messages.
  • Loading branch information
yhabteab authored and julianbrost committed Nov 20, 2023
1 parent cd55a99 commit 1acb259
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 11 deletions.
17 changes: 9 additions & 8 deletions internal/incident/incident.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func (i *Incident) ProcessEvent(ctx context.Context, ev *event.Event, created bo
}

// Re-evaluate escalations based on the newly evaluated rules.
escalations, err := i.evaluateEscalations(ev.Time)
escalations, err := i.evaluateEscalations(ev)
if err != nil {
return err
}
Expand Down Expand Up @@ -196,7 +196,7 @@ func (i *Incident) RetriggerEscalations(ev *event.Event) {
return
}

escalations, err := i.evaluateEscalations(ev.Time)
escalations, err := i.evaluateEscalations(ev)
if err != nil {
i.logger.Errorw("Reevaluating time-based escalations failed", zap.Error(err))
return
Expand Down Expand Up @@ -393,7 +393,7 @@ func (i *Incident) evaluateRules(ctx context.Context, tx *sqlx.Tx, eventID int64

// evaluateEscalations evaluates this incidents rule escalations to be triggered if they aren't already.
// Returns the newly evaluated escalations to be triggered or an error on database failure.
func (i *Incident) evaluateEscalations(eventTime time.Time) ([]*rule.Escalation, error) {
func (i *Incident) evaluateEscalations(e *event.Event) ([]*rule.Escalation, error) {
if i.EscalationState == nil {
i.EscalationState = make(map[int64]*EscalationState)
}
Expand All @@ -406,7 +406,7 @@ func (i *Incident) evaluateEscalations(eventTime time.Time) ([]*rule.Escalation,
i.timer = nil
}

filterContext := &rule.EscalationFilter{IncidentAge: eventTime.Sub(i.StartedAt), IncidentSeverity: i.Severity}
filterContext := &rule.EscalationFilter{IncidentAge: e.Time.Sub(i.StartedAt), IncidentSeverity: i.Severity}

var escalations []*rule.Escalation
retryAfter := rule.RetryNever
Expand Down Expand Up @@ -453,16 +453,17 @@ func (i *Incident) evaluateEscalations(eventTime time.Time) ([]*rule.Escalation,
// i.e. if an incident is 15m old and an escalation rule evaluates incident_age>=1h the retryAfter would
// contain 45m (1h - incident age (15m)). Therefore, we have to use the event time instead of the incident
// start time here.
nextEvalAt := eventTime.Add(retryAfter)
nextEvalAt := e.Time.Add(retryAfter)

i.logger.Infow("Scheduling escalation reevaluation", zap.Duration("after", retryAfter), zap.Time("at", nextEvalAt))
i.timer = time.AfterFunc(retryAfter, func() {
i.logger.Info("Reevaluating escalations")

i.RetriggerEscalations(&event.Event{
Type: event.TypeInternal,
Time: nextEvalAt,
Message: fmt.Sprintf("Incident reached age %v", nextEvalAt.Sub(i.StartedAt)),
Type: event.TypeInternal,
Time: nextEvalAt,
Severity: e.Severity,
Message: fmt.Sprintf("Incident reached age %v", nextEvalAt.Sub(i.StartedAt)),
})
})
}
Expand Down
7 changes: 4 additions & 3 deletions internal/incident/incidents.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,10 @@ func LoadOpenIncidents(ctx context.Context, db *icingadb.DB, logger *logging.Log
}

incident.RetriggerEscalations(&event.Event{
Time: time.Now(),
Type: event.TypeInternal,
Message: "Incident reevaluation at daemon startup",
Time: time.Now(),
Type: event.TypeInternal,
Severity: incident.Severity,
Message: "Incident reevaluation at daemon startup",
})
}

Expand Down

0 comments on commit 1acb259

Please sign in to comment.