Skip to content

Commit

Permalink
Add support to add labels on SLI/SLOs
Browse files Browse the repository at this point in the history
  • Loading branch information
@wpjunior
wpjunior committed Oct 17, 2019
1 parent 4ee344a commit c099d98
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 19 deletions.
36 changes: 17 additions & 19 deletions slo/slo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,17 +122,24 @@ func (slo SLO) GenerateGroupRules() []rulefmt.RuleGroup {
return rules
}

func (slo SLO) labels() map[string]string {
labels := map[string]string{}
if !slo.HonorLabels {
labels["service"] = slo.Name
}
for key, value := range slo.Labels {
labels[key] = value
}
return labels
}

func (slo SLO) generateRules(bucket string) []rulefmt.Rule {
rules := []rulefmt.Rule{}
if slo.TrafficRateRecord.Expr != "" {
trafficRateRecord := rulefmt.Rule{
Record: "slo:service_traffic:ratio_rate_" + bucket,
Expr: slo.TrafficRateRecord.ComputeExpr(bucket, ""),
Labels: map[string]string{},
}

if !slo.HonorLabels {
trafficRateRecord.Labels["service"] = slo.Name
Labels: slo.labels(),
}

rules = append(rules, trafficRateRecord)
Expand All @@ -142,11 +149,7 @@ func (slo SLO) generateRules(bucket string) []rulefmt.Rule {
errorRateRecord := rulefmt.Rule{
Record: "slo:service_errors_total:ratio_rate_" + bucket,
Expr: slo.ErrorRateRecord.ComputeExpr(bucket, ""),
Labels: map[string]string{},
}

if !slo.HonorLabels {
errorRateRecord.Labels["service"] = slo.Name
Labels: slo.labels(),
}

rules = append(rules, errorRateRecord)
Expand All @@ -157,11 +160,7 @@ func (slo SLO) generateRules(bucket string) []rulefmt.Rule {
latencyQuantileRecord := rulefmt.Rule{
Record: "slo:service_latency:" + quantile.name + "_" + bucket,
Expr: slo.LatencyQuantileRecord.ComputeQuantile(bucket, quantile.quantile),
Labels: map[string]string{},
}

if !slo.HonorLabels {
latencyQuantileRecord.Labels["service"] = slo.Name
Labels: slo.labels(),
}

rules = append(rules, latencyQuantileRecord)
Expand All @@ -172,12 +171,11 @@ func (slo SLO) generateRules(bucket string) []rulefmt.Rule {
latencyRateRecord := rulefmt.Rule{
Record: "slo:service_latency:ratio_rate_" + bucket,
Expr: slo.LatencyRecord.ComputeExpr(bucket, latencyBucket.LE),
Labels: map[string]string{
"service": slo.Name,
"le": latencyBucket.LE,
},
Labels: slo.labels(),
}

latencyRateRecord.Labels["le"] = latencyBucket.LE

rules = append(rules, latencyRateRecord)
}

Expand Down
31 changes: 31 additions & 0 deletions slo/slo_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ func TestSLOGenerateGroupRules(t *testing.T) {
AlertMethod: "multi-window",
Expr: "sum(rate(http_bucket{le=\"$le\"}[$window]))/sum(rate(http_total[$window]))",
},
Labels: map[string]string{
"team": "team-avengers",
},
Annotations: map[string]string{
"message": "Service A has lower SLI",
"link": "http://wiki.ops/1234",
Expand All @@ -57,20 +60,23 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_total[5m]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_errors_total:ratio_rate_5m",
Expr: "sum(rate(http_errors[5m]))/sum(rate(http_total[5m]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_latency:ratio_rate_5m",
Expr: "sum(rate(http_bucket{le=\"0.1\"}[5m]))/sum(rate(http_total[5m]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
"le": "0.1",
},
},
Expand All @@ -79,6 +85,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_bucket{le=\"0.5\"}[5m]))/sum(rate(http_total[5m]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
"le": "0.5",
},
},
Expand All @@ -89,13 +96,15 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_total[30m]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_errors_total:ratio_rate_30m",
Expr: "sum(rate(http_errors[30m]))/sum(rate(http_total[30m]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Expand All @@ -104,6 +113,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Labels: map[string]string{
"service": "my-team.my-service.payment",
"le": "0.1",
"team": "team-avengers",
},
},
{
Expand All @@ -112,6 +122,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Labels: map[string]string{
"service": "my-team.my-service.payment",
"le": "0.5",
"team": "team-avengers",
},
},

Expand All @@ -121,20 +132,23 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_total[1h]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_errors_total:ratio_rate_1h",
Expr: "sum(rate(http_errors[1h]))/sum(rate(http_total[1h]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_latency:ratio_rate_1h",
Expr: "sum(rate(http_bucket{le=\"0.1\"}[1h]))/sum(rate(http_total[1h]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
"le": "0.1",
},
},
Expand All @@ -144,6 +158,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Labels: map[string]string{
"service": "my-team.my-service.payment",
"le": "0.5",
"team": "team-avengers",
},
},
},
Expand All @@ -159,13 +174,15 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_total[2h]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_errors_total:ratio_rate_2h",
Expr: "sum(rate(http_errors[2h]))/sum(rate(http_total[2h]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Expand All @@ -174,6 +191,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Labels: map[string]string{
"service": "my-team.my-service.payment",
"le": "0.1",
"team": "team-avengers",
},
},
{
Expand All @@ -182,6 +200,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Labels: map[string]string{
"service": "my-team.my-service.payment",
"le": "0.5",
"team": "team-avengers",
},
},

Expand All @@ -191,13 +210,15 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_total[6h]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_errors_total:ratio_rate_6h",
Expr: "sum(rate(http_errors[6h]))/sum(rate(http_total[6h]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Expand All @@ -206,12 +227,14 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Labels: map[string]string{
"service": "my-team.my-service.payment",
"le": "0.1",
"team": "team-avengers",
},
},
{
Record: "slo:service_latency:ratio_rate_6h",
Expr: "sum(rate(http_bucket{le=\"0.5\"}[6h]))/sum(rate(http_total[6h]))",
Labels: map[string]string{
"team": "team-avengers",
"service": "my-team.my-service.payment",
"le": "0.5",
},
Expand All @@ -229,20 +252,23 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_total[1d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_errors_total:ratio_rate_1d",
Expr: "sum(rate(http_errors[1d]))/sum(rate(http_total[1d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_latency:ratio_rate_1d",
Expr: "sum(rate(http_bucket{le=\"0.1\"}[1d]))/sum(rate(http_total[1d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
"le": "0.1",
},
},
Expand All @@ -251,6 +277,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_bucket{le=\"0.5\"}[1d]))/sum(rate(http_total[1d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
"le": "0.5",
},
},
Expand All @@ -261,20 +288,23 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_total[3d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_errors_total:ratio_rate_3d",
Expr: "sum(rate(http_errors[3d]))/sum(rate(http_total[3d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
},
},
{
Record: "slo:service_latency:ratio_rate_3d",
Expr: "sum(rate(http_bucket{le=\"0.1\"}[3d]))/sum(rate(http_total[3d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
"le": "0.1",
},
},
Expand All @@ -283,6 +313,7 @@ func TestSLOGenerateGroupRules(t *testing.T) {
Expr: "sum(rate(http_bucket{le=\"0.5\"}[3d]))/sum(rate(http_total[3d]))",
Labels: map[string]string{
"service": "my-team.my-service.payment",
"team": "team-avengers",
"le": "0.5",
},
},
Expand Down

0 comments on commit c099d98

Please sign in to comment.