From 4cbf4abc12b9da5917d6d4007baa6c154e1667f0 Mon Sep 17 00:00:00 2001 From: Farasath Ahamed Date: Sun, 20 Oct 2024 12:56:57 +0200 Subject: [PATCH] Measure percentiles in benchmarks Signed-off-by: Farasath Ahamed --- .../opaauthorizerequest_test.go | 41 ++++++++++--------- go.mod | 2 + go.sum | 4 ++ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go b/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go index dfdc49b34a..6f9c4a3b87 100644 --- a/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go +++ b/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/benburkert/pbench" "github.com/golang-jwt/jwt/v4" opasdktest "github.com/open-policy-agent/opa/sdk/test" "github.com/stretchr/testify/assert" @@ -630,7 +631,7 @@ const ( // Usage: // To run a specific scenario with or without decision logging: // -// go test -bench=BenchmarkAuthorizeRequest//{without-decision-logging|with-decision-logging} +// go test -bench=BenchmarkAuthorizeRequest//{no-decision-logs|with-decision-logs} // // Example: // @@ -639,13 +640,14 @@ const ( // Running with multiple GOMAXPROCS values (parallel CPU execution): // Use the -cpu flag to simulate the benchmark under different levels of CPU parallelism: // -// go test -bench 'BenchmarkAuthorizeRequest/minimal/without-decision-logging' -cpu 1,2,4,8 +// go test -bench 'BenchmarkAuthorizeRequest/minimal/no-decision-logs' -cpu 1,2,4,8 // // Note: Refer to the code for the latest available scenarios. func BenchmarkAuthorizeRequest(b *testing.B) { + scenarios := []struct { name string - benchmarkFunc func(b *testing.B, decisionLogging bool) + benchmarkFunc func(pb *pbench.B, decisionLogging bool) }{ {"minimal", benchmarkMinimal}, {"with-body", benchmarkAllowWithReqBody}, @@ -653,19 +655,24 @@ func BenchmarkAuthorizeRequest(b *testing.B) { } for _, scenario := range scenarios { - b.Run(scenario.name, func(b *testing.B) { - b.Run("without-decision-logging", func(b *testing.B) { - scenario.benchmarkFunc(b, false) + b.Run(scenario.name, func(tb *testing.B) { + pb := pbench.New(tb) + pb.ReportPercentile(0.5) + pb.ReportPercentile(0.95) + pb.ReportPercentile(0.99) + + pb.Run("no-decision-logs", func(pb *pbench.B) { + scenario.benchmarkFunc(pb, false) }) - b.Run("with-decision-logging", func(b *testing.B) { - scenario.benchmarkFunc(b, true) + pb.Run("with-decision-logs", func(pb *pbench.B) { + scenario.benchmarkFunc(pb, true) }) }) } } -func benchmarkMinimal(b *testing.B, decisionLogging bool) { +func benchmarkMinimal(b *pbench.B, decisionLogging bool) { opaControlPlane := opasdktest.MustNewServer( opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ "main.rego": ` @@ -702,16 +709,14 @@ func benchmarkMinimal(b *testing.B, decisionLogging bool) { } b.ResetTimer() - b.ReportAllocs() - - b.RunParallel(func(pb *testing.PB) { + b.RunParallel(func(pb *pbench.PB) { for pb.Next() { f.Request(ctx) } }) } -func benchmarkAllowWithReqBody(b *testing.B, decisionLogging bool) { +func benchmarkAllowWithReqBody(b *pbench.B, decisionLogging bool) { opaControlPlane := opasdktest.MustNewServer( opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{ "main.rego": ` @@ -755,16 +760,14 @@ func benchmarkAllowWithReqBody(b *testing.B, decisionLogging bool) { } b.ResetTimer() - b.ReportAllocs() - - b.RunParallel(func(pb *testing.PB) { + b.RunParallel(func(pb *pbench.PB) { for pb.Next() { f.Request(ctx) } }) } -func benchmarkJwtValidation(b *testing.B, decisionLogging bool) { +func benchmarkJwtValidation(b *pbench.B, decisionLogging bool) { publicKey, err := os.ReadFile(certPath) if err != nil { @@ -851,9 +854,7 @@ func benchmarkJwtValidation(b *testing.B, decisionLogging bool) { } b.ResetTimer() - b.ReportAllocs() - - b.RunParallel(func(pb *testing.PB) { + b.RunParallel(func(pb *pbench.PB) { for pb.Next() { f.Request(ctx) assert.False(b, ctx.FServed) diff --git a/go.mod b/go.mod index ecaba54f5d..cf3d438f6d 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/abbot/go-http-auth v0.4.0 github.com/andybalholm/brotli v1.1.0 github.com/aryszka/jobqueue v0.0.3 + github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef github.com/cenkalti/backoff v2.2.1+incompatible github.com/cespare/xxhash/v2 v2.3.0 github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9 @@ -94,6 +95,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.0.2 // indirect github.com/go-logr/logr v1.4.2 // indirect diff --git a/go.sum b/go.sum index 9356ad0578..45b677a0e9 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/aryszka/jobqueue v0.0.3 h1:O5YbgzQCjRomudwnDTY5BrHUNJhvPHQHq7GfGpE+ybs= github.com/aryszka/jobqueue v0.0.3/go.mod h1:SdxqI6HZ4E1Lss94tey5OfjcAu3bdCDWS1AQzzIN4m4= +github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef h1:+7ZJvJGiV4hUBdjhEDhfGdjBCOmhVi0YQ5n+6g/ei+k= +github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef/go.mod h1:hrhDSsc41bBqGejYXbvMh6qexfcC2vXjodP5gufwWyI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -151,6 +153,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 h1:Vh7rylVZRZCj6W41lRlP17xPk4Nq260H4Xo/DDYmEZk= +github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=