From aea16b8bde8d6df1f92101b5f560deb741dee97e Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Thu, 8 Jun 2023 17:53:19 +0200 Subject: [PATCH 1/2] benchmarks without stats because the samples are not normally distributed --- test/benchmark.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/benchmark.jl b/test/benchmark.jl index 8dcb4a4..4cfac3d 100644 --- a/test/benchmark.jl +++ b/test/benchmark.jl @@ -80,6 +80,10 @@ const TEST_BENCHMARK = true # and we have an admissible error of 2.5% @test tdiff < 2*σdiff + + # NO because the samples are not normally distributed + # so let's just do a percentage again + @test ratio < 1.2 end end From c482ce643609253d31a1b658acc3dd5b457bc84f Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Thu, 8 Jun 2023 17:56:20 +0200 Subject: [PATCH 2/2] compare median not mean --- test/benchmark.jl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/benchmark.jl b/test/benchmark.jl index 4cfac3d..276c6ee 100644 --- a/test/benchmark.jl +++ b/test/benchmark.jl @@ -61,18 +61,22 @@ const TEST_BENCHMARK = true b1 = run(bench1) b2 = run(bench2) - t1 = mean(b1).time - t2 = mean(b2).time + mean1 = mean(b1).time + mean2 = mean(b2).time + + median1 = BenchmarkTools.median(b1).time + median2 = BenchmarkTools.median(b2).time σ1 = BenchmarkTools.std(b1).time σ2 = BenchmarkTools.std(b2).time - tdiff = t1 - t2 + tdiff = mean1 - mean2 σdiff = sqrt(σ1^2 + σ2^2) - ratio = t1 / t2 + ratio_mean = mean1 / mean2 + ratio_median = median1 / median2 - @info "Expr $i" t1 t2 ratio diff=Text("$round(Int64, tdiff) ± $round(Int64, σdiff))") b1 b2 + @info "Expr $i" mean1 mean2 ratio_mean ratio_median diff=Text("$round(Int64, tdiff) ± $round(Int64, σdiff))") b1 b2 if TEST_BENCHMARK # we should be faster, i.e. @@ -83,7 +87,7 @@ const TEST_BENCHMARK = true # NO because the samples are not normally distributed # so let's just do a percentage again - @test ratio < 1.2 + @test ratio_median < 1.2 end end