Merge pull request #43 from JuliaConstraints/output

Save and load tables as CSV and UUID for local machines

.gitignore

@@ -1,12 +1,12 @@
 *.jl.*.cov
 *.jl.cov
-*.jl.mem
+*.jl.*.mem
 *Manifest.toml
 
 /docs/build/
 
-/test/benchmarks/
-/test/mallocs/
+/test/output/
+/test/metadata/
 
 *~
 *.swp

Project.toml

@@ -6,40 +6,43 @@ version = "0.2.0"
 [deps]
 CoverageTools = "c36e975a-824b-4404-a568-ef97ca766997"
 CpuId = "adafc99b-e345-5852-983c-f28acb93d879"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Malt = "36869731-bdee-424d-aa32-cab38c994e3b"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [weakdeps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
 Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
 
 [extensions]
 BenchmarkToolsExt = "BenchmarkTools"
-CSVExt = "CSV"
 ChairmarksExt = "Chairmarks"
 MakieExt = "Makie"
 
 [compat]
-BenchmarkTools = "1.5"
+BenchmarkTools = "1"
 CSV = "0.10"
 Chairmarks = "1"
 CoverageTools = "1.3.1"
-CpuId = "0.3.1"
+CpuId = "0.3"
+JSON = "0.21"
 Makie = "0.21"
-Malt = "1.1.1"
+Malt = "1"
 TypedTables = "1"
 julia = "1.9"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
 PatternFolds = "c18a7f1d-76ad-4ce4-950d-5419b888513b"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "BenchmarkTools", "PatternFolds", "Test"]
+test = ["Aqua", "BenchmarkTools", "Chairmarks", "PatternFolds", "Test"]

ext/MakieExt/allocs.jl

@@ -21,6 +21,10 @@ function PerfChecker.table_to_pie(x::Table, ::Val{:alloc}; pkg_name = "")
 end
 
 function PerfChecker.checkres_to_pie(x::PerfChecker.CheckerResult, ::Val{:alloc})
+    name(i) = x.pkgs[i].name * "_v" * string(x.pkgs[i].version)
+    return map(
+        i -> (name(i) => table_to_pie(x.tables[i], Val(:alloc), pkg_name = name(i))),
+        eachindex(x.tables))
 end
 
 function PerfChecker.checkres_to_scatterlines(

perf/GLM/Project.toml

@@ -1,5 +1,7 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
 GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a"
 PerfChecker = "6309bf6b-a531-4b08-891e-8ee981e5c424"
 StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d"

perf/GLM/README.md

@@ -2,6 +2,8 @@
 
 `GLM.jl` is a Julia library for Generalized Linear Model.
 
+<!-- TODO: update with new interface with new figures -->
+
 ### Getting Started with `PerfChecker.jl`
 Before getting started with any checks, we must ensure that we are evaluating the correct version. Because `PerfChecker.jl` lacks a few features at the time of writing this, this has to be done manually.
 

perf/GLM/allocs.jl

@@ -1,6 +1,17 @@
 using PerfChecker
 
-result = @check :alloc Dict(:targets => ["GLM"], :path => @__DIR__) begin
+d = Dict(:targets => ["GLM"],
+    :path => @__DIR__,
+    :pkgs => ("GLM",
+        :custom,
+        [
+            v"1.3.9", v"1.3.10", v"1.3.11", v"1.4.0",
+            v"1.5.0", v"1.6.0", v"1.7.0", v"1.8.0",
+            v"1.9.0"],
+        true),
+    :tags => [:bernoulli])
+
+x = @check :alloc d begin
     using GLM, Random, StatsModels
 end begin
     n = 2_500_000
@@ -21,4 +32,4 @@ end begin
     glm(pred, resp, Bernoulli())
 end
 
-@info result
+@info x

perf/GLM/bench.jl

@@ -1,6 +1,17 @@
-using PerfChecker, BenchmarkTools
+using PerfChecker, BenchmarkTools, CairoMakie
 
-t = @check :benchmark Dict(:path => @__DIR__, :evals => 1, :samples => 100, :seconds => 100) begin
+d = Dict(:targets => ["GLM"],
+    :path => @__DIR__, :evals => 1, :samples => 100, :seconds => 100,
+    :pkgs => ("GLM",
+        :custom,
+        [
+            v"1.3.9", v"1.3.10", v"1.3.11", v"1.4.0",
+            v"1.5.0", v"1.6.0", v"1.7.0", v"1.8.0",
+            v"1.9.0"],
+        true),
+    :tags => [:bernoulli])
+
+x = @check :benchmark d begin
     using GLM, Random, StatsModels
 end begin
     n = 2_500_000
@@ -19,7 +30,16 @@ end begin
     logistic(x::Real) = inv(1 + exp(-x))
     resp .= rand(rng, n) .< logistic.(pred * B)
     glm(pred, resp, Bernoulli())
-    return nothing
 end
 
-@info t
+@info x
+
+mkpath(joinpath(@__DIR__, "visuals"))
+
+c = checkres_to_scatterlines(x, Val(:benchmark))
+save(joinpath(@__DIR__, "visuals", "bench_evolution.png"), c)
+
+for kwarg in [:times, :gctimes, :memory, :allocs]
+    c2 = checkres_to_boxplots(x, Val(:benchmark); kwarg)
+    save(joinpath(@__DIR__, "visuals", "bench_boxplots_$kwarg.png"), c2)
+end