Add support for scatterlines and boxplots for Chairmarks and BenchmarkTools

ext/MakieExt/MakieExt.jl

@@ -7,5 +7,6 @@ using PerfChecker
 include("plotutils.jl")
 include("allocs.jl")
 include("bench.jl")
+include("chair.jl")
 
 end

ext/MakieExt/bench.jl

@@ -1,44 +1,60 @@
 function PerfChecker.checkres_to_scatterlines(
         x::PerfChecker.CheckerResult, ::Val{:benchmark})
-    println("Hello!")
-    @warn "Here!"
+    data = []
+    props = TypedTables.columnnames(x.tables[1])
+    for i in eachindex(x.tables)
+        t = x.tables[i]
+        m = [map(TypedTables.GetProperty{i}(), t) for i in props]
+        g = minimum.(m)
+        push!(data, g)
+    end
+
+    d = [[data[i][j] for i in eachindex(data)] for j in eachindex(data[1])]
+    r = minimum.(d)
+
+    versionnums = [x.pkgs[i].version for i in eachindex(x.pkgs)]
+
+    f = Figure()
+    ax = f[1, 1] = Axis(f)
+    colors = make_colors(length(props))
+    max = 2
+    for i in eachindex(data[1])
+        xs = collect(eachindex(versionnums))
+        ys = d[i] ./ r[i]
+        if max < maximum(ys)
+            max = maximum(ys)
+        end
+        scatterlines!(xs, ys, label = string(props[i]), color = (colors[i], 0.4))
+    end
+    ax.xticks = (eachindex(versionnums), string.(versionnums))
+    ax.xlabel = "versions"
+    ax.ylabel = "ratio"
+    ax.title = "Evolution for $(x.pkgs[1].name) (via BenchmarkTools.jl)"
+    ylims!(; low = 0, high = max)
+    axislegend()
+    return f
 end
 
 function PerfChecker.checkres_to_boxplots(
-        x::PerfChecker.CheckerResult, ::Val{:benchmark}; kwarg = :times)
-    di = Dict()
-    data = []
-    #for i in eachindex(x.tables)
-    #j = x.tables[i]
-    #p = x.pkgs[i]
-    #g = getproperties(j[i], (kwargs,))
-    #g = [g[k][1] for k in eachindex(g)]
-    #push!((fill(i, length(g)), g))
-    #end
+        x::PerfChecker.CheckerResult, ::Val{:benchmark}; kwarg::Symbol = :times)
+    datax, datay = [], []
 
-    #=
-    w = getproperties(j[1], (:allocs,))
-    versions = Dict()
-    for i in eachindex(x.pkgs)
-    versions[x.pkgs[i].version] = i
+    for i in eachindex(x.tables)
+        j = x.tables[i]
+        p = x.pkgs[i]
+        g = map(TypedTables.GetProperty{kwarg}(), j)
+        append!(datax, fill(i, length(g)))
+        append!(datay, g)
     end
 
-    versionnums = [x.pkgs[i].version for i in eachindex(x.pkgs)] 
+    versionnums = [x.pkgs[i].version for i in eachindex(x.pkgs)]
     f = Figure()
-    ax = Axis(f[1, 1])
+    ax = f[1, 1] = Axis(f)
     ax.xticks = (eachindex(versionnums), string.(versionnums))
     ax.xlabel = "versions"
-    ax.ylabel = "bytes"
-    colors = make_colors(length(keys(di)))
-    i = 1
-    for (keys, values) in di
-    xs = [values[i][1] for i in eachindex(values)]
-    ys = [versions[values[i][2]] for i in eachindex(values)]
-    scatterlines!(f[1,1], ys, xs, label = keys, color = (colors[i], 0.6)); i += 1
-    end
+    ax.ylabel = string(kwarg)
+    boxplot!(datax, datay, label = string(kwarg))
     ax.title = x.pkgs[1].name
-    Legend(f[1,2], ax)
+    axislegend()
     return f
-    =#
-    return data
 end

ext/MakieExt/chair.jl

@@ -0,0 +1,61 @@
+function PerfChecker.checkres_to_scatterlines(
+        x::PerfChecker.CheckerResult, ::Val{:chairmark})
+    data = []
+    props = TypedTables.columnnames(x.tables[1])
+    for i in eachindex(x.tables)
+        t = x.tables[i]
+        m = [map(TypedTables.GetProperty{i}(), t) for i in props]
+        g = minimum.(m)
+        push!(data, g)
+    end
+
+    d = [[data[i][j] for i in eachindex(data)] for j in eachindex(data[1])]
+    r = minimum.(d)
+
+    versionnums = [x.pkgs[i].version for i in eachindex(x.pkgs)]
+
+    f = Figure()
+    ax = f[1, 1] = Axis(f)
+    colors = make_colors(length(props))
+    max = 2
+    for i in eachindex(data[1])
+        xs = collect(eachindex(versionnums))
+        ys = d[i] ./ r[i]
+        if max < maximum(ys)
+            max = maximum(ys)
+        end
+        scatterlines!(xs, ys, label = string(props[i]), color = (colors[i], 0.4))
+    end
+    ax.xticks = (eachindex(versionnums), string.(versionnums))
+    ax.xlabel = "versions"
+    ax.ylabel = "ratio"
+    ax.title = "Evolution for $(x.pkgs[1].name) (via Chairmarks.jl)"
+    ylims!(; low = 0, high = max)
+    axislegend()
+    return f
+end
+
+function PerfChecker.checkres_to_boxplots(
+        x::PerfChecker.CheckerResult, ::Val{:chairmark}; kwarg::Symbol = :times)
+    di = Dict()
+    datax, datay = [], []
+
+    for i in eachindex(x.tables)
+        j = x.tables[i]
+        p = x.pkgs[i]
+        g = map(TypedTables.GetProperty{kwarg}(), j)
+        append!(datax, fill(i, length(g)))
+        append!(datay, g)
+    end
+
+    versionnums = [x.pkgs[i].version for i in eachindex(x.pkgs)]
+    f = Figure()
+    ax = f[1, 1] = Axis(f)
+    ax.xticks = (eachindex(versionnums), string.(versionnums))
+    ax.xlabel = "versions"
+    ax.ylabel = string(kwarg)
+    boxplot!(datax, datay, label = string(kwarg))
+    ax.title = x.pkgs[1].name
+    axislegend()
+    return f
+end

perf/PatternFolds/bench.jl

@@ -2,7 +2,8 @@ using PerfChecker, BenchmarkTools, CairoMakie
 
 d = Dict(:path => @__DIR__, :evals => 1, :samples => 100,
     :seconds => 100, :tags => [:patterns, :intervals],
-    :pkgs => ("PatternFolds", :custom, [v"0.2.2", v"0.2.3"], true))
+    :pkgs => ("PatternFolds", :custom, [v"0.2.2", v"0.2.3"], true),
+    :devops => "PatternFolds")
 
 t = @check :benchmark d begin
     using PatternFolds
@@ -32,4 +33,6 @@ end
 
 @info t
 
-#@info checkres_to_boxplots(t, Val(:benchmark))
+checkres_to_boxplots(t, Val(:benchmark))
+c = checkres_to_scatterlines(t, Val(:benchmark))
+save(joinpath(homedir(), "ubab.png"), c)

perf/PatternFolds/chair.jl

@@ -1,7 +1,7 @@
-using PerfChecker, Chairmarks
+using PerfChecker, Chairmarks, CairoMakie
 
 d = Dict(:path => @__DIR__, :evals => 1, :tags => [:patterns, :intervals],
-    :pkgs => ("PatternFolds", :custom, [v"0.2.2", v"0.2.3"], true))
+    :pkgs => ("PatternFolds", :custom, [v"0.2.2", v"0.2.3"], true), :devops => "PatternFolds")
 
 t = @check :chairmark d begin
     using PatternFolds
@@ -30,3 +30,6 @@ end begin
 end
 
 @info t
+checkres_to_boxplots(t, Val(:chairmark))
+c = checkres_to_scatterlines(t, Val(:chairmark))
+save(joinpath(homedir(), "ubac.png"), c)

src/check.jl

@@ -92,6 +92,11 @@ function check_function(x::Symbol, d::Dict, block1, block2)
         push!(results.tables, res |> to_table)
         if !(devop && i == len)
             push!(results.pkgs, pkgs[i])
+        else
+            pkg = d[:devops]
+            p = pkg isa Tuple ? pkg[1] : pkg
+            p = p isa Pkg.PackageSpec ? p.name : p
+            push!(results.pkgs, Pkg.PackageSpec(name = p, version = "dev"))
         end
     end