WIP: Tag system + Version System

Project.toml

@@ -5,10 +5,11 @@ version = "0.2.0"
 
 [deps]
 CoverageTools = "c36e975a-824b-4404-a568-ef97ca766997"
+CpuId = "adafc99b-e345-5852-983c-f28acb93d879"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9"
 
 [weakdeps]

perf/GLM/allocs.jl

@@ -1,6 +1,6 @@
 using PerfChecker, PrettyTables
 
-x = @check :alloc Dict(:targets => ["GLM"], :path => @__DIR__) begin
+result = @check :alloc Dict(:targets => ["GLM"], :path => @__DIR__) begin
     using GLM, Random, StatsModels
     end begin
     n = 2_500_000
@@ -21,5 +21,4 @@ x = @check :alloc Dict(:targets => ["GLM"], :path => @__DIR__) begin
     glm(pred, resp, Bernoulli())
 end
 
-
-pretty_table(x |> to_table)
+@info result

perf/GLM/bench.jl

@@ -1,4 +1,4 @@
-using PerfChecker, BenchmarkTools, PrettyTables
+using PerfChecker, BenchmarkTools
 
 t = @check :benchmark Dict(:path => @__DIR__, :evals => 1, :samples => 100, :seconds => 100) begin
     using GLM, Random, StatsModels
@@ -22,6 +22,4 @@ t = @check :benchmark Dict(:path => @__DIR__, :evals => 1, :samples => 100, :sec
     return nothing
 end
 
-Base.show(stdout, "text/plain", t)
-println()
-pretty_table(t |> to_table)
+@info t

perf/PatternFolds/allocs.jl

@@ -1,7 +1,7 @@
 using PerfChecker
 using PrettyTables
 
-x = @check :alloc Dict(:targets => ["PatternFolds"], :path => @__DIR__) begin
+x = @check :alloc Dict(:targets => ["PatternFolds"], :path => @__DIR__, :tags => [:patterns, :intervals]) begin
     using PatternFolds
     end begin
     itv = Interval{Open,Closed}(0.0, 1.0)
@@ -24,4 +24,4 @@ x = @check :alloc Dict(:targets => ["PatternFolds"], :path => @__DIR__) begin
     rand(vf, 1000)
 end
 
-pretty_table(x |> to_table)
+@info x

perf/PatternFolds/bench.jl

@@ -26,6 +26,4 @@ t = @check :benchmark Dict(:path => @__DIR__, :evals => 1, :samples => 100, :sec
     return nothing
 end
 
-Base.show(stdout, "text/plain", t)
-println()
-pretty_table(t |> to_table)
+@info t

src/PerfChecker.jl

@@ -2,17 +2,60 @@ module PerfChecker
 
 # SECTION - Imports
 using Pkg
+using Pkg.Types
+import TOML: parse
 using Profile
 import TypedTables: Table
 import Distributed: remotecall_fetch, addprocs, rmprocs
 import CoverageTools: analyze_malloc_files, find_malloc_files, MallocInfo
+import Base.Sys: CPUinfo, CPU_NAME, cpu_info, WORD_SIZE
+import CpuId: simdbytes, cpucores, cpucores_total, cputhreads_per_core
+
+struct HwInfo
+    cpus::Vector{CPUinfo}
+    machine::String
+    word::Int
+    simdbytes::Int
+    corecount::Tuple{Int, Int, Int}
+end
+
+struct CheckerResult
+    tables::Vector{Table}
+    hwinfo::Union{HwInfo,Nothing}
+    tags::Union{Nothing,Vector{Symbol}}
+    pkgs::Vector{PackageSpec}
+end
+
+function Base.show(io::IO, v::PerfChecker.CheckerResult)
+    println(io, "Tables:")
+    for i in v.tables
+        println(io, '\t', Base.display(i))
+    end
+
+    println(io, "Hardware Info:")
+    println(io, "CPU Information:")
+    println(io, '\t', v.hwinfo.cpus)
+    println(io, "Machine name: ", v.hwinfo.machine)
+    println(io, "Word Size: ", v.hwinfo.word)
+    println(io, "SIMD Bytes: ", v.hwinfo.simdbytes)
+    println(io, "Core count (physical, total and threads per core): ", v.hwinfo.corecount)
+
+    println(io, "Tags used: ", v.tags)
+
+    println(io, "Package versions tested (if provided): ")
+    println(io, Base.display(v.pkgs))
+end
+
+find_by_tags(tags::Vector{Symbol}, results::CheckerResult; exact_match = true) = findall(x -> exact_match ? (tags == x.tags) : (!isempty(x.tags ∩ tags)), results)
 
 # SECTION - Exports
 export @check
 export to_table
+export find_by_tags
+export get_versions
 
 # SECTION - Includes
-
+include("versions.jl")
 include("check.jl")
 include("alloc.jl")
 

src/alloc.jl

@@ -1,6 +1,4 @@
 prep(d::Dict, block::Expr, ::Val{:alloc}) = quote
-    import Pkg
-    Pkg.instantiate()
     import Profile
     $block
     nothing

src/check.jl

@@ -18,26 +18,57 @@ macro check(x, d, block1, block2)
         di = default_options($d, $x)
         g = prep(di, $block1, $x)
         h = check(di, $block2, $x)
+        results = CheckerResult(
+            Table[],
+            HwInfo(
+                cpu_info(),
+                CPU_NAME,
+                WORD_SIZE,
+                simdbytes(),
+                (cpucores(), cpucores_total(), cputhreads_per_core())
+            ),
+            haskey(di, :tags) ? di[:tags] : Symbol[:none],
+            PackageSpec[]
+        )
+
         p = remotecall_fetch(Core.eval, 1, Main,
             Expr(:toplevel, quote
                 import Distributed
                 d = $di
-                Distributed.addprocs(1; exeflags=["--track-allocation=$(d[:track])", "--project=$(d[:path])", "-t $(d[:threads])"])
+                t = tempname()
+                cp(d[:path], t)
+                Distributed.addprocs(1; exeflags=["--track-allocation=$(d[:track])", "--project=$t", "-t $(d[:threads])"])
             end).args...) |> first
 
-        di[:prep_result] = remotecall_fetch(Core.eval, p, Main,
-            Expr(:toplevel, g.args...))
+        pkgs = haskey(di, :pkgs) ? [PackageSpec(name=di[:pkgs][1], version=i) for i in get_versions(di[:pkgs])] : PackageSpec[PackageSpec()]
 
-        di[:check_result] = remotecall_fetch(Core.eval, p, Main,
-            Expr(:toplevel, h.args...))
+        for i in pkgs
+            remotecall_fetch(Core.eval, p, Main,
+                    Expr(:toplevel, quote
+                        import Pkg;
+                        d = $di
+                        Pkg.instantiate();
+                        $pkgs != [Pkg.PackageSpec()] && Pkg.add($i)
+                        haskey(d, :extra_pkgs) && Pkg.add(d[:extra_pkgs])
+                end).args...)
 
-        remotecall_fetch(Core.eval, 1, Main,
-            Expr(:toplevel, quote
-                import Distributed
-                Distributed.rmprocs($p)
-            end).args...)
+            di[:prep_result] = remotecall_fetch(Core.eval, p, Main,
+                Expr(:toplevel, g.args...))
+
+            di[:check_result] = remotecall_fetch(Core.eval, p, Main,
+                Expr(:toplevel, h.args...))
+
+            remotecall_fetch(Core.eval, 1, Main,
+                Expr(:toplevel, quote
+                    import Distributed
+                    Distributed.rmprocs($p)
+                end).args...)
 
-        $post(di, $x)
+            res = $post(di, $x)
+            push!(results.tables, res |> to_table)
+            push!(results.pkgs, i)
+        end
+        results
     end
 end
 

src/versions.jl

@@ -0,0 +1,75 @@
+"""
+Finds all versions of a package in all the installed registries and returns it as a vector.
+
+Example:
+
+```julia-repl
+julia> get_pkg_versions("ConstraintLearning")
+7-element Vector{VersionNumber}:
+ v"0.1.4"
+ v"0.1.5"
+ v"0.1.0"
+ v"0.1.6"
+ v"0.1.1"
+ v"0.1.3"
+ v"0.1.2"
+```
+"""
+function get_pkg_versions(name::String, regname::Union{Nothing,Vector{String}} = nothing)::Vector{VersionNumber}
+	regs = Types.Context().registries
+    indexes = isnothing(regname) ? collect(1:length(regs)) : findall(x -> x.name in regname, regs)
+
+	versions::Set{String} = Set([])
+	for i in indexes
+		push!(versions, keys(parse(regs[i].in_memory_registry[join([first(name),name,"Versions.toml"], '/')]))...)
+	end
+	return VersionNumber.(versions)
+end
+
+const VerConfig = Tuple{String, Symbol, Vector{VersionNumber}, Bool}
+
+"""
+Outputs the last patch or first patch of a version. 
+If the input is 1.2.3, then the output is 1.2.0 or 1.2.9 (assuming both exist, and both are the first and last patch of the version)
+"""
+function arrange_patches(a::VersionNumber, v::Vector{VersionNumber}, maxo::Bool)
+	a = filter(x -> a.minor == x.minor && a.major == x.major, v)
+	isempty(a) && error("No matching version found")
+	return maxo ? maximum(a) : minimum(a)
+end
+
+"""
+Outputs the last breaking or next breaking version. 
+If the input is 1.2.3, then the output is 1.2.0 or 1.3.0 (assuming both exist)
+"""
+function arrange_breaking(a::VersionNumber, v::Vector{VersionNumber}, maxo::Bool)
+	p = !maxo && filter(x -> a.major == x.major && a.minor == x.minor, v)
+	q = maxo && filter(x -> a.major == x.major && a.minor < x.minor, v)
+	(isempty(p) || isempty(q)) && error("No matching version found.")
+	return maxo ? minimum(q) : minimum(p)
+end
+
+"""
+Outputs the earlier or next major version.
+"""
+function arrange_major(a::VersionNumber, v::Vector{VersionNumber}, maxo::Bool)
+	p = maxo && filter(x -> a.major < x.major, v)
+	q = !maxo && filter(x -> a.major > x.major, v)
+	(isempty(p) || isempty(q)) && error("No matching version found.")
+	return maxo ? minimum(p) : maximum(q)
+end
+
+function get_versions(name::String, pkgconf::VerConfig, head::Bool = true, regname::Union{Nothing, Vector{String}} = nothing)
+	versions = get_pkg_versions(name, regname)
+	s = pkgconf[2]
+	f = if s == :patches
+		arrange_patches
+	elseif s == :breaking
+		arrange_breaking
+	elseif s == :major
+		arrange_major
+	else
+		error("Unknown option provided $s")
+	end
+	return name, map(x -> f(x, versions, pkgconf[4]), pkgconf[3])
+end

test/pattern_folds.jl

@@ -22,7 +22,7 @@
         rand(vf, 1000)
     end
 
-    pretty_table(x |> to_table)
+    @info x
 
     t = @check :benchmark Dict(:path => @__DIR__, :evals => 1, :samples => 100, :seconds => 100) begin
         using PatternFolds
@@ -50,7 +50,5 @@
         return nothing
     end
 
-    Base.show(stdout, "text/plain", t)
-    println()
-    pretty_table(t |> to_table)
+    @info t
 end