From e5f8e3584ff798c5a5fab0273e15f9e69fe4508a Mon Sep 17 00:00:00 2001 From: Azzaare Date: Tue, 28 May 2024 08:23:44 +0000 Subject: [PATCH] Fix output and medata as csv --- Project.toml | 10 +- ext/CSVExt/CSVExt.jl | 15 -- perf/PatternFolds/Project.toml | 1 + perf/PatternFolds/allocs.jl | 48 ++-- perf/PatternFolds/metadata/metadata.csv | 2 + .../300280de-258c-52ff-ac69-4bbbbb93d570.csv | 5 + .../d67e26e8-2a3a-5155-989c-29aac013e915.csv | 5 + src/PerfChecker.jl | 40 ++- src/check.jl | 231 ++++++++++-------- src/csv.jl | 11 + src/utils.jl | 49 ++-- src/versions.jl | 2 +- test/Aqua.jl | 2 +- 13 files changed, 227 insertions(+), 194 deletions(-) delete mode 100644 ext/CSVExt/CSVExt.jl create mode 100644 perf/PatternFolds/metadata/metadata.csv create mode 100644 perf/PatternFolds/output/300280de-258c-52ff-ac69-4bbbbb93d570.csv create mode 100644 perf/PatternFolds/output/d67e26e8-2a3a-5155-989c-29aac013e915.csv create mode 100644 src/csv.jl diff --git a/Project.toml b/Project.toml index 6a72ee8..6378696 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.2.0" [deps] CoverageTools = "c36e975a-824b-4404-a568-ef97ca766997" CpuId = "adafc99b-e345-5852-983c-f28acb93d879" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Malt = "36869731-bdee-424d-aa32-cab38c994e3b" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -16,24 +17,23 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [weakdeps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" [extensions] BenchmarkToolsExt = "BenchmarkTools" -CSVExt = "CSV" ChairmarksExt = "Chairmarks" MakieExt = "Makie" [compat] -BenchmarkTools = "1.5" +BenchmarkTools = "1" CSV = "0.10" Chairmarks = "1" CoverageTools = "1.3.1" -CpuId = "0.3.1" +CpuId = "0.3" +JSON = "0.21" Makie = "0.21" -Malt = "1.1.1" +Malt = "1" TypedTables = "1" julia = "1.9" diff --git a/ext/CSVExt/CSVExt.jl b/ext/CSVExt/CSVExt.jl deleted file mode 100644 index 52fc3e5..0000000 --- a/ext/CSVExt/CSVExt.jl +++ /dev/null @@ -1,15 +0,0 @@ -module CSVExt - -using CSV -import TypedTables: Table - -table_to_csv(t::Table, path::String) = CSV.write(path, t) - -csv_to_table(path::String) = CSV.read(path, Table) - -function check_to_metadata_csv( - x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}; metadata = "") - check_to_metadata(x, pkg, version, tags; metadata) -end - -end diff --git a/perf/PatternFolds/Project.toml b/perf/PatternFolds/Project.toml index 5077243..973d4ea 100644 --- a/perf/PatternFolds/Project.toml +++ b/perf/PatternFolds/Project.toml @@ -3,6 +3,7 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" +Coverage = "a2441757-f6aa-5fb2-8edb-039e3f45d037" PatternFolds = "c18a7f1d-76ad-4ce4-950d-5419b888513b" PerfChecker = "6309bf6b-a531-4b08-891e-8ee981e5c424" TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" diff --git a/perf/PatternFolds/allocs.jl b/perf/PatternFolds/allocs.jl index 7b6e287..9149e1c 100644 --- a/perf/PatternFolds/allocs.jl +++ b/perf/PatternFolds/allocs.jl @@ -1,41 +1,41 @@ using PerfChecker, CairoMakie, CSV d = Dict(:targets => ["PatternFolds"], :path => @__DIR__, :tags => [:patterns, :intervals], - :pkgs => ("PatternFolds", :custom, [v"0.2.3", v"0.2.2"], true)) + :pkgs => ("PatternFolds", :custom, [v"0.2.2", v"0.2.3"], true)) x = @check :alloc d begin - using PatternFolds + using PatternFolds end begin - itv = Interval{Open, Closed}(0.0, 1.0) - i = IntervalsFold(itv, 2.0, 1000) + itv = Interval{Open, Closed}(0.0, 1.0) + i = IntervalsFold(itv, 2.0, 1000) - @info "Checking IntervalsFold" i pattern(i) gap(i) folds(i) size(i) length(i) + @info "Checking IntervalsFold" i pattern(i) gap(i) folds(i) size(i) length(i) - unfold(i) - collect(i) - reverse(collect(i)) + unfold(i) + collect(i) + reverse(collect(i)) - # Vectors - vf = make_vector_fold([0, 1], 2, 1000) - @info "Checking VectorFold" vf pattern(vf) gap(vf) folds(vf) length(vf) + # Vectors + vf = make_vector_fold([0, 1], 2, 1000) + @info "Checking VectorFold" vf pattern(vf) gap(vf) folds(vf) length(vf) - unfold(vf) - collect(vf) - reverse(collect(vf)) + unfold(vf) + collect(vf) + reverse(collect(vf)) - rand(vf, 1000) + rand(vf, 1000) end @info x -for (i, t) in enumerate(x.tables) - p = d[:pkgs] - @info "debug" p[1] p[2] p[3] p[4] - mkpath("perf/PatternFolds/output") - display(table_to_pie(t, Val(:alloc); pkg_name = "PatternFolds.jl")) - path = joinpath( - d[:path], "perf", "PatternFolds", "output", string(p[1], "_v$(p[3][i])", ".png")) - @info path -end +# for (i, t) in enumerate(x.tables) +# p = d[:pkgs] +# @info "debug" p[1] p[2] p[3] p[4] +# mkpath("perf/PatternFolds/output") +# display(table_to_pie(t, Val(:alloc); pkg_name = "PatternFolds.jl")) +# path = joinpath( +# d[:path], "perf", "PatternFolds", "output", string(p[1], "_v$(p[3][i])", ".png")) +# @info path +# end # checkres_to_scatterlines(x, Val(:alloc)) diff --git a/perf/PatternFolds/metadata/metadata.csv b/perf/PatternFolds/metadata/metadata.csv new file mode 100644 index 0000000..5a76da5 --- /dev/null +++ b/perf/PatternFolds/metadata/metadata.csv @@ -0,0 +1,2 @@ +alloc_PatternFolds_v0.2.2_patterns_intervals,36c9068f-2e67-410b-b8bb-d57c1b476f62 +alloc_PatternFolds_v0.2.3_patterns_intervals,36c9068f-2e67-410b-b8bb-d57c1b476f62 diff --git a/perf/PatternFolds/output/300280de-258c-52ff-ac69-4bbbbb93d570.csv b/perf/PatternFolds/output/300280de-258c-52ff-ac69-4bbbbb93d570.csv new file mode 100644 index 0000000..ce70501 --- /dev/null +++ b/perf/PatternFolds/output/300280de-258c-52ff-ac69-4bbbbb93d570.csv @@ -0,0 +1,5 @@ +bytes,percentage,filenames,linenumbers +16128,97.49,/home/baffier/.julia/packages/PatternFolds/3DAei/src/vector.jl,87 +320,1.93,/home/baffier/.julia/packages/PatternFolds/3DAei/src/vector.jl,19 +48,0.29,/home/baffier/.julia/packages/PatternFolds/3DAei/src/intervals.jl,32 +48,0.29,/home/baffier/.julia/packages/PatternFolds/3DAei/src/common.jl,100 diff --git a/perf/PatternFolds/output/d67e26e8-2a3a-5155-989c-29aac013e915.csv b/perf/PatternFolds/output/d67e26e8-2a3a-5155-989c-29aac013e915.csv new file mode 100644 index 0000000..2630dd1 --- /dev/null +++ b/perf/PatternFolds/output/d67e26e8-2a3a-5155-989c-29aac013e915.csv @@ -0,0 +1,5 @@ +bytes,percentage,filenames,linenumbers +16128,97.49,/home/baffier/.julia/packages/PatternFolds/KzXjV/src/vector.jl,87 +320,1.93,/home/baffier/.julia/packages/PatternFolds/KzXjV/src/vector.jl,19 +48,0.29,/home/baffier/.julia/packages/PatternFolds/KzXjV/src/intervals.jl,32 +48,0.29,/home/baffier/.julia/packages/PatternFolds/KzXjV/src/common.jl,100 diff --git a/src/PerfChecker.jl b/src/PerfChecker.jl index 609630e..572ac49 100644 --- a/src/PerfChecker.jl +++ b/src/PerfChecker.jl @@ -1,45 +1,39 @@ module PerfChecker # SECTION - Imports -using Pkg -using Pkg.Types -import TOML: parse -using Profile -import TypedTables: Table -import Malt: remote_eval_wait, Worker, remote_eval_fetch, stop, fetch -import CoverageTools: analyze_malloc_files, find_malloc_files, MallocInfo import Base.Sys: CPUinfo, CPU_NAME, cpu_info, WORD_SIZE +import CoverageTools: analyze_malloc_files, find_malloc_files, MallocInfo import CpuId: simdbytes, cpucores, cputhreads, cputhreads_per_core -import UUIDs: uuid4, uuid5 +import CSV import JSON - -function csv_to_table(path) - # @warn "CSV module not loaded. Please load it before using this function." -end - -function table_to_csv(t::Table, path::String) - # @warn "CSV module not loaded. Please load it before using this function." -end +import Malt: remote_eval_wait, Worker, remote_eval_fetch, stop, fetch +import Pkg +import Pkg.Types: PackageSpec, Context +import Profile +import TOML: parse +import TypedTables: Table +import UUIDs: UUID, uuid4, uuid5 # SECTION - Exports export @check -export to_table -export find_by_tags -export get_versions -export table_to_pie +export check_to_metadata_csv +export checkres_to_boxplots export checkres_to_pie export checkres_to_scatterlines -export saveplot export csv_to_table +export find_by_tags +export get_versions +export saveplot export table_to_csv -export checkres_to_boxplots -export check_to_metadata_csv +export table_to_pie +export to_table # SECTION - Includes include("init.jl") include("hwinfo.jl") include("checker_results.jl") include("utils.jl") +include("csv.jl") include("versions.jl") include("check.jl") include("alloc.jl") diff --git a/src/check.jl b/src/check.jl index bfa8113..e17b0ff 100644 --- a/src/check.jl +++ b/src/check.jl @@ -1,11 +1,11 @@ initpkgs(x) = quote - nothing + nothing end prep(d, b, v) = quote - nothing + nothing end check(d, b, v) = quote - nothing + nothing end post(d, v) = nothing default_options(v) = Dict() @@ -16,112 +16,137 @@ check(d::Dict, b::Expr, v::Symbol) = check(d, b, Val(v)) post(d::Dict, v::Symbol) = post(d, Val(v)) function default_options(d::Dict, v::Symbol) - di = default_options(Val(v)) - return merge(di, d) + di = default_options(Val(v)) + return merge(di, d) end function check_function(x::Symbol, d::Dict, block1, block2) - di = default_options(d, x) - g = prep(di, block1, x) - h = check(di, block2, x) - initpkg = initpkgs(x) - - results = CheckerResult( - Table[], - HwInfo( - cpu_info(), - CPU_NAME, - WORD_SIZE, - simdbytes(), - (cpucores(), cputhreads(), cputhreads_per_core()) - ), - haskey(di, :tags) ? di[:tags] : Symbol[:none], - PackageSpec[] - ) - - pkgs = if haskey(di, :pkgs) - [PackageSpec(name = di[:pkgs][1], version = i) for i in get_versions(di[:pkgs])[2]] - else - PackageSpec[PackageSpec()] - end - - devop = haskey(di, :devops) - - len = length(pkgs) + devop - - t = [tempname() for _ in 1:len] - cp.(Ref(di[:path]), t) - - procs = @sync begin - fetch.([@async(Worker(; - exeflags = ["--track-allocation=$(di[:track])", - "-t $(di[:threads])", "--project=$(t[i])"])) for i in 1:len]) - end - - for i in 1:len - remote_eval_wait(Main, procs[i], quote - import Pkg - let - i = $i - @info "Worker No.: $i" - end - Pkg.instantiate(; io = stderr) - end) - - remote_eval_wait(Main, procs[i], initpkg) - - remote_eval_wait(Main, procs[i], - quote - d = $di - pkgs = $pkgs - if !($i == $len && $devop) - pkgs != [Pkg.PackageSpec()] && Pkg.add(getindex(pkgs, $i)) - else - pkg = d[:devops] - pkg isa Tuple ? Pkg.develop(pkg[1]; pkg[2]...) : Pkg.develop(pkg) - end - haskey(d, :extra_pkgs) && Pkg.add(d[:extra_pkgs]) - end) - - di[:prep_result] = remote_eval_fetch(Main, procs[i], g) - di[:check_result] = remote_eval_fetch(Main, procs[i], h) - - stop(procs[i]) - - res = post(di, x) - push!(results.tables, res |> to_table) - if !(devop && i == len) - push!(results.pkgs, pkgs[i]) - else - pkg = d[:devops] - p = pkg isa Tuple ? pkg[1] : pkg - p = p isa Pkg.PackageSpec ? p.name : p - push!(results.pkgs, Pkg.PackageSpec(name = p, version = "dev")) - end - end - - for (k, t) in enumerate(results.tables) - tags = results.tags - ps = results.pkgs[k] - pkg = ps.name - v = ps.version - name = filename(x, pkg, v, tags; ext = "csv") - path = joinpath(d[:path], "output", name) - table_to_csv(t, path) - name = joinpath(d[:path], "metadata", "metadata.csv") - check_to_metadata_csv(x, pkg, v, tags; metadata = name) - end - - return results + di = default_options(d, x) + g = prep(di, block1, x) + h = check(di, block2, x) + initpkg = initpkgs(x) + + results = CheckerResult( + Table[], + HwInfo( + cpu_info(), + CPU_NAME, + WORD_SIZE, + simdbytes(), + (cpucores(), cputhreads(), cputhreads_per_core()) + ), + haskey(di, :tags) ? di[:tags] : Symbol[:none], + PackageSpec[] + ) + + pkgs = if haskey(di, :pkgs) + [PackageSpec(name = di[:pkgs][1], version = i) for i in get_versions(di[:pkgs])[2]] + else + PackageSpec[PackageSpec()] + end + + devop = haskey(di, :devops) + + len = length(pkgs) + devop + + t = [tempname() for _ in 1:len] + cp.(Ref(di[:path]), t) + + procs = @sync begin + fetch.([@async(Worker(; + exeflags = ["--track-allocation=$(di[:track])", + "-t $(di[:threads])", "--project=$(t[i])"])) for i in 1:len]) + end + + for i in 1:len + is_loaded = false + if i ≤ length(pkgs) + path = joinpath(di[:path], "metadata", "metadata.csv") + fp = flatten_parameters(x, pkgs[i].name, pkgs[i].version, d[:tags]) + u = get_uuid() |> Base.UUID + if in_metadata(path, fp, u) + is_loaded = true + end + end + + if !is_loaded + remote_eval_wait(Main, procs[i], quote + import Pkg + let + i = $i + @info "Worker No.: $i" + end + Pkg.instantiate(; io = stderr) + end) + + remote_eval_wait(Main, procs[i], initpkg) + + remote_eval_wait(Main, procs[i], + quote + d = $di + pkgs = $pkgs + if !($i == $len && $devop) + pkgs != [Pkg.PackageSpec()] && Pkg.add(getindex(pkgs, $i)) + else + pkg = d[:devops] + pkg isa Tuple ? Pkg.develop(pkg[1]; pkg[2]...) : Pkg.develop(pkg) + end + haskey(d, :extra_pkgs) && Pkg.add(d[:extra_pkgs]) + end) + + di[:prep_result] = remote_eval_fetch(Main, procs[i], g) + di[:check_result] = remote_eval_fetch(Main, procs[i], h) + + stop(procs[i]) + end + + res = if is_loaded + fp = flatten_parameters(x, pkgs[i].name, pkgs[i].version, d[:tags]) + u = uuid5(get_uuid() |> Base.UUID, fp) + path = joinpath(di[:path], "output", string(u)) * ".csv" + csv_to_table(path) + else + post(di, x) |> to_table + end + push!(results.tables, res) + if !(devop && i == len) + push!(results.pkgs, pkgs[i]) + else + pkg = d[:devops] + p = pkg isa Tuple ? pkg[1] : pkg + p = p isa Pkg.PackageSpec ? p.name : p + push!(results.pkgs, Pkg.PackageSpec(name = p, version = "dev")) + end + end + + for (k, t) in enumerate(results.tables) + tags = results.tags + ps = results.pkgs[k] + pkg = ps.name + v = ps.version + isnothing(pkg) && continue + name = filename(x, pkg, v, tags; ext = "csv") + path = joinpath(d[:path], "output", name) + metadata = joinpath(d[:path], "metadata", "metadata.csv") + fp = flatten_parameters(x, pkg, v, tags) + u = get_uuid() |> Base.UUID + if in_metadata(metadata, fp, u) + continue + end + table_to_csv(t, path) + check_to_metadata_csv(x, pkg, v, tags; metadata) + end + + return results end macro check(x, d, block1, block2) - block1, block2 = Expr(:quote, block1), Expr(:quote, block2) - quote - x = $(esc(x)) - d = $(esc(d)) - check_function(x, d, $block1, $block2) - end + block1, block2 = Expr(:quote, block1), Expr(:quote, block2) + quote + x = $(esc(x)) + d = $(esc(d)) + check_function(x, d, $block1, $block2) + end end function perf_table end diff --git a/src/csv.jl b/src/csv.jl new file mode 100644 index 0000000..a12ffef --- /dev/null +++ b/src/csv.jl @@ -0,0 +1,11 @@ +function table_to_csv(t::Table, path::String) + mkpath(dirname(path)) + CSV.write(path, t) +end + +csv_to_table(path::String) = CSV.read(path, Table) + +function check_to_metadata_csv( + x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}; metadata = "") + check_to_metadata(x, pkg, version, tags; metadata) +end diff --git a/src/utils.jl b/src/utils.jl index 3bbd3e1..a805713 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,39 +1,44 @@ get_uuid() = ENV["PERFCHECKER_UUID"] function flatten_parameters( - x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}) - return join(vcat([x, pkg, string("v", version)], tags), "_") + x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}) + return join(vcat([x, pkg, string("v", version)], tags), "_") end function file_uuid( - x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}) - return uuid5(get_uuid() |> Base.UUID, flatten_parameters(x, pkg, version, tags)) + x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}) + return uuid5(get_uuid() |> Base.UUID, flatten_parameters(x, pkg, version, tags)) end function filename(x::Symbol, pkg::AbstractString, version, - tags::Vector{Symbol}; ext::AbstractString) - return "$(file_uuid(x, pkg, version, tags)).$ext" + tags::Vector{Symbol}; ext::AbstractString) + return "$(file_uuid(x, pkg, version, tags)).$ext" end function check_to_metadata( - x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}; metadata = "") - fp = flatten_parameters(x, pkg, version, tags) - u = get_uuid() |> Base.UUID + x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}; metadata = "") + fp = flatten_parameters(x, pkg, version, tags) + u = get_uuid() |> Base.UUID - if !isempty(metadata) - if !isfile(metadata) - mkpath(dirname(metadata)) - open(metadata, "a") do f - write(f, string(flatten_parameters(x, pkg, version, tags), ",", u, "\n")) - end - end - end + if !isempty(metadata) + f = isfile(metadata) + f || mkpath(dirname(metadata)) + if !f || !in_metadata(metadata, fp, u) + open(metadata, "a") do f + @info "Writing metada" metadata + write(f, string(fp, ",", u, "\n")) + end + end + end - return fp, u + return fp, u end -function check_to_metadata_csv( - x::Symbol, pkg::AbstractString, version, tags::Vector{Symbol}; metadata = "") - @info "should not be here" - return nothing +function in_metadata(metadata, fp, u) + isfile(metadata) && for l in eachline(metadata) + if l == string(fp, ",", u) + return true + end + end + return false end diff --git a/src/versions.jl b/src/versions.jl index 7d08b48..64cf1fb 100644 --- a/src/versions.jl +++ b/src/versions.jl @@ -17,7 +17,7 @@ julia> get_pkg_versions("ConstraintLearning") """ function get_pkg_versions(name::String, regname::Union{Nothing, Vector{String}} = nothing)::Vector{VersionNumber} - regs = Types.Context().registries + regs = Context().registries indexes = isnothing(regname) ? collect(1:length(regs)) : findall(x -> x.name in regname, regs) diff --git a/test/Aqua.jl b/test/Aqua.jl index 4affd12..228581e 100644 --- a/test/Aqua.jl +++ b/test/Aqua.jl @@ -23,7 +23,7 @@ @testset "Dependencies compatibility (no extras)" begin Aqua.test_deps_compat(PerfChecker; check_extras = false, - ignore = [:Distributed, :Pkg, :Profile, :TOML] + ignore = [:Pkg, :Profile, :TOML, :UUIDs] ) end end