From d1d93840d12dd8ef1ccedebf46e23a2e09e2d215 Mon Sep 17 00:00:00 2001 From: Chetan Vardhan Date: Mon, 18 Dec 2023 00:24:05 +0900 Subject: [PATCH 1/8] Refactor allocations.jl Fix some typos Some bug fixing Repetition Convert GLM example Revert absolute path New abstract syntax Change GLM again mmmmmmmmmmmmmmmmmmmmm Bugfix --- perf/GLM/allocs.jl | 31 ++++++++------------- src/PerfChecker.jl | 12 ++++++-- src/allocations.jl | 69 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 84 insertions(+), 28 deletions(-) diff --git a/perf/GLM/allocs.jl b/perf/GLM/allocs.jl index 9006d36..efb614d 100644 --- a/perf/GLM/allocs.jl +++ b/perf/GLM/allocs.jl @@ -1,25 +1,17 @@ using PerfChecker -using Test -using GLM -using Random -using StatsModels - -@testset "GLM.jl" begin - title = "Basic GLM computation" - dependencies = [GLM] - targets = [GLM] - - function alloc() +j = @prep :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :track => "user") begin + using GLM, Random, StatsModels + @check begin n = 2_500_000 rng = Random.MersenneTwister(1234321) tbl = ( - x1 = randn(rng, n), - x2 = Random.randexp(rng, n), - ss = rand(rng, string.(50:99), n), - y = zeros(n), - ) - f = @formula(y ~ 1 + x1 + x2 + ss) + x1=randn(rng, n), + x2=Random.randexp(rng, n), + ss=rand(rng, string.(50:99), n), + y=zeros(n), + ) + f = @eval @formula(y ~ 1 + x1 + x2 + ss) f = apply_schema(f, schema(f, tbl)) resp, pred = modelcols(f, tbl) B = randn(rng, size(pred, 2)) @@ -27,8 +19,7 @@ using StatsModels logistic(x::Real) = inv(1 + exp(-x)) resp .= rand(rng, n) .< logistic.(pred * B) glm(pred, resp, Bernoulli()) - return nothing end - - alloc_check(title, dependencies, targets, alloc, alloc; path=@__DIR__) end + +println(j) diff --git a/src/PerfChecker.jl b/src/PerfChecker.jl index ac0c38a..78e32c6 100644 --- a/src/PerfChecker.jl +++ b/src/PerfChecker.jl @@ -2,10 +2,8 @@ module PerfChecker # SECTION - Imports using BenchmarkTools -using CoverageTools using CSV using DataFrames -using Distributed using LibGit2 using OrderedCollections using PGFPlotsX @@ -16,12 +14,22 @@ using Random using StatsPlots using Term using TypedTables +import Distributed: remotecall_fetch, addprocs, rmprocs +import CoverageTools: analyze_malloc_files, find_malloc_files + +struct CheckerOptions + threads::Int + path::AbstractString + targets::Union{String, Vector{String}} +end # SECTION - Exports export alloc_check export alloc_plot export bench_plot export store_benchmark +export @prep +export CheckerOptions # SECTION - Includes diff --git a/src/allocations.jl b/src/allocations.jl index f0a0d7e..828f1fb 100644 --- a/src/allocations.jl +++ b/src/allocations.jl @@ -1,3 +1,59 @@ +prep(d::Dict, block::Expr, ::Val{:alloc}) = quote + import Pkg + Pkg.instantiate() + import Profile + macro check(repeat, block) + repeat && block + Profile.clear_malloc_data() + block + end + macro check(block) + block + Profile.clear_malloc_data() + block + end + $block + targets = eval(Meta.parse("[" * (typeof($d[:targets]) == String ? $d[:targets] : join($d[:targets], ", ")) * "]")) + @warn targets + return map(dirname ∘ pathof ∘ eval, targets) +end + +prep(d::Dict, b::Expr, v::Symbol) = prep(d, b, Val(v)) + +macro prep(x, d, block) + block = Expr(:quote, block) + quote + g = prep($d, $block, $x) + + p = remotecall_fetch(Core.eval, 1, Main, + Expr(:toplevel, quote + import Distributed + d = $($d) + Distributed.addprocs(1; exeflags=["--track-allocation=$(d[:track])", "--project=$(d[:path])", "-t $(d[:threads])"]) + end).args...) |> first + + j = remotecall_fetch(Core.eval, p, Main, + Expr(:toplevel, g.args...)) + + remotecall_fetch(Core.eval, 1, Main, + Expr(:toplevel, quote + import Distributed + Distributed.rmprocs($p) + end).args...) + + files = find_malloc_files(j) + + myallocs = analyze_malloc_files(files) + if isempty(myallocs) + @error "No allocation files found in $($d[:targets])" + else + rm.(files) + return myallocs + end + end +end + +#= function alloc_check( title, dependencies, @@ -15,20 +71,20 @@ function alloc_check( # add a proc (id == p) that track allocations p = first(isnothing(threads) ? addprocs(1; exeflags=["--track-allocation=user", "--project=$path"]) : addprocs(1; exeflags=["--track-allocation=user", "--project=$path", "-t $threads"])) - + remotecall_fetch(Core.eval, p, Main, Expr(:toplevel, (quote import Pkg; Pkg.instantiate() import Profile; nothing end).args...)) - + for d in dependencies remotecall_fetch(Core.eval, p, Main, Expr(:toplevel, (quote using $(Symbol(d)) nothing end).args...)) end - + @eval @everywhere $p $pre_alloc() @eval @everywhere $p Profile.clear_malloc_data() @eval @everywhere $p $alloc() @@ -50,13 +106,13 @@ function alloc_check( end # Smart paths - + # common, specifics = smart_paths(map(a -> a.filename, Iterators.reverse(myallocs))) # @info "sizes" map(a -> a.filename, Iterators.reverse(myallocs)) specifics # slash = Sys.iswindows() ? "\\" : "/" # Make the allocations data readable through a dataframe - + #= df = DataFrame() df.bytes = map(a -> a.bytes, Iterators.reverse(myallocs)) @@ -71,7 +127,7 @@ function alloc_check( l = map(a -> a.linenumber, Iterators.reverse(myallocs)) TypedTables.Table(bytes = b, percentage = r, filenames = f, linenumbers = l) end - + # Save it as a CSV file label = "" if labeller == :oid @@ -149,3 +205,4 @@ function alloc_plot( end end end +=# From 682b9faa3915c6f286923e9013a34e3633a90cef Mon Sep 17 00:00:00 2001 From: azzaare Date: Sat, 30 Dec 2023 13:49:50 +0900 Subject: [PATCH 2/8] Fix GLM allocs script --- perf/GLM/allocs.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf/GLM/allocs.jl b/perf/GLM/allocs.jl index efb614d..e96c950 100644 --- a/perf/GLM/allocs.jl +++ b/perf/GLM/allocs.jl @@ -1,6 +1,6 @@ using PerfChecker -j = @prep :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :track => "user") begin +@prep :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :track => "user") begin using GLM, Random, StatsModels @check begin n = 2_500_000 @@ -22,4 +22,4 @@ j = @prep :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :tr end end -println(j) +# println(j) From e2cd7b8d2a87a0e52304bbc5f8644523b837b541 Mon Sep 17 00:00:00 2001 From: Chetan Vardhan Date: Sun, 31 Dec 2023 00:41:03 +0900 Subject: [PATCH 3/8] New syntax again with GLM example --- perf/GLM/allocs.jl | 40 ++++++++++---------- src/PerfChecker.jl | 14 ++----- src/{allocations.jl => check.jl} | 65 +++++++++++++++++++------------- 3 files changed, 62 insertions(+), 57 deletions(-) rename src/{allocations.jl => check.jl} (83%) diff --git a/perf/GLM/allocs.jl b/perf/GLM/allocs.jl index e96c950..d81a63f 100644 --- a/perf/GLM/allocs.jl +++ b/perf/GLM/allocs.jl @@ -1,25 +1,25 @@ using PerfChecker -@prep :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :track => "user") begin +x = @check :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :track => "user", :repeat => true) begin using GLM, Random, StatsModels - @check begin - n = 2_500_000 - rng = Random.MersenneTwister(1234321) - tbl = ( - x1=randn(rng, n), - x2=Random.randexp(rng, n), - ss=rand(rng, string.(50:99), n), - y=zeros(n), - ) - f = @eval @formula(y ~ 1 + x1 + x2 + ss) - f = apply_schema(f, schema(f, tbl)) - resp, pred = modelcols(f, tbl) - B = randn(rng, size(pred, 2)) - B[1] = 0.5 - logistic(x::Real) = inv(1 + exp(-x)) - resp .= rand(rng, n) .< logistic.(pred * B) - glm(pred, resp, Bernoulli()) - end + end begin + n = 2_500_000 + rng = Random.MersenneTwister(1234321) + tbl = ( + x1=randn(rng, n), + x2=Random.randexp(rng, n), + ss=rand(rng, string.(50:99), n), + y=zeros(n), + ) + f = @formula(y ~ 1 + x1 + x2 + ss) + f = apply_schema(f, schema(f, tbl)) + resp, pred = modelcols(f, tbl) + B = randn(rng, size(pred, 2)) + B[1] = 0.5 + logistic(x::Real) = inv(1 + exp(-x)) + resp .= rand(rng, n) .< logistic.(pred * B) + glm(pred, resp, Bernoulli()) end -# println(j) + +@info x diff --git a/src/PerfChecker.jl b/src/PerfChecker.jl index 78e32c6..19e84bf 100644 --- a/src/PerfChecker.jl +++ b/src/PerfChecker.jl @@ -17,26 +17,18 @@ using TypedTables import Distributed: remotecall_fetch, addprocs, rmprocs import CoverageTools: analyze_malloc_files, find_malloc_files -struct CheckerOptions - threads::Int - path::AbstractString - targets::Union{String, Vector{String}} -end - # SECTION - Exports -export alloc_check -export alloc_plot export bench_plot export store_benchmark -export @prep -export CheckerOptions +export @check # SECTION - Includes include("init.jl") -include("allocations.jl") +include("check.jl") include("benchmarks.jl") include("utils.jl") end + diff --git a/src/allocations.jl b/src/check.jl similarity index 83% rename from src/allocations.jl rename to src/check.jl index 828f1fb..4b48b43 100644 --- a/src/allocations.jl +++ b/src/check.jl @@ -2,29 +2,46 @@ prep(d::Dict, block::Expr, ::Val{:alloc}) = quote import Pkg Pkg.instantiate() import Profile - macro check(repeat, block) - repeat && block + $block + nothing +end + +function check(d::Dict, block::Expr, ::Val{:alloc}) + + j = haskey(d, :repeat) && d[:repeat] ? block : nothing + + quote + $j Profile.clear_malloc_data() - block + $block + targets = eval(Meta.parse("[" * join($(d[:targets]), ", ") * "]")) + rmstuff = Base.loaded_modules_array() + return dirname.(filter(!isnothing, pathof.(targets))), dirname.(filter(!isnothing, pathof.(rmstuff))) end - macro check(block) - block - Profile.clear_malloc_data() - block +end + +function post(result, d::Dict, ::Val{:alloc}) + files = find_malloc_files(result[1]) + delete_files = find_malloc_files(result[2]) + myallocs = analyze_malloc_files(files) + if !isempty(myallocs) + rm.(delete_files) + else + @error "No allocation files found in $(d[:targets])" end - $block - targets = eval(Meta.parse("[" * (typeof($d[:targets]) == String ? $d[:targets] : join($d[:targets], ", ")) * "]")) - @warn targets - return map(dirname ∘ pathof ∘ eval, targets) + myallocs end + prep(d::Dict, b::Expr, v::Symbol) = prep(d, b, Val(v)) +check(d::Dict, b::Expr, v::Symbol) = check(d, b, Val(v)) +post(result, d::Dict, v::Symbol) = post(result, d, Val(v)) -macro prep(x, d, block) - block = Expr(:quote, block) +macro check(x, d, block1, block2) + block1, block2 = Expr(:quote, block1), Expr(:quote, block2) quote - g = prep($d, $block, $x) - + g = prep($d, $block1, $x) + h = check($d, $block2, $x) p = remotecall_fetch(Core.eval, 1, Main, Expr(:toplevel, quote import Distributed @@ -32,24 +49,19 @@ macro prep(x, d, block) Distributed.addprocs(1; exeflags=["--track-allocation=$(d[:track])", "--project=$(d[:path])", "-t $(d[:threads])"]) end).args...) |> first - j = remotecall_fetch(Core.eval, p, Main, + remotecall_fetch(Core.eval, p, Main, Expr(:toplevel, g.args...)) + j = remotecall_fetch(Core.eval, p, Main, + Expr(:toplevel, h.args...)) + remotecall_fetch(Core.eval, 1, Main, Expr(:toplevel, quote - import Distributed + import Distributed Distributed.rmprocs($p) end).args...) - files = find_malloc_files(j) - - myallocs = analyze_malloc_files(files) - if isempty(myallocs) - @error "No allocation files found in $($d[:targets])" - else - rm.(files) - return myallocs - end + $post(j, $d, $x) end end @@ -206,3 +218,4 @@ function alloc_plot( end end =# + From 14a43d5572b059b413ab429484d0c2429672d9ac Mon Sep 17 00:00:00 2001 From: azzaare Date: Sun, 31 Dec 2023 09:29:29 +0900 Subject: [PATCH 4/8] Update methods args. Clean code --- perf/GLM/allocs.jl | 4 +- src/PerfChecker.jl | 3 +- src/alloc.jl | 36 +++++++++++ src/check.jl | 151 +++++++++++++++++++++++++++------------------ 4 files changed, 132 insertions(+), 62 deletions(-) create mode 100644 src/alloc.jl diff --git a/perf/GLM/allocs.jl b/perf/GLM/allocs.jl index d81a63f..498909c 100644 --- a/perf/GLM/allocs.jl +++ b/perf/GLM/allocs.jl @@ -2,7 +2,7 @@ using PerfChecker x = @check :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :track => "user", :repeat => true) begin using GLM, Random, StatsModels - end begin +end begin n = 2_500_000 rng = Random.MersenneTwister(1234321) tbl = ( @@ -22,4 +22,4 @@ x = @check :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :t end -@info x +# @info x diff --git a/src/PerfChecker.jl b/src/PerfChecker.jl index 19e84bf..3cd73be 100644 --- a/src/PerfChecker.jl +++ b/src/PerfChecker.jl @@ -25,8 +25,9 @@ export @check # SECTION - Includes include("init.jl") - include("check.jl") +include("alloc.jl") + include("benchmarks.jl") include("utils.jl") diff --git a/src/alloc.jl b/src/alloc.jl new file mode 100644 index 0000000..d822d2c --- /dev/null +++ b/src/alloc.jl @@ -0,0 +1,36 @@ + +prep(::Val{:alloc}, d::Dict, block::Expr, ) = quote + import Pkg + Pkg.instantiate() + import Profile + $block + return nothing +end + +function check(::Val{:alloc}, d::Dict, block::Expr) + + j = haskey(d, :repeat) && d[:repeat] ? block : nothing + + quote + $j + Profile.clear_malloc_data() + $block + targets = eval(Meta.parse("[" * join($(d[:targets]), ", ") * "]")) + rmstuff = Base.loaded_modules_array() + return dirname.(filter(!isnothing, pathof.(targets))), dirname.(filter(!isnothing, pathof.(rmstuff))) + end +end + +function post(::Val{:alloc}, d::Dict, result) + @info "debug" d + # result = d[:check_result] + files = find_malloc_files(result[1]) + delete_files = find_malloc_files(result[2]) + myallocs = analyze_malloc_files(files) + if !isempty(myallocs) + rm.(delete_files) + else + @error "No allocation files found in $(d[:targets])" + end + return myallocs +end \ No newline at end of file diff --git a/src/check.jl b/src/check.jl index 4b48b43..726ea08 100644 --- a/src/check.jl +++ b/src/check.jl @@ -1,67 +1,100 @@ -prep(d::Dict, block::Expr, ::Val{:alloc}) = quote - import Pkg - Pkg.instantiate() - import Profile - $block - nothing -end - -function check(d::Dict, block::Expr, ::Val{:alloc}) - - j = haskey(d, :repeat) && d[:repeat] ? block : nothing - +## Check macro and related functions + +#SECTION - Generic dispatch methods +prep(::Val, args...) = nothing +check(::Val, args...) = nothing +post(::Val, args...) = nothing + +#SECTION - Multiple dispatch methods +prep(tool::Symbol, d, b) = prep(Val(tool), d, b) +check(tool::Symbol, d, b) = check(Val(tool), d, b ) +post(tool::Symbol, d, r) = post(Val(tool), d, r) + +#SECTION - Check macro +macro check(tool, d, block1, block2) + block1 = Expr(:quote, block1) + block2 = Expr(:quote, block2) quote - $j - Profile.clear_malloc_data() - $block - targets = eval(Meta.parse("[" * join($(d[:targets]), ", ") * "]")) - rmstuff = Base.loaded_modules_array() - return dirname.(filter(!isnothing, pathof.(targets))), dirname.(filter(!isnothing, pathof.(rmstuff))) - end -end + g = prep($tool, $d, $block1) + h = check($tool, $d, $block2) + p = remotecall_fetch( + Core.eval, + 1, + Main, + Expr( + :toplevel, + quote + import Distributed + d = $($d) + Distributed.addprocs( + 1; + exeflags=["--track-allocation=$(d[:track])", + "--project=$(d[:path])", + "-t $(d[:threads])"] + ) + end + ).args... + ) |> first + + r1 = remotecall_fetch( + Core.eval, + p, + Main, + Expr(:toplevel, g.args...), + ) -function post(result, d::Dict, ::Val{:alloc}) - files = find_malloc_files(result[1]) - delete_files = find_malloc_files(result[2]) - myallocs = analyze_malloc_files(files) - if !isempty(myallocs) - rm.(delete_files) - else - @error "No allocation files found in $(d[:targets])" - end - myallocs -end + $d[:prep_result] = r1 + @warn "debug" $d + + # push!( + # $d, + # :prep_result => remotecall_fetch( + # Core.eval, + # p, + # Main, + # Expr(:toplevel, g.args...), + # ) + # ) + + r2 = remotecall_fetch( + Core.eval, + p, + Main, + Expr(:toplevel, h.args...), + ) + $d[:check_result] = r2 + + # push!( + # $d, + # :check_result => remotecall_fetch( + # Core.eval, + # p, + # Main, + # Expr(:toplevel, h.args...), + # ) + # ) + + remotecall_fetch( + Core.eval, + 1, + Main, + Expr( + :toplevel, + quote + import Distributed + Distributed.rmprocs($p) + end + ).args... + ) -prep(d::Dict, b::Expr, v::Symbol) = prep(d, b, Val(v)) -check(d::Dict, b::Expr, v::Symbol) = check(d, b, Val(v)) -post(result, d::Dict, v::Symbol) = post(result, d, Val(v)) + results = $post($tool, $d, r2) + # results = $post($tool, $d) -macro check(x, d, block1, block2) - block1, block2 = Expr(:quote, block1), Expr(:quote, block2) - quote - g = prep($d, $block1, $x) - h = check($d, $block2, $x) - p = remotecall_fetch(Core.eval, 1, Main, - Expr(:toplevel, quote - import Distributed - d = $($d) - Distributed.addprocs(1; exeflags=["--track-allocation=$(d[:track])", "--project=$(d[:path])", "-t $(d[:threads])"]) - end).args...) |> first - - remotecall_fetch(Core.eval, p, Main, - Expr(:toplevel, g.args...)) - - j = remotecall_fetch(Core.eval, p, Main, - Expr(:toplevel, h.args...)) - - remotecall_fetch(Core.eval, 1, Main, - Expr(:toplevel, quote - import Distributed - Distributed.rmprocs($p) - end).args...) - - $post(j, $d, $x) + # pop!($d, :prep_result) + # pop!($d, :check_result) + + return results end end From 548783fdc2e1d6d4e54c651e7a842b50cc578369 Mon Sep 17 00:00:00 2001 From: Chetan Vardhan Date: Sun, 31 Dec 2023 15:48:55 +0900 Subject: [PATCH 5/8] default options and some other cleaning Better defaults Table and pretty printing of stuff --- perf/GLM/Project.toml | 1 + perf/GLM/allocs.jl | 8 +- src/PerfChecker.jl | 9 +- src/alloc.jl | 33 +++-- src/check.jl | 288 ++++++------------------------------------ test/pattern_folds.jl | 23 +--- 6 files changed, 76 insertions(+), 286 deletions(-) diff --git a/perf/GLM/Project.toml b/perf/GLM/Project.toml index cf0cca4..8fbc80b 100644 --- a/perf/GLM/Project.toml +++ b/perf/GLM/Project.toml @@ -3,5 +3,6 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a" PGFPlotsX = "8314cec4-20b6-5062-9cdb-752b83310925" PerfChecker = "6309bf6b-a531-4b08-891e-8ee981e5c424" +PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/perf/GLM/allocs.jl b/perf/GLM/allocs.jl index 498909c..70405a5 100644 --- a/perf/GLM/allocs.jl +++ b/perf/GLM/allocs.jl @@ -1,8 +1,8 @@ -using PerfChecker +using PerfChecker, PrettyTables -x = @check :alloc Dict(:threads => 1, :targets => ["GLM"], :path => @__DIR__, :track => "user", :repeat => true) begin +x = @check :alloc Dict(:targets => ["GLM"], :path => @__DIR__) begin using GLM, Random, StatsModels -end begin + end begin n = 2_500_000 rng = Random.MersenneTwister(1234321) tbl = ( @@ -22,4 +22,4 @@ end begin end -# @info x +pretty_table(x |> to_table) diff --git a/src/PerfChecker.jl b/src/PerfChecker.jl index 3cd73be..0780dfe 100644 --- a/src/PerfChecker.jl +++ b/src/PerfChecker.jl @@ -13,23 +13,24 @@ using Profile using Random using StatsPlots using Term -using TypedTables +import TypedTables: Table import Distributed: remotecall_fetch, addprocs, rmprocs -import CoverageTools: analyze_malloc_files, find_malloc_files +import CoverageTools: analyze_malloc_files, find_malloc_files, MallocInfo # SECTION - Exports +export alloc_check +export alloc_plot export bench_plot export store_benchmark export @check +export to_table # SECTION - Includes include("init.jl") include("check.jl") include("alloc.jl") - include("benchmarks.jl") include("utils.jl") end - diff --git a/src/alloc.jl b/src/alloc.jl index d822d2c..9afb80b 100644 --- a/src/alloc.jl +++ b/src/alloc.jl @@ -1,13 +1,16 @@ - -prep(::Val{:alloc}, d::Dict, block::Expr, ) = quote +prep(d::Dict, block::Expr, ::Val{:alloc}) = quote import Pkg Pkg.instantiate() import Profile $block - return nothing + nothing +end + +function default_options(::Val{:alloc}) + return Dict(:threads => 1, :targets => [], :track => "user", :repeat => true) end -function check(::Val{:alloc}, d::Dict, block::Expr) +function check(d::Dict, block::Expr, ::Val{:alloc}) j = haskey(d, :repeat) && d[:repeat] ? block : nothing @@ -15,15 +18,17 @@ function check(::Val{:alloc}, d::Dict, block::Expr) $j Profile.clear_malloc_data() $block - targets = eval(Meta.parse("[" * join($(d[:targets]), ", ") * "]")) + targets = eval(Meta.parse("[" * join($(d[:targets]), ", ") * "]")) rmstuff = Base.loaded_modules_array() + if isempty(targets) + targets = Base.loaded_modules_array() + end return dirname.(filter(!isnothing, pathof.(targets))), dirname.(filter(!isnothing, pathof.(rmstuff))) end end -function post(::Val{:alloc}, d::Dict, result) - @info "debug" d - # result = d[:check_result] +function post(d::Dict, ::Val{:alloc}) + result = d[:check_result] files = find_malloc_files(result[1]) delete_files = find_malloc_files(result[2]) myallocs = analyze_malloc_files(files) @@ -32,5 +37,13 @@ function post(::Val{:alloc}, d::Dict, result) else @error "No allocation files found in $(d[:targets])" end - return myallocs -end \ No newline at end of file + myallocs +end + +function to_table(myallocs::Vector{MallocInfo}) + b = map(a -> a.bytes, Iterators.reverse(myallocs)) + r = round.(b / sum(b) * 100; digits=2) + f = map(first ∘ splitext ∘ first ∘ splitext, map(a -> a.filename, Iterators.reverse(myallocs))) + l = map(a -> a.linenumber, Iterators.reverse(myallocs)) + Table(bytes=b, percentage=r, filenames=f, linenumbers=l) +end diff --git a/src/check.jl b/src/check.jl index 726ea08..2ea98e3 100644 --- a/src/check.jl +++ b/src/check.jl @@ -1,254 +1,42 @@ -## Check macro and related functions - -#SECTION - Generic dispatch methods -prep(::Val, args...) = nothing -check(::Val, args...) = nothing -post(::Val, args...) = nothing - -#SECTION - Multiple dispatch methods -prep(tool::Symbol, d, b) = prep(Val(tool), d, b) -check(tool::Symbol, d, b) = check(Val(tool), d, b ) -post(tool::Symbol, d, r) = post(Val(tool), d, r) - -#SECTION - Check macro -macro check(tool, d, block1, block2) - block1 = Expr(:quote, block1) - block2 = Expr(:quote, block2) - quote - g = prep($tool, $d, $block1) - h = check($tool, $d, $block2) - p = remotecall_fetch( - Core.eval, - 1, - Main, - Expr( - :toplevel, - quote - import Distributed - d = $($d) - Distributed.addprocs( - 1; - exeflags=["--track-allocation=$(d[:track])", - "--project=$(d[:path])", - "-t $(d[:threads])"] - ) - end - ).args... - ) |> first - - r1 = remotecall_fetch( - Core.eval, - p, - Main, - Expr(:toplevel, g.args...), - ) - - $d[:prep_result] = r1 - @warn "debug" $d - - # push!( - # $d, - # :prep_result => remotecall_fetch( - # Core.eval, - # p, - # Main, - # Expr(:toplevel, g.args...), - # ) - # ) - - r2 = remotecall_fetch( - Core.eval, - p, - Main, - Expr(:toplevel, h.args...), - ) - - $d[:check_result] = r2 - - # push!( - # $d, - # :check_result => remotecall_fetch( - # Core.eval, - # p, - # Main, - # Expr(:toplevel, h.args...), - # ) - # ) - - remotecall_fetch( - Core.eval, - 1, - Main, - Expr( - :toplevel, - quote - import Distributed - Distributed.rmprocs($p) - end - ).args... - ) - - results = $post($tool, $d, r2) - # results = $post($tool, $d) - - # pop!($d, :prep_result) - # pop!($d, :check_result) - - return results - end +prep(d, b, v) = quote nothing end +check(d, b, v) = quote nothing end +post(d, v) = nothing +default_options(v) = Dict() + +prep(d::Dict, b::Expr, v::Symbol) = prep(d, b, Val(v)) +check(d::Dict, b::Expr, v::Symbol) = check(d, b, Val(v)) +post(d::Dict, v::Symbol) = post(d, Val(v)) + +function default_options(d::Dict, v::Symbol) + di = default_options(Val(v)) + return merge(di, d) end -#= -function alloc_check( - title, - dependencies, - targets, - pre_alloc, - alloc; - path=pwd(), - labeller=:version, - threads=nothing, -) - @info "Tracking allocations: $title" - - # cd to path if valid - isdir(path) && cd(path) - - # add a proc (id == p) that track allocations - p = first(isnothing(threads) ? addprocs(1; exeflags=["--track-allocation=user", "--project=$path"]) : addprocs(1; exeflags=["--track-allocation=user", "--project=$path", "-t $threads"])) - - remotecall_fetch(Core.eval, p, Main, Expr(:toplevel, (quote - import Pkg; Pkg.instantiate() - import Profile; - nothing - end).args...)) - - for d in dependencies - remotecall_fetch(Core.eval, p, Main, Expr(:toplevel, (quote - using $(Symbol(d)) - nothing - end).args...)) - end - - @eval @everywhere $p $pre_alloc() - @eval @everywhere $p Profile.clear_malloc_data() - @eval @everywhere $p $alloc() - - rmprocs(p) - - myallocs = CoverageTools.analyze_malloc(map(dirname ∘ pathof ∘ eval, targets)) - - for t in dependencies, d in walkdir(dirname(pathof(t))), f in d[end] - splitext(f)[2] == ".mem" && rm(joinpath(d[1], f)) - end - for d in walkdir(path), f in d[end] - splitext(f)[2] == ".mem" && rm(joinpath(d[1], f)) - end - - if isempty(myallocs) - @warn "No allocations was found in " targets - return nothing - end - - # Smart paths - - # common, specifics = smart_paths(map(a -> a.filename, Iterators.reverse(myallocs))) - # @info "sizes" map(a -> a.filename, Iterators.reverse(myallocs)) specifics - # slash = Sys.iswindows() ? "\\" : "/" - - # Make the allocations data readable through a dataframe - - #= - df = DataFrame() - df.bytes = map(a -> a.bytes, Iterators.reverse(myallocs)) - df[!, "ratio (%)"] = round.(df.bytes / sum(df.bytes) * 100; digits=2) - df[!, "filename: [$common$slash"] = map(first ∘ splitext ∘ first ∘ splitext, specifics) - df.linenumber = map(a -> a.linenumber, Iterators.reverse(myallocs)) - =# - t = let - b = map(a -> a.bytes, Iterators.reverse(myallocs)) - r = round.(b / sum(b) * 100; digits=2) - f = map(first ∘ splitext ∘ first ∘ splitext, map(a -> a.filename, Iterators.reverse(myallocs))) - l = map(a -> a.linenumber, Iterators.reverse(myallocs)) - TypedTables.Table(bytes = b, percentage = r, filenames = f, linenumbers = l) - end - - # Save it as a CSV file - label = "" - if labeller == :oid - label = oid2string(map(p -> joinpath(dirname(pathof(p)), ".."), targets)) - elseif labeller == :version - label = version2string(map(p -> joinpath(dirname(pathof(p)), ".."), targets)) - end - mkpath("mallocs") - CSV.write(joinpath(path, "mallocs/mallocs$label.csv"), t) - - # Visualize a pretty table - return t -end - -function pie_filter(df, threshold=5.0) - i = findfirst(x -> x < threshold, df[!, 2]) - X = df[1:(i - 1), 3] .* " line " .* map(string, df[1:(i - 1), 4]) - push!(X, "others .< $threshold") - Y = df[1:(i - 1), 2] - push!(Y, sum(df[i:end, 2])) - return X, Y -end - -function alloc_plot( - targets; formats=["pdf", "svg", "png"], backend=Plots.GRBackend, seriestype=:step -) - backend() - for target in targets - path = normpath(joinpath(dirname(pathof(target)), "..", "perf", "mallocs")) - versions = Vector{VersionNumber}() - for f in readdir(path; join=true) - st = splitext(basename(f)) - last(st) == ".csv" || continue - push!(versions, VersionNumber(first(st)[9:end])) - end - sort!(versions) - - bytes = Dict{String,Vector{Int}}() - for (i, version) in enumerate(versions) - csv_path = joinpath(path, "mallocs-$(string(version)).csv") - df = DataFrame(CSV.File(csv_path)) - v = get!(bytes, "Total", zeros(Int, length(versions))) - v[i] = sum(df.bytes) - for (b, f) in zip(df[:, 1], df[:, 3]) - w = get!(bytes, "$f", zeros(Int, length(versions))) - w[i] += b - end - - X, Y = pie_filter(df) - pie(X, Y; title="Mallocs for $target.jl@v$(string(version))", l=0.5) - for format in formats - savefig(joinpath(path, "mallocs-$version.$format")) - end - end - - X = map(string, versions) - Y = reshape( - collect(Iterators.flatten(values(bytes))), length(versions), length(bytes) - ) - L = reshape(collect(keys(bytes)), 1, length(bytes)) - plot( - X, - Y; - xlabel="version", - ylabel="bytes", - markershape=:circle, - # seriestype, - title="Mallocs evolution in\n$target.jl", - l=(0.5, 2), - label=L, - # yaxis=:log, - ) - for format in formats - savefig(joinpath(path, "mallocs-evolutions.$format")) - end +macro check(x, d, block1, block2) + block1, block2 = Expr(:quote, block1), Expr(:quote, block2) + quote + di = default_options($d, $x) + g = prep(di, $block1, $x) + h = check(di, $block2, $x) + p = remotecall_fetch(Core.eval, 1, Main, + Expr(:toplevel, quote + import Distributed + d = $di + Distributed.addprocs(1; exeflags=["--track-allocation=$(d[:track])", "--project=$(d[:path])", "-t $(d[:threads])"]) + end).args...) |> first + + di[:prep_result] = remotecall_fetch(Core.eval, p, Main, + Expr(:toplevel, g.args...)) + + di[:check_result] = remotecall_fetch(Core.eval, p, Main, + Expr(:toplevel, h.args...)) + + remotecall_fetch(Core.eval, 1, Main, + Expr(:toplevel, quote + import Distributed + Distributed.rmprocs($p) + end).args...) + + $post(di, $x) end end -=# - diff --git a/test/pattern_folds.jl b/test/pattern_folds.jl index a3c8355..e2ba57d 100644 --- a/test/pattern_folds.jl +++ b/test/pattern_folds.jl @@ -1,17 +1,9 @@ using PatternFolds @testset "PatternFolds.jl" begin - # Title of the alloc check (for logging purpose) - title = "Basic intervals and vectors folds operation" - - # Dependencies needed to execute pre_alloc and alloc - dependencies = [PatternFolds] - - # Target of the alloc check - targets = [PatternFolds] - - function alloc() # 0.2.x - # Intervals + @check :alloc Dict(:target => ["PatternFolds"]) begin + using PatternFolds + end begin itv = Interval{Open,Closed}(0.0, 1.0) i = IntervalsFold(itv, 2.0, 1000) @@ -30,15 +22,9 @@ using PatternFolds reverse(collect(vf)) rand(vf, 1000) - - return nothing end - - # Actual call to PerfChecker - alloc_check(title, dependencies, targets, alloc, alloc; path=@__DIR__) - end - +#= target = PatternFolds function bench() # 0.2.x @@ -69,3 +55,4 @@ t = @benchmark bench() evals = 1 samples = 1000 seconds = 3600 # Actual call to PerfChecker store_benchmark(t, target; path=@__DIR__) +=# From 165c1dee7b99cafbddf65f87f86cbfe9bffb7bf2 Mon Sep 17 00:00:00 2001 From: azzaare Date: Sun, 31 Dec 2023 17:27:21 +0900 Subject: [PATCH 6/8] Fix tests: PatternFolds --- test/Project.toml | 1 + test/pattern_folds.jl | 4 ++-- test/runtests.jl | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index e9fb8b2..983e519 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CompatHelperLocal = "5224ae11-6099-4aaa-941d-3aab004bd678" ConstraintDomains = "5800fd60-8556-4464-8d61-84ebf7a0bedb" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" PatternFolds = "c18a7f1d-76ad-4ce4-950d-5419b888513b" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" diff --git a/test/pattern_folds.jl b/test/pattern_folds.jl index e2ba57d..42cde96 100644 --- a/test/pattern_folds.jl +++ b/test/pattern_folds.jl @@ -1,8 +1,8 @@ using PatternFolds @testset "PatternFolds.jl" begin - @check :alloc Dict(:target => ["PatternFolds"]) begin - using PatternFolds + @check :alloc Dict(:target => ["PatternFolds"], :path => pwd()) begin + using PatternFolds end begin itv = Interval{Open,Closed}(0.0, 1.0) i = IntervalsFold(itv, 2.0, 1000) diff --git a/test/runtests.jl b/test/runtests.jl index f5119dd..0de52df 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ using BenchmarkTools +using Distributed using PerfChecker using Test From 85c5f236e204c888718f9c2e8bb142f0481b72ed Mon Sep 17 00:00:00 2001 From: Chetan Vardhan <51269425+VarLad@users.noreply.github.com> Date: Sun, 31 Dec 2023 17:27:04 +0900 Subject: [PATCH 7/8] Update pattern_folds.jl --- test/pattern_folds.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pattern_folds.jl b/test/pattern_folds.jl index 42cde96..472787b 100644 --- a/test/pattern_folds.jl +++ b/test/pattern_folds.jl @@ -1,8 +1,8 @@ using PatternFolds @testset "PatternFolds.jl" begin - @check :alloc Dict(:target => ["PatternFolds"], :path => pwd()) begin - using PatternFolds + @check :alloc Dict(:target => ["PatternFolds"], :path => @__DIR__) begin + using PatternFolds end begin itv = Interval{Open,Closed}(0.0, 1.0) i = IntervalsFold(itv, 2.0, 1000) From 217fa275f8fb6461a464d25cb4482e26df7d1568 Mon Sep 17 00:00:00 2001 From: azzaare Date: Sun, 31 Dec 2023 17:39:32 +0900 Subject: [PATCH 8/8] Update compat and CI --- .github/workflows/CI.yml | 6 +----- Project.toml | 5 ++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3349f20..0cee6db 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,7 +1,6 @@ name: CI on: - push - - pull_request jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} @@ -10,10 +9,7 @@ jobs: fail-fast: false matrix: version: - - '1.6' - - '1.7' - - '1.8' - - "^1.9.0-0" + - "1.10" - 'nightly' os: - ubuntu-latest diff --git a/Project.toml b/Project.toml index 1ad70fa..8444a62 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PerfChecker" uuid = "6309bf6b-a531-4b08-891e-8ee981e5c424" authors = ["Azzaare "] -version = "0.1.4" +version = "0.2.0" [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" @@ -9,7 +9,6 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CoverageTools = "c36e975a-824b-4404-a568-ef97ca766997" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" -GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" PGFPlotsX = "8314cec4-20b6-5062-9cdb-752b83310925" @@ -33,7 +32,7 @@ Plots = "1" StatsPlots = "0.15" Term = "2" TypedTables = "1" -julia = "1.6" +julia = "1.10" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"