From 6affabf46c59e395f8787ddcd280b1ea994dceb6 Mon Sep 17 00:00:00 2001 From: Chetan Vardhan Date: Tue, 11 Jun 2024 12:09:24 +0900 Subject: [PATCH 1/5] Add some docs --- docs/Manifest.toml | 4 +- docs/make.jl | 6 +- docs/src/perf/benchmark_ext.md | 35 ++++++++++- docs/src/perf/chairmarks_ext.md | 33 +++++++++++ docs/src/perf/perf_interface.md | 102 +++++++++++++++++++++++++++++++- 5 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 docs/src/perf/chairmarks_ext.md diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 0cff207..a574052 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -930,9 +930,7 @@ version = "0.2.4" [[deps.PerfChecker]] deps = ["CSV", "CoverageTools", "CpuId", "JSON", "Malt", "Pkg", "Profile", "TOML", "TypedTables", "UUIDs"] -git-tree-sha1 = "113b402af7d1d4d1d1a4fe176f7d4881822872b1" -repo-rev = "main" -repo-url = "https://github.com/JuliaConstraints/PerfChecker.jl.git" +path = "/var/home/varlad/.julia/dev/PerfChecker" uuid = "6309bf6b-a531-4b08-891e-8ee981e5c424" version = "0.2.1" diff --git a/docs/make.jl b/docs/make.jl index 4bffbc8..96576af 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -38,7 +38,8 @@ makedocs(; repo = "https://github.com/JuliaConstraints/JuliaConstraints.github.io", devurl = "dev", deploy_url = "https://juliaconstraints.github.io/", - description = "Resources about Constraint Programming in Julia" + description = "Resources about Constraint Programming in Julia", + build_vitepress = false ), pages=[ "Home" => "index.md", @@ -106,8 +107,9 @@ makedocs(; ], "Performance Checking" => [ "PerfChecker.jl" => "perf/perf_checker.md", - "BenchmarkToolsExt" => "perf/benchmark_ext.md", "Interface" => "perf/perf_interface.md", + "BenchmarkToolsExt" => "perf/benchmark_ext.md", + "ChairmarksExt" => "perf/chairmarks_ext.md" ], "API" => [ "Public" => "public_api.md", diff --git a/docs/src/perf/benchmark_ext.md b/docs/src/perf/benchmark_ext.md index bc7dbb8..ab6a780 100644 --- a/docs/src/perf/benchmark_ext.md +++ b/docs/src/perf/benchmark_ext.md @@ -1,3 +1,36 @@ # BenchmarkTools Extension -A benchmarking extension, based on `BenchmarkTools.jl`, has been interfaced with `PerfChecker.jl`. This section (will) provides some usage examples, documentation, and links to related notebooks. +A benchmarking extension, based on `BenchmarkTools.jl`, has been interfaced with `PerfChecker.jl`. +This section will provide some usage examples, documentation, and links to related notebooks. + +## Usage + +Like all other extensions, `BenchmarkTools` extension can be used in the following way: + +```julia +julia> using BenchmarkTools, PerfChecker + +julia> @check :benchmark Dict(:option1 => "value1", :option2 => "value2", :PATH => @__DIR__) begin + # the prelimnary code goes here + using Example +end begin + # the code you want to be benchmarked + Example.domath(10) # returns x + 5, 15 in this case. +end +``` + +## Options + +Options specific to this backend with their default values are defined as: +```julia +:threads => 1 +:track => "none" +:samples => BenchmarkTools.DEFAULT_PARAMETERS.samples +:seconds => BenchmarkTools.DEFAULT_PARAMETERS.seconds +:evals => BenchmarkTools.DEFAULT_PARAMETERS.evals +:overhead => BenchmarkTools.DEFAULT_PARAMETERS.overhead +:gctrial => BenchmarkTools.DEFAULT_PARAMETERS.gctrial +:gcsample => BenchmarkTools.DEFAULT_PARAMETERS.gcsample +:time_tolerance => BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance +:memory_tolerance => BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance +``` diff --git a/docs/src/perf/chairmarks_ext.md b/docs/src/perf/chairmarks_ext.md new file mode 100644 index 0000000..02cf9fe --- /dev/null +++ b/docs/src/perf/chairmarks_ext.md @@ -0,0 +1,33 @@ +# Chairmarks Extension + +A benchmarking extension, based on `Chairmarks.jl`, has been interfaced with `PerfChecker.jl`. +This section will provide some usage examples, documentation, and links to related notebooks. + +## Usage + +Like all other extensions, `BenchmarkTools` extension can be used in the following way: + +```julia +julia> using Chairmarks, PerfChecker + +julia> @check :chairmark Dict(:option1 => "value1", :option2 => "value2", :PATH => @__DIR__) begin + # the prelimnary code goes here + using Example +end begin + # the code you want to be benchmarked + Example.domath(10) # returns x + 5, 15 in this case. +end +``` + +## Options + +Options specific to this backend with their default values are defined as: +```julia +:threads => 1 +:track => "none" +:evals => nothing +:seconds => 1, +:samples => nothing +:gc => true +``` + diff --git a/docs/src/perf/perf_interface.md b/docs/src/perf/perf_interface.md index 986949c..b59d826 100644 --- a/docs/src/perf/perf_interface.md +++ b/docs/src/perf/perf_interface.md @@ -1,3 +1,101 @@ -# Interfacing PerfChecker +# Extending PerfChecker -PerfChecker was build as an easy to extend interface. This section will cover the few method required. +PerfChecker was build as an easy to extend interface. A good reference example for this is the Chairmarks extension. + +Extending PerfChecker works via PkgExtensions feature in Julia. There are 6 essential functions that need to be extended inside the Pkg extension. +Each extension has a keyword symbol for it, which users can input to use the extension. + +## The Default Options + +Method to be overloaded: `PerfChecker.default_options(::Val{:myperfextension})::Dict` + +PerfChecker works via a config dictionary. Users can populate this dictionary with options and provide it to the main `check` macro to customize the performance testing to their liking. + +For Chairmarks.jl, it looks like this: +```julia +function PerfChecker.default_options(::Val{:chairmark}) + return Dict( + :threads => 1, + :track => "none", + :evals => nothing, + :seconds => 1, + :samples => nothing, + :gc => true + ) +end +``` + + +## Package initialization +Method to be overloaded: `PerfChecker.initpkgs(::Val{:myperfextension})::Expr` + +This method is plainly to load the main package(s) associated with the custom backend. In case of Chairmarks.jl, it looks like this: +```julia +PerfChecker.initpkgs(::Val{:chairmark}) = quote + using Chairmarks +end +``` + +## Preparatory Code: +Method to be overloaded: `PerfChecker.prep(config_dict::Dict, block::Expr, ::Val{:myperfextension})::Expr` + +This method exists to run arbitrary "preparatory" code (represented by `block` parameter here) before running the code to be performance tested for. + +The output from here are stored inside the `:prep_result` key of the configuration dictionary. + +Example for Chairmarks.jl: +```julia +PerfChecker.prep(::Dict, block::Expr, ::Val{:chairmark}) = quote + $block + nothing +end +``` + +This just runs the code in `block` provided by the user. + +## Main Code to be Performance Tested +Method to be overloaded: `PerfChecker.check(config_dict::Dict, block::Expr, ::Val{:myperfextension})::Expr`. + +Runs the appropriate code to run the performance testing on user's code. For Chairmarks.jl, it looks like this: +```julia +function PerfChecker.check(d::Dict, block::Expr, ::Val{:chairmark}) + quote + d = $d + return @be $block evals=d[:evals] seconds=d[:seconds] samples=d[:samples] gc=d[:gc] + end +end +``` + +The output from here are stored inside the `:check_result` key of the configuration dictionary. + +## Post Performance Testing Code +Method to be overloaded: `PerfChecker.post(config_dict::Dict, ::Val{:myperfextension})` + +The code to be run after the performance testing is done. The output from here is converted into a table via the `to_table` method overloading. + +In the case of Chairmarks.jl: +```julia +PerfChecker.post(d::Dict, ::Val{:chairmark}) = d[:check_result] +``` + +## Converting the result into a Table +Method to be overloaded: `PerfChecker.to_table` + +Convert the output from `post` function into an appropriate table. + +In the case of Chairmarks.jl +```julia +function PerfChecker.to_table(chair::Chairmarks.Benchmark) + l = length(chair.samples) + times = [chair.samples[i].time for i in 1:l] + gctimes = [chair.samples[i].gc_fraction for i in 1:l] + bytes = [chair.samples[i].bytes for i in 1:l] + allocs = [chair.samples[i].allocs for i in 1:l] + return Table(times = times, gctimes = gctimes, bytes = bytes, allocs = allocs) +end +``` + + +--- + +There are also other functions that can be overloaded, mostly related to plotting but these are the basic functions to extend PerfChecker for a custom backend. From be990017e05eb53027c6f7caa401e1ff8417a151 Mon Sep 17 00:00:00 2001 From: Chetan Vardhan Date: Wed, 12 Jun 2024 16:32:06 +0900 Subject: [PATCH 2/5] Add usage docs for PerfChecker --- docs/make.jl | 3 ++- docs/src/perf/api.md | 7 +++++ docs/src/perf/chairmarks_ext.md | 1 - docs/src/perf/perf_checker.md | 47 ++++++++++++++++++++++++++++++--- 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 docs/src/perf/api.md diff --git a/docs/make.jl b/docs/make.jl index 96576af..8e0c09c 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -109,7 +109,8 @@ makedocs(; "PerfChecker.jl" => "perf/perf_checker.md", "Interface" => "perf/perf_interface.md", "BenchmarkToolsExt" => "perf/benchmark_ext.md", - "ChairmarksExt" => "perf/chairmarks_ext.md" + "ChairmarksExt" => "perf/chairmarks_ext.md", + "API" => "perf/api.md" ], "API" => [ "Public" => "public_api.md", diff --git a/docs/src/perf/api.md b/docs/src/perf/api.md new file mode 100644 index 0000000..8695b70 --- /dev/null +++ b/docs/src/perf/api.md @@ -0,0 +1,7 @@ +# API + +Here's the API for PerfChecker.jl + +```@autodocs +Modules=[PerfChecker] +``` diff --git a/docs/src/perf/chairmarks_ext.md b/docs/src/perf/chairmarks_ext.md index 02cf9fe..166a17a 100644 --- a/docs/src/perf/chairmarks_ext.md +++ b/docs/src/perf/chairmarks_ext.md @@ -30,4 +30,3 @@ Options specific to this backend with their default values are defined as: :samples => nothing :gc => true ``` - diff --git a/docs/src/perf/perf_checker.md b/docs/src/perf/perf_checker.md index 8a8f40a..aedf08a 100644 --- a/docs/src/perf/perf_checker.md +++ b/docs/src/perf/perf_checker.md @@ -1,7 +1,48 @@ # PerfChecker.jl -Documentation for `PerfChecker.jl`. +PerfChecker.jl is a package designed for package authors to easily performance test their packages. +To achieve that, it provides the follwing features: -```@autodocs -Modules=[PerfChecker] +- The main macro `@check`, which provides an easy-to-use interface over various interfaces, configurable for various backends via a dictionary. +- A CI for reproducible performance testing. +- Visualization of different metrics from `@check` using Makie.jl + +## Usage + +The primary usage of PerfChecker.jl looks like this: + +```julia + using PerfChecker + # optional using custom backend like BenchmarkTools, Chairmark etc + config = Dict(:option1 => "value1", :option2 => :value2) + + results = @check :name_of_backend config begin + # preparatory code goes here + end begin + # the code block to be performance tested goes here + end + + # Visualization of the results + using Makie + checkres_to_scatterlines(results) ``` + +The config dictionary can take many options, depending on the backend. + +Some of the commonly used options are: +- `:PATH` => The path where to the default environment of julia when creating a new process. +- `:pkgs` => A list of versions to test performance for. Its defined as the `Tuple`, `(name::String, option::Symbol, versions::Vector{VersionNumber}, last_or_first::Bool)` Can be given as follows: + - `name` is the name of the package. + - `option` is one of the 5 symbols: + - `:patches`: last patch or first patch of a version + - `:breaking`: last breaking or next breaking version + - `:major`: previous or next major version + - `:minor`: previous or next minor version + - `:custom`: custom version numbers (provide any boolean value for `last_or_first` in this case as it doesn't matter) + - `versions`: The input for the provided `option` + - `last_or_first`: Input for the provided `option` +- `:tags` => A list of tags (a vector of symbols) to easily tag performance tests. +- `:devops` => Giving a custom input to `Pkg.develop`. Intended to be used to test performance of a local development branch of a pacakge with previous versions. Often can be used as simply as `:devops => "MyPackageName"` +- `:threads` => An integer to select the number of threads to start Julia with. + +Checkout the documentation of the other backends for more default options and the default values. From 5df0e8d7e2f1fd0015289063992a8c04c8f7c4e5 Mon Sep 17 00:00:00 2001 From: Chetan Vardhan Date: Wed, 12 Jun 2024 17:48:52 +0900 Subject: [PATCH 3/5] Add tutorial --- docs/make.jl | 1 + docs/src/perf/tutorial.md | 68 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 docs/src/perf/tutorial.md diff --git a/docs/make.jl b/docs/make.jl index 8e0c09c..c64b869 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -107,6 +107,7 @@ makedocs(; ], "Performance Checking" => [ "PerfChecker.jl" => "perf/perf_checker.md", + "Tutorial" => "perf/tutorial.md", "Interface" => "perf/perf_interface.md", "BenchmarkToolsExt" => "perf/benchmark_ext.md", "ChairmarksExt" => "perf/chairmarks_ext.md", diff --git a/docs/src/perf/tutorial.md b/docs/src/perf/tutorial.md new file mode 100644 index 0000000..f72dbcf --- /dev/null +++ b/docs/src/perf/tutorial.md @@ -0,0 +1,68 @@ +# Tutorial + +Taken from PerfChecker.jl examples, this is a guide for performance testing of PatterFolds.jl package using Chairmarks.jl + +Using PerfChecker.jl requires an environment with the dependencies present in it. + +The actual script looks like this: + +```julia +using PerfChecker, Chairmarks, CairoMakie + +d = Dict(:path => @__DIR__, :evals => 10, :samples => 1000, + :seconds => 100, :tags => [:patterns, :intervals], + :pkgs => ( + "PatternFolds", :custom, [v"0.2.0", v"0.2.1", v"0.2.2", v"0.2.3", v"0.2.4"], true), + :devops => "PatternFolds") + +x = @check :chairmark d begin + using PatternFolds +end begin + # Intervals + itv = Interval{Open, Closed}(0.0, 1.0) + i = IntervalsFold(itv, 2.0, 1000) + + unfold(i) + collect(i) + reverse(collect(i)) + + # Vectors + vf = make_vector_fold([0, 1], 2, 1000) + + unfold(vf) + collect(vf) + reverse(collect(vf)) + + rand(vf, 1000) + + return nothing +end + +mkpath(joinpath(@__DIR__, "visuals")) + +c = checkres_to_scatterlines(x, Val(:chairmark)) +save(joinpath(@__DIR__, "visuals", "chair_evolution.png"), c) + +for kwarg in [:times, :gctimes, :bytes, :allocs] + c2 = checkres_to_boxplots(x, Val(:chairmark); kwarg) + save(joinpath(@__DIR__, "visuals", "chair_boxplots_$kwarg.png"), c2) +end +``` + +`d` here is the configuration dictionary. +`x` stores the results from performance testing + +The code below the macro call is for plotting and storing the plots. It creates the visuals folder and stores the following plots in the folder: + +Boxplots from Chairmarks for allocations: + +![chair_boxplots](https://github.com/JuliaConstraints/PerfChecker.jl/blob/main/perf/PatternFolds/visuals/chair_boxplots_allocs.png?raw=true) + +Boxplots from Chairmarks for times: + +![chair_times](https://github.com/JuliaConstraints/PerfChecker.jl/blob/main/perf/PatternFolds/visuals/chair_boxplots_times.png?raw=true) + +Evolution of different metrics across versions according to Chairmarks: + +![chair_evolution](https://github.com/JuliaConstraints/PerfChecker.jl/blob/main/perf/PatternFolds/visuals/chair_evolution.png?raw=true) + From f5cfe91c0434cc8b5f54c22c8fd010d7f4755e60 Mon Sep 17 00:00:00 2001 From: Chetan Vardhan Date: Wed, 12 Jun 2024 17:52:23 +0900 Subject: [PATCH 4/5] Add WIP --- docs/src/perf/perf_checker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/perf/perf_checker.md b/docs/src/perf/perf_checker.md index aedf08a..4906f5a 100644 --- a/docs/src/perf/perf_checker.md +++ b/docs/src/perf/perf_checker.md @@ -4,7 +4,7 @@ PerfChecker.jl is a package designed for package authors to easily performance t To achieve that, it provides the follwing features: - The main macro `@check`, which provides an easy-to-use interface over various interfaces, configurable for various backends via a dictionary. -- A CI for reproducible performance testing. +- (WIP) A CI for reproducible performance testing. - Visualization of different metrics from `@check` using Makie.jl ## Usage From cd06c996e632abc40e087947000e5c0de69d23c2 Mon Sep 17 00:00:00 2001 From: Azzaare Date: Tue, 18 Jun 2024 14:27:35 +0900 Subject: [PATCH 5/5] Fix manifest --- docs/Manifest.toml | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/docs/Manifest.toml b/docs/Manifest.toml index a574052..e07bcfc 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -216,15 +216,15 @@ version = "0.1.2" [[deps.ConstraintCommons]] deps = ["Dictionaries", "TestItemRunner", "TestItems"] -git-tree-sha1 = "866bd8d55f57037d7542c9f1d8d17285f44b78dd" +git-tree-sha1 = "4698469c1292bea5573dc294a180c364d6ebfa81" repo-rev = "main" repo-url = "https://github.com/JuliaConstraints/ConstraintCommons.jl.git" uuid = "e37357d9-0691-492f-a822-e5ea6a920954" -version = "0.2.0" +version = "0.2.1" [[deps.ConstraintDomains]] deps = ["ConstraintCommons", "Dictionaries", "Intervals", "PatternFolds", "StatsBase", "TestItemRunner", "TestItems"] -git-tree-sha1 = "0d087d2580392660948480a0484dbb17b0141e83" +git-tree-sha1 = "50801b71002e4b8c7d943980a8b3008dcbc2aa33" repo-rev = "main" repo-url = "https://github.com/JuliaConstraints/ConstraintDomains.jl.git" uuid = "5800fd60-8556-4464-8d61-84ebf7a0bedb" @@ -374,9 +374,9 @@ version = "1.4.1" [[deps.DocumenterVitepress]] deps = ["ANSIColoredPrinters", "Base64", "DocStringExtensions", "Documenter", "IOCapture", "Markdown", "NodeJS_20_jll", "REPL"] -git-tree-sha1 = "733cf5d8819a583b2753a59e820e9de130c2df4e" +git-tree-sha1 = "a58b1681c9d3ef9f5842ac86c13f9f5b8ff2eee6" uuid = "4710194d-e776-4893-9690-8d956a29c365" -version = "0.1.0" +version = "0.1.1" [deps.DocumenterVitepress.extensions] DocumenterVitepressDocumenterCitationsExt = "DocumenterCitations" @@ -461,9 +461,9 @@ version = "2.23.1" [[deps.Flux]] deps = ["Adapt", "ChainRulesCore", "Compat", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote"] -git-tree-sha1 = "a5475163b611812d073171583982c42ea48d22b0" +git-tree-sha1 = "edacf029ed6276301e455e34d7ceeba8cc34078a" uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" -version = "0.14.15" +version = "0.14.16" [deps.Flux.extensions] FluxAMDGPUExt = "AMDGPU" @@ -499,9 +499,9 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[deps.GPUArrays]] deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] -git-tree-sha1 = "38cb19b8a3e600e509dc36a6396ac74266d108c1" +git-tree-sha1 = "c154546e322a9c73364e8a60430b0f79b812d320" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "10.1.1" +version = "10.2.0" [[deps.GPUArraysCore]] deps = ["Adapt"] @@ -528,9 +528,9 @@ version = "1.3.1" [[deps.IOCapture]] deps = ["Logging", "Random"] -git-tree-sha1 = "8b72179abc660bfab5e28472e019392b97d0985c" +git-tree-sha1 = "b6d6bfdd7ce25b0f9b2f6b3dd56b2673a66c8770" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" -version = "0.2.4" +version = "0.2.5" [[deps.IRTools]] deps = ["InteractiveUtils", "MacroTools"] @@ -550,9 +550,15 @@ version = "0.3.1" [[deps.InlineStrings]] deps = ["Parsers"] -git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461" +git-tree-sha1 = "86356004f30f8e737eff143d57d41bd580e437aa" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.4.0" +version = "1.4.1" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -593,9 +599,9 @@ version = "0.21.4" [[deps.JuMP]] deps = ["LinearAlgebra", "MacroTools", "MathOptInterface", "MutableArithmetics", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays"] -git-tree-sha1 = "28f9313ba6603e0d2850fc3eae617e769c99bf83" +git-tree-sha1 = "7e10a0d8b534f2d8e9f712b33488584254624fb1" uuid = "4076af6c-e467-56ae-b986-b466b2749572" -version = "1.22.1" +version = "1.22.2" [deps.JuMP.extensions] JuMPDimensionalDataExt = "DimensionalData" @@ -813,9 +819,9 @@ version = "2023.1.10" [[deps.MutableArithmetics]] deps = ["LinearAlgebra", "SparseArrays", "Test"] -git-tree-sha1 = "a3589efe0005fc4718775d8641b2de9060d23f73" +git-tree-sha1 = "898c56fbf8bf71afb0c02146ef26f3a454e88873" uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" -version = "1.4.4" +version = "1.4.5" [[deps.NLSolversBase]] deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"] @@ -930,7 +936,9 @@ version = "0.2.4" [[deps.PerfChecker]] deps = ["CSV", "CoverageTools", "CpuId", "JSON", "Malt", "Pkg", "Profile", "TOML", "TypedTables", "UUIDs"] -path = "/var/home/varlad/.julia/dev/PerfChecker" +git-tree-sha1 = "58979ea0b3de20e82626de04aca6328e1a81c3fd" +repo-rev = "docstuff" +repo-url = "https://github.com/JuliaConstraints/PerfChecker.jl.git" uuid = "6309bf6b-a531-4b08-891e-8ee981e5c424" version = "0.2.1"