From f479d3798702110ac41ea58ad1d8e4fa43f61406 Mon Sep 17 00:00:00 2001 From: Seth Axen Date: Tue, 19 Dec 2023 00:13:15 -0800 Subject: [PATCH] Run extension tests in separate workflow (#317) * Use correct imports in helpers * Define extension tests * Delete old tests * Remove weakdeps from extras * Define workflow for testing extensions * Run extension tests on older Julia versions * Run MCMCChains tests separately * Create Project for MCMCChains tests * Run ArviZMCMCChainsExt tests * Remove unused test deps * Add missing DimensionalData compats * Don't include helpers if unneeded * Update extension test deps with CompatHelper * Add back weak deps to extras --- .github/workflows/CompatHelper.yml | 3 +- .github/workflows/ExtensionTest.yml | 36 ++++++++++++++++++ Project.toml | 5 +-- test/ext/ArviZMCMCChainsExt/Project.toml | 15 ++++++++ .../ArviZMCMCChainsExt/runtests.jl} | 1 + .../Project.toml | 15 ++++++++ .../runtests.jl | 38 +++++++++++++++++++ test/ext/ArviZSampleChainsExt/Project.toml | 13 +++++++ .../ArviZSampleChainsExt/runtests.jl} | 34 +---------------- test/helpers.jl | 3 +- test/runtests.jl | 3 -- 11 files changed, 125 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/ExtensionTest.yml create mode 100644 test/ext/ArviZMCMCChainsExt/Project.toml rename test/{test_mcmcchains.jl => ext/ArviZMCMCChainsExt/runtests.jl} (99%) create mode 100644 test/ext/ArviZSampleChainsDynamicHMCExt/Project.toml create mode 100644 test/ext/ArviZSampleChainsDynamicHMCExt/runtests.jl create mode 100644 test/ext/ArviZSampleChainsExt/Project.toml rename test/{test_samplechains.jl => ext/ArviZSampleChainsExt/runtests.jl} (60%) diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index f390a7e6..1e16b386 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -16,4 +16,5 @@ jobs: COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} run: | using CompatHelper - CompatHelper.main(; subdirs=["", "docs"]) + subdirs = ["", "docs", readdir("test/ext"; join=true)...] + CompatHelper.main(; subdirs) diff --git a/.github/workflows/ExtensionTest.yml b/.github/workflows/ExtensionTest.yml new file mode 100644 index 00000000..70fcd300 --- /dev/null +++ b/.github/workflows/ExtensionTest.yml @@ -0,0 +1,36 @@ +name: ExtensionTest +on: + push: + branches: [main] + tags: [v*] + pull_request: +jobs: + test: + name: ${{ matrix.extension }} - Julia ${{ matrix.julia-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + julia-version: [1.6, 1] + extension: + - ArviZMCMCChainsExt + - ArviZSampleChainsExt + - ArviZSampleChainsDynamicHMCExt + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.julia-version }} + arch: x64 + - uses: julia-actions/julia-buildpkg@v1 + - run: | + julia --code-coverage=user -e ' + using Pkg + test_path = joinpath(pwd(), "test", "ext", "${{ matrix.extension }}") + Pkg.activate(test_path) + Pkg.develop(PackageSpec(path=pwd())) + include(joinpath(test_path, "runtests.jl"))' + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v2 + with: + files: lcov.info diff --git a/Project.toml b/Project.toml index 994754a7..7ac803f2 100644 --- a/Project.toml +++ b/Project.toml @@ -36,13 +36,10 @@ StatsBase = "0.32, 0.33, 0.34" julia = "1.6" [extras] -DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" -OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SampleChains = "754583d1-7fc4-4dab-93b5-5eaca5c9622e" SampleChainsDynamicHMC = "6d9fd711-e8b2-4778-9c70-c1dfb499d4c4" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["DimensionalData", "MCMCChains", "OrderedCollections", "Random", "SampleChains", "SampleChainsDynamicHMC", "Test"] +test = ["Test"] diff --git a/test/ext/ArviZMCMCChainsExt/Project.toml b/test/ext/ArviZMCMCChainsExt/Project.toml new file mode 100644 index 00000000..b5c741f5 --- /dev/null +++ b/test/ext/ArviZMCMCChainsExt/Project.toml @@ -0,0 +1,15 @@ +[deps] +ArviZ = "131c737c-5715-5e2e-ad31-c244f01c1dc7" +DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" +MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +ArviZ = "0.10.3" +DimensionalData = "0.24, 0.25" +MCMCChains = "6" +OrderedCollections = "1" +Random = "1.6" +Test = "1.6" diff --git a/test/test_mcmcchains.jl b/test/ext/ArviZMCMCChainsExt/runtests.jl similarity index 99% rename from test/test_mcmcchains.jl rename to test/ext/ArviZMCMCChainsExt/runtests.jl index c0a8e41d..99a9f4a1 100644 --- a/test/test_mcmcchains.jl +++ b/test/ext/ArviZMCMCChainsExt/runtests.jl @@ -3,6 +3,7 @@ using ArviZ using DimensionalData using MCMCChains: MCMCChains using OrderedCollections +using Random function makechains( names, ndraws, nchains, domains=[Float64 for _ in names]; seed=42, internal_names=[] diff --git a/test/ext/ArviZSampleChainsDynamicHMCExt/Project.toml b/test/ext/ArviZSampleChainsDynamicHMCExt/Project.toml new file mode 100644 index 00000000..bb3be8ad --- /dev/null +++ b/test/ext/ArviZSampleChainsDynamicHMCExt/Project.toml @@ -0,0 +1,15 @@ +[deps] +ArviZ = "131c737c-5715-5e2e-ad31-c244f01c1dc7" +DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SampleChains = "754583d1-7fc4-4dab-93b5-5eaca5c9622e" +SampleChainsDynamicHMC = "6d9fd711-e8b2-4778-9c70-c1dfb499d4c4" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +ArviZ = "0.10.3" +DimensionalData = "0.24, 0.25" +Random = "1.6" +SampleChains = "0.5" +SampleChainsDynamicHMC = "0.3" +Test = "1.6" diff --git a/test/ext/ArviZSampleChainsDynamicHMCExt/runtests.jl b/test/ext/ArviZSampleChainsDynamicHMCExt/runtests.jl new file mode 100644 index 00000000..f041631c --- /dev/null +++ b/test/ext/ArviZSampleChainsDynamicHMCExt/runtests.jl @@ -0,0 +1,38 @@ +using ArviZ +using SampleChains: SampleChains +using SampleChains.TupleVectors: TupleVectors +using SampleChainsDynamicHMC: SampleChainsDynamicHMC +using SampleChainsDynamicHMC.TransformVariables +using Test + +include("../../helpers.jl") + +function samplechains_dynamichmc_sample(nchains, ndraws) + # μ ~ Normal(0, 1) |> iid(2, 3), σ ~ HalfNormal(0, 1), y[i] ~ Normal(μ[i], σ) + y = [0.74 0.15 -1.08; -0.42 1.08 -0.52] + function ℓ(nt) + μ = nt.μ + σ = nt.σ + return -(sum(abs2, μ) + σ^2 + sum((y .- μ) .^ 2) / σ^2) / 2 - length(y) * log(σ) + end + t = as((μ=as(Array, 2, 3), σ=asℝ₊)) + chain = SampleChains.newchain(nchains, SampleChainsDynamicHMC.dynamichmc(), ℓ, t) + return SampleChains.sample!(chain, ndraws - 1) +end + +@testset "SampleChainsDynamicHMC" begin + expected_stats_vars = ( + :acceptance_rate, :n_steps, :diverging, :energy, :tree_depth, :turning + ) + + multichain = samplechains_dynamichmc_sample(4, 10) + idata = convert_to_inference_data(multichain) + @test InferenceObjects.groupnames(idata) === (:posterior, :sample_stats) + @test issubset(expected_stats_vars, keys(idata.sample_stats)) + @test size(idata.posterior.μ) == (10, 4, 2, 3) + @test size(idata.posterior.σ) == (10, 4) + + idata = convert_to_inference_data(multichain; group=:prior) + @test InferenceObjects.groupnames(idata) === (:prior, :sample_stats_prior) + @test issubset(expected_stats_vars, keys(idata.sample_stats_prior)) +end diff --git a/test/ext/ArviZSampleChainsExt/Project.toml b/test/ext/ArviZSampleChainsExt/Project.toml new file mode 100644 index 00000000..181b2dd0 --- /dev/null +++ b/test/ext/ArviZSampleChainsExt/Project.toml @@ -0,0 +1,13 @@ +[deps] +ArviZ = "131c737c-5715-5e2e-ad31-c244f01c1dc7" +DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SampleChains = "754583d1-7fc4-4dab-93b5-5eaca5c9622e" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +ArviZ = "0.10.3" +DimensionalData = "0.24, 0.25" +Random = "1.6" +SampleChains = "0.5" +Test = "1.6" diff --git a/test/test_samplechains.jl b/test/ext/ArviZSampleChainsExt/runtests.jl similarity index 60% rename from test/test_samplechains.jl rename to test/ext/ArviZSampleChainsExt/runtests.jl index adc13135..cae7c86f 100644 --- a/test/test_samplechains.jl +++ b/test/ext/ArviZSampleChainsExt/runtests.jl @@ -1,11 +1,10 @@ using ArviZ -using DimensionalData using SampleChains: SampleChains using SampleChains.TupleVectors: TupleVectors -using SampleChainsDynamicHMC: SampleChainsDynamicHMC -using SampleChainsDynamicHMC.TransformVariables using Test +include("../../helpers.jl") + # minimal AbstractChain implementation struct TestChain{T} <: SampleChains.AbstractChain{T} samples::TupleVectors.TupleVector{T} @@ -14,19 +13,6 @@ end SampleChains.samples(chain::TestChain) = getfield(chain, :samples) SampleChains.info(chain::TestChain) = getfield(chain, :info) -function samplechains_dynamichmc_sample(nchains, ndraws) - # μ ~ Normal(0, 1) |> iid(2, 3), σ ~ HalfNormal(0, 1), y[i] ~ Normal(μ[i], σ) - y = [0.74 0.15 -1.08; -0.42 1.08 -0.52] - function ℓ(nt) - μ = nt.μ - σ = nt.σ - return -(sum(abs2, μ) + σ^2 + sum((y .- μ) .^ 2) / σ^2) / 2 - length(y) * log(σ) - end - t = as((μ=as(Array, 2, 3), σ=asℝ₊)) - chain = SampleChains.newchain(nchains, SampleChainsDynamicHMC.dynamichmc(), ℓ, t) - return SampleChains.sample!(chain, ndraws - 1) -end - @testset "SampleChains" begin @testset "TestChain with $nchains chains" for nchains in (1, 4) ndraws = 10 @@ -70,20 +56,4 @@ end end end end - @testset "SampleChainsDynamicHMC" begin - expected_stats_vars = ( - :acceptance_rate, :n_steps, :diverging, :energy, :tree_depth, :turning - ) - - multichain = samplechains_dynamichmc_sample(4, 10) - idata = convert_to_inference_data(multichain) - @test InferenceObjects.groupnames(idata) === (:posterior, :sample_stats) - @test issubset(expected_stats_vars, keys(idata.sample_stats)) - @test size(idata.posterior.μ) == (10, 4, 2, 3) - @test size(idata.posterior.σ) == (10, 4) - - idata = convert_to_inference_data(multichain; group=:prior) - @test InferenceObjects.groupnames(idata) === (:prior, :sample_stats_prior) - @test issubset(expected_stats_vars, keys(idata.sample_stats_prior)) - end end diff --git a/test/helpers.jl b/test/helpers.jl index 11c7c5f2..e6d1d2a1 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -1,5 +1,6 @@ -using Random using ArviZ +using DimensionalData +using Random function random_dim_array(var_name, dims, coords, default_dims=()) _dims = (default_dims..., dims...) diff --git a/test/runtests.jl b/test/runtests.jl index 14b03173..153b9e30 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,8 +2,5 @@ using ArviZ using Test @testset "ArviZ" begin - include("helpers.jl") include("test_utils.jl") - include("test_samplechains.jl") - include("test_mcmcchains.jl") end