From 564af93598a671c89722ef4c0795f196418c56d5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 15 Jul 2021 01:16:54 +0100 Subject: [PATCH] Improved testing suite (#270) * move away from using extras in Project.toml * added integration tests for Turing.jl * removed usage of Turing.jl and MCMCDebugging.jl in main testsuite * fixed bug in deprecated HMCDA constructor * allow specification of which testing suites to run * added Turing.jl integration tests to CI * fixed name for integration tests * added using AdvancedHMC in runtests.jl * removed some now unnecessary usings * fixed a bug in the downstream testing * give integration tests a separate CI * forgot to remove the continue-on-error from CI * renamed env variable for test groups --- .github/workflows/CI.yml | 2 +- .github/workflows/IntegrationTests.yml | 40 +++++++++++++ .gitignore | 1 - Project.toml | 18 ------ src/AdvancedHMC.jl | 4 +- test/Project.toml | 17 ++++++ test/common.jl | 39 ------------- test/integrator.jl | 11 ---- test/runtests.jl | 79 +++++++++++++++++++------- test/sampler.jl | 8 +-- test/turing/Project.toml | 12 ++++ test/turing/runtests.jl | 19 +++++++ 12 files changed, 150 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/IntegrationTests.yml create mode 100644 test/Project.toml create mode 100644 test/turing/Project.toml create mode 100644 test/turing/runtests.jl diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c26e8d04..1e2956dc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -39,4 +39,4 @@ jobs: - name: Run tests uses: julia-actions/julia-runtest@latest env: - GEWEKE_TEST: 1 + AHMC_TEST_GROUP: AdvancedHMC diff --git a/.github/workflows/IntegrationTests.yml b/.github/workflows/IntegrationTests.yml new file mode 100644 index 00000000..1eb8053a --- /dev/null +++ b/.github/workflows/IntegrationTests.yml @@ -0,0 +1,40 @@ +name: IntegrationTests + +on: + push: + branches: + - master + pull_request: + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + version: + - '1' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x86 + - x64 + exclude: + - os: ubuntu-latest + arch: x86 + - os: macOS-latest + arch: x86 + - os: windows-latest + arch: x86 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/julia-buildpkg@latest + - name: Run integration tests + uses: julia-actions/julia-runtest@latest + env: + AHMC_TEST_GROUP: Downstream diff --git a/.gitignore b/.gitignore index cae5cb87..7f554567 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ .history .DS_Store Manifest.toml -test/Project.toml diff --git a/Project.toml b/Project.toml index 039bba1c..89250b2c 100644 --- a/Project.toml +++ b/Project.toml @@ -27,21 +27,3 @@ StatsBase = "0.31, 0.32, 0.33" StatsFuns = "0.8, 0.9" UnPack = "1" julia = "1" - -[extras] -Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" -CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" -ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" -Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" -Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" -ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" -MCMCDebugging = "6d524b87-5f90-4494-b601-374a5b87a94b" -OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" -Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -Turing = "fce5fe82-541a-59a6-adf8-730c64b5f9a0" -UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" -Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" - -[targets] -test = ["CUDA", "Distributed", "Distributions", "ComponentArrays", "ForwardDiff", "Plots", "MCMCDebugging", "Test", "Turing", "UnicodePlots", "Bijectors", "OrdinaryDiffEq", "Zygote"] diff --git a/src/AdvancedHMC.jl b/src/AdvancedHMC.jl index a7cdf05a..9c90bff0 100644 --- a/src/AdvancedHMC.jl +++ b/src/AdvancedHMC.jl @@ -78,8 +78,8 @@ struct StaticTrajectory{TS} end struct HMCDA{TS} end @deprecate HMCDA{TS}(int::AbstractIntegrator, λ) where {TS} HMCKernel(Trajectory{TS}(int, FixedIntegrationTime(λ))) -@deprecate HMCDA(int::AbstractIntegrator, λ) HMCKernel(Trajectory{MetropolisTS}(int, FixedIntegrationTime(λ))) -@deprecate HMCDA(ϵ::AbstractScalarOrVec{<:Real}, λ) HMCKernel(Trajectory{MetropolisTS}(Leapfrog(ϵ), FixedIntegrationTime(λ))) +@deprecate HMCDA(int::AbstractIntegrator, λ) HMCKernel(Trajectory{EndPointTS}(int, FixedIntegrationTime(λ))) +@deprecate HMCDA(ϵ::AbstractScalarOrVec{<:Real}, λ) HMCKernel(Trajectory{EndPointTS}(Leapfrog(ϵ), FixedIntegrationTime(λ))) @deprecate find_good_eps find_good_stepsize diff --git a/test/Project.toml b/test/Project.toml new file mode 100644 index 00000000..8927e4ba --- /dev/null +++ b/test/Project.toml @@ -0,0 +1,17 @@ +[deps] +Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/test/common.jl b/test/common.jl index 3af536a4..e6caa99f 100644 --- a/test/common.jl +++ b/test/common.jl @@ -79,45 +79,6 @@ function ℓπ_gdemo(θ) return logprior + loglikelihood end -using Distributions: MvNormal -import Turing - -Turing.@model function mvntest(θ=missing, x=missing) - θ ~ MvNormal(zeros(D), 2) - x ~ Normal(sum(θ), 1) - return θ, x -end - -function get_primitives(x, modelgen) - spl_prior = Turing.SampleFromPrior() - function ℓπ(θ) - vi = Turing.VarInfo(model) - vi[spl_prior] = θ - model(vi, spl_prior) - Turing.getlogp(vi) - end - adbackend = Turing.Core.ForwardDiffAD{40} - alg_ad = Turing.HMC{adbackend}(0.1, 1) - model = modelgen(missing, x) - vi = Turing.VarInfo(model) - spl = Turing.Sampler(alg_ad, model) - Turing.Core.link!(vi, spl) - ∂ℓπ∂θ = θ -> Turing.Core.gradient_logp(adbackend(), θ, vi, model, spl) - θ₀ = Turing.VarInfo(model)[Turing.SampleFromPrior()] - return ℓπ, ∂ℓπ∂θ, θ₀ -end - -function rand_θ_given(x, modelgen, metric, κ; n_samples=20) - ℓπ, ∂ℓπ∂θ, θ₀ = get_primitives(x, modelgen) - h = Hamiltonian(metric, ℓπ, ∂ℓπ∂θ) - samples, stats = sample(h, κ, θ₀, n_samples; verbose=false, progress=false) - s = samples[end] - return length(s) == 1 ? s[1] : s -end - -# Test function -geweke_g(θ, x) = cat(θ, x; dims=1) - test_show(x) = test_show(s -> length(s) > 0, x) function test_show(pred, x) io = IOBuffer(; append = true) diff --git a/test/integrator.jl b/test/integrator.jl index acffd8f0..3430f21b 100644 --- a/test/integrator.jl +++ b/test/integrator.jl @@ -26,17 +26,6 @@ n_steps = 10 @test z_step.r ≈ z_steps.r atol=DETATOL end -# using Turing: Inference -# @testset "step(::Leapfrog) against Turing.Inference._leapfrog()" begin -# z = AdvancedHMC.phasepoint(h, θ_init, r_init) -# t_Turing = @elapsed θ_Turing, r_Turing, _ = Inference._leapfrog(θ_init, r_init, n_steps, ϵ, x -> (nothing, ∂logπ∂θ(x))) -# t_AHMC = @elapsed z_AHMC = AdvancedHMC.step(lf, h, z, n_steps) -# @info "Performance of leapfrog of AdvancedHMC v.s. Turing" n_steps t_Turing t_AHMC t_Turing / t_AHMC -# -# @test θ_Turing ≈ z_AHMC.θ atol=DETATOL -# @test r_Turing ≈ z_AHMC.r atol=DETATOL -# end - @testset "jitter" begin @testset "Leapfrog" begin ϵ0 = 0.1 diff --git a/test/runtests.jl b/test/runtests.jl index 0bfe015f..873b682b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,33 +1,70 @@ -using Distributed, Test, CUDA +using Distributed, Test, CUDA, Pkg + +using AdvancedHMC: AdvancedHMC println("Environment variables for testing") println(ENV) +const DIRECTORY_AdvancedHMC = dirname(dirname(pathof(AdvancedHMC))) +const DIRECTORY_Turing_tests = joinpath(DIRECTORY_AdvancedHMC, "test", "turing") +const GROUP = get(ENV, "AHMC_TEST_GROUP", "All") + @testset "AdvancedHMC" begin - tests = [ - "metric", - "hamiltonian", - "integrator", - "trajectory", - "adaptation", - "sampler", - "sampler-vec", - "demo", - "models", - ] - - if CUDA.functional() - @eval module TestCUDA + if GROUP == "All" || GROUP == "AdvancedHMC" + tests = [ + "metric", + "hamiltonian", + "integrator", + "trajectory", + "adaptation", + "sampler", + "sampler-vec", + "demo", + "models", + ] + + if CUDA.functional() + @eval module TestCUDA include("cuda.jl") + end + else + @warn "Skipping GPU tests because no GPU available." end - else - @warn "Skipping GPU tests because no GPU available." - end - res = map(tests) do t - @eval module $(Symbol("Test_", t)) + res = map(tests) do t + @eval module $(Symbol("Test_", t)) include($t * ".jl") + end + return + end + end + + if GROUP == "All" || GROUP == "Downstream" + @testset "turing" begin + try + # activate separate test environment + Pkg.activate(DIRECTORY_Turing_tests) + Pkg.develop(PackageSpec(; path=DIRECTORY_AdvancedHMC)) + Pkg.instantiate() + + # make sure that the new environment is considered `using` and `import` statements + # (not added automatically on Julia 1.3, see e.g. PR #209) + if !(joinpath(DIRECTORY_Turing_tests, "Project.toml") in Base.load_path()) + pushfirst!(LOAD_PATH, DIRECTORY_Turing_tests) + end + + # Avoids conflicting namespaces, e.g. `NUTS` used in Turing.jl's tests + # refers to `Turing.NUTS` not `AdvancedHMC.NUTS`. + @eval module TuringIntegrationTests + include(joinpath("turing", "runtests.jl")) + end + catch err + err isa Pkg.Resolve.ResolverError || rethrow() + # If we can't resolve that means this is incompatible by SemVer and this is fine + # It means we marked this as a breaking change, so we don't need to worry about + # Mistakenly introducing a breaking change, as we have intentionally made one + @info "Not compatible with this release. No problem." exception = err + end end - return end end diff --git a/test/sampler.jl b/test/sampler.jl index 937ea3f6..40177c0c 100644 --- a/test/sampler.jl +++ b/test/sampler.jl @@ -1,7 +1,7 @@ # Allow pass --progress when running this script individually to turn on progress meter const PROGRESS = length(ARGS) > 0 && ARGS[1] == "--progress" ? true : false -using Test, AdvancedHMC, LinearAlgebra, Random, MCMCDebugging, Plots +using Test, AdvancedHMC, LinearAlgebra, Random, Plots using AdvancedHMC: StaticTerminationCriterion, DynamicTerminationCriterion using Setfield using Statistics: mean, var, cov @@ -61,12 +61,6 @@ end Random.seed!(1) samples, stats = sample(h, HMCKernel(τ), θ_init, n_samples; verbose=false, progress=PROGRESS) @test mean(samples[n_adapts+1:end]) ≈ zeros(D) atol=RNDATOL - if "GEWEKE_TEST" in keys(ENV) && ENV["GEWEKE_TEST"] == "1" - res = perform(GewekeTest(5_000), mvntest, x -> rand_θ_given(x, mvntest, metric, HMCKernel(τ)); g=geweke_g, progress=false) - p = plot(res, mvntest()) - display(p) - println() - end end # Skip adaptation tests with tempering diff --git a/test/turing/Project.toml b/test/turing/Project.toml new file mode 100644 index 00000000..7241c9c6 --- /dev/null +++ b/test/turing/Project.toml @@ -0,0 +1,12 @@ +[deps] +AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Turing = "fce5fe82-541a-59a6-adf8-730c64b5f9a0" + +[compat] +StatsFuns = "0.9.5" +Turing = "0.16" +julia = "1.3" diff --git a/test/turing/runtests.jl b/test/turing/runtests.jl new file mode 100644 index 00000000..b8eacb33 --- /dev/null +++ b/test/turing/runtests.jl @@ -0,0 +1,19 @@ +using Random +using Test +using LinearAlgebra + +using Turing +using Turing.DynamicPPL + +using StatsFuns: logistic + +Turing.setprogress!(false) + +Random.seed!(100) + +# Load test utilities. +testdir(args...) = joinpath(pathof(Turing), "..", "..", "test", args...) +include(testdir("test_utils", "AllUtils.jl")) + +# Test HMC. +include(testdir("inference", "hmc.jl"))