Improved testing suite (#270)

* move away from using extras in Project.toml * added integration tests for Turing.jl * removed usage of Turing.jl and MCMCDebugging.jl in main testsuite * fixed bug in deprecated HMCDA constructor * allow specification of which testing suites to run * added Turing.jl integration tests to CI * fixed name for integration tests * added using AdvancedHMC in runtests.jl * removed some now unnecessary usings * fixed a bug in the downstream testing * give integration tests a separate CI * forgot to remove the continue-on-error from CI * renamed env variable for test groups
TuringLang · Jul 15, 2021 · 564af93 · 564af93
1 parent 6cec3da
commit 564af93
Show file tree

Hide file tree

Showing 12 changed files with 150 additions and 100 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -39,4 +39,4 @@ jobs:
       - name: Run tests
         uses: julia-actions/julia-runtest@latest
         env:
-          GEWEKE_TEST: 1
+          AHMC_TEST_GROUP: AdvancedHMC
diff --git a/.github/workflows/IntegrationTests.yml b/.github/workflows/IntegrationTests.yml
@@ -0,0 +1,40 @@
+name: IntegrationTests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        version:
+          - '1'
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        arch:
+          - x86
+          - x64
+        exclude:
+          - os: ubuntu-latest
+            arch: x86
+          - os: macOS-latest
+            arch: x86
+          - os: windows-latest
+            arch: x86
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: Run integration tests
+        uses: julia-actions/julia-runtest@latest
+        env:
+          AHMC_TEST_GROUP: Downstream
diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,3 @@
 .history
 .DS_Store
 Manifest.toml
-test/Project.toml
diff --git a/Project.toml b/Project.toml
@@ -27,21 +27,3 @@ StatsBase = "0.31, 0.32, 0.33"
 StatsFuns = "0.8, 0.9"
 UnPack = "1"
 julia = "1"
-
-[extras]
-Bijectors = "76274a88-744f-5084-9051-94815aaf08c4"
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
-ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
-Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
-Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
-ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
-MCMCDebugging = "6d524b87-5f90-4494-b601-374a5b87a94b"
-OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
-Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-Turing = "fce5fe82-541a-59a6-adf8-730c64b5f9a0"
-UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228"
-Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
-
-[targets]
-test = ["CUDA", "Distributed", "Distributions", "ComponentArrays", "ForwardDiff", "Plots", "MCMCDebugging", "Test", "Turing", "UnicodePlots", "Bijectors", "OrdinaryDiffEq", "Zygote"]
diff --git a/src/AdvancedHMC.jl b/src/AdvancedHMC.jl
@@ -78,8 +78,8 @@ struct StaticTrajectory{TS} end
 
 struct HMCDA{TS} end
 @deprecate HMCDA{TS}(int::AbstractIntegrator, λ) where {TS} HMCKernel(Trajectory{TS}(int, FixedIntegrationTime(λ)))
-@deprecate HMCDA(int::AbstractIntegrator, λ) HMCKernel(Trajectory{MetropolisTS}(int, FixedIntegrationTime(λ)))
-@deprecate HMCDA(ϵ::AbstractScalarOrVec{<:Real}, λ) HMCKernel(Trajectory{MetropolisTS}(Leapfrog(ϵ), FixedIntegrationTime(λ)))
+@deprecate HMCDA(int::AbstractIntegrator, λ) HMCKernel(Trajectory{EndPointTS}(int, FixedIntegrationTime(λ)))
+@deprecate HMCDA(ϵ::AbstractScalarOrVec{<:Real}, λ) HMCKernel(Trajectory{EndPointTS}(Leapfrog(ϵ), FixedIntegrationTime(λ)))
 
 @deprecate find_good_eps find_good_stepsize
 

diff --git a/test/Project.toml b/test/Project.toml
@@ -0,0 +1,17 @@
+[deps]
+Bijectors = "76274a88-744f-5084-9051-94815aaf08c4"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
diff --git a/test/common.jl b/test/common.jl
@@ -79,45 +79,6 @@ function ℓπ_gdemo(θ)
     return logprior + loglikelihood
 end
 
-using Distributions: MvNormal
-import Turing
-
-Turing.@model function mvntest(θ=missing, x=missing)
-    θ ~ MvNormal(zeros(D), 2)
-    x ~ Normal(sum(θ), 1)
-    return θ, x
-end
-
-function get_primitives(x, modelgen)
-    spl_prior = Turing.SampleFromPrior()
-    function ℓπ(θ)
-        vi = Turing.VarInfo(model)
-        vi[spl_prior] = θ
-        model(vi, spl_prior)
-        Turing.getlogp(vi)
-    end
-    adbackend = Turing.Core.ForwardDiffAD{40}
-    alg_ad = Turing.HMC{adbackend}(0.1, 1)
-    model = modelgen(missing, x)
-    vi = Turing.VarInfo(model)
-    spl = Turing.Sampler(alg_ad, model)
-    Turing.Core.link!(vi, spl)
-    ∂ℓπ∂θ = θ -> Turing.Core.gradient_logp(adbackend(), θ, vi, model, spl)
-    θ₀ = Turing.VarInfo(model)[Turing.SampleFromPrior()]
-    return ℓπ, ∂ℓπ∂θ, θ₀
-end
-
-function rand_θ_given(x, modelgen, metric, κ; n_samples=20)
-    ℓπ, ∂ℓπ∂θ, θ₀ = get_primitives(x, modelgen)
-    h = Hamiltonian(metric, ℓπ, ∂ℓπ∂θ)
-    samples, stats = sample(h, κ, θ₀, n_samples; verbose=false, progress=false)
-    s = samples[end]
-    return length(s) == 1 ? s[1] : s
-end
-
-# Test function
-geweke_g(θ, x) = cat(θ, x; dims=1)
-
 test_show(x) = test_show(s -> length(s) > 0, x)
 function test_show(pred, x)
     io = IOBuffer(; append = true)

diff --git a/test/integrator.jl b/test/integrator.jl
@@ -26,17 +26,6 @@ n_steps = 10
     @test z_step.r ≈ z_steps.r atol=DETATOL
 end
 
-# using Turing: Inference
-# @testset "step(::Leapfrog) against Turing.Inference._leapfrog()" begin
-#     z = AdvancedHMC.phasepoint(h, θ_init, r_init)
-#     t_Turing = @elapsed θ_Turing, r_Turing, _ = Inference._leapfrog(θ_init, r_init, n_steps, ϵ, x -> (nothing, ∂logπ∂θ(x)))
-#     t_AHMC = @elapsed z_AHMC = AdvancedHMC.step(lf, h, z, n_steps)
-#     @info "Performance of leapfrog of AdvancedHMC v.s. Turing" n_steps t_Turing t_AHMC t_Turing / t_AHMC
-#
-#     @test θ_Turing ≈ z_AHMC.θ atol=DETATOL
-#     @test r_Turing ≈ z_AHMC.r atol=DETATOL
-# end
-
 @testset "jitter" begin
     @testset "Leapfrog" begin
         ϵ0 = 0.1

diff --git a/test/runtests.jl b/test/runtests.jl
@@ -1,33 +1,70 @@
-using Distributed, Test, CUDA
+using Distributed, Test, CUDA, Pkg
+
+using AdvancedHMC: AdvancedHMC
 
 println("Environment variables for testing")
 println(ENV)
 
+const DIRECTORY_AdvancedHMC = dirname(dirname(pathof(AdvancedHMC)))
+const DIRECTORY_Turing_tests = joinpath(DIRECTORY_AdvancedHMC, "test", "turing")
+const GROUP = get(ENV, "AHMC_TEST_GROUP", "All")
+
 @testset "AdvancedHMC" begin
-    tests = [
-        "metric",
-        "hamiltonian",
-        "integrator",
-        "trajectory",
-        "adaptation",
-        "sampler",
-        "sampler-vec",
-        "demo",
-        "models",
-    ]
-
-    if CUDA.functional()
-        @eval module TestCUDA
+    if GROUP == "All" || GROUP == "AdvancedHMC"
+        tests = [
+            "metric",
+            "hamiltonian",
+            "integrator",
+            "trajectory",
+            "adaptation",
+            "sampler",
+            "sampler-vec",
+            "demo",
+            "models",
+        ]
+
+        if CUDA.functional()
+            @eval module TestCUDA
             include("cuda.jl")
+            end
+        else
+            @warn "Skipping GPU tests because no GPU available."
         end
-    else
-        @warn "Skipping GPU tests because no GPU available."
-    end
 
-    res = map(tests) do t
-        @eval module $(Symbol("Test_", t))
+        res = map(tests) do t
+            @eval module $(Symbol("Test_", t))
             include($t * ".jl")
+            end
+            return
+        end
+    end
+
+    if GROUP == "All" || GROUP == "Downstream"
+        @testset "turing" begin
+            try
+                # activate separate test environment
+                Pkg.activate(DIRECTORY_Turing_tests)
+                Pkg.develop(PackageSpec(; path=DIRECTORY_AdvancedHMC))
+                Pkg.instantiate()
+
+                # make sure that the new environment is considered `using` and `import` statements
+                # (not added automatically on Julia 1.3, see e.g. PR #209)
+                if !(joinpath(DIRECTORY_Turing_tests, "Project.toml") in Base.load_path())
+                    pushfirst!(LOAD_PATH, DIRECTORY_Turing_tests)
+                end
+
+                # Avoids conflicting namespaces, e.g. `NUTS` used in Turing.jl's tests
+                # refers to `Turing.NUTS` not `AdvancedHMC.NUTS`.
+                @eval module TuringIntegrationTests
+                include(joinpath("turing", "runtests.jl"))
+                end
+            catch err
+                err isa Pkg.Resolve.ResolverError || rethrow()
+                # If we can't resolve that means this is incompatible by SemVer and this is fine
+                # It means we marked this as a breaking change, so we don't need to worry about
+                # Mistakenly introducing a breaking change, as we have intentionally made one
+                @info "Not compatible with this release. No problem." exception = err
+            end
         end
-        return
     end
 end
diff --git a/test/sampler.jl b/test/sampler.jl
@@ -1,7 +1,7 @@
 # Allow pass --progress when running this script individually to turn on progress meter
 const PROGRESS = length(ARGS) > 0 && ARGS[1] == "--progress" ? true : false
 
-using Test, AdvancedHMC, LinearAlgebra, Random, MCMCDebugging, Plots
+using Test, AdvancedHMC, LinearAlgebra, Random, Plots
 using AdvancedHMC: StaticTerminationCriterion, DynamicTerminationCriterion
 using Setfield
 using Statistics: mean, var, cov
@@ -61,12 +61,6 @@ end
                     Random.seed!(1)
                     samples, stats = sample(h, HMCKernel(τ), θ_init, n_samples; verbose=false, progress=PROGRESS)
                     @test mean(samples[n_adapts+1:end]) ≈ zeros(D) atol=RNDATOL
-                    if "GEWEKE_TEST" in keys(ENV) && ENV["GEWEKE_TEST"] == "1"
-                        res = perform(GewekeTest(5_000), mvntest, x -> rand_θ_given(x, mvntest, metric, HMCKernel(τ)); g=geweke_g, progress=false)
-                        p = plot(res, mvntest())
-                        display(p)
-                        println()
-                    end
                 end
 
                 # Skip adaptation tests with tempering

diff --git a/test/turing/Project.toml b/test/turing/Project.toml
@@ -0,0 +1,12 @@
+[deps]
+AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Turing = "fce5fe82-541a-59a6-adf8-730c64b5f9a0"
+
+[compat]
+StatsFuns = "0.9.5"
+Turing = "0.16"
+julia = "1.3"
diff --git a/test/turing/runtests.jl b/test/turing/runtests.jl
@@ -0,0 +1,19 @@
+using Random
+using Test
+using LinearAlgebra
+
+using Turing
+using Turing.DynamicPPL
+
+using StatsFuns: logistic
+
+Turing.setprogress!(false)
+
+Random.seed!(100)
+
+# Load test utilities.
+testdir(args...) = joinpath(pathof(Turing), "..", "..", "test", args...)
+include(testdir("test_utils", "AllUtils.jl"))
+
+# Test HMC.
+include(testdir("inference", "hmc.jl"))