diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f07e71b..9b72628 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: version: - - '1.9' + - 'lts' - '1' os: - ubuntu-latest diff --git a/Project.toml b/Project.toml index 099648e..8314503 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,8 @@ authors = ["Kai Xu ", "Júlio Hoffimann 1 We use the naming convention of prefixing the type name with `u` for the unconstrained variant of the corresponding estimator. -The fourth argument `optlib` specifies the optimization package used to implement +The keyword argument `optlib` specifies the optimization package used to implement the estimator. Some estimators are implemented with different optimization packages to facilitate the usage in different environments. In the example above, users that already have the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) package in @@ -83,12 +83,22 @@ and to find out the available implementations, please use available_optlib(KLIEP) ``` +In the case of `ConvexLib` and `JuMPLib`, specific optimizer packages must be loaded +besides the optimization library itself. For example, one must load `Convex` *and* `ECOS` +to use `optlib=ConvexLib` with the `KLIEP` estimator, or `JuMP` *and* `Ipopt` to use +`optlib=JuMPLib` with the `LSIF` estimator. +design. + +### Density ratio function + Some methods support the evaluation of the density ratio at all `x`, besides the -denominator samples. In this case, the following line returns a function `r(x)` +denominator samples. In this case, `densratiofunc` returns a function `r` that can be evaluated at new unseen samples: ```julia r = densratiofunc(x_nu, x_de, KLIEP()) + +r(x) # evaluate at new x ``` ### Hyperparameter tuning diff --git a/ext/DensityRatioEstimationChainRulesCoreExt.jl b/ext/DensityRatioEstimationChainRulesCoreExt.jl deleted file mode 100644 index a7949e0..0000000 --- a/ext/DensityRatioEstimationChainRulesCoreExt.jl +++ /dev/null @@ -1,12 +0,0 @@ -# ------------------------------------------------------------------ -# Licensed under the MIT License. See LICENSE in the project root. -# ------------------------------------------------------------------ - -module DensityRatioEstimationChainRulesCoreExt - -using DensityRatioEstimation -using ChainRulesCore - -ChainRulesCore.@non_differentiable DensityRatioEstimation.safe_diagm(::Any, ::Any) - -end #module diff --git a/ext/DensityRatioEstimationConvexExt.jl b/ext/DensityRatioEstimationConvexExt/DensityRatioEstimationConvexExt.jl similarity index 80% rename from ext/DensityRatioEstimationConvexExt.jl rename to ext/DensityRatioEstimationConvexExt/DensityRatioEstimationConvexExt.jl index df177c7..148a2ec 100644 --- a/ext/DensityRatioEstimationConvexExt.jl +++ b/ext/DensityRatioEstimationConvexExt/DensityRatioEstimationConvexExt.jl @@ -4,11 +4,12 @@ module DensityRatioEstimationConvexExt -using DensityRatioEstimation using DensityRatioEstimation: KLIEP, ConvexLib using Convex using ECOS -include("../src/kliep/convex.jl") +import DensityRatioEstimation -end #module +include("kliep.jl") + +end diff --git a/src/kliep/convex.jl b/ext/DensityRatioEstimationConvexExt/kliep.jl similarity index 93% rename from src/kliep/convex.jl rename to ext/DensityRatioEstimationConvexExt/kliep.jl index 1fedb95..b5bbb90 100644 --- a/src/kliep/convex.jl +++ b/ext/DensityRatioEstimationConvexExt/kliep.jl @@ -2,8 +2,6 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -# This file is part of the module DensityRatioEstimationConvexExt. - function DensityRatioEstimation._kliep_coeffs(K_nu, K_de, dre::KLIEP, optlib::Type{ConvexLib}) # retrieve parameters σ, b = dre.σ, size(K_de, 2) diff --git a/ext/DensityRatioEstimationGPUArraysExt.jl b/ext/DensityRatioEstimationGPUArraysExt.jl deleted file mode 100644 index 4b5ab02..0000000 --- a/ext/DensityRatioEstimationGPUArraysExt.jl +++ /dev/null @@ -1,18 +0,0 @@ -# ------------------------------------------------------------------ -# Licensed under the MIT License. See LICENSE in the project root. -# ------------------------------------------------------------------ - -module DensityRatioEstimationGPUArraysExt - -using DensityRatioEstimation -using GPUArrays -using LinearAlgebra - -# Avoid `mat + a * I` with CUDA which involves scalar operations and is slow -function DensityRatioEstimation.safe_diagm(mat::M, a::T) where {M<:GPUArrays.AbstractGPUArray{T,2}} where {T} - diag = similar(mat, size(m, 1)) - fill!(diag, a) - LinearAlgebra.Diagonal(diag) -end - -end #module diff --git a/ext/DensityRatioEstimationJuMPExt.jl b/ext/DensityRatioEstimationJuMPExt/DensityRatioEstimationJuMPExt.jl similarity index 80% rename from ext/DensityRatioEstimationJuMPExt.jl rename to ext/DensityRatioEstimationJuMPExt/DensityRatioEstimationJuMPExt.jl index 4626dfc..45bcfe8 100644 --- a/ext/DensityRatioEstimationJuMPExt.jl +++ b/ext/DensityRatioEstimationJuMPExt/DensityRatioEstimationJuMPExt.jl @@ -4,7 +4,6 @@ module DensityRatioEstimationJuMPExt -using DensityRatioEstimation using DensityRatioEstimation: LSIF, JuMPLib, AbstractKMM, uKMM, KMM using DensityRatioEstimation.Parameters using JuMP @@ -12,7 +11,9 @@ using Ipopt using LinearAlgebra using Statistics -include("../src/kmm/jump.jl") -include("../src/lsif/jump.jl") +import DensityRatioEstimation -end #module +include("kmm.jl") +include("lsif.jl") + +end diff --git a/src/kmm/jump.jl b/ext/DensityRatioEstimationJuMPExt/kmm.jl similarity index 95% rename from src/kmm/jump.jl rename to ext/DensityRatioEstimationJuMPExt/kmm.jl index f213480..9744adf 100644 --- a/src/kmm/jump.jl +++ b/ext/DensityRatioEstimationJuMPExt/kmm.jl @@ -2,8 +2,6 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -# This file is part of the module DensityRatioEstimationJuMPExt. - function _kmm_jump_model(K, κ, dre::AbstractKMM, optlib::Type{JuMPLib}) # number of denominator samples m = length(κ) diff --git a/src/lsif/jump.jl b/ext/DensityRatioEstimationJuMPExt/lsif.jl similarity index 90% rename from src/lsif/jump.jl rename to ext/DensityRatioEstimationJuMPExt/lsif.jl index bdb4045..51f1686 100644 --- a/src/lsif/jump.jl +++ b/ext/DensityRatioEstimationJuMPExt/lsif.jl @@ -2,8 +2,6 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -# This file is part of the module DensityRatioEstimationJuMPExt. - function DensityRatioEstimation._lsif_coeffs(H, h, dre::LSIF, optlib::Type{JuMPLib}) # retrieve parameters λ, b = dre.λ, length(h) diff --git a/ext/DensityRatioEstimationOptimExt.jl b/ext/DensityRatioEstimationOptimExt/DensityRatioEstimationOptimExt.jl similarity index 75% rename from ext/DensityRatioEstimationOptimExt.jl rename to ext/DensityRatioEstimationOptimExt/DensityRatioEstimationOptimExt.jl index 77578fb..6c86caa 100644 --- a/ext/DensityRatioEstimationOptimExt.jl +++ b/ext/DensityRatioEstimationOptimExt/DensityRatioEstimationOptimExt.jl @@ -4,13 +4,13 @@ module DensityRatioEstimationOptimExt -using DensityRatioEstimation using DensityRatioEstimation: KLIEP, LSIF, OptimLib +using LinearAlgebra using Optim -using LinearAlgebra +import DensityRatioEstimation -include("../src/kliep/optim.jl") -include("../src/lsif/optim.jl") +include("kliep.jl") +include("lsif.jl") -end #module +end diff --git a/src/kliep/optim.jl b/ext/DensityRatioEstimationOptimExt/kliep.jl similarity index 95% rename from src/kliep/optim.jl rename to ext/DensityRatioEstimationOptimExt/kliep.jl index 8074ea8..b0eb038 100644 --- a/src/kliep/optim.jl +++ b/ext/DensityRatioEstimationOptimExt/kliep.jl @@ -2,8 +2,6 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -# This file is part of the module DensityRatioEstimationOptimExt. - function DensityRatioEstimation._kliep_coeffs(K_nu, K_de, dre::KLIEP, optlib::Type{OptimLib}) # retrieve parameters σ, b = dre.σ, size(K_de, 2) diff --git a/src/lsif/optim.jl b/ext/DensityRatioEstimationOptimExt/lsif.jl similarity index 93% rename from src/lsif/optim.jl rename to ext/DensityRatioEstimationOptimExt/lsif.jl index dc90fc6..93654d4 100644 --- a/src/lsif/optim.jl +++ b/ext/DensityRatioEstimationOptimExt/lsif.jl @@ -2,8 +2,6 @@ # Licensed under the MIT License. See LICENSE in the project root. # ------------------------------------------------------------------ -# This file is part of the module DensityRatioEstimationOptimExt. - function DensityRatioEstimation._lsif_coeffs(H, h, dre::LSIF, optlib::Type{OptimLib}) # retrieve parameters λ, b = dre.λ, length(h) diff --git a/src/DensityRatioEstimation.jl b/src/DensityRatioEstimation.jl index 6f9e24b..5a81df2 100644 --- a/src/DensityRatioEstimation.jl +++ b/src/DensityRatioEstimation.jl @@ -10,7 +10,9 @@ using LinearAlgebra using Parameters using Random -# implement fit for estimators +using GPUArraysCore: AbstractGPUMatrix +using ChainRulesCore: @non_differentiable + import StatsBase: fit # API for density ratio estimation @@ -29,9 +31,6 @@ include("lsif.jl") # available estimator fitters include("lcv.jl") -# pure Julia implementations -include("kmm/julia.jl") - export # optim libs OptimizationLibrary, diff --git a/src/kmm.jl b/src/kmm.jl index 84da514..b208ae8 100644 --- a/src/kmm.jl +++ b/src/kmm.jl @@ -88,3 +88,9 @@ default_optlib(dre::Type{<:KMM}) = JuMPLib available_optlib(dre::Type{<:KMM}) = [JuMPLib] _kmm_ratios(K, κ, dre::AbstractKMM, optlib::Type{<:OptimizationLibrary}) = _throw_opt_error(dre, optlib) + +# pure Julia implementation +function _kmm_ratios(K, κ, dre::uKMM, optlib::Type{JuliaLib}) + # density ratio via solver + K \ vec(κ) +end diff --git a/src/kmm/julia.jl b/src/kmm/julia.jl deleted file mode 100644 index 1dd3d37..0000000 --- a/src/kmm/julia.jl +++ /dev/null @@ -1,8 +0,0 @@ -# ------------------------------------------------------------------ -# Licensed under the MIT License. See LICENSE in the project root. -# ------------------------------------------------------------------ - -function _kmm_ratios(K, κ, dre::uKMM, optlib::Type{JuliaLib}) - # density ratio via solver - K \ vec(κ) -end diff --git a/src/utils.jl b/src/utils.jl index f6aad8b..fbdbc8b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -9,8 +9,6 @@ Return the Euclidean distance between two indexable objects. """ euclidsq(x, y) = sum((x[i] - y[i])^2 for i in eachindex(x)) -# Support matrix data in a GPU and AD compatible way - """ euclidsq(X::T, Y::T) where {T<:AbstractMatrix} @@ -55,13 +53,18 @@ gaussian_gramian(esq, σ::AbstractFloat) = exp.(-esq ./ 2σ^2) Generate a squared matrix whose diagonal is `a` that is compatible to perform addition on `mat`. It behaves differently based on whether `mat` is on a CPU or GPU. - -It is compatible with -- CuArrays.jl (see lib/cuarrays.jl) -- Zygote.jl (see lib/zygote.jl) """ safe_diagm(mat, a) = a * I +# avoid `mat + a * I` on GPU which involves scalar operations and is slow +function safe_diagm(mat::AbstractGPUMatrix, a) + diag = similar(mat, size(m, 1)) + fill!(diag, a) + Diagonal(diag) +end + +@non_differentiable safe_diagm(::Any, ::Any) + ################################################### ## Functions and objects for throwing errors ## ################################################### diff --git a/test/data/KLIEP-ConvexLib-1.png b/test/data/KLIEP-ConvexLib-1.png index 8cdd648..91b8633 100644 Binary files a/test/data/KLIEP-ConvexLib-1.png and b/test/data/KLIEP-ConvexLib-1.png differ diff --git a/test/data/KLIEP-ConvexLib-2.png b/test/data/KLIEP-ConvexLib-2.png index f797c3a..3167b58 100644 Binary files a/test/data/KLIEP-ConvexLib-2.png and b/test/data/KLIEP-ConvexLib-2.png differ diff --git a/test/data/KLIEP-OptimLib-1.png b/test/data/KLIEP-OptimLib-1.png index 0878e5a..d6bebe2 100644 Binary files a/test/data/KLIEP-OptimLib-1.png and b/test/data/KLIEP-OptimLib-1.png differ diff --git a/test/data/KLIEP-OptimLib-2.png b/test/data/KLIEP-OptimLib-2.png index 7686355..7faa385 100644 Binary files a/test/data/KLIEP-OptimLib-2.png and b/test/data/KLIEP-OptimLib-2.png differ diff --git a/test/data/KMM-JuMPLib-1.png b/test/data/KMM-JuMPLib-1.png index 08be638..5de76b0 100644 Binary files a/test/data/KMM-JuMPLib-1.png and b/test/data/KMM-JuMPLib-1.png differ diff --git a/test/data/KMM-JuMPLib-2.png b/test/data/KMM-JuMPLib-2.png index 049bca2..74331a8 100644 Binary files a/test/data/KMM-JuMPLib-2.png and b/test/data/KMM-JuMPLib-2.png differ diff --git a/test/data/LSIF-JuMPLib-1.png b/test/data/LSIF-JuMPLib-1.png index 8d01749..0b4721c 100644 Binary files a/test/data/LSIF-JuMPLib-1.png and b/test/data/LSIF-JuMPLib-1.png differ diff --git a/test/data/LSIF-JuMPLib-2.png b/test/data/LSIF-JuMPLib-2.png index 4897e48..b2ccff8 100644 Binary files a/test/data/LSIF-JuMPLib-2.png and b/test/data/LSIF-JuMPLib-2.png differ diff --git a/test/data/LSIF-OptimLib-1.png b/test/data/LSIF-OptimLib-1.png index 9972203..a9de406 100644 Binary files a/test/data/LSIF-OptimLib-1.png and b/test/data/LSIF-OptimLib-1.png differ diff --git a/test/data/LSIF-OptimLib-2.png b/test/data/LSIF-OptimLib-2.png index 5b8922e..2e9df85 100644 Binary files a/test/data/LSIF-OptimLib-2.png and b/test/data/LSIF-OptimLib-2.png differ diff --git a/test/data/uKMM-JuMPLib-1.png b/test/data/uKMM-JuMPLib-1.png index 5aee149..a17ea90 100644 Binary files a/test/data/uKMM-JuMPLib-1.png and b/test/data/uKMM-JuMPLib-1.png differ diff --git a/test/data/uKMM-JuMPLib-2.png b/test/data/uKMM-JuMPLib-2.png index bf3c1a2..69b73f7 100644 Binary files a/test/data/uKMM-JuMPLib-2.png and b/test/data/uKMM-JuMPLib-2.png differ diff --git a/test/data/uKMM-JuliaLib-1.png b/test/data/uKMM-JuliaLib-1.png index 5aee149..a17ea90 100644 Binary files a/test/data/uKMM-JuliaLib-1.png and b/test/data/uKMM-JuliaLib-1.png differ diff --git a/test/data/uKMM-JuliaLib-2.png b/test/data/uKMM-JuliaLib-2.png index bf3c1a2..69b73f7 100644 Binary files a/test/data/uKMM-JuliaLib-2.png and b/test/data/uKMM-JuliaLib-2.png differ