From a96b5dfdf5ee27336c5a25718173043635e92082 Mon Sep 17 00:00:00 2001 From: Antonello Lobianco Date: Thu, 25 Jan 2024 11:26:20 +0100 Subject: [PATCH] Large rewrite of MoxelXHyperPArametersSet and ModelXLearnableParameters to shorter versions --- .github/workflows/ci.yml | 4 +- README.md | 4 +- docs/src/Api_v2_developer.md | 10 ++-- docs/src/index.md | 3 ++ src/Api.jl | 8 +-- src/Clustering/Clustering.jl | 2 +- src/Clustering/Clustering_hard.jl | 34 ++++++------- src/GMM/GMM.jl | 2 +- src/GMM/GMM_clustering.jl | 22 ++++---- src/GMM/GMM_regression.jl | 32 ++++++------ src/Imputation/Imputation.jl | 76 ++++++++++++++-------------- src/Nn/Nn.jl | 24 ++++----- src/Perceptron/Perceptron.jl | 2 +- src/Perceptron/Perceptron_classic.jl | 16 +++--- src/Perceptron/Perceptron_kernel.jl | 16 +++--- src/Perceptron/Perceptron_pegasos.jl | 16 +++--- src/Trees/DecisionTrees.jl | 16 +++--- src/Trees/RandomForests.jl | 16 +++--- src/Trees/Trees.jl | 4 +- src/Utils/Measures.jl | 16 +++--- src/Utils/Processing.jl | 72 +++++++++++++------------- src/Utils/Utils.jl | 8 +-- src/Utils/Utils_extra.jl | 19 +++---- 23 files changed, 214 insertions(+), 208 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e5d110d..8ecef94d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: name: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: '1.6' @@ -71,4 +71,4 @@ jobs: DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} GKSwstype: "100" # https://discourse.julialang.org/t/generation-of-documentation-fails-qt-qpa-xcb-could-not-connect-to-display/60988 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v3 diff --git a/README.md b/README.md index 3f747497..19f69fa8 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ ML toolkits/pipelines | [ScikitLearn.jl](https://github.com/cstjean/ScikitLearn. Neural Networks | [Flux.jl](https://fluxml.ai/), [Knet](https://github.com/denizyuret/Knet.jl) Decision Trees | [DecisionTree.jl](https://github.com/bensadeghi/DecisionTree.jl) Clustering | [Clustering.jl](https://github.com/JuliaStats/Clustering.jl), [GaussianMixtures.jl](https://github.com/davidavdav/GaussianMixtures.jl) -Missing imputation | [Impute.jl](https://github.com/invenia/Impute.jl) +Missing imputation | [Impute.jl](https://github.com/invenia/Impute.jl), [Mice.jl](https://github.com/tom-metherell/Mice.jl) @@ -181,6 +181,8 @@ Missing imputation | [Impute.jl](https://github.com/invenia/Impute.jl) - Add RNN support and improve convolutional layers speed - Reinforcement learning (Markov decision processes) +- Standardize data sampling in training +- Convert to GPU ## Contribute diff --git a/docs/src/Api_v2_developer.md b/docs/src/Api_v2_developer.md index d1923eb9..9249e4a0 100644 --- a/docs/src/Api_v2_developer.md +++ b/docs/src/Api_v2_developer.md @@ -12,9 +12,9 @@ The model struct is composed of the following elements: ``` mutable struct DecisionTreeEstimator <: BetaMLSupervisedModel - hpar::DTHyperParametersSet # Hyper-pharameters - opt::BetaMLDefaultOptionsSet # Option sets, default or a specific one for the model - par::DTLearnableParameters # Model learnable parameters (needed for predictions) + hpar::DecisionTreeE_hp # Hyper-pharameters + opt::BML_options # Option sets, default or a specific one for the model + par::DT_lp # Model learnable parameters (needed for predictions) cres::T # Cached results trained::Bool # Trained flag info # Complementary information, but not needed to make predictions @@ -26,7 +26,7 @@ Each specific model hyperparameter set and learnable parameter set are childs of While hyperparameters are elements that control the learning process, i.e. would influence the model training and prediction, the options have a more general meaning and do not directly affect the training (they can do indirectly, like the rng). The default option set is implemented as: ``` -Base.@kwdef mutable struct BetaMLDefaultOptionsSet +Base.@kwdef mutable struct BML_options "Cache the results of the fitting stage, as to allow predict(mod) [default: `true`]. Set it to `false` to save memory for large data." cache::Bool = true "An optional title and/or description for this model" @@ -44,7 +44,7 @@ Note that the user doesn't generally need to make a difference between an hyperp ``` function KMedoidsClusterer(;kwargs...) - m = KMedoidsClusterer(KMeansMedoidsHyperParametersSet(),BetaMLDefaultOptionsSet(),KMeansMedoidsLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = KMedoidsClusterer(KMeansMedoidsHyperParametersSet(),BML_options(),KMeansMedoids_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false diff --git a/docs/src/index.md b/docs/src/index.md index 520f8c5f..cc2bbc14 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -11,6 +11,9 @@ Most models have an interface for the [`MLJ`](https://github.com/alan-turing-ins Aside Julia, BetaML can be accessed in R or Python using respectively [JuliaCall](https://github.com/Non-Contradiction/JuliaCall) and [PyJulia](https://github.com/JuliaPy/pyjulia). See [the tutorial](@ref using_betaml_from_other_languages) for details. +!!! Warning + Version 0.11 brings homogenization in the models' names and put some order on other stuff, but at the cost of severe breaking changes. Follow the updated documentation. + ## Installation The BetaML package is included in the standard Julia register, install it with: diff --git a/src/Api.jl b/src/Api.jl index 7b515bfe..1d048aaf 100644 --- a/src/Api.jl +++ b/src/Api.jl @@ -25,7 +25,7 @@ import JLD2 export Verbosity, NONE, LOW, STD, HIGH, FULL, FIXEDSEED, FIXEDRNG, BetaMLModel, BetaMLSupervisedModel, BetaMLUnsupervisedModel, - BetaMLOptionsSet, BetaMLDefaultOptionsSet, BetaMLHyperParametersSet, BetaMLLearnableParametersSet, + BetaMLOptionsSet, BML_options, BetaMLHyperParametersSet, BetaMLLearnableParametersSet, AutoTuneMethod, predict, inverse_predict, fit!, fit_ex, info, reset!, reset_ex, parameters,hyperparameters, options, sethp!, model_save, model_load @@ -84,14 +84,14 @@ A struct defining the options used by default by the algorithms that do not over $(TYPEDFIELDS) # Notes: -- even if a model doesn't override `BetaMLDefaultOptionsSet`, may not use all its options, for example deterministic models would not make use of the `rng` parameter. Passing such parameters in these cases would simply have no influence. +- even if a model doesn't override `BML_options`, may not use all its options, for example deterministic models would not make use of the `rng` parameter. Passing such parameters in these cases would simply have no influence. # Example: ``` -julia> options = BetaMLDefaultOptionsSet(cache=false,descr="My model") +julia> options = BML_options(cache=false,descr="My model") ``` """ -Base.@kwdef mutable struct BetaMLDefaultOptionsSet <: BetaMLOptionsSet +Base.@kwdef mutable struct BML_options <: BetaMLOptionsSet "Cache the results of the fitting stage, as to allow predict(mod) [default: `true`]. Set it to `false` to save memory for large data." cache::Bool = true "An optional title and/or description for this model" diff --git a/src/Clustering/Clustering.jl b/src/Clustering/Clustering.jl index cd8dba72..f138cd37 100644 --- a/src/Clustering/Clustering.jl +++ b/src/Clustering/Clustering.jl @@ -29,7 +29,7 @@ import Base.print import Base.show # export kmeans, kmedoids -export KMeansHyperParametersSet, KMedoidsHyperParametersSet, KMeansClusterer, KMedoidsClusterer +export KMeansC_hp, KMedoidsC_hp, KMeansClusterer, KMedoidsClusterer include("Clustering_hard.jl") # K-means and k-medoids diff --git a/src/Clustering/Clustering_hard.jl b/src/Clustering/Clustering_hard.jl index 033ceed4..1ba6336b 100644 --- a/src/Clustering/Clustering_hard.jl +++ b/src/Clustering/Clustering_hard.jl @@ -247,7 +247,7 @@ Hyperparameters for the [`KMeansClusterer`](@ref) model # Parameters: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct KMeansHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct KMeansC_hp <: BetaMLHyperParametersSet "Number of classes to discriminate the data [def: 3]" n_classes::Int64 = 3 "Function to employ as distance. Default to the Euclidean distance. Can be one of the predefined distances (`l1_distance`, `l2_distance`, `l2squared_distance`), `cosine_distance`), any user defined function accepting two vectors and returning a scalar or an anonymous function with the same characteristics. Attention that the `KMeansClusterer` algorithm is not guaranteed to converge with other distances than the Euclidean one." @@ -273,7 +273,7 @@ Hyperparameters for the and [`KMedoidsClusterer`](@ref) models # Parameters: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct KMedoidsHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct KMedoidsC_hp <: BetaMLHyperParametersSet "Number of classes to discriminate the data [def: 3]" n_classes::Int64 = 3 "Function to employ as distance. Default to the Euclidean distance. Can be one of the predefined distances (`l1_distance`, `l2_distance`, `l2squared_distance`), `cosine_distance`), any user defined function accepting two vectors and returning a scalar or an anonymous function with the same characteristics. Attention that the `KMeansClusterer` algorithm is not guaranteed to converge with other distances than the Euclidean one." @@ -291,7 +291,7 @@ Base.@kwdef mutable struct KMedoidsHyperParametersSet <: BetaMLHyperParametersSe initial_representatives::Union{Nothing,Matrix{Float64}} = nothing end -Base.@kwdef mutable struct KMeansMedoidsLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct KMeansMedoids_lp <: BetaMLLearnableParametersSet representatives::Union{Nothing,Matrix{Float64}} = nothing end @@ -302,7 +302,7 @@ The classical "K-Means" clustering algorithm (unsupervised). Learn to partition the data and assign each record to one of the `n_classes` classes according to a distance metric (default Euclidean). -For the parameters see [`?KMeansHyperParametersSet`](@ref KMeansHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet). +For the parameters see [`?KMeansC_hp`](@ref KMeansC_hp) and [`?BML_options`](@ref BML_options). # Notes: - data must be numerical @@ -343,15 +343,15 @@ Dict{String, Any} with 2 entries: "xndims" => 2 julia> parameters(mod) -BetaML.Clustering.KMeansMedoidsLearnableParameters (a BetaMLLearnableParametersSet struct) +BetaML.Clustering.KMeansMedoids_lp (a BetaMLLearnableParametersSet struct) - representatives: [1.13366 9.7209; 11.0 0.9] ``` """ mutable struct KMeansClusterer <: BetaMLUnsupervisedModel - hpar::KMeansHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,KMeansMedoidsLearnableParameters} + hpar::KMeansC_hp + opt::BML_options + par::Union{Nothing,KMeansMedoids_lp} cres::Union{Nothing,Vector{Int64}} fitted::Bool info::Dict{String,Any} @@ -364,7 +364,7 @@ The classical "K-Medoids" clustering algorithm (unsupervised). Similar to K-Means, learn to partition the data and assign each record to one of the `n_classes` classes according to a distance metric, but the "representatives" (the cetroids) are guaranteed to be one of the training points. The algorithm work with any arbitrary distance measure (default Euclidean). -For the parameters see [`?KMedoidsHyperParametersSet`](@ref KMedoidsHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet). +For the parameters see [`?KMedoidsC_hp`](@ref KMedoidsC_hp) and [`?BML_options`](@ref BML_options). # Notes: - data must be numerical @@ -405,14 +405,14 @@ Dict{String, Any} with 2 entries: "xndims" => 2 julia> parameters(mod) -BetaML.Clustering.KMeansMedoidsLearnableParameters (a BetaMLLearnableParametersSet struct) +BetaML.Clustering.KMeansMedoids_lp (a BetaMLLearnableParametersSet struct) - representatives: [0.9 9.8; 11.0 0.9] ``` """ mutable struct KMedoidsClusterer <: BetaMLUnsupervisedModel - hpar::KMedoidsHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,KMeansMedoidsLearnableParameters} + hpar::KMedoidsC_hp + opt::BML_options + par::Union{Nothing,KMeansMedoids_lp} cres::Union{Nothing,Vector{Int64}} fitted::Bool info::Dict{String,Any} @@ -420,7 +420,7 @@ end function KMeansClusterer(;kwargs...) - m = KMeansClusterer(KMeansHyperParametersSet(),BetaMLDefaultOptionsSet(),KMeansMedoidsLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = KMeansClusterer(KMeansC_hp(),BML_options(),KMeansMedoids_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -437,7 +437,7 @@ function KMeansClusterer(;kwargs...) end function KMedoidsClusterer(;kwargs...) - m = KMedoidsClusterer(KMedoidsHyperParametersSet(),BetaMLDefaultOptionsSet(),KMeansMedoidsLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = KMedoidsClusterer(KMedoidsC_hp(),BML_options(),KMeansMedoids_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -479,7 +479,7 @@ function fit!(m::KMeansClusterer,x) else (clIdx,Z) = kmeans(x,K,dist=dist,initialisation_strategy=initialisation_strategy,initial_representatives=initial_representatives,verbosity=verbosity,rng=rng) end - m.par = KMeansMedoidsLearnableParameters(representatives=Z) + m.par = KMeansMedoids_lp(representatives=Z) m.cres = cache ? clIdx : nothing m.info["fitted_records"] = get(m.info,"fitted_records",0) + size(x,1) m.info["xndims"] = size(x,2) @@ -514,7 +514,7 @@ function fit!(m::KMedoidsClusterer,x) else (clIdx,Z) = kmedoids(x,K,dist=dist,initialisation_strategy=initialisation_strategy,initial_representatives=initial_representatives,verbosity=verbosity,rng=rng) end - m.par = KMeansMedoidsLearnableParameters(representatives=Z) + m.par = KMeansMedoids_lp(representatives=Z) m.cres = cache ? clIdx : nothing m.info["fitted_records"] = get(m.info,"fitted_records",0) + size(x,1) m.info["xndims"] = size(x,2) diff --git a/src/GMM/GMM.jl b/src/GMM/GMM.jl index 414910af..1ba1f182 100644 --- a/src/GMM/GMM.jl +++ b/src/GMM/GMM.jl @@ -43,7 +43,7 @@ import Base.show export AbstractMixture, GaussianMixtureClusterer, GaussianMixtureRegressor2, GaussianMixtureRegressor, - GMMHyperParametersSet + GaussianMixture_hp abstract type AbstractMixture end diff --git a/src/GMM/GMM_clustering.jl b/src/GMM/GMM_clustering.jl index 73bdbc43..941a9809 100644 --- a/src/GMM/GMM_clustering.jl +++ b/src/GMM/GMM_clustering.jl @@ -173,7 +173,7 @@ Hyperparameters for GMM clusters and other GMM-based algorithms $(FIELDS) """ -mutable struct GMMHyperParametersSet <: BetaMLHyperParametersSet +mutable struct GaussianMixture_hp <: BetaMLHyperParametersSet "Number of mixtures (latent classes) to consider [def: 3]" n_classes::Int64 "Initial probabilities of the categorical distribution (n_classes x 1) [default: `[]`]" @@ -208,7 +208,7 @@ mutable struct GMMHyperParametersSet <: BetaMLHyperParametersSet """ tunemethod::AutoTuneMethod - function GMMHyperParametersSet(; + function GaussianMixture_hp(; n_classes::Union{Nothing,Int64} = nothing, # def: 3 initial_probmixtures::Vector{Float64} = Float64[], mixtures::Union{Type,Vector{<: AbstractMixture},Nothing} = nothing, # DiagonalGaussian @@ -246,7 +246,7 @@ mutable struct GMMHyperParametersSet <: BetaMLHyperParametersSet end -Base.@kwdef mutable struct GMMClusterLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct GMMCluster_lp <: BetaMLLearnableParametersSet mixtures::Union{Type,Vector{<: AbstractMixture}} = AbstractMixture[] # attention that this is set up at model construction, as it has the same name as the hyperparameter initial_probmixtures::Vector{Float64} = [] #probRecords::Union{Nothing,Matrix{Float64}} = nothing @@ -257,7 +257,7 @@ $(TYPEDEF) Assign class probabilities to records (i.e. _soft_ clustering) assuming a probabilistic generative model of observed data using mixtures. -For the parameters see [`?GMMHyperParametersSet`](@ref GMMHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet). +For the parameters see [`?GaussianMixture_hp`](@ref GaussianMixture_hp) and [`?BML_options`](@ref BML_options). # Notes: - Data must be numerical @@ -298,22 +298,22 @@ Dict{String, Any} with 6 entries: "BIC" => -2.21571 julia> parameters(mod) -BetaML.GMM.GMMClusterLearnableParameters (a BetaMLLearnableParametersSet struct) +BetaML.GMM.GMMCluster_lp (a BetaMLLearnableParametersSet struct) - mixtures: DiagonalGaussian{Float64}[DiagonalGaussian{Float64}([0.9333333333333332, 9.9], [0.05, 0.05]), DiagonalGaussian{Float64}([11.05, 0.9500000000000001], [0.05, 0.05])] - initial_probmixtures: [0.0, 1.0] ``` """ mutable struct GaussianMixtureClusterer <: BetaMLUnsupervisedModel - hpar::GMMHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,GMMClusterLearnableParameters} + hpar::GaussianMixture_hp + opt::BML_options + par::Union{Nothing,GMMCluster_lp} cres::Union{Nothing,Matrix{Float64}} fitted::Bool info::Dict{String,Any} end function GaussianMixtureClusterer(;kwargs...) - m = GaussianMixtureClusterer(GMMHyperParametersSet(),BetaMLDefaultOptionsSet(),GMMClusterLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = GaussianMixtureClusterer(GaussianMixture_hp(),BML_options(),GMMCluster_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -327,7 +327,7 @@ function GaussianMixtureClusterer(;kwargs...) found || error("Keyword \"$kw\" is not part of this model.") end - # Special correction for GMMHyperParametersSet + # Special correction for GaussianMixture_hp kwkeys = keys(kwargs) #in(2,[1,2,3]) if !in(:mixtures,kwkeys) && !in(:n_classes,kwkeys) m.hpar.n_classes = 3 @@ -381,7 +381,7 @@ function fit!(m::GaussianMixtureClusterer,x) gmmOut = gmm(x,K;initial_probmixtures=initial_probmixtures,mixtures=mixtures,tol=tol,verbosity=verbosity,minimum_variance=minimum_variance,minimum_covariance=minimum_covariance,initialisation_strategy=initialisation_strategy,maximum_iterations=maximum_iterations,rng = rng) end probRecords = gmmOut.pₙₖ - m.par = GMMClusterLearnableParameters(mixtures = gmmOut.mixtures, initial_probmixtures=makecolvector(gmmOut.pₖ)) + m.par = GMMCluster_lp(mixtures = gmmOut.mixtures, initial_probmixtures=makecolvector(gmmOut.pₖ)) m.cres = cache ? probRecords : nothing m.info["error"] = gmmOut.ϵ diff --git a/src/GMM/GMM_regression.jl b/src/GMM/GMM_regression.jl index e2ba2bd3..1569b63f 100644 --- a/src/GMM/GMM_regression.jl +++ b/src/GMM/GMM_regression.jl @@ -5,7 +5,7 @@ import BetaML.Utils.allowmissing # ------------------------------------------------------------------------------ # GaussianMixtureRegressor2 -Base.@kwdef mutable struct GaussianMixtureRegressor2LearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct GaussianMixtureRegressor2_lp <: BetaMLLearnableParametersSet mixtures::Union{Type,Vector{<: AbstractMixture}} = DiagonalGaussian[] # The type is only temporary, it should always be replaced by an actual mixture initial_probmixtures::Vector{Float64} = [] #probRecords::Union{Nothing,Matrix{Float64}} = nothing @@ -19,7 +19,7 @@ A multi-dimensional, missing data friendly non-linear regressor based on Generat The training data is used to fit a probabilistic model with latent mixtures (Gaussian distributions with different covariances are already implemented) and then predictions of new data is obtained by fitting the new data to the mixtures. -For hyperparameters see [`GMMHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref). +For hyperparameters see [`GaussianMixture_hp`](@ref) and [`BML_options`](@ref). This strategy (`GaussianMixtureRegressor2`) works by fitting the EM algorithm on the feature matrix X. Once the data has been probabilistically assigned to the various classes, a mean value of fitting values Y is computed for each cluster (using the probabilities as weigths). @@ -70,16 +70,16 @@ Dict{String, Any} with 6 entries: """ mutable struct GaussianMixtureRegressor2 <: BetaMLUnsupervisedModel - hpar::GMMHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,GaussianMixtureRegressor2LearnableParameters} + hpar::GaussianMixture_hp + opt::BML_options + par::Union{Nothing,GaussianMixtureRegressor2_lp} cres::Union{Nothing,Matrix{Float64}} fitted::Bool info::Dict{String,Any} end function GaussianMixtureRegressor2(;kwargs...) - m = GaussianMixtureRegressor2(GMMHyperParametersSet(),BetaMLDefaultOptionsSet(),GaussianMixtureRegressor2LearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = GaussianMixtureRegressor2(GaussianMixture_hp(),BML_options(),GaussianMixtureRegressor2_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -93,7 +93,7 @@ function GaussianMixtureRegressor2(;kwargs...) found || error("Keyword \"$kw\" is not part of this model.") end - # Special correction for GMMHyperParametersSet + # Special correction for GaussianMixture_hp kwkeys = keys(kwargs) #in(2,[1,2,3]) if !in(:mixtures,kwkeys) && !in(:n_classes,kwkeys) m.hpar.n_classes = 3 @@ -156,7 +156,7 @@ function fit!(m::GaussianMixtureRegressor2,x,y) ysum = probRecords' * y ymean = vcat(transpose([ysum[r,:] / sumProbrecords[1,r] for r in 1:size(ysum,1)])...) - m.par = GaussianMixtureRegressor2LearnableParameters(mixtures = gmmOut.mixtures, initial_probmixtures=makecolvector(gmmOut.pₖ), meanYByMixture = ymean) + m.par = GaussianMixtureRegressor2_lp(mixtures = gmmOut.mixtures, initial_probmixtures=makecolvector(gmmOut.pₖ), meanYByMixture = ymean) m.cres = cache ? probRecords * ymean : nothing @@ -217,7 +217,7 @@ A multi-dimensional, missing data friendly non-linear regressor based on Generat The training data is used to fit a probabilistic model with latent mixtures (Gaussian distributions with different covariances are already implemented) and then predictions of new data is obtained by fitting the new data to the mixtures. -For hyperparameters see [`GMMHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref). +For hyperparameters see [`GaussianMixture_hp`](@ref) and [`BML_options`](@ref). Thsi strategy (`GaussianMixtureRegressor`) works by training the EM algorithm on a combined (hcat) matrix of X and Y. At predict time, the new data is first fitted to the learned mixtures using the e-step part of the EM algorithm (and using missing values for the dimensions belonging to Y) to obtain the probabilistic assignment of each record to the various mixtures. Then these probabilities are multiplied to the mixture averages for the Y dimensions to obtain the predicted value(s) for each record. @@ -262,22 +262,22 @@ Dict{String, Any} with 6 entries: "BIC" => 54.9911 julia> parameters(mod) -BetaML.GMM.GMMClusterLearnableParameters (a BetaMLLearnableParametersSet struct) +BetaML.GMM.GMMCluster_lp (a BetaMLLearnableParametersSet struct) - mixtures: DiagonalGaussian{Float64}[DiagonalGaussian{Float64}([0.9333333333333332, 9.9, -8.033333333333333], [1.1024999999999996, 0.05, 5.0625]), DiagonalGaussian{Float64}([11.05, 0.9500000000000001, 21.15], [1.1024999999999996, 0.05, 5.0625])] - initial_probmixtures: [0.6, 0.4] ``` """ mutable struct GaussianMixtureRegressor <: BetaMLUnsupervisedModel - hpar::GMMHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,GMMClusterLearnableParameters} + hpar::GaussianMixture_hp + opt::BML_options + par::Union{Nothing,GMMCluster_lp} cres::Union{Nothing,Matrix{Float64}} fitted::Bool info::Dict{String,Any} end function GaussianMixtureRegressor(;kwargs...) - m = GaussianMixtureRegressor(GMMHyperParametersSet(),BetaMLDefaultOptionsSet(),GMMClusterLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = GaussianMixtureRegressor(GaussianMixture_hp(),BML_options(),GMMCluster_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -290,7 +290,7 @@ function GaussianMixtureRegressor(;kwargs...) end found || error("Keyword \"$kw\" is not part of this model.") end - # Special correction for GMMHyperParametersSet + # Special correction for GaussianMixture_hp kwkeys = keys(kwargs) #in(2,[1,2,3]) if !in(:mixtures,kwkeys) && !in(:n_classes,kwkeys) m.hpar.n_classes = 3 @@ -351,7 +351,7 @@ function fit!(m::GaussianMixtureRegressor,x,y) gmmOut = gmm(x,K;initial_probmixtures=initial_probmixtures,mixtures=mixtures,tol=tol,verbosity=verbosity,minimum_variance=minimum_variance,minimum_covariance=minimum_covariance,initialisation_strategy=initialisation_strategy,maximum_iterations=maximum_iterations,rng = rng) end probRecords = gmmOut.pₙₖ - m.par = GMMClusterLearnableParameters(mixtures = gmmOut.mixtures, initial_probmixtures=makecolvector(gmmOut.pₖ)) + m.par = GMMCluster_lp(mixtures = gmmOut.mixtures, initial_probmixtures=makecolvector(gmmOut.pₖ)) m.cres = cache ? probRecords * [gmmOut.mixtures[k].μ[d] for k in 1:K, d in DX+1:DFull] : nothing m.info["error"] = gmmOut.ϵ diff --git a/src/Imputation/Imputation.jl b/src/Imputation/Imputation.jl index 12c0291d..e663d96c 100644 --- a/src/Imputation/Imputation.jl +++ b/src/Imputation/Imputation.jl @@ -94,7 +94,7 @@ import Base.print import Base.show #export predictMissing, -export SimpleImputerHyperParametersSet, RandomForestImputerHyperParametersSet,GeneralImputerHyperParametersSet, +export SimpleI_hp,RandomForestI_hp,GeneralI_hp, Imputer, SimpleImputer, GaussianMixtureImputer, RandomForestImputer, GeneralImputer #fit!, predict, info @@ -110,13 +110,13 @@ Hyperparameters for the [`SimpleImputer`](@ref) model # Parameters: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct SimpleImputerHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct SimpleI_hp <: BetaMLHyperParametersSet "The descriptive statistic of the column (feature) to use as imputed value [def: `mean`]" statistic::Function = mean "Normalise the feature mean by l-`norm` norm of the records [default: `nothing`]. Use it (e.g. `norm=1` to use the l-1 norm) if the records are highly heterogeneus (e.g. quantity exports of different countries)." norm::Union{Nothing,Int64} = nothing end -Base.@kwdef mutable struct SimpleImputerLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct SimpleImputer_lp <: BetaMLLearnableParametersSet cStats::Vector{Float64} = [] norms::Vector{Float64} = [] @@ -157,23 +157,23 @@ Dict{String, Any} with 1 entry: "n_imputed_values" => 1 julia> parameters(mod) -BetaML.Imputation.SimpleImputerLearnableParameters (a BetaMLLearnableParametersSet struct) +BetaML.Imputation.SimpleImputer_lp (a BetaMLLearnableParametersSet struct) - cStats: [11.0, 40.0, 55.0] - norms: [6.0, 53.333333333333336] ``` """ mutable struct SimpleImputer <: Imputer - hpar::SimpleImputerHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,SimpleImputerLearnableParameters} + hpar::SimpleI_hp + opt::BML_options + par::Union{Nothing,SimpleImputer_lp} cres::Union{Nothing,Matrix{Float64}} fitted::Bool info::Dict{String,Any} end function SimpleImputer(;kwargs...) - m = SimpleImputer(SimpleImputerHyperParametersSet(),BetaMLDefaultOptionsSet(),SimpleImputerLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = SimpleImputer(SimpleI_hp(),BML_options(),SimpleImputer_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -214,7 +214,7 @@ function fit!(imputer::SimpleImputer,X) adjNorms[ismissing.(adjNorms)] .= adjNormsMean X̂ = [missingMask[r,c] ? cStats[c]*adjNorms[r]/sum(adjNorms) : X[r,c] for r in 1:nR, c in 1:nC] end - imputer.par = SimpleImputerLearnableParameters(cStats,adjNorms) + imputer.par = SimpleImputer_lp(cStats,adjNorms) imputer.cres = cache ? X̂ : nothing imputer.info["n_imputed_values"] = sum(missingMask) imputer.fitted = true @@ -262,7 +262,7 @@ end # ------------------------------------------------------------------------------ # GaussianMixtureImputer -Base.@kwdef mutable struct GaussianMixtureImputerLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct GaussianMixtureImputer_lp <: BetaMLLearnableParametersSet mixtures::Union{Type,Vector{<: AbstractMixture}} = DiagonalGaussian[] # The type is only temporary, it should always be replaced by an actual mixture initial_probmixtures::Vector{Float64} = [] probRecords::Union{Nothing,Matrix{Float64}} = nothing @@ -275,7 +275,7 @@ $(TYPEDEF) Missing data imputer that uses a Generative (Gaussian) Mixture Model. -For the parameters (`n_classes`,`mixtures`,..) see [`GMMHyperParametersSet`](@ref). +For the parameters (`n_classes`,`mixtures`,..) see [`GaussianMixture_hp`](@ref). # Limitations: - data must be numerical @@ -312,23 +312,23 @@ Dict{String, Any} with 7 entries: "BIC" => 56.3403 julia> parameters(mod) -BetaML.Imputation.GaussianMixtureImputerLearnableParameters (a BetaMLLearnableParametersSet struct) +BetaML.Imputation.GaussianMixtureImputer_lp (a BetaMLLearnableParametersSet struct) - mixtures: AbstractMixture[SphericalGaussian{Float64}([1.0179819950570768, 3.0999990977255845], 0.2865287884295908), SphericalGaussian{Float64}([6.149053737674149, 20.43331198167713], 15.18664378248651)] - initial_probmixtures: [0.48544987084082347, 0.5145501291591764] - probRecords: [0.9999996039918224 3.9600817749531375e-7; 2.3866922376272767e-229 1.0; … ; 0.9127030246369684 0.08729697536303167; 0.9999965964161501 3.403583849794472e-6] ``` """ mutable struct GaussianMixtureImputer <: Imputer - hpar::GMMHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{GaussianMixtureImputerLearnableParameters,Nothing} + hpar::GaussianMixture_hp + opt::BML_options + par::Union{GaussianMixtureImputer_lp,Nothing} cres::Union{Nothing,Matrix{Float64}} fitted::Bool info::Dict{String,Any} end function GaussianMixtureImputer(;kwargs...) - m = GaussianMixtureImputer(GMMHyperParametersSet(),BetaMLDefaultOptionsSet(),GaussianMixtureImputerLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = GaussianMixtureImputer(GaussianMixture_hp(),BML_options(),GaussianMixtureImputer_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -341,7 +341,7 @@ function GaussianMixtureImputer(;kwargs...) end found || error("Keyword \"$kw\" is not part of this model.") end - # Special correction for GMMHyperParametersSet + # Special correction for GaussianMixture_hp kwkeys = keys(kwargs) #in(2,[1,2,3]) if !in(:mixtures,kwkeys) && !in(:n_classes,kwkeys) m.hpar.n_classes = 3 @@ -404,7 +404,7 @@ function fit!(m::GaussianMixtureImputer,X) n_imputed_values = nFill - m.par = GaussianMixtureImputerLearnableParameters(mixtures = emOut.mixtures, initial_probmixtures=makecolvector(emOut.pₖ), probRecords = emOut.pₙₖ) + m.par = GaussianMixtureImputer_lp(mixtures = emOut.mixtures, initial_probmixtures=makecolvector(emOut.pₖ), probRecords = emOut.pₙₖ) if cache X̂ = [XMask[n,d] ? X[n,d] : sum([emOut.mixtures[k].μ[d] * emOut.pₙₖ[n,k] for k in 1:K]) for n in 1:N, d in 1:D ] @@ -476,9 +476,9 @@ $(TYPEDFIELDS) julia>mod = RandomForestImputer(n_trees=20,max_depth=10,recursive_passages=3) ``` """ -Base.@kwdef mutable struct RandomForestImputerHyperParametersSet <: BetaMLHyperParametersSet - "For the underlying random forest algorithm parameters (`n_trees`,`max_depth`,`min_gain`,`min_records`,`max_features:`,`splitting_criterion`,`β`,`initialisation_strategy`, `oob` and `rng`) see [`RFHyperParametersSet`](@ref) for the specific RF algorithm parameters" - rfhpar = RFHyperParametersSet() +Base.@kwdef mutable struct RandomForestI_hp <: BetaMLHyperParametersSet + "For the underlying random forest algorithm parameters (`n_trees`,`max_depth`,`min_gain`,`min_records`,`max_features:`,`splitting_criterion`,`β`,`initialisation_strategy`, `oob` and `rng`) see [`RandomForestE_hp`](@ref) for the specific RF algorithm parameters" + rfhpar = RandomForestE_hp() "Specify the positions of the integer columns to treat as categorical instead of cardinal. [Default: empty vector (all numerical cols are treated as cardinal by default and the others as categorical)]" forced_categorical_cols::Vector{Int64} = Int64[] # like in RF, normally integers are considered ordinal "Define the times to go trough the various columns to impute their data. Useful when there are data to impute on multiple columns. The order of the first passage is given by the decreasing number of missing values per column, the other passages are random [default: `1`]." @@ -489,7 +489,7 @@ Base.@kwdef mutable struct RandomForestImputerHyperParametersSet <: BetaMLHyperP cols_to_impute::Union{String,Vector{Int64}} = "auto" end -Base.@kwdef struct RandomForestImputerLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef struct RandomForestImputer_lp <: BetaMLLearnableParametersSet forests = nothing cols_to_impute_actual = Int64[] #imputedValues = nothing @@ -502,7 +502,7 @@ $(TYPEDEF) Impute missing data using Random Forests, with optional replicable multiple imputations. -See [`RandomForestImputerHyperParametersSet`](@ref), [`RFHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref) for the parameters. +See [`RandomForestI_hp`](@ref), [`RandomForestE_hp`](@ref) and [`BML_options`](@ref) for the parameters. # Notes: - Given a certain RNG and its status (e.g. `RandomForestImputer(...,rng=StableRNG(FIXEDSEED))`), the algorithm is completely deterministic, i.e. replicable. @@ -537,9 +537,9 @@ julia> X_full = fit!(mod,X) """ mutable struct RandomForestImputer <: Imputer - hpar::RandomForestImputerHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{RandomForestImputerLearnableParameters,Nothing} + hpar::RandomForestI_hp + opt::BML_options + par::Union{RandomForestImputer_lp,Nothing} cres fitted::Bool info::Dict{String,Any} @@ -547,8 +547,8 @@ end function RandomForestImputer(;kwargs...) - hps = RandomForestImputerHyperParametersSet() - m = RandomForestImputer(hps,BetaMLDefaultOptionsSet(),RandomForestImputerLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + hps =RandomForestI_hp() + m = RandomForestImputer(hps,BML_options(),RandomForestImputer_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -699,7 +699,7 @@ function fit!(m::RandomForestImputer,X) ooberrors[imputation] = ooberrorsImputation end # end individual imputation - m.par = RandomForestImputerLearnableParameters(forests,cols2imp) + m.par = RandomForestImputer_lp(forests,cols2imp) if cache if multiple_imputations == 1 m.cres = Utils.disallowmissing(imputed[1]) @@ -804,7 +804,7 @@ Hyperparameters for [`GeneralImputer`](@ref) # Parameters: $(FIELDS) """ -Base.@kwdef mutable struct GeneralImputerHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct GeneralI_hp <: BetaMLHyperParametersSet "Columns in the matrix for which to create an imputation model, i.e. to impute. It can be a vector of columns IDs (positions), or the keywords \"auto\" (default) or \"all\". With \"auto\" the model automatically detects the columns with missing data and impute only them. You may manually specify the columns or use \"all\" if you want to create a imputation model for that columns during training even if all training data are non-missing to apply then the training model to further data with possibly missing values." cols_to_impute::Union{String,Vector{Int64}} = "auto" "An entimator model (regressor or classifier), with eventually its options (hyper-parameters), to be used to impute the various columns of the matrix. It can also be a `cols_to_impute`-length vector of different estimators to consider a different estimator for each column (dimension) to impute, for example when some columns are categorical (and will hence require a classifier) and some others are numerical (hence requiring a regressor). [default: `nothing`, i.e. use BetaML random forests, handling classification and regression jobs automatically]." @@ -821,7 +821,7 @@ Base.@kwdef mutable struct GeneralImputerHyperParametersSet <: BetaMLHyperParame multiple_imputations::Int64 = 1 end -Base.@kwdef struct GeneralImputerLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef struct GeneralImputer_lp <: BetaMLLearnableParametersSet fittedModels = nothing # by cols_to_imute only cols_to_impute_actual = Int64[] x_used_cols = Vector{Int64}[] # by all columns @@ -837,7 +837,7 @@ Impute missing values using any arbitrary learning model (classifier or regresso If needed (for example when some columns with missing data are categorical and some numerical) different models can be specified for each column. Multiple imputations and multiple "passages" trought the various colums for a single imputation are supported. -See [`GeneralImputerHyperParametersSet`](@ref) for all the hyper-parameters. +See [`GeneralI_hp`](@ref) for all the hyper-parameters. # Examples: @@ -912,9 +912,9 @@ julia> X_full = fit!(mod,X) ``` """ mutable struct GeneralImputer <: Imputer - hpar::GeneralImputerHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{GeneralImputerLearnableParameters,Nothing} + hpar::GeneralI_hp + opt::BML_options + par::Union{GeneralImputer_lp,Nothing} cres fitted::Bool info::Dict{String,Any} @@ -922,8 +922,8 @@ end function GeneralImputer(;kwargs...) - hps = GeneralImputerHyperParametersSet() - m = GeneralImputer(hps,BetaMLDefaultOptionsSet(),GeneralImputerLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + hps = GeneralI_hp() + m = GeneralImputer(hps,BML_options(),GeneralImputer_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -1093,7 +1093,7 @@ function fit!(m::GeneralImputer,X) end # end recursive passage pass imputed[imputation] = Xout end # end individual imputation - m.par = GeneralImputerLearnableParameters(estimators,cols2imp,x_used_cols) + m.par = GeneralImputer_lp(estimators,cols2imp,x_used_cols) if cache if multiple_imputations == 1 m.cres = Utils.disallowmissing(imputed[1]) diff --git a/src/Nn/Nn.jl b/src/Nn/Nn.jl index 4fec6cdb..d6f46e76 100644 --- a/src/Nn/Nn.jl +++ b/src/Nn/Nn.jl @@ -72,7 +72,7 @@ export GroupedLayer export ConvLayer, ReshaperLayer, PoolingLayer export init_optalg!, single_update! # Optimizers API export SGD,ADAM, DebugOptAlg # Available optimizers -export Learnable, fitting_info, NeuralNetworkEstimator, NNHyperParametersSet, NeuralNetworkEstimatorOptionsSet # NN API +export Learnable, fitting_info, NeuralNetworkEstimator, NeuralNetworkE_hp, NeuralNetworkE_options # NN API # export get_nparams, NN, buildNetwork, predict, loss, train!, getindex, show # old @@ -813,7 +813,7 @@ $(FIELDS) To know the available layers type `subtypes(AbstractLayer)`) and then type `?LayerName` for information on how to use each layer. """ -Base.@kwdef mutable struct NNHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct NeuralNetworkE_hp <: BetaMLHyperParametersSet "Array of layer objects [def: `nothing`, i.e. basic network]. See `subtypes(BetaML.AbstractLayer)` for supported layers" layers::Union{Array{AbstractLayer,1},Nothing} = nothing """Loss (cost) function [def: `squared_cost`] @@ -839,14 +839,14 @@ Base.@kwdef mutable struct NNHyperParametersSet <: BetaMLHyperParametersSet end """ -NeuralNetworkEstimatorOptionsSet +NeuralNetworkE_options A struct defining the options used by the Feedforward neural network model ## Parameters: $(FIELDS) """ -Base.@kwdef mutable struct NeuralNetworkEstimatorOptionsSet +Base.@kwdef mutable struct NeuralNetworkE_options "Cache the results of the fitting stage, as to allow predict(mod) [default: `true`]. Set it to `false` to save memory for large data." cache::Bool = true "An optional title and/or description for this model" @@ -863,7 +863,7 @@ Base.@kwdef mutable struct NeuralNetworkEstimatorOptionsSet rng::AbstractRNG = Random.GLOBAL_RNG end -Base.@kwdef mutable struct NeuralNetworkEstimatorLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct NeuralNetworkEstimator_lp <: BetaMLLearnableParametersSet nnstruct::Union{Nothing,NN} = nothing end @@ -873,7 +873,7 @@ end A "feedforward" (but also multi-branch) neural network (supervised). -For the parameters see [`NNHyperParametersSet`](@ref) and for the training options [`NeuralNetworkEstimatorOptionsSet`](@ref) (we have a few more options for this specific estimator). +For the parameters see [`NeuralNetworkE_hp`](@ref) and for the training options [`NeuralNetworkE_options`](@ref) (we have a few more options for this specific estimator). # Notes: - data must be numerical @@ -962,16 +962,16 @@ julia> hcat(y,ŷ) ``` """ mutable struct NeuralNetworkEstimator <: BetaMLSupervisedModel - hpar::NNHyperParametersSet - opt::NeuralNetworkEstimatorOptionsSet - par::Union{Nothing,NeuralNetworkEstimatorLearnableParameters} + hpar::NeuralNetworkE_hp + opt::NeuralNetworkE_options + par::Union{Nothing,NeuralNetworkEstimator_lp} cres::Union{Nothing,AbstractArray} fitted::Bool info::Dict{String,Any} end function NeuralNetworkEstimator(;kwargs...) - m = NeuralNetworkEstimator(NNHyperParametersSet(),NeuralNetworkEstimatorOptionsSet(),NeuralNetworkEstimatorLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = NeuralNetworkEstimator(NeuralNetworkE_hp(),NeuralNetworkE_options(),NeuralNetworkEstimator_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -984,7 +984,7 @@ function NeuralNetworkEstimator(;kwargs...) end found || error("Keyword \"$kw\" is not part of this model.") end - # Special correction for NNHyperParametersSet + # Special correction for NeuralNetworkE_hp kwkeys = keys(kwargs) #in(2,[1,2,3]) #if !in(:dloss,kwkeys) # if dloss in not explicitly provided # if (in(:loss,kwkeys) && kwargs[:loss] == squared_cost ) || # loss is explicitly provided and it is equal to squared_loss @@ -1069,7 +1069,7 @@ function fit!(m::NeuralNetworkEstimator,X,Y) nn_isize == nD || error("The first layer of the network must have the ndims of the input data ($nD) instead of $(nn_isize).") nn_osize == nDy || error("The last layer of the network must have the ndims of the output data ($nDy) instead of $(nn_osize). For classification tasks, this is normally the number of possible categories.") - m.par = NeuralNetworkEstimatorLearnableParameters(NN(deepcopy(layers),loss,dloss,false,descr)) + m.par = NeuralNetworkEstimator_lp(NN(deepcopy(layers),loss,dloss,false,descr)) m.info["nepochs_ran"] = 0 m.info["loss_per_epoch"] = Float64[] m.info["par_per_epoch"] = [] diff --git a/src/Perceptron/Perceptron.jl b/src/Perceptron/Perceptron.jl index c074c772..c2fe279b 100644 --- a/src/Perceptron/Perceptron.jl +++ b/src/Perceptron/Perceptron.jl @@ -28,7 +28,7 @@ import Base.show # export perceptron, perceptronBinary, KernelPerceptronClassifier, KernelPerceptronClassifierBinary, pegasos, pegasosBinary, predict export PerceptronClassifier, KernelPerceptronClassifier, PegasosClassifier -export PerceptronClassifierHyperParametersSet, KernelPerceptronClassifierHyperParametersSet, PegasosClassifierHyperParametersSet +export PerceptronC_hp, KernelPerceptronC_hp, PegasosC_hp include("Perceptron_classic.jl") diff --git a/src/Perceptron/Perceptron_classic.jl b/src/Perceptron/Perceptron_classic.jl index 7605ec55..27ca8486 100644 --- a/src/Perceptron/Perceptron_classic.jl +++ b/src/Perceptron/Perceptron_classic.jl @@ -284,7 +284,7 @@ Hyperparameters for the [`PerceptronClassifier`](@ref) model # Parameters: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct PerceptronClassifierHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct PerceptronC_hp <: BetaMLHyperParametersSet "Initial parameters. If given, should be a matrix of n-classes by feature dimension + 1 (to include the constant term as the first element) [def: `nothing`, i.e. zeros]" initial_parameters::Union{Nothing,Matrix{Float64}} = nothing "Maximum number of epochs, i.e. passages trough the whole training sample [def: `1000`]" @@ -303,7 +303,7 @@ Base.@kwdef mutable struct PerceptronClassifierHyperParametersSet <: BetaMLHyper tunemethod::AutoTuneMethod = SuccessiveHalvingSearch(hpranges=Dict("epochs" =>[50,100,1000,10000], "shuffle"=>[true,false], "force_origin"=>[true,false],"return_mean_hyperplane"=>[true,false]),multithreads=true) end -Base.@kwdef mutable struct PerceptronClassifierLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct PerceptronClassifier_lp <: BetaMLLearnableParametersSet weigths::Union{Nothing,Matrix{Float64}} = nothing classes::Vector = [] end @@ -313,7 +313,7 @@ $(TYPEDEF) The classical "perceptron" linear classifier (supervised). -For the parameters see [`?PerceptronClassifierHyperParametersSet`](@ref PerceptronClassifierHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet). +For the parameters see [`?PerceptronC_hp`](@ref PerceptronC_hp) and [`?BML_options`](@ref BML_options). # Notes: - data must be numerical @@ -347,16 +347,16 @@ Avg. error after iteration 1 : 0.5 ``` """ mutable struct PerceptronClassifier <: BetaMLSupervisedModel - hpar::PerceptronClassifierHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,PerceptronClassifierLearnableParameters} + hpar::PerceptronC_hp + opt::BML_options + par::Union{Nothing,PerceptronClassifier_lp} cres::Union{Nothing,Vector} fitted::Bool info::Dict{String,Any} end function PerceptronClassifier(;kwargs...) - m = PerceptronClassifier(PerceptronClassifierHyperParametersSet(),BetaMLDefaultOptionsSet(),PerceptronClassifierLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = PerceptronClassifier(PerceptronC_hp(),BML_options(),PerceptronClassifier_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -412,7 +412,7 @@ function fit!(m::PerceptronClassifier,X,Y) out = perceptron(X,Y; θ₀=initial_parameters[:,1], θ=[initial_parameters[c,2:end] for c in 1:nCl], T=epochs, nMsgs=nMsgs, shuffle=shuffle, force_origin=force_origin, return_mean_hyperplane=return_mean_hyperplane, rng = rng, verbosity=verbosity) weights = hcat(out.θ₀,vcat(out.θ' ...)) - m.par = PerceptronClassifierLearnableParameters(weights,out.classes) + m.par = PerceptronClassifier_lp(weights,out.classes) if cache out = predict(X,out.θ,out.θ₀,out.classes) m.cres = cache ? out : nothing diff --git a/src/Perceptron/Perceptron_kernel.jl b/src/Perceptron/Perceptron_kernel.jl index d304e718..68c32819 100644 --- a/src/Perceptron/Perceptron_kernel.jl +++ b/src/Perceptron/Perceptron_kernel.jl @@ -313,7 +313,7 @@ Hyperparameters for the [`KernelPerceptronClassifier`](@ref) model # Parameters: $(FIELDS) """ -Base.@kwdef mutable struct KernelPerceptronClassifierHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct KernelPerceptronC_hp <: BetaMLHyperParametersSet "Kernel function to employ. See `?radial_kernel` or `?polynomial_kernel` for details or check `?BetaML.Utils` to verify if other kernels are defined (you can alsways define your own kernel) [def: [`radial_kernel`](@ref)]" kernel::Function = radial_kernel "Initial distribution of the number of errors errors [def: `nothing`, i.e. zeros]. If provided, this should be a nModels-lenght vector of nRecords integer values vectors , where nModels is computed as `(n_classes * (n_classes - 1)) / 2`" @@ -330,7 +330,7 @@ Base.@kwdef mutable struct KernelPerceptronClassifierHyperParametersSet <: BetaM tunemethod::AutoTuneMethod = SuccessiveHalvingSearch(hpranges=Dict("kernel" =>[radial_kernel,polynomial_kernel, (x,y) -> polynomial_kernel(x,y,degree=3)], "epochs" =>[50,100,1000,10000], "shuffle"=>[true,false]),multithreads=true) end -Base.@kwdef mutable struct KernelPerceptronClassifierLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct KernelPerceptronClassifier_lp <: BetaMLLearnableParametersSet xtrain::Union{Nothing,Vector{Matrix{Float64}}} = nothing ytrain::Union{Nothing,Vector{Vector{Int64}}} = nothing errors::Union{Nothing,Vector{Vector{Int64}}} = nothing @@ -342,7 +342,7 @@ $(TYPEDEF) A "kernel" version of the `Perceptron` model (supervised) with user configurable kernel function. -For the parameters see [`?KernelPerceptronClassifierHyperParametersSet`](@ref KernelPerceptronClassifierHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet) +For the parameters see [`? KernelPerceptronC_hp`](@ref KernelPerceptronC_hp) and [`?BML_options`](@ref BML_options) # Limitations: - data must be numerical @@ -379,16 +379,16 @@ Avg. error after iteration 10 : 0.16666666666666666 ``` """ mutable struct KernelPerceptronClassifier <: BetaMLSupervisedModel - hpar::KernelPerceptronClassifierHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,KernelPerceptronClassifierLearnableParameters} + hpar:: KernelPerceptronC_hp + opt::BML_options + par::Union{Nothing,KernelPerceptronClassifier_lp} cres::Union{Nothing,Vector} fitted::Bool info::Dict{String,Any} end function KernelPerceptronClassifier(;kwargs...) - m = KernelPerceptronClassifier(KernelPerceptronClassifierHyperParametersSet(),BetaMLDefaultOptionsSet(),KernelPerceptronClassifierLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = KernelPerceptronClassifier( KernelPerceptronC_hp(),BML_options(),KernelPerceptronClassifier_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -444,7 +444,7 @@ function fit!(m::KernelPerceptronClassifier,X,Y) out = kernel_perceptron_classifier(X, Y; K=kernel, T=epochs, α=initial_errors, nMsgs=nMsgs, shuffle=shuffle, rng = rng, verbosity=verbosity) - m.par = KernelPerceptronClassifierLearnableParameters(out.x,out.y,out.α,out.classes) + m.par = KernelPerceptronClassifier_lp(out.x,out.y,out.α,out.classes) if cache diff --git a/src/Perceptron/Perceptron_pegasos.jl b/src/Perceptron/Perceptron_pegasos.jl index c8b77cf2..dd00ab26 100644 --- a/src/Perceptron/Perceptron_pegasos.jl +++ b/src/Perceptron/Perceptron_pegasos.jl @@ -203,7 +203,7 @@ Hyperparameters for the [`PegasosClassifier`](@ref) model. ## Parameters: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct PegasosClassifierHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct PegasosC_hp <: BetaMLHyperParametersSet "Learning rate [def: (epoch -> 1/sqrt(epoch))]" learning_rate::Function = (epoch -> 1/sqrt(epoch)) "Multiplicative term of the learning rate [def: `0.5`]" @@ -226,7 +226,7 @@ Base.@kwdef mutable struct PegasosClassifierHyperParametersSet <: BetaMLHyperPar tunemethod::AutoTuneMethod = SuccessiveHalvingSearch(hpranges=Dict("learning_rate" =>[(epoch -> 1/sqrt(epoch)),(epoch -> 1/epoch),(epoch -> 1)], "epochs" =>[50,100,1000,10000], "shuffle"=>[true,false], "force_origin"=>[true,false],"return_mean_hyperplane"=>[true,false]),multithreads=true) end -Base.@kwdef mutable struct PegasosClassifierLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct PegasosClassifier_lp <: BetaMLLearnableParametersSet weigths::Union{Nothing,Matrix{Float64}} = nothing classes::Vector = [] end @@ -236,7 +236,7 @@ $(TYPEDEF) The `PegasosClassifier` model, a _linear_, gradient-based classifier. Multiclass is supported using a one-vs-all approach. -See [`?PegasosClassifierHyperParametersSet`](@ref PegasosClassifierHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet) for applicable hyperparameters and options. +See [`?PegasosC_hp`](@ref PegasosC_hp) and [`?BML_options`](@ref BML_options) for applicable hyperparameters and options. # Example: ```julia @@ -265,16 +265,16 @@ Avg. error after iteration 1 : 0.5 """ mutable struct PegasosClassifier <: BetaMLSupervisedModel - hpar::PegasosClassifierHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,PegasosClassifierLearnableParameters} + hpar::PegasosC_hp + opt::BML_options + par::Union{Nothing,PegasosClassifier_lp} cres::Union{Nothing,Vector} fitted::Bool info::Dict{String,Any} end function PegasosClassifier(;kwargs...) - m = PegasosClassifier(PegasosClassifierHyperParametersSet(),BetaMLDefaultOptionsSet(),PegasosClassifierLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = PegasosClassifier(PegasosC_hp(),BML_options(),PegasosClassifier_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -332,7 +332,7 @@ function fit!(m::PegasosClassifier,X,Y) out = pegasos(X,Y; θ₀=initial_parameters[:,1], θ=[initial_parameters[c,2:end] for c in 1:nCl], λ=learning_rate_multiplicative, η=learning_rate, T=epochs, nMsgs=nMsgs, shuffle=shuffle, force_origin=force_origin, return_mean_hyperplane=return_mean_hyperplane, rng = rng, verbosity=verbosity) weights = hcat(out.θ₀,vcat(out.θ' ...)) - m.par = PegasosClassifierLearnableParameters(weights,out.classes) + m.par = PegasosClassifier_lp(weights,out.classes) if cache out = predict(X,out.θ,out.θ₀,out.classes) m.cres = cache ? out : nothing diff --git a/src/Trees/DecisionTrees.jl b/src/Trees/DecisionTrees.jl index 8f4163c9..1255c8e5 100644 --- a/src/Trees/DecisionTrees.jl +++ b/src/Trees/DecisionTrees.jl @@ -117,7 +117,7 @@ Hyperparameters for [`DecisionTreeEstimator`](@ref) (Decision Tree). ## Parameters: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct DTHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct DecisionTreeE_hp <: BetaMLHyperParametersSet "The maximum depth the tree is allowed to reach. When this is reached the node is forced to become a leaf [def: `nothing`, i.e. no limits]" max_depth::Union{Nothing,Int64} = nothing "The minimum information gain to allow for a node's partition [def: `0`]" @@ -143,7 +143,7 @@ Base.@kwdef mutable struct DTHyperParametersSet <: BetaMLHyperParametersSet end -Base.@kwdef mutable struct DTLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct DT_lp <: BetaMLLearnableParametersSet tree::Union{Nothing,AbstractNode} = nothing Ty::DataType = Any end @@ -155,7 +155,7 @@ A Decision Tree classifier and regressor (supervised). Decision Tree works by finding the "best" question to split the fitting data (according to the metric specified by the parameter `splitting_criterion` on the associated labels) untill either all the dataset is separated or a terminal condition is reached. -For the parameters see [`?DTHyperParametersSet`](@ref DTHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet). +For the parameters see [`?DecisionTreeE_hp`](@ref DecisionTreeE_hp) and [`?BML_options`](@ref BML_options). # Notes: - Online fitting (re-fitting with new data) is not supported @@ -273,16 +273,16 @@ julia> plot(wrapped_tree) """ mutable struct DecisionTreeEstimator <: BetaMLSupervisedModel - hpar::DTHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,DTLearnableParameters} + hpar::DecisionTreeE_hp + opt::BML_options + par::Union{Nothing,DT_lp} cres fitted::Bool info::Dict{String,Any} end function DecisionTreeEstimator(;kwargs...) - m = DecisionTreeEstimator(DTHyperParametersSet(),BetaMLDefaultOptionsSet(),DTLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = DecisionTreeEstimator(DecisionTreeE_hp(),BML_options(),DT_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -790,7 +790,7 @@ function fit!(m::DecisionTreeEstimator,x,y::AbstractArray{Ty,1}) where {Ty} tree = buildTree(x, y; max_depth = max_depth, min_gain=min_gain, min_records=min_records, max_features=max_features, force_classification=force_classification, splitting_criterion = splitting_criterion, mCols=nothing, fast_algorithm=fast_algorithm, integer_encoded_cols=integer_encoded_cols, rng = rng) - m.par = DTLearnableParameters(tree,Tynm) + m.par = DT_lp(tree,Tynm) if cache #println(Tynm) #println("zzz") diff --git a/src/Trees/RandomForests.jl b/src/Trees/RandomForests.jl index 7c58e9aa..48ada66f 100644 --- a/src/Trees/RandomForests.jl +++ b/src/Trees/RandomForests.jl @@ -36,7 +36,7 @@ Hyperparameters for [`RandomForestEstimator`](@ref) (Random Forest). ## Parameters: $(TYPEDFIELDS) """ -Base.@kwdef mutable struct RFHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct RandomForestE_hp <: BetaMLHyperParametersSet "Number of (decision) trees in the forest [def: `30`]" n_trees::Int64 = 30 "The maximum depth the tree is allowed to reach. When this is reached the node is forced to become a leaf [def: `nothing`, i.e. no limits]" @@ -67,7 +67,7 @@ Base.@kwdef mutable struct RFHyperParametersSet <: BetaMLHyperParametersSet tunemethod::AutoTuneMethod = SuccessiveHalvingSearch(hpranges=Dict("n_trees" => [10, 20, 30, 40], "max_depth" =>[5,10,nothing], "min_gain"=>[0.0, 0.1, 0.5], "min_records"=>[2,3,5],"max_features"=>[nothing,5,10,30],"beta"=>[0,0.01,0.1]),multithreads=false) # RF are already MT end -Base.@kwdef mutable struct RFLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct RF_lp <: BetaMLLearnableParametersSet forest::Union{Nothing,Forest} = nothing #TODO: Forest contain info that is actualy in report. Currently we duplicate, we should just remove them from par by making a dedicated struct instead of Forest Ty::DataType = Any end @@ -80,7 +80,7 @@ A Random Forest classifier and regressor (supervised). Random forests are _ensemble_ of Decision Trees models (see [`?DecisionTreeEstimator`](@ref DecisionTreeEstimator)). -For the parameters see [`?RFHyperParametersSet`](@ref RFHyperParametersSet) and [`?BetaMLDefaultOptionsSet`](@ref BetaMLDefaultOptionsSet). +For the parameters see [`?RandomForestE_hp`](@ref RandomForestE_hp) and [`?BML_options`](@ref BML_options). # Notes : - Each individual decision tree is built using bootstrap over the data, i.e. "sampling N records with replacement" (hence, some records appear multiple times and some records do not appear in the specific tree training). The `maxx_feature` injects further variability and reduces the correlation between the forest trees. @@ -145,16 +145,16 @@ Dict{String, Any}("job_is_regression" => 1, "fitted_records" => 6, "avg_avg_dept ``` """ mutable struct RandomForestEstimator <: BetaMLSupervisedModel - hpar::RFHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,RFLearnableParameters} + hpar::RandomForestE_hp + opt::BML_options + par::Union{Nothing,RF_lp} cres fitted::Bool info::Dict{String,Any} end function RandomForestEstimator(;kwargs...) -m = RandomForestEstimator(RFHyperParametersSet(),BetaMLDefaultOptionsSet(),RFLearnableParameters(),nothing,false,Dict{Symbol,Any}()) +m = RandomForestEstimator(RandomForestE_hp(),BML_options(),RF_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -292,7 +292,7 @@ function fit!(m::RandomForestEstimator,x,y::AbstractArray{Ty,1}) where {Ty} forest = buildForest(x, y, n_trees; max_depth = max_depth, min_gain=min_gain, min_records=min_records, max_features=max_features, force_classification=force_classification, splitting_criterion = splitting_criterion, fast_algorithm=fast_algorithm, integer_encoded_cols=integer_encoded_cols, β=β, oob=false, rng = rng) - m.par = RFLearnableParameters(forest,Tynm) + m.par = RF_lp(forest,Tynm) if cache rawout = predictSingle.(Ref(forest),eachrow(x),rng=rng) diff --git a/src/Trees/Trees.jl b/src/Trees/Trees.jl index f0c4e458..78ff7451 100644 --- a/src/Trees/Trees.jl +++ b/src/Trees/Trees.jl @@ -33,12 +33,12 @@ import Base.print import Base.show import Base.convert -export DecisionTreeEstimator, DTHyperParametersSet +export DecisionTreeEstimator, DecisionTreeE_hp # export AbstractDecisionNode,Leaf, DecisionNode, # export buildTree #predictSingle # TODO: to remove -export RandomForestEstimator, RFHyperParametersSet +export RandomForestEstimator, RandomForestE_hp #export Forest # export buildForest # updateTreesWeights! # TODO:to remove diff --git a/src/Utils/Measures.jl b/src/Utils/Measures.jl index 537eee0d..42990c82 100644 --- a/src/Utils/Measures.jl +++ b/src/Utils/Measures.jl @@ -304,7 +304,7 @@ Hyperparameters for [`ConfusionMatrix`](@ref) $(FIELDS) """ -Base.@kwdef mutable struct ConfusionMatrixHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct ConfusionMatrix_hp <: BetaMLHyperParametersSet "The categories (aka \"levels\") to represent. [def: `nothing`, i.e. unique ground true values]." categories::Union{Vector,Nothing} = nothing "How to handle categories not seen in the ground true values or not present in the provided `categories` array? \"error\" (default) rises an error, \"infrequent\" adds a specific category for these values." @@ -319,7 +319,7 @@ Base.@kwdef mutable struct ConfusionMatrixHyperParametersSet <: BetaMLHyperParam normalise_scores = true end -Base.@kwdef mutable struct ConfusionMatrixLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct ConfusionMatrix_lp <: BetaMLLearnableParametersSet categories_applied::Vector = [] original_vector_eltype::Union{Type,Nothing} = nothing scores::Union{Nothing,Matrix{Int64}} = nothing @@ -330,7 +330,7 @@ $(TYPEDEF) Compute a confusion matrix detailing the mismatch between observations and predictions of a categorical variable -For the parameters see [`ConfusionMatrixHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref). +For the parameters see [`ConfusionMatrix_hp`](@ref) and [`BML_options`](@ref). The "predicted" values are either the scores or the normalised scores (depending on the parameter `normalise_scores` [def: `true`]). @@ -450,16 +450,16 @@ julia> heatmap(string.(res["categories"]),string.(res["categories"]),res["normal """ mutable struct ConfusionMatrix <: BetaMLUnsupervisedModel - hpar::ConfusionMatrixHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,ConfusionMatrixLearnableParameters} + hpar::ConfusionMatrix_hp + opt::BML_options + par::Union{Nothing,ConfusionMatrix_lp} cres::Union{Nothing,Matrix{Int64},Matrix{Float64}} fitted::Bool info::Dict{String,Any} end function ConfusionMatrix(;kwargs...) - m = ConfusionMatrix(ConfusionMatrixHyperParametersSet(),BetaMLDefaultOptionsSet(),ConfusionMatrixLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = ConfusionMatrix(ConfusionMatrix_hp(),BML_options(),ConfusionMatrix_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -587,7 +587,7 @@ function fit!(m::ConfusionMatrix,Y,Ŷ) cache && (m.cres = normalise_scores ? normalised_scores : scores) - m.par = ConfusionMatrixLearnableParameters(categories_applied,vtype,scores) + m.par = ConfusionMatrix_lp(categories_applied,vtype,scores) m.info["accuracy"] = accuracy # Overall accuracy rate m.info["misclassification"] = misclassification # Overall misclassification rate diff --git a/src/Utils/Processing.jl b/src/Utils/Processing.jl index 36da1aac..3ad90bcd 100644 --- a/src/Utils/Processing.jl +++ b/src/Utils/Processing.jl @@ -85,7 +85,7 @@ Hyperparameters for both [`OneHotEncoder`](@ref) and [`OrdinalEncoder`](@ref) $(FIELDS) """ -Base.@kwdef mutable struct OneHotEncoderHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct OneHotE_hp <: BetaMLHyperParametersSet "The categories to represent as columns. [def: `nothing`, i.e. unique training values or range for integers]. Do not include `missing` in this list." categories::Union{Vector,Nothing} = nothing "How to handle categories not seen in training or not present in the provided `categories` array? \"error\" (default) rises an error, \"missing\" labels the whole output with missing values, \"infrequent\" adds a specific column for these categories in one-hot encoding or a single new category for ordinal one." @@ -94,7 +94,7 @@ Base.@kwdef mutable struct OneHotEncoderHyperParametersSet <: BetaMLHyperParamet other_categories_name = nothing end -Base.@kwdef mutable struct OneHotEncoderLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct OneHotEncoder_lp <: BetaMLLearnableParametersSet categories_applied::Vector = [] original_vector_eltype::Union{Type,Nothing} = nothing end @@ -106,7 +106,7 @@ Encode a vector of categorical values as one-hot columns. The algorithm distinguishes between _missing_ values, for which it returns a one-hot encoded row of missing values, and _other_ categories not in the provided list or not seen during training that are handled according to the `handle_unknown` parameter. -For the parameters see [`OneHotEncoderHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref). This model supports `inverse_predict`. +For the parameters see [`OneHotE_hp`](@ref) and [`BML_options`](@ref). This model supports `inverse_predict`. # Example: ```julia @@ -142,9 +142,9 @@ julia> x2_back = inverse_predict(mod,x2_oh) """ mutable struct OneHotEncoder <: BetaMLUnsupervisedModel - hpar::OneHotEncoderHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,OneHotEncoderLearnableParameters} + hpar::OneHotE_hp + opt::BML_options + par::Union{Nothing,OneHotEncoder_lp} cres::Union{Nothing,Matrix{Bool},Matrix{Union{Bool,Missing}}} fitted::Bool info::Dict{String,Any} @@ -157,7 +157,7 @@ Encode a vector of categorical values as integers. The algorithm distinguishes between _missing_ values, for which it propagate the missing, and _other_ categories not in the provided list or not seen during training that are handled according to the `handle_unknown` parameter. -For the parameters see [`OneHotEncoderHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref). This model supports `inverse_predict`. +For the parameters see [`OneHotE_hp`](@ref) and [`BML_options`](@ref). This model supports `inverse_predict`. # Example: ```julia @@ -194,16 +194,16 @@ julia> x2_back = inverse_predict(mod,x2_oh) ``` """ mutable struct OrdinalEncoder <: BetaMLUnsupervisedModel - hpar::OneHotEncoderHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,OneHotEncoderLearnableParameters} + hpar::OneHotE_hp + opt::BML_options + par::Union{Nothing,OneHotEncoder_lp} cres::Union{Nothing,Vector{Int64},Vector{Union{Int64,Missing}}} fitted::Bool info::Dict{String,Any} end function OneHotEncoder(;kwargs...) - m = OneHotEncoder(OneHotEncoderHyperParametersSet(),BetaMLDefaultOptionsSet(),OneHotEncoderLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = OneHotEncoder(OneHotE_hp(),BML_options(),OneHotEncoder_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -220,7 +220,7 @@ function OneHotEncoder(;kwargs...) end function OrdinalEncoder(;kwargs...) - m = OrdinalEncoder(OneHotEncoderHyperParametersSet(),BetaMLDefaultOptionsSet(),OneHotEncoderLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = OrdinalEncoder(OneHotE_hp(),BML_options(),OneHotEncoder_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -258,7 +258,7 @@ function _fit!(m::Union{OneHotEncoder,OrdinalEncoder},x,enctype::Symbol) if nonmissingtype(vtype) <: Number && !(nonmissingtype(vtype) <: Integer) # continuous column: we just apply identity - m.par = OneHotEncoderLearnableParameters([],vtype) + m.par = OneHotEncoder_lp([],vtype) return cache ? nothing : x end @@ -275,7 +275,7 @@ function _fit!(m::Union{OneHotEncoder,OrdinalEncoder},x,enctype::Symbol) end handle_unknown == "infrequent" && push!(categories_applied,other_categories_name) - m.par = OneHotEncoderLearnableParameters(categories_applied,vtype) + m.par = OneHotEncoder_lp(categories_applied,vtype) if cache if enctype == :onehot @@ -563,7 +563,7 @@ Base.@kwdef mutable struct MinMaxScaler <: AbstractScaler "The range of the scaled output [def: (0,1)]" outputRange::Tuple{Real,Real} = (0,1) end -Base.@kwdef mutable struct MinMaxScalerLearnableParameters <: AbstractScalerLearnableParameter +Base.@kwdef mutable struct MinMaxScaler_lp <: AbstractScalerLearnableParameter inputRangeApplied::Vector{Tuple{Float64,Float64}} = [(-Inf,+Inf)] end @@ -621,7 +621,7 @@ Base.@kwdef mutable struct StandardScaler <: AbstractScaler center::Bool=true end -Base.@kwdef mutable struct StandardScalerLearnableParameters <: AbstractScalerLearnableParameter +Base.@kwdef mutable struct StandardScaler_lp <: AbstractScalerLearnableParameter sfμ::Vector{Float64} = Float64[] # scale factor of mean sfσ::Vector{Float64} = Float64[] # scale vector of st.dev. end @@ -641,7 +641,7 @@ function _fit(m::MinMaxScaler,skip,X,cache) push!(actualRanges,(-Inf,+Inf)) end end - return X_scaled, MinMaxScalerLearnableParameters(actualRanges) + return X_scaled, MinMaxScaler_lp(actualRanges) end function _fit(m::StandardScaler,skip,X::AbstractArray,cache) nDims = ndims(X) @@ -662,10 +662,10 @@ function _fit(m::StandardScaler,skip,X::AbstractArray,cache) end end - return X_scaled, StandardScalerLearnableParameters(sfμ,sfσ) + return X_scaled, StandardScaler_lp(sfμ,sfσ) end -function _predict(m::MinMaxScaler,pars::MinMaxScalerLearnableParameters,skip,X;inverse=false) +function _predict(m::MinMaxScaler,pars::MinMaxScaler_lp,skip,X;inverse=false) if !inverse xnew = float.(X) for (ic,c) in enumerate(eachcol(X)) @@ -688,7 +688,7 @@ function _predict(m::MinMaxScaler,pars::MinMaxScalerLearnableParameters,skip,X;i return xorig end end -function _predict(m::StandardScaler,pars::StandardScalerLearnableParameters,skip,X;inverse=false) +function _predict(m::StandardScaler,pars::StandardScaler_lp,skip,X;inverse=false) if !inverse xnew = float.(X) for (ic,c) in enumerate(eachcol(X)) @@ -716,15 +716,15 @@ Hyperparameters for the Scaler transformer ## Parameters $(FIELDS) """ -Base.@kwdef mutable struct ScalerHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct Scaler_hp <: BetaMLHyperParametersSet "The specific scaler method to employ with its own parameters. See [`StandardScaler`](@ref) [def] or [`MinMaxScaler`](@ref)." method::AbstractScaler = StandardScaler() "The positional ids of the columns to skip scaling (eg. categorical columns, dummies,...) [def: `[]`]" skip::Vector{Int64} = Int64[] end -Base.@kwdef mutable struct ScalerLearnableParameters <: BetaMLLearnableParametersSet - scalerpars::AbstractScalerLearnableParameter = StandardScalerLearnableParameters() +Base.@kwdef mutable struct Scaler_lp <: BetaMLLearnableParametersSet + scalerpars::AbstractScalerLearnableParameter = StandardScaler_lp() end """ @@ -732,7 +732,7 @@ $(TYPEDEF) Scale the data according to the specific chosen method (def: `StandardScaler`) -For the parameters see [`ScalerHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref) +For the parameters see [`Scaler_hp`](@ref) and [`BML_options`](@ref) # Examples: @@ -806,16 +806,16 @@ julia> xback = inverse_predict(mod,xscaled) ``` """ mutable struct Scaler <: BetaMLUnsupervisedModel - hpar::ScalerHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,ScalerLearnableParameters} + hpar::Scaler_hp + opt::BML_options + par::Union{Nothing,Scaler_lp} cres::Union{Nothing,Array} fitted::Bool info::Dict{String,Any} end function Scaler(;kwargs...) - m = Scaler(ScalerHyperParametersSet(),BetaMLDefaultOptionsSet(),ScalerLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = Scaler(Scaler_hp(),BML_options(),Scaler_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -832,7 +832,7 @@ function Scaler(;kwargs...) end function Scaler(method;kwargs...) - m = Scaler(ScalerHyperParametersSet(method=method),BetaMLDefaultOptionsSet(),ScalerLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = Scaler(Scaler_hp(method=method),BML_options(),Scaler_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -885,14 +885,14 @@ Hyperparameters for the PCAEncoder transformer $(FIELDS) """ -Base.@kwdef mutable struct PCAHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct PCAE_hp <: BetaMLHyperParametersSet "The number of dimensions to maintain (with `outdims <= size(X,2)` ) [def: `nothing`, i.e. the number of output dimensions is determined from the parameter `max_unexplained_var`]" outdims::Union{Nothing,Int64} = nothing "The maximum proportion of variance that we are willing to accept when reducing the number of dimensions in our data [def: 0.05]. It doesn't have any effect when the output number of dimensions is explicitly chosen with the parameter `outdims`" max_unexplained_var::Float64 = 0.05 end -Base.@kwdef mutable struct PCALearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct PCA_lp <: BetaMLLearnableParametersSet eigen_out::Union{Eigen,Nothing} =nothing outdims_actual::Union{Int64,Nothing}=nothing end @@ -904,7 +904,7 @@ Perform a Principal Component Analysis, a dimensionality reduction tecnique empl PCAEncoder returns the matrix reprojected among the dimensions of maximum variance. -For the parameters see [`PCAHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref) +For the parameters see [`PCAE_hp`](@ref) and [`BML_options`](@ref) # Notes: - PCAEncoder doesn't automatically scale the data. It is suggested to apply the [`Scaler`](@ref) model before running it. @@ -946,16 +946,16 @@ julia> xtest_reproj = predict(mod,xtest) ``` """ mutable struct PCAEncoder <: BetaMLUnsupervisedModel - hpar::PCAHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,PCALearnableParameters} + hpar::PCAE_hp + opt::BML_options + par::Union{Nothing,PCA_lp} cres::Union{Nothing,Matrix} fitted::Bool info::Dict{String,Any} end function PCAEncoder(;kwargs...) - m = PCAEncoder(PCAHyperParametersSet(),BetaMLDefaultOptionsSet(),PCALearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = PCAEncoder(PCAE_hp(),BML_options(),PCA_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 289f23fc..388bb162 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -55,16 +55,16 @@ export @codelocation, generate_parallel_rngs, autojacobian, match_known_derivatives, squared_cost, dsquared_cost, mse, crossentropy, dcrossentropy, class_counts, class_counts_with_labels, mean_dicts, mode, gini, entropy, variance, error, accuracy, relative_mean_error, - ConfusionMatrix, ConfusionMatrixHyperParametersSet, silhouette, + ConfusionMatrix, ConfusionMatrix_hp, silhouette, cross_validation, AbstractDataSampler, SamplerWithData, KFold, autotune!, GridSearch, SuccessiveHalvingSearch, l2loss_by_cv, l1_distance,l2_distance, l2squared_distance, cosine_distance, pairwise, lse, sterling, radial_kernel, polynomial_kernel, Scaler, MinMaxScaler, StandardScaler, - ScalerHyperParametersSet, MinMaxScaler,StandardScaler, - PCAEncoder, PCAHyperParametersSet, - OneHotEncoder, OrdinalEncoder, OneHotEncoderHyperParametersSet, + Scaler_hp, MinMaxScaler,StandardScaler, + PCAEncoder, PCAE_hp, + OneHotEncoder, OrdinalEncoder, OneHotE_hp, @threadsif, get_parametric_types, isinteger_bml diff --git a/src/Utils/Utils_extra.jl b/src/Utils/Utils_extra.jl index 0aa75e48..85917742 100644 --- a/src/Utils/Utils_extra.jl +++ b/src/Utils/Utils_extra.jl @@ -1,11 +1,12 @@ @eval Utils begin -export AutoEncoder, AutoEncoderHyperParametersSet +export AutoEncoder, AutoE_hp @force using ..Nn import ..Nn: AbstractLayer, ADAM, SGD, NeuralNetworkEstimator, OptimisationAlgorithm, DenseLayer, NN +import Imputation """ @@ -17,7 +18,7 @@ Hyperparameters for the AutoEncoder transformer $(FIELDS) """ -Base.@kwdef mutable struct AutoEncoderHyperParametersSet <: BetaMLHyperParametersSet +Base.@kwdef mutable struct AutoE_hp <: BetaMLHyperParametersSet "The layers (vector of `AbstractLayer`s) responsable of the encoding of the data [def: `nothing`, i.e. two dense layers with the inner one of `innerdims`]" e_layers::Union{Nothing,Vector{AbstractLayer}} = nothing "The layers (vector of `AbstractLayer`s) responsable of the decoding of the data [def: `nothing`, i.e. two dense layers with the inner one of `innerdims`]" @@ -48,7 +49,7 @@ Base.@kwdef mutable struct AutoEncoderHyperParametersSet <: BetaMLHyperParameter tunemethod::AutoTuneMethod = SuccessiveHalvingSearch(hpranges = Dict("epochs"=>[100,200,400],"batch_size"=>[8,16],"outdims"=>[0.2,0.3,0.5],"innerdims"=>[1.3,2.0,5.0,10.0,nothing]),multithreads=true) end -Base.@kwdef mutable struct AutoEncoderLearnableParameters <: BetaMLLearnableParametersSet +Base.@kwdef mutable struct AutoEncoder_lp <: BetaMLLearnableParametersSet outdims_actual::Union{Int64,Nothing} = nothing fullnn::Union{NeuralNetworkEstimator,Nothing} = nothing n_el::Union{Nothing,Int64} = nothing @@ -64,7 +65,7 @@ A neural network is trained to first transform the data (ofter "compress") to a `predict(mod::AutoEncoder,x)` returns the encoded data, `inverse_predict(mod::AutoEncoder,xtransformed)` performs the decoding. -For the parameters see [`AutoEncoderHyperParametersSet`](@ref) and [`BetaMLDefaultOptionsSet`](@ref) +For the parameters see [`AutoE_hp`](@ref) and [`BML_options`](@ref) # Notes: - AutoEncoder doesn't automatically scale the data. It is suggested to apply the [`Scaler`](@ref) model before running it. @@ -120,16 +121,16 @@ julia> hcat(x,x̂) ``` """ mutable struct AutoEncoder <: BetaMLUnsupervisedModel - hpar::AutoEncoderHyperParametersSet - opt::BetaMLDefaultOptionsSet - par::Union{Nothing,AutoEncoderLearnableParameters} + hpar::AutoE_hp + opt::BML_options + par::Union{Nothing,AutoEncoder_lp} cres::Union{Nothing,Matrix} fitted::Bool info::Dict{String,Any} end function AutoEncoder(;kwargs...) - m = AutoEncoder(AutoEncoderHyperParametersSet(),BetaMLDefaultOptionsSet(),AutoEncoderLearnableParameters(),nothing,false,Dict{Symbol,Any}()) + m = AutoEncoder(AutoE_hp(),BML_options(),AutoEncoder_lp(),nothing,false,Dict{Symbol,Any}()) thisobjfields = fieldnames(nonmissingtype(typeof(m))) for (kw,kwv) in kwargs found = false @@ -215,7 +216,7 @@ function fit!(m::AutoEncoder,X) x̂ = fit!(fullnn,X,X) - par = AutoEncoderLearnableParameters() + par = AutoEncoder_lp() par.outdims_actual = outdims_actual par.fullnn = fullnn par.n_el = n_el