diff --git a/src/Utils/Measures.jl b/src/Utils/Measures.jl index ea40c3bd..537eee0d 100644 --- a/src/Utils/Measures.jl +++ b/src/Utils/Measures.jl @@ -182,25 +182,41 @@ $(TYPEDEF) Compute the loss of a given model over a given (x,y) dataset running cross-validation """ function l2loss_by_cv(m,data;nsplits=5,rng=Random.GLOBAL_RNG) - x,y = data[1],data[2] - sampler = KFold(nsplits=nsplits,rng=rng) - if (ndims(y) == 1) - ohm = OneHotEncoder(handle_unknown="infrequent",cache=false) - fit!(ohm,y) - end - (μ,σ) = cross_validation([x,y],sampler) do trainData,valData,rng - (xtrain,ytrain) = trainData; (xval,yval) = valData - fit!(m,xtrain,ytrain) - ŷval = predict(m,xval) - if (eltype(ŷval) <: Dict) - yval = predict(ohm,yval) - ŷval = predict(ohm,ŷval) + if length(data) == 2 # supervised model + x,y = data[1],data[2] + sampler = KFold(nsplits=nsplits,rng=rng) + if (ndims(y) == 1) + ohm = OneHotEncoder(handle_unknown="infrequent",cache=false) + fit!(ohm,y) + end + (μ,σ) = cross_validation([x,y],sampler) do trainData,valData,rng + (xtrain,ytrain) = trainData; (xval,yval) = valData + fit!(m,xtrain,ytrain) + ŷval = predict(m,xval) + if (eltype(ŷval) <: Dict) + yval = predict(ohm,yval) + ŷval = predict(ohm,ŷval) + end + ϵ = norm(yval-ŷval)/size(yval,1) + reset!(m) + return ismissing(ϵ) ? Inf : ϵ end - ϵ = norm(yval-ŷval)/size(yval,1) - reset!(m) - return ismissing(ϵ) ? Inf : ϵ - end - return μ + return μ + elseif length(data) == 1 # unsupervised model with inverse_predict + x= data[1] + sampler = KFold(nsplits=nsplits,rng=rng) + (μ,σ) = cross_validation([x],sampler) do trainData,valData,rng + (xtrain,) = trainData; (xval,) = valData + fit!(m,xtrain) + x̂val = inverse_predict(m,xval) + ϵ = norm(xval .- x̂val)/size(xval,1) + reset!(m) + return ismissing(ϵ) ? Inf : ϵ + end + return μ + else + @error "Function `l2loss_by_cv` accepts only 1-lenght or 2-length data for respectivelly unsupervised and supervised models" + end end """ error(y,ŷ) - Categorical error with probabilistic prediction of a single datapoint (Int vs PMF). """ diff --git a/src/Utils/Processing.jl b/src/Utils/Processing.jl index 06c3cd4a..a88316e7 100644 --- a/src/Utils/Processing.jl +++ b/src/Utils/Processing.jl @@ -1174,12 +1174,12 @@ Hyperparameter autotuning using the [`GridSearch`](@ref) method. """ function tune!(m,method::GridSearch,data) - options(m).verbosity >= STD && println("Starting hp autotuning (could take a while..)") + options(m).verbosity >= STD && println("Starting hyper-parameters autotuning (this could take a while..)") options(m).verbosity >= HIGH && println(method) hpranges = method.hpranges candidates = _hpranges_2_candidates(hpranges) rng = options(m).rng - multithreads = method.multithreads && Threads.nthreads() > 1 + multithreads = method.multithreads && Threads.nthreads() > 1 compLock = ReentrantLock() best_candidate = Dict() lowest_loss = Inf @@ -1228,7 +1228,7 @@ Hyperparameter autotuning using the [`SuccessiveHalvingSearch`](@ref) method. """ function tune!(m,method::SuccessiveHalvingSearch,data) - options(m).verbosity >= STD && println("Starting hp autotuning (could take a while..)") + options(m).verbosity >= STD && println("Starting hyper-parameters autotuning (this could take a while..)") options(m).verbosity >= HIGH && println(method) hpranges = method.hpranges res_shares = method.res_shares @@ -1250,7 +1250,7 @@ function tune!(m,method::SuccessiveHalvingSearch,data) epochdata = (collect([esubs[i][1] for i in 1:length(esubs)])...,) ncandidates_thisepoch = Int(round(ncandidates/shrinkfactor^(e-1))) ncandidates_tokeep = Int(round(ncandidates/shrinkfactor^e)) - options(m).verbosity >= STD && println("(e $e / $epochs) N data / candidates / candidates to retain : $(n_orig * res_share) \t $ncandidates_thisepoch $ncandidates_tokeep") + options(m).verbosity >= STD && println("(e $e / $epochs) N data / n candidates / n candidates to retain : $(n_orig * res_share) \t $ncandidates_thisepoch $ncandidates_tokeep") scores = Vector{Tuple{Float64,Dict}}(undef,ncandidates_thisepoch) masterSeed = rand(rng,100:typemax(Int64)) rngs = generate_parallel_rngs(rng,Threads.nthreads()) diff --git a/src/Utils/Utils_extra.jl b/src/Utils/Utils_extra.jl index a21870a0..3a31bb71 100644 --- a/src/Utils/Utils_extra.jl +++ b/src/Utils/Utils_extra.jl @@ -45,7 +45,7 @@ Base.@kwdef mutable struct AutoEncoderHyperParametersSet <: BetaMLHyperParameter See [`SuccessiveHalvingSearch`](@ref) for the default method. To implement automatic hyperparameter tuning during the (first) `fit!` call simply set `autotune=true` and eventually change the default `tunemethod` options (including the parameter ranges, the resources to employ and the loss function to adopt). """ - tunemethod::AutoTuneMethod = SuccessiveHalvingSearch(hpranges = Dict("epochs"=>[100,150,200],"batch_size"=>[8,16,32],"outdims"=>[0.2,0.3,0.5],"innerdims"=>[1.3,2.0,5.0]),multithreads=false) + tunemethod::AutoTuneMethod = SuccessiveHalvingSearch(hpranges = Dict("epochs"=>[100,200,400],"batch_size"=>[8,16],"outdims"=>[0.2,0.3,0.5],"innerdims"=>[1.3,2.0,5.0,10.0,nothing]),multithreads=true) end Base.@kwdef mutable struct AutoEncoderLearnableParameters <: BetaMLLearnableParametersSet @@ -172,6 +172,8 @@ function fit!(m::AutoEncoder,X) verbosity >= HIGH && @info "Re-fitting of the model on new data" outdims_actual = m.par.outdims_actual fullnn = m.par.fullnn + n_el = m.par.n_el + n_dl = m.par.n_dl else typeof(outdims) <: Integer ? outdims_actual = outdims : outdims_actual = max(1,Int(round(D * outdims))) if isnothing(innerdims) @@ -209,15 +211,19 @@ function fit!(m::AutoEncoder,X) fullnn = NeuralNetworkEstimator(layers=[e_layers_actual...,d_layers_actual...],loss=loss,dloss=dloss,epochs=epochs,batch_size=batch_size,opt_alg=opt_alg,shuffle=shuffle,cache=cache,descr=descr,verbosity=verbosity,rng=rng ) n_el = length(e_layers_actual) n_dl = length(d_layers_actual) - m.par.n_el = n_el - m.par.n_dl = n_dl end x̂ = fit!(fullnn,X,X) - m.par.outdims_actual = outdims_actual - m.par.fullnn = fullnn + par = AutoEncoderLearnableParameters() + par.outdims_actual = outdims_actual + par.fullnn = fullnn + par.n_el = n_el + par.n_dl = n_dl + m.par = par + m.fitted=true + rme = cache ? relative_mean_error(X,x̂) : missing m.info["nepochs_ran"] = info(fullnn)["nepochs_ran"] diff --git a/test/Utils_tests.jl b/test/Utils_tests.jl index e539a7a7..f0ad49e5 100644 --- a/test/Utils_tests.jl +++ b/test/Utils_tests.jl @@ -316,8 +316,16 @@ println("** Testing AutoEncoder...") iris = readdlm(joinpath(@__DIR__,"data","iris_shuffled.csv"),',',skipstart=1) x = convert(Array{Float64,2}, iris[:,1:4]) y = convert(Array{String,1}, iris[:,5]) - -m = AutoEncoder(outdims=2,rng=copy(TESTRNG)) +tuning_method = SuccessiveHalvingSearch( + hpranges = Dict( + "innerdims"=>[2.0,5.0,nothing] + ), + res_shares = [0.2, 0.3], + multithreads = true + ) +m = AutoEncoder(epochs=400,outdims=2,autotune=true, +tunemethod=tuning_method, verbosity=NONE, rng=copy(TESTRNG) ) +#m = AutoEncoder(outdims=2,rng=copy(TESTRNG)) x2 = fit!(m,x) x2b = predict(m) x2c = predict(m,x) @@ -349,7 +357,7 @@ x = [0.12 0.31 0.29 3.21 0.21; 0.22 0.61 0.58 6.43 0.42; 0.12 0.31 0.29 3.21 0.21; 0.44 1.21 1.18 13.54 0.85]; -m = AutoEncoder(outdims=1,epochs=400,autotune=false) #TODO: check why autotune is broken here +m = AutoEncoder(outdims=1,epochs=400,autotune=false) x_reduced = fit!(m,x) x̂ = inverse_predict(m,x_reduced) info(m)["rme"]