From ef6bd6627cb06578a10a45072ebd07f80a4aeb13 Mon Sep 17 00:00:00 2001 From: phajy Date: Wed, 22 Nov 2023 16:14:42 +0000 Subject: [PATCH 1/9] Attempt to add data_mask to InjectiveData --- src/datasets/injectivedata.jl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/datasets/injectivedata.jl b/src/datasets/injectivedata.jl index 4c97a3ab..660404a1 100644 --- a/src/datasets/injectivedata.jl +++ b/src/datasets/injectivedata.jl @@ -5,6 +5,7 @@ struct InjectiveData{V} <: AbstractDataset domain_variance::Union{Nothing,V} codomain_variance::Union{Nothing,V} name::String + data_mask::BitVector end function InjectiveData( @@ -13,8 +14,9 @@ function InjectiveData( domain_variance = nothing, codomain_variance = nothing, name = "[no-name]", + data_mask = BitVector(fill(true, size(codomain))) ) - InjectiveData(domain, codomain, domain_variance, codomain_variance, name) + InjectiveData(domain, codomain, domain_variance, codomain_variance, name, data_mask) end supports_contiguosly_binned(::Type{<:InjectiveData}) = true @@ -27,20 +29,20 @@ function make_model_domain(::ContiguouslyBinned, dataset::InjectiveData) push!(domain, domain[end] + Δ) domain end -make_objective(::ContiguouslyBinned, dataset::InjectiveData) = dataset.codomain +make_objective(::ContiguouslyBinned, dataset::InjectiveData) = dataset.codomain[data_mask] -make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain -make_objective(::OneToOne, dataset::InjectiveData) = dataset.codomain +make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain[data_mask] +make_objective(::OneToOne, dataset::InjectiveData) = dataset.codomain[data_mask] function make_objective_variance( ::AbstractDataLayout, dataset::InjectiveData{V}, )::V where {V} if !isnothing(dataset.domain_variance) - dataset.codomain_variance + dataset.codomain_variance[dataset.data_mask] else # todo: i dunno just something - 1e-8 .* dataset.codomain + 1e-8 .* dataset.codomain[dataset.data_mask] end end From c58533dd93c2d48f46cff8c7462042d88da8299f Mon Sep 17 00:00:00 2001 From: Andy Young Date: Thu, 23 Nov 2023 10:26:07 +0000 Subject: [PATCH 2/9] Update src/datasets/injectivedata.jl Co-authored-by: Fergus Baker --- src/datasets/injectivedata.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/injectivedata.jl b/src/datasets/injectivedata.jl index 660404a1..df40ece4 100644 --- a/src/datasets/injectivedata.jl +++ b/src/datasets/injectivedata.jl @@ -29,7 +29,7 @@ function make_model_domain(::ContiguouslyBinned, dataset::InjectiveData) push!(domain, domain[end] + Δ) domain end -make_objective(::ContiguouslyBinned, dataset::InjectiveData) = dataset.codomain[data_mask] +make_objective(::ContiguouslyBinned, dataset::InjectiveData) = dataset.codomain[dataset.data_mask] make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain[data_mask] make_objective(::OneToOne, dataset::InjectiveData) = dataset.codomain[data_mask] From a0308ecb8d1418d0a759367a900541eeec35484d Mon Sep 17 00:00:00 2001 From: Andy Young Date: Thu, 23 Nov 2023 10:26:18 +0000 Subject: [PATCH 3/9] Update src/datasets/injectivedata.jl Co-authored-by: Fergus Baker --- src/datasets/injectivedata.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/injectivedata.jl b/src/datasets/injectivedata.jl index df40ece4..ddae0685 100644 --- a/src/datasets/injectivedata.jl +++ b/src/datasets/injectivedata.jl @@ -31,7 +31,7 @@ function make_model_domain(::ContiguouslyBinned, dataset::InjectiveData) end make_objective(::ContiguouslyBinned, dataset::InjectiveData) = dataset.codomain[dataset.data_mask] -make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain[data_mask] +make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain[dataset.data_mask] make_objective(::OneToOne, dataset::InjectiveData) = dataset.codomain[data_mask] function make_objective_variance( From 32b34b58cb5190698fcb9dea3e6c19ddca8a796d Mon Sep 17 00:00:00 2001 From: phajy Date: Thu, 23 Nov 2023 10:53:53 +0000 Subject: [PATCH 4/9] Objective transformer --- src/datasets/injectivedata.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/datasets/injectivedata.jl b/src/datasets/injectivedata.jl index ddae0685..758866f3 100644 --- a/src/datasets/injectivedata.jl +++ b/src/datasets/injectivedata.jl @@ -32,7 +32,7 @@ end make_objective(::ContiguouslyBinned, dataset::InjectiveData) = dataset.codomain[dataset.data_mask] make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain[dataset.data_mask] -make_objective(::OneToOne, dataset::InjectiveData) = dataset.codomain[data_mask] +make_objective(::OneToOne, dataset::InjectiveData) = dataset.codomain[dataset.data_mask] function make_objective_variance( ::AbstractDataLayout, @@ -46,7 +46,15 @@ function make_objective_variance( end end -objective_transformer(::AbstractDataLayout, dataset::InjectiveData) = _DEFAULT_TRANSFORMER() +function objective_transformer(::AbstractDataLayout, dataset::InjectiveData) + function _transformer!!(domain, objective) + @views objective[dataset.data_mask] + end + function _transformer!!(output, domain, objective) + @. output = objective[dataset.data_mask] + end + _transformer!! +end make_label(dataset::InjectiveData) = dataset.name From 1e7380ad7186b34a1a46a0f6c7a01de6480d31ee Mon Sep 17 00:00:00 2001 From: phajy Date: Thu, 23 Nov 2023 11:55:51 +0000 Subject: [PATCH 5/9] Make objective variance for injective data --- src/datasets/injectivedata.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/injectivedata.jl b/src/datasets/injectivedata.jl index 758866f3..31ca6ad8 100644 --- a/src/datasets/injectivedata.jl +++ b/src/datasets/injectivedata.jl @@ -38,7 +38,7 @@ function make_objective_variance( ::AbstractDataLayout, dataset::InjectiveData{V}, )::V where {V} - if !isnothing(dataset.domain_variance) + if !isnothing(dataset.codomain_variance) dataset.codomain_variance[dataset.data_mask] else # todo: i dunno just something From c55049fed967ee895d07660318b261579c634338 Mon Sep 17 00:00:00 2001 From: Andy Young Date: Thu, 23 Nov 2023 12:04:57 +0000 Subject: [PATCH 6/9] Update src/datasets/injectivedata.jl Co-authored-by: Fergus Baker --- src/datasets/injectivedata.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/injectivedata.jl b/src/datasets/injectivedata.jl index 31ca6ad8..15780a28 100644 --- a/src/datasets/injectivedata.jl +++ b/src/datasets/injectivedata.jl @@ -31,7 +31,7 @@ function make_model_domain(::ContiguouslyBinned, dataset::InjectiveData) end make_objective(::ContiguouslyBinned, dataset::InjectiveData) = dataset.codomain[dataset.data_mask] -make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain[dataset.data_mask] +make_model_domain(::OneToOne, dataset::InjectiveData) = dataset.domain make_objective(::OneToOne, dataset::InjectiveData) = dataset.codomain[dataset.data_mask] function make_objective_variance( From d048f1676124a2265c85b63c6474a914a4f97e01 Mon Sep 17 00:00:00 2001 From: phajy Date: Fri, 24 Nov 2023 10:25:37 +0000 Subject: [PATCH 7/9] Add test for fitting a contiguously binned dataset with masked bins --- test/fitting/test-fit-simple-dataset.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/fitting/test-fit-simple-dataset.jl b/test/fitting/test-fit-simple-dataset.jl index 277f92ce..5e02333c 100644 --- a/test/fitting/test-fit-simple-dataset.jl +++ b/test/fitting/test-fit-simple-dataset.jl @@ -44,3 +44,20 @@ prob = FittingProblem(model => data) result = fit(prob, LevenbergMarquadt()) @test result.u[2] ≈ 6.1 atol = 0.1 + +# fitting a contiguously binned dataset with some masked bins +x = 10 .^ collect(range(-1, 2, 10)) +y = energy_grid .^ -2.0 +y_err = 0.1 .* int_f_E +# introduce some bogus data points to ignore +int_f_E[2:5] .= 2.0 +data = InjectiveData(energy_grid, int_f_E, codomain_variance=int_f_E_var) +# mask out the bogus data points +data.data_mask[2:5] .= false +model = XS_PowerLaw(K=FitParam(1.0E-5), a=FitParam(2.0)) +prob = FittingProblem(model => data) +@test SpectralFitting.common_support(model, data) isa SpectralFitting.ContiguouslyBinned +result = fit(prob, LevenbergMarquadt()) +@test result.u[1] ≈ 2.55 atol = 0.01 +@test result.u[2] ≈ 3.0 atol = 0.05 +# note best fit photon index, u[2] should be 3 not 2 becuase y contains bin integrated values not the density From 2a06faae138a89701fa74c8e65b5c40df0da26f3 Mon Sep 17 00:00:00 2001 From: phajy Date: Fri, 24 Nov 2023 10:32:37 +0000 Subject: [PATCH 8/9] Fix dataset fitting bug --- test/fitting/test-fit-simple-dataset.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fitting/test-fit-simple-dataset.jl b/test/fitting/test-fit-simple-dataset.jl index 5e02333c..b95436ba 100644 --- a/test/fitting/test-fit-simple-dataset.jl +++ b/test/fitting/test-fit-simple-dataset.jl @@ -47,11 +47,11 @@ result = fit(prob, LevenbergMarquadt()) # fitting a contiguously binned dataset with some masked bins x = 10 .^ collect(range(-1, 2, 10)) -y = energy_grid .^ -2.0 +y = x .^ -2.0 y_err = 0.1 .* int_f_E # introduce some bogus data points to ignore int_f_E[2:5] .= 2.0 -data = InjectiveData(energy_grid, int_f_E, codomain_variance=int_f_E_var) +data = InjectiveData(x, int_f_E, codomain_variance=int_f_E_var) # mask out the bogus data points data.data_mask[2:5] .= false model = XS_PowerLaw(K=FitParam(1.0E-5), a=FitParam(2.0)) From 56f0ba4b2b1b35bc16cada741064ea69438df718 Mon Sep 17 00:00:00 2001 From: phajy Date: Fri, 24 Nov 2023 10:46:56 +0000 Subject: [PATCH 9/9] Fix contiguously binned with masked data test --- test/fitting/test-fit-simple-dataset.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/fitting/test-fit-simple-dataset.jl b/test/fitting/test-fit-simple-dataset.jl index b95436ba..f8bbc0f1 100644 --- a/test/fitting/test-fit-simple-dataset.jl +++ b/test/fitting/test-fit-simple-dataset.jl @@ -48,10 +48,10 @@ result = fit(prob, LevenbergMarquadt()) # fitting a contiguously binned dataset with some masked bins x = 10 .^ collect(range(-1, 2, 10)) y = x .^ -2.0 -y_err = 0.1 .* int_f_E +y_err = 0.1 .* y # introduce some bogus data points to ignore -int_f_E[2:5] .= 2.0 -data = InjectiveData(x, int_f_E, codomain_variance=int_f_E_var) +y[2:5] .= 2.0 +data = InjectiveData(x, y, codomain_variance=y_err) # mask out the bogus data points data.data_mask[2:5] .= false model = XS_PowerLaw(K=FitParam(1.0E-5), a=FitParam(2.0))