From be930e8a16fd896d33ca1aabeaf7fa3909cba996 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 20 Sep 2023 11:32:28 -0600 Subject: [PATCH 01/90] add fix for resolution mismatches --- src/parameters/update_parameters.jl | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 671c1b59d7..52b7bed3ff 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -171,16 +171,15 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -211,16 +210,15 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -251,17 +249,16 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix From 676293471a8d83475b7800984ab10390c85c3755 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 20 Sep 2023 11:32:28 -0600 Subject: [PATCH 02/90] add fix for resolution mismatches --- src/parameters/update_parameters.jl | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 671c1b59d7..52b7bed3ff 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -171,16 +171,15 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -211,16 +210,15 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -251,17 +249,16 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix From 6e661bbcdb8144488bcd817ab1605b5671d65b49 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 20 Sep 2023 18:36:01 -0400 Subject: [PATCH 03/90] fix t_step calc --- src/operation/operation_model_interface.jl | 5 +++-- src/parameters/update_parameters.jl | 24 +++++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index e3511a754c..97c9faa313 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -87,10 +87,11 @@ function solve_impl!(model::OperationModel) if status != RunStatus.SUCCESSFUL settings = get_settings(model) model_name = get_name(model) + ts = get_current_timestamp(model) if !get_allow_fails(settings) - error("Solving model $(model_name) failed") + error("Solving model $(model_name) failed at $(ts)") else - @error "Solving model $(model_name) failed. Failure Allowed" + @error "Solving model $(model_name) failed at $(ts). Failure Allowed" end end return diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 52b7bed3ff..088dfd1fca 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -175,7 +175,13 @@ function _update_parameter_values!( state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - t_step = model_resolution ÷ state_data.resolution + if model_resolution < state_data.resolution + t_step = 1 + elseif model_resolution >= state_data.resolution + t_step = model_resolution ÷ state_data.resolution + else + @assert false + end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time @@ -214,7 +220,13 @@ function _update_parameter_values!( state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - t_step = model_resolution ÷ state_data.resolution + if model_resolution < state_data.resolution + t_step = 1 + elseif model_resolution >= state_data.resolution + t_step = model_resolution ÷ state_data.resolution + else + @assert false + end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time @@ -253,7 +265,13 @@ function _update_parameter_values!( state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - t_step = model_resolution ÷ state_data.resolution + if model_resolution < state_data.resolution + t_step = 1 + elseif model_resolution >= state_data.resolution + t_step = model_resolution ÷ state_data.resolution + else + @assert false + end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) From 731f0b1073da1378a86c5cf74e48be2b989bcf9f Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 16:19:33 -0600 Subject: [PATCH 04/90] allow 3D datasets --- src/core/dataset.jl | 58 +++++++++++++++----- src/core/optimizer_stats.jl | 2 +- src/core/results_by_time.jl | 25 ++++++--- src/operation/decision_model_store.jl | 4 +- src/operation/emulation_model_store.jl | 4 +- src/simulation/hdf_simulation_store.jl | 14 ++++- src/simulation/in_memory_simulation_store.jl | 2 +- src/simulation/simulation.jl | 11 ++-- src/simulation/simulation_state.jl | 28 ++++------ src/utils/dataframes_utils.jl | 6 +- src/utils/jump_utils.jl | 21 ++++--- 11 files changed, 112 insertions(+), 63 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 6340551e1f..be6e377783 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -20,9 +20,9 @@ end # Values field is accessed with dot syntax to avoid type instability -mutable struct InMemoryDataset <: AbstractDataset - "Data with dimensions (column names, row indexes)" - values::DenseAxisArray{Float64, 2} +mutable struct InMemoryDataset{N} <: AbstractDataset + "Data with dimensions (N column names, row indexes)" + values::DenseAxisArray{Float64, N} # We use Array here to allow for overwrites when updating the state timestamps::Vector{Dates.DateTime} # Resolution is needed because AbstractDataset might have just one row @@ -33,12 +33,12 @@ mutable struct InMemoryDataset <: AbstractDataset end function InMemoryDataset( - values::DenseAxisArray{Float64, 2}, + values::DenseAxisArray{Float64, N}, timestamps::Vector{Dates.DateTime}, resolution::Dates.Millisecond, end_of_step_index::Int, -) - return InMemoryDataset( +) where {N} + return InMemoryDataset{N}( values, timestamps, resolution, @@ -48,8 +48,8 @@ function InMemoryDataset( ) end -function InMemoryDataset(values::DenseAxisArray{Float64, 2}) - return InMemoryDataset( +function InMemoryDataset(values::DenseAxisArray{Float64, N}) where {N} + return InMemoryDataset{N}( values, Vector{Dates.DateTime}(), Dates.Second(0.0), @@ -59,14 +59,38 @@ function InMemoryDataset(values::DenseAxisArray{Float64, 2}) ) end -get_num_rows(s::InMemoryDataset) = size(s.values)[2] +function InMemoryDataset( + fill_val::Float64, + initial_time::Dates.DateTime, + resolution::Dates.Millisecond, + end_of_step_index::Int, + row_count::Int, + column_names::NTuple{N, <:Any}) where {N} + return InMemoryDataset( + fill!( + DenseAxisArray{Float64}(undef, column_names..., 1:row_count), + fill_val, + ), + collect( + range( + initial_time; + step = resolution, + length = row_count, + ), + ), + resolution, + end_of_step_index, + ) +end + +get_num_rows(s::InMemoryDataset{N}) where {N} = size(s.values)[N] function make_system_state( - values::DenseAxisArray{Float64, 2}, timestamp::Dates.DateTime, resolution::Dates.Millisecond, -) - return InMemoryDataset(values, [timestamp], resolution, 0, 1, UNSET_INI_TIME) + columns::NTuple{N, <:Any}, +) where {N} + return InMemoryDataset(NaN, timestamp, resolution, 0, 1, columns) end function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) @@ -77,8 +101,9 @@ function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) return s.values[:, s_index] end -get_column_names(s::InMemoryDataset) = axes(s.values)[1] -get_column_names(::OptimizationContainerKey, s::InMemoryDataset) = get_column_names(s) +function get_column_names(k::OptimizationContainerKey, s::InMemoryDataset) + return get_column_names(k, s.values) +end function get_last_recorded_value(s::InMemoryDataset) if get_last_recorded_row(s) == 0 @@ -120,6 +145,11 @@ function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 1}, index: return end +function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 3}, index::Int) + s.values[:, :, index] = vals + return +end + # HDF5Dataset does not account of overwrites in the data. Values are written sequentially. mutable struct HDF5Dataset <: AbstractDataset values::HDF5.Dataset diff --git a/src/core/optimizer_stats.jl b/src/core/optimizer_stats.jl index 6e2c98806a..19475381c6 100644 --- a/src/core/optimizer_stats.jl +++ b/src/core/optimizer_stats.jl @@ -100,5 +100,5 @@ function to_dict(stats::OptimizerStats) end function get_column_names(::Type{OptimizerStats}) - return collect(string.(fieldnames(OptimizerStats))) + return (collect(string.(fieldnames(OptimizerStats))),) end diff --git a/src/core/results_by_time.jl b/src/core/results_by_time.jl index 827b32d4c4..e71084234b 100644 --- a/src/core/results_by_time.jl +++ b/src/core/results_by_time.jl @@ -1,21 +1,26 @@ -mutable struct ResultsByTime{T} +mutable struct ResultsByTime{T, N} key::OptimizationContainerKey data::SortedDict{Dates.DateTime, T} resolution::Dates.Period - column_names::Vector{String} + column_names::NTuple{N, Vector{String}} end -function ResultsByTime(key, data, resolution, column_names) +function ResultsByTime( + key::OptimizationContainerKey, + data::SortedDict{Dates.DateTime, T}, + resolution::Dates.Period, + column_names, +) where {T} _check_column_consistency(data, column_names) ResultsByTime(key, data, resolution, column_names) end function _check_column_consistency( data::SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}, - cols::Vector{String}, + cols::Tuple{Vector{String}}, ) for val in values(data) - if axes(val)[1] != cols + if axes(val)[1] != cols[1] error("Mismatch in DenseAxisArray column names: $(axes(val)[1]) $cols") end end @@ -23,15 +28,19 @@ end function _check_column_consistency( data::SortedDict{Dates.DateTime, Matrix{Float64}}, - cols::Vector{String}, + cols::Tuple{Vector{String}}, ) for val in values(data) - if size(val)[2] != length(cols) - error("Mismatch in length of Matrix columns: $(size(val)[2]) $(length(cols))") + if size(val)[2] != length(cols[1]) + error( + "Mismatch in length of Matrix columns: $(size(val)[2]) $(length(cols[1]))", + ) end end end +# TODO: Implement consistency check for other sizes + # This struct behaves like a dict, delegating to its 'data' field. Base.length(res::ResultsByTime) = length(res.data) Base.iterate(res::ResultsByTime) = iterate(res.data) diff --git a/src/operation/decision_model_store.jl b/src/operation/decision_model_store.jl index bf10abcf42..b8c2591164 100644 --- a/src/operation/decision_model_store.jl +++ b/src/operation/decision_model_store.jl @@ -49,7 +49,7 @@ function initialize_storage!( for timestamp in range(initial_time; step = model_interval, length = num_of_executions) data[timestamp] = fill!( - DenseAxisArray{Float64}(undef, column_names, 1:time_steps_count), + DenseAxisArray{Float64}(undef, column_names..., 1:time_steps_count), NaN, ) end @@ -133,5 +133,5 @@ end function get_column_names(store::DecisionModelStore, key::OptimizationContainerKey) container = getfield(store, get_store_container_type(key)) - return axes(first(values(container[key])))[1] + return get_column_names(key, first(values(container[key]))) end diff --git a/src/operation/emulation_model_store.jl b/src/operation/emulation_model_store.jl index fe468b6483..deca8dbca6 100644 --- a/src/operation/emulation_model_store.jl +++ b/src/operation/emulation_model_store.jl @@ -77,7 +77,7 @@ function initialize_storage!( column_names = get_column_names(key, field_container) results_container[key] = InMemoryDataset( fill!( - DenseAxisArray{Float64}(undef, column_names, 1:num_of_executions), + DenseAxisArray{Float64}(undef, column_names..., 1:num_of_executions), NaN, ), ) @@ -139,7 +139,7 @@ end function get_column_names(store::EmulationModelStore, key::OptimizationContainerKey) container = get_data_field(store, get_store_container_type(key)) - return axes(container[key].values)[1] + return get_column_names(key, container[key].values) end function get_dataset_size(store::EmulationModelStore, key::OptimizationContainerKey) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 223407bea0..e0c45843ef 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -261,7 +261,12 @@ function initialize_problem_storage!( # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. col = _make_column_name(name) - HDF5.write_dataset(group, col, string.(reqs["columns"])) + if length(reqs["columns"]) == 1 + HDF5.write_dataset(group, col, string.(reqs["columns"][1])) + else + col_vals = vcat(reqs["columns"]...) + HDF5.write_dataset(group, col, string.(col_vals)) + end column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) datasets[key] = HDF5Dataset( @@ -305,7 +310,12 @@ function initialize_problem_storage!( # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. col = _make_column_name(name) - HDF5.write_dataset(group, col, string.(reqs["columns"])) + if length(reqs["columns"]) == 1 + HDF5.write_dataset(group, col, string.(reqs["columns"][1])) + else + col_vals = vcat(reqs["columns"]...) + HDF5.write_dataset(group, col, string.(col_vals)) + end column_dataset = group[col] datasets = getfield(store.em_data, type) datasets[key] = HDF5Dataset( diff --git a/src/simulation/in_memory_simulation_store.jl b/src/simulation/in_memory_simulation_store.jl index cff3ef5b60..59cd94de1a 100644 --- a/src/simulation/in_memory_simulation_store.jl +++ b/src/simulation/in_memory_simulation_store.jl @@ -143,7 +143,7 @@ function initialize_problem_storage!( container = get_data_field(get_em_data(store), type) container[key] = InMemoryDataset( fill!( - DenseAxisArray{Float64}(undef, reqs["columns"], 1:reqs["dims"][1]), + DenseAxisArray{Float64}(undef, reqs["columns"]..., 1:reqs["dims"][1]), NaN, ), ) diff --git a/src/simulation/simulation.jl b/src/simulation/simulation.jl index 0b2985a9ec..aa0a77754f 100644 --- a/src/simulation/simulation.jl +++ b/src/simulation/simulation.jl @@ -384,35 +384,36 @@ function _get_emulation_store_requirements(sim::Simulation) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.duals[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.duals[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_parameters_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.parameters[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.parameters[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_variables_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.variables[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.variables[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_aux_variables_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.aux_variables[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.aux_variables[key] = + Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_expression_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.expressions[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.expressions[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end return reqs end diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index daaa02fcfa..b7e2932ef5 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -90,20 +90,12 @@ function _initialize_model_states!( column_names = get_column_names(key, value) if !haskey(field_states, key) || get_num_rows(field_states[key]) < value_counts field_states[key] = InMemoryDataset( - fill!( - DenseAxisArray{Float64}(undef, column_names, 1:value_counts), - NaN, - ), - collect( - range( - simulation_initial_time; - step = params[key].resolution, - length = value_counts, - ), - ), + NaN, + simulation_initial_time, params[key].resolution, Int(simulation_step / params[key].resolution), - ) + value_counts, + column_names) end end end @@ -125,9 +117,9 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, cols, 1:1), NaN), simulation_initial_time, min_res, + cols, ), ) end @@ -153,9 +145,9 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, column_names, 1:1), NaN), simulation_initial_time, - get_resolution(emulation_model), + min_res, + cols, ), ) end @@ -173,9 +165,9 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, cols, 1:1), NaN), simulation_initial_time, - get_resolution(emulation_model), + min_res, + cols, ), ) end @@ -212,7 +204,7 @@ function update_decision_state!( model_params::ModelStoreParams, ) state_data = get_decision_state_data(state, key) - column_names = get_column_names(state_data) + column_names = get_column_names(key, state_data)[1] model_resolution = get_resolution(model_params) state_resolution = get_data_resolution(state_data) resolution_ratio = model_resolution ÷ state_resolution diff --git a/src/utils/dataframes_utils.jl b/src/utils/dataframes_utils.jl index 4f78172b67..6db302134c 100644 --- a/src/utils/dataframes_utils.jl +++ b/src/utils/dataframes_utils.jl @@ -7,12 +7,12 @@ Creates a DataFrame from a JuMP DenseAxisArray or SparseAxisArray. - `array`: JuMP DenseAxisArray or SparseAxisArray to convert - `key::OptimizationContainerKey`: """ -function to_dataframe(array::DenseAxisArray, key::OptimizationContainerKey) - return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)) +function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T} + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end function to_dataframe(array::SparseAxisArray, key::OptimizationContainerKey) - return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)) + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end function to_matrix(df::DataFrames.DataFrame) diff --git a/src/utils/jump_utils.jl b/src/utils/jump_utils.jl index 5f1b2aa2ce..8d790e732c 100644 --- a/src/utils/jump_utils.jl +++ b/src/utils/jump_utils.jl @@ -61,15 +61,22 @@ end function get_column_names( key::OptimizationContainerKey, ::DenseAxisArray{T, 1, K}, -) where {T, K <: NTuple{1, Any}} +) where {T, K <: NTuple{1, Int}} return get_column_names(key) end function get_column_names( - ::OptimizationContainerKey, + k::OptimizationContainerKey, array::DenseAxisArray{T, 2, K}, ) where {T, K <: NTuple{2, Any}} - return string.(axes(array)[1]) + return (string.(axes(array)[1]),) +end + +function get_column_names( + k::OptimizationContainerKey, + array::DenseAxisArray{T, 3, K}, +) where {T, K <: NTuple{3, Any}} + return (string.(axes(array)[1]), string.(axes(array)[2])) end function _get_column_names(arr::SparseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} @@ -80,7 +87,7 @@ function get_column_names( ::OptimizationContainerKey, array::SparseAxisArray{T, N, K}, ) where {T, N, K <: NTuple{N, Any}} - return get_column_names(array) + return (get_column_names(array),) end function get_column_names(array::SparseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} @@ -198,12 +205,12 @@ function _calc_dimensions( if length(ax[2]) != horizon @debug "$(encode_key_as_string(key)) has length $(length(ax[1])). Different than horizon $horizon." end - dims = (length(ax[2]), length(columns), num_rows) + dims = (length(ax[2]), length(columns[1]), num_rows) elseif length(ax) == 3 if length(ax[3]) != horizon @debug "$(encode_key_as_string(key)) has length $(length(ax[1])). Different than horizon $horizon." end - dims = (length(ax[2]), length(ax[3]), length(columns), num_rows) + dims = (length(ax[3]), length(columns[1]), length(columns[2]), num_rows) else error("unsupported data size $(length(ax))") end @@ -218,7 +225,7 @@ function _calc_dimensions( horizon::Int, ) columns = get_column_names(key, array) - dims = (horizon, length(columns), num_rows) + dims = (horizon, length.(columns)..., num_rows) return Dict("columns" => columns, "dims" => dims) end From 64509f4f15a9ebb75974656e3176b11bf1fbd37d Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 16:40:36 -0600 Subject: [PATCH 05/90] fix get_columns --- src/utils/jump_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/jump_utils.jl b/src/utils/jump_utils.jl index 8d790e732c..c3aa86fe60 100644 --- a/src/utils/jump_utils.jl +++ b/src/utils/jump_utils.jl @@ -61,7 +61,7 @@ end function get_column_names( key::OptimizationContainerKey, ::DenseAxisArray{T, 1, K}, -) where {T, K <: NTuple{1, Int}} +) where {T, K <: NTuple{1, Any}} return get_column_names(key) end From 36c1d4fed1e5c9e569e6a7de009efbe2853045aa Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 16:49:27 -0600 Subject: [PATCH 06/90] add method for 1 d data set --- src/core/dataset.jl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index be6e377783..199c5eb6ad 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -59,6 +59,24 @@ function InMemoryDataset(values::DenseAxisArray{Float64, N}) where {N} ) end +# Helper method for one dimensional cases +function InMemoryDataset( + fill_val::Float64, + initial_time::Dates.DateTime, + resolution::Dates.Millisecond, + end_of_step_index::Int, + row_count::Int, + column_names::Vector{String}) + return InMemoryDataset( + fill_val, + initial_time, + resolution, + end_of_step_index, + row_count, + (column_names,), + ) +end + function InMemoryDataset( fill_val::Float64, initial_time::Dates.DateTime, From e3ecbfd33006a8a891a55217e92d570039d14f3e Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 17:23:11 -0600 Subject: [PATCH 07/90] fix return --- src/utils/jump_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/jump_utils.jl b/src/utils/jump_utils.jl index c3aa86fe60..c3360c5da5 100644 --- a/src/utils/jump_utils.jl +++ b/src/utils/jump_utils.jl @@ -55,7 +55,7 @@ function to_matrix(::DenseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} end function get_column_names(key::OptimizationContainerKey) - return [encode_key_as_string(key)] + return ([encode_key_as_string(key)],) end function get_column_names( From b23c10817fbef74c8ab9476fb1446704e7a71381 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 18:35:32 -0600 Subject: [PATCH 08/90] fix results processing --- src/core/dataset.jl | 12 +++- src/core/results_by_time.jl | 7 +++ .../decision_model_simulation_results.jl | 2 +- src/simulation/hdf_simulation_store.jl | 5 +- src/simulation/realized_meta.jl | 56 +++++++++++-------- src/simulation/simulation_state.jl | 6 +- 6 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 199c5eb6ad..9d996d3375 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -201,7 +201,17 @@ HDF5Dataset(values, column_dataset, resolution, initial_time) = column_dataset[:], ) -get_column_names(::OptimizationContainerKey, s::HDF5Dataset) = s.column_names +function _make_column_names(vals::Vector{String}) + return (vals,) +end + +function _make_column_names(vals::Matrix{String}) + return (vals[:, 1], vals[:, 2]) +end + +function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) + return _make_column_names(s.column_names) +end """ Return the timestamp from most recent data row updated in the dataset. This value may not be the same as the result from `get_update_timestamp` diff --git a/src/core/results_by_time.jl b/src/core/results_by_time.jl index e71084234b..32d77ede00 100644 --- a/src/core/results_by_time.jl +++ b/src/core/results_by_time.jl @@ -39,6 +39,13 @@ function _check_column_consistency( end end +function _check_column_consistency( + data::SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}, + cols::NTuple{N, Vector{String}}, +) where {N} + # TODO: +end + # TODO: Implement consistency check for other sizes # This struct behaves like a dict, delegating to its 'data' field. diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 38e6bb958c..96cbd04289 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -186,7 +186,7 @@ function _get_store_value( resolution = get_resolution(sim_results) for key in container_keys - results_by_time = ResultsByTime{Matrix{Float64}}( + results_by_time = ResultsByTime{Matrix{Float64}, 1}( key, SortedDict{Dates.DateTime, Matrix{Float64}}(), resolution, diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index e0c45843ef..6372ccb457 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -371,7 +371,7 @@ function read_result( else data, columns = _read_result(store, model_name, key, index) end - return DenseAxisArray(permutedims(data), columns, 1:size(data)[1]) + return DenseAxisArray(permutedims(data), columns..., 1:size(data)[1]) end function read_result( @@ -407,8 +407,7 @@ function read_results( data = dataset.values[index:(index + len - 1), :] end columns = get_column_names(key, dataset) - @assert_op size(data)[2] == length(columns) - return DenseAxisArray(permutedims(data), columns, 1:size(data)[1]) + return DenseAxisArray(permutedims(data), columns..., 1:size(data)[1]) end function get_column_names( diff --git a/src/simulation/realized_meta.jl b/src/simulation/realized_meta.jl index 3b8f9c34ea..4dbe971bc6 100644 --- a/src/simulation/realized_meta.jl +++ b/src/simulation/realized_meta.jl @@ -46,6 +46,38 @@ function RealizedMeta( ) end +function _make_dataframe( + columns::Tuple{Vector{String}}, + results_by_time::ResultsByTime{Matrix{Float64}, 1}, + num_rows::Int, + meta::RealizedMeta, +) + num_cols = length(columns[1]) + matrix = Matrix{Float64}(undef, num_rows, num_cols) + row_index = 1 + for (step, (_, array)) in enumerate(results_by_time) + first_id = step > 1 ? 1 : meta.start_offset + last_id = + step == meta.len ? meta.interval_len - meta.end_offset : meta.interval_len + if last_id - first_id > size(array, 1) + error( + "Variable $(encode_key_as_string(key)) has $(size(array, 1)) number of steps, that is different than the default problem horizon. \ + Can't calculate the realized variables. Use `read_variables` instead and write your own concatenation", + ) + end + row_end = row_index + last_id - first_id + matrix[row_index:row_end, :] = array[first_id:last_id, :] + row_index += last_id - first_id + 1 + end + df = DataFrames.DataFrame(matrix, collect(columns[1]); copycols = false) + DataFrames.insertcols!( + df, + 1, + :DateTime => meta.realized_timestamps, + ) + return df +end + function get_realization( results::Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}, meta::RealizedMeta, @@ -57,29 +89,7 @@ function get_realization( Threads.@threads for key in collect(keys(results)) results_by_time = results[key] columns = get_column_names(results_by_time) - num_cols = length(columns) - matrix = Matrix{Float64}(undef, num_rows, num_cols) - row_index = 1 - for (step, (_, array)) in enumerate(results_by_time) - first_id = step > 1 ? 1 : meta.start_offset - last_id = - step == meta.len ? meta.interval_len - meta.end_offset : meta.interval_len - if last_id - first_id > size(array, 1) - error( - "Variable $(encode_key_as_string(key)) has $(size(array, 1)) number of steps, that is different than the default problem horizon. \ - Can't calculate the realized variables. Use `read_variables` instead and write your own concatenation", - ) - end - row_end = row_index + last_id - first_id - matrix[row_index:row_end, :] = array[first_id:last_id, :] - row_index += last_id - first_id + 1 - end - df = DataFrames.DataFrame(matrix, collect(columns); copycols = false) - DataFrames.insertcols!( - df, - 1, - :DateTime => meta.realized_timestamps, - ) + df = _make_dataframe(columns, results_by_time, num_rows, meta) lock(lk) do realized_values[key] = df end diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index b7e2932ef5..05b5c3d482 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -199,7 +199,7 @@ end function update_decision_state!( state::SimulationState, key::OptimizationContainerKey, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) @@ -241,7 +241,7 @@ end function update_decision_state!( state::SimulationState, key::AuxVarKey{EnergyOutput, T}, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) where {T <: PSY.Component} @@ -283,7 +283,7 @@ end function update_decision_state!( state::SimulationState, key::AuxVarKey{S, T}, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) where {T <: PSY.Component, S <: Union{TimeDurationOff, TimeDurationOn}} From 318d9fca389a7786f7ffcdf6b56fe3fe61e28285 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 21:43:15 -0600 Subject: [PATCH 09/90] fix results testing --- src/simulation/hdf_simulation_store.jl | 23 ++++++++++++++++------- test/test_simulation_store.jl | 10 +++++----- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 6372ccb457..986d9404fc 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -340,6 +340,14 @@ end log_cache_hit_percentages(x::HdfSimulationStore) = log_cache_hit_percentages(x.cache) + +function _make_dataframe(data::Matrix{Float64}, columns::Tuple{Vector{String}}) + if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] + data = reshape(data, length(data), 1) + end + return DataFrames.DataFrame(data, columns[1]; copycols = false) +end + """ Return DataFrame, DenseAxisArray, or Array for a model result at a timestamp. """ @@ -351,11 +359,7 @@ function read_result( index::Union{DecisionModelIndexType, EmulationModelIndexType}, ) data, columns = _read_data_columns(store, model_name, key, index) - - if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] - data = reshape(data, length(data), 1) - end - return DataFrames.DataFrame(data, columns; copycols = false) + return _make_dataframe(data, columns) end function read_result( @@ -865,11 +869,16 @@ function _read_data_columns( if is_cached(store.cache, model_name, key, index) data = read_result(store.cache, model_name, key, index) column_dataset = _get_dm_dataset(store, model_name, key).column_dataset - columns = column_dataset[:] + if ndims(column_dataset) == 1 + columns = (column_dataset[:],) + elseif ndims(column_dataset) == 2 + columns = (column_dataset[:, 1], column_dataset[:, 2]) + else + error("Datasets with $(ndims(column_dataset)) columns not supported") + end else data, columns = _read_result(store, model_name, key, index) end - return data, columns end diff --git a/test/test_simulation_store.jl b/test/test_simulation_store.jl index 5a29a38d35..bb7a27f4c1 100644 --- a/test/test_simulation_store.jl +++ b/test/test_simulation_store.jl @@ -39,7 +39,7 @@ function _initialize!(store, sim, variables, model_defs, cache_rules) for (key, array) in model_defs[model]["variables"] reqs.variables[key] = Dict( "columns" => model_defs[model]["names"], - "dims" => (horizon, length(model_defs[model]["names"]), num_rows), + "dims" => (horizon, length(model_defs[model]["names"][1]), num_rows), ) keep_in_cache = variables[key]["keep_in_cache"] add_rule!(cache_rules, model, key, keep_in_cache) @@ -146,8 +146,8 @@ function _verify_read_results(path, sim, variables, model_defs, seed) end end -function _verify_data(expected, store, model, name, time, columns) - expected_df = DataFrames.DataFrame(expected, columns) +function _verify_data(expected, store, model, name, time, columns::Tuple{Vector{Symbol}}) + expected_df = DataFrames.DataFrame(expected, columns[1]) df = read_result(DataFrames.DataFrame, store, model, name, time) @test expected_df == df end @@ -172,7 +172,7 @@ end :ED => Dict( "execution_count" => 24, "horizon" => 12, - "names" => [:dev1, :dev2, :dev3, :dev4, :dev5], + "names" => ([:dev1, :dev2, :dev3, :dev4, :dev5],), "variables" => Dict(x => ones(12, 5) for x in keys(variables)), "interval" => Dates.Hour(1), "resolution" => Dates.Hour(1), @@ -182,7 +182,7 @@ end :UC => Dict( "execution_count" => 1, "horizon" => 24, - "names" => [:dev1, :dev2, :dev3], + "names" => ([:dev1, :dev2, :dev3],), "variables" => Dict(x => ones(24, 3) for x in keys(variables)), "interval" => Dates.Hour(1), "resolution" => Dates.Hour(24), From cde7af7cc59fd6c3e0de71061f8091c9bb5ecb66 Mon Sep 17 00:00:00 2001 From: rodrigomha Date: Thu, 28 Sep 2023 11:42:06 -0700 Subject: [PATCH 10/90] update hdf debug for 3 dims Co-authored-by: jd-lara --- src/core/dataset.jl | 7 ++- src/simulation/hdf_simulation_store.jl | 68 ++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 9d996d3375..f0eee616bb 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -189,7 +189,8 @@ mutable struct HDF5Dataset <: AbstractDataset end end -HDF5Dataset(values, column_dataset, resolution, initial_time) = +function HDF5Dataset(values, column_dataset, resolution, initial_time) + @show column_dataset[:] HDF5Dataset( values, column_dataset, @@ -200,16 +201,20 @@ HDF5Dataset(values, column_dataset, resolution, initial_time) = UNSET_INI_TIME, column_dataset[:], ) +end function _make_column_names(vals::Vector{String}) + @show "col vector" return (vals,) end function _make_column_names(vals::Matrix{String}) + @show "col matrix" return (vals[:, 1], vals[:, 2]) end function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) + @show s.column_names return _make_column_names(s.column_names) end diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 6372ccb457..ba48ee7e5b 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -358,6 +358,22 @@ function read_result( return DataFrames.DataFrame(data, columns; copycols = false) end +function _make_denseaxisarray( + data::Matrix{Float64}, + columns::Tuple{Vector{String}}, +) + return DenseAxisArray(permutedims(data), columns[1], 1:size(data)[1]) +end + +function _make_denseaxisarray( + data::Array{Float64, 3}, + columns::NTuple{2, <:Any}, +) + @show data + @show columns + return DenseAxisArray(permutedims(data), columns[1], columns[2], 1:size(data)[1]) +end + function read_result( ::Type{DenseAxisArray}, store::HdfSimulationStore, @@ -365,13 +381,17 @@ function read_result( key::OptimizationContainerKey, index::Union{DecisionModelIndexType, EmulationModelIndexType}, ) + @show model_name if is_cached(store.cache, model_name, key, index) + @show "in cache" data = read_result(store.cache, model_name, key, index) columns = get_column_names(store, DecisionModelIndexType, model_name, key) else + @show "not in cache" data, columns = _read_result(store, model_name, key, index) end - return DenseAxisArray(permutedims(data), columns..., 1:size(data)[1]) + @show columns + return _make_denseaxisarray(data, columns) end function read_result( @@ -502,15 +522,15 @@ function _read_result( dataset = _get_dm_dataset(store, model_name, key) dset = dataset.values row_index = (simulation_step - 1) * num_executions + execution_index - columns = get_column_names(key, dataset) + @show columns = get_column_names(key, dataset) # Uncomment for performance checking #TimerOutputs.@timeit RUN_SIMULATION_TIMER "Read dataset" begin num_dims = ndims(dset) if num_dims == 3 data = dset[:, :, row_index] - #elseif num_dims == 4 - # data = dset[:, :, :, row_index] + elseif num_dims == 4 + data = dset[:, :, :, row_index] else error("unsupported dims: $num_dims") end @@ -519,6 +539,7 @@ function _read_result( return data, columns end + """ Write a decision model result for a timestamp to the store. """ @@ -553,6 +574,45 @@ function write_result!( return end +""" +Write a decision model result for a timestamp to the store. +""" +function write_result!( + store::HdfSimulationStore, + model_name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + ::Dates.DateTime, + data::DenseAxisArray{Float64, 3, <:NTuple{3, Any}}, +) + @show "Here" + #= + output_cache = get_output_cache(store.cache, model_name, key) + cur_size = get_size(store.cache) + add_result!(output_cache, index, to_matrix(data), is_full(store.cache, cur_size)) + + if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) + discard = !should_keep_in_cache(output_cache) + + # PERF: A potentially significant performance improvement would be to queue several + # flushes and submit them in parallel. + size_flushed = _flush_data!(output_cache, store, model_name, key, discard) + + @debug "flushed data" LOG_GROUP_SIMULATION_STORE key size_flushed discard cur_size + end + + # Disabled because this is currently a noop. + #if is_full(store.cache) + # _flush_data!(store.cache, store) + #end + + @debug "write_result" get_size(store.cache) encode_key_as_string(key) + =# + return +end + + + """ Write an emulation model result for an execution index value and the timestamp of the update """ From c061ec637d2646e01ae7e8351f3fde642cad0744 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:00:00 -0600 Subject: [PATCH 11/90] support 3D HDFD dataset --- src/core/dataset.jl | 61 +++++++++++++++++--------- src/simulation/hdf_simulation_store.jl | 13 +++--- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index f0eee616bb..6228a811f1 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -169,7 +169,7 @@ function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 3}, index: end # HDF5Dataset does not account of overwrites in the data. Values are written sequentially. -mutable struct HDF5Dataset <: AbstractDataset +mutable struct HDF5Dataset{N} <: AbstractDataset values::HDF5.Dataset column_dataset::HDF5.Dataset write_index::Int @@ -177,21 +177,31 @@ mutable struct HDF5Dataset <: AbstractDataset resolution::Dates.Millisecond initial_timestamp::Dates.DateTime update_timestamp::Dates.DateTime - column_names::Vector{String} + column_names::NTuple{N, Vector{String}} - function HDF5Dataset(values, column_dataset, write_index, last_recorded_row, resolution, + function HDF5Dataset{N}(values, + column_dataset, + write_index, + last_recorded_row, + resolution, initial_timestamp, - update_timestamp, column_names, - ) - new(values, column_dataset, write_index, last_recorded_row, resolution, + update_timestamp, + column_names::NTuple{N, Vector{String}}, + ) where {N} + new{N}(values, column_dataset, write_index, last_recorded_row, resolution, initial_timestamp, update_timestamp, column_names) end end -function HDF5Dataset(values, column_dataset, resolution, initial_time) - @show column_dataset[:] - HDF5Dataset( +function HDF5Dataset{1}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + ::Tuple, + resolution::Dates.Millisecond, + initial_time::Dates.DateTime, +) + HDF5Dataset{1}( values, column_dataset, 1, @@ -199,23 +209,34 @@ function HDF5Dataset(values, column_dataset, resolution, initial_time) resolution, initial_time, UNSET_INI_TIME, - column_dataset[:], + (column_dataset[:],), ) end -function _make_column_names(vals::Vector{String}) - @show "col vector" - return (vals,) -end - -function _make_column_names(vals::Matrix{String}) - @show "col matrix" - return (vals[:, 1], vals[:, 2]) +function HDF5Dataset{2}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + dims::NTuple{4, Int}, + resolution::Dates.Period, + initial_time::Dates.DateTime, +) + col1 = column_dataset[1:dims[2]] + col2 = column_dataset[(dims[2] + 1):end] + HDF5Dataset{2}( + values, + column_dataset, + 1, + 0, + resolution, + initial_time, + UNSET_INI_TIME, + (col1, col2), + ) end function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) - @show s.column_names - return _make_column_names(s.column_names) + s.column_names + return s.column_names end """ diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index b947623f61..7881831b6d 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -269,9 +269,12 @@ function initialize_problem_storage!( end column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) - datasets[key] = HDF5Dataset( + # First dim is Horizon, Last number of steps + n_dims = length(reqs["dims"]) - 2 + datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, + reqs["dims"], get_resolution(problem_params), initial_time, ) @@ -318,9 +321,11 @@ function initialize_problem_storage!( end column_dataset = group[col] datasets = getfield(store.em_data, type) - datasets[key] = HDF5Dataset( + n_dims = length(reqs["dims"]) - 1 + datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, + reqs["dims"], get_resolution(emulation_params), initial_time, ) @@ -340,7 +345,6 @@ end log_cache_hit_percentages(x::HdfSimulationStore) = log_cache_hit_percentages(x.cache) - function _make_dataframe(data::Matrix{Float64}, columns::Tuple{Vector{String}}) if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] data = reshape(data, length(data), 1) @@ -543,7 +547,6 @@ function _read_result( return data, columns end - """ Write a decision model result for a timestamp to the store. """ @@ -615,8 +618,6 @@ function write_result!( return end - - """ Write an emulation model result for an execution index value and the timestamp of the update """ From 7ad9adfee26ab29c51ee479bfe61903261ceb527 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:13:33 -0600 Subject: [PATCH 12/90] fix dims count --- src/simulation/hdf_simulation_store.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 7881831b6d..bec4992144 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -321,7 +321,8 @@ function initialize_problem_storage!( end column_dataset = group[col] datasets = getfield(store.em_data, type) - n_dims = length(reqs["dims"]) - 1 + # First dim is Horizon, Last number of steps + n_dims = length(reqs["dims"]) - 2 datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, From 26ce1358ed09580758102655351fa56a7961c581 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:14:58 -0600 Subject: [PATCH 13/90] fix to dims --- src/simulation/hdf_simulation_store.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index bec4992144..d6d51c660e 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -270,7 +270,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) # First dim is Horizon, Last number of steps - n_dims = length(reqs["dims"]) - 2 + n_dims = min(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, @@ -322,7 +322,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(store.em_data, type) # First dim is Horizon, Last number of steps - n_dims = length(reqs["dims"]) - 2 + n_dims = min(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, From bf7c6a33055705433a71ed42fdf3c0cd59baf720 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:28:32 -0600 Subject: [PATCH 14/90] use max --- src/simulation/hdf_simulation_store.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index d6d51c660e..3519f03f54 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -270,7 +270,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) # First dim is Horizon, Last number of steps - n_dims = min(1, length(reqs["dims"]) - 2) + n_dims = max(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, @@ -322,7 +322,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(store.em_data, type) # First dim is Horizon, Last number of steps - n_dims = min(1, length(reqs["dims"]) - 2) + n_dims = max(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, From d9b65a3a1d69d88e2b9ed5cb994a0d4096febe54 Mon Sep 17 00:00:00 2001 From: rodrigomha Date: Thu, 28 Sep 2023 14:38:10 -0700 Subject: [PATCH 15/90] force permute order --- src/simulation/hdf_simulation_store.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 3519f03f54..510f1ac395 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -378,9 +378,9 @@ function _make_denseaxisarray( data::Array{Float64, 3}, columns::NTuple{2, <:Any}, ) - @show data + @show size(data) @show columns - return DenseAxisArray(permutedims(data), columns[1], columns[2], 1:size(data)[1]) + return DenseAxisArray(permutedims(data, (2, 3, 1)), columns[1], columns[2], 1:size(data)[1]) end function read_result( From 6936835a1dc697029ab9c461e4bc4a975b3d53b1 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 17:18:48 -0600 Subject: [PATCH 16/90] fix results --- src/simulation/hdf_simulation_store.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 510f1ac395..7cbd461967 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -378,8 +378,6 @@ function _make_denseaxisarray( data::Array{Float64, 3}, columns::NTuple{2, <:Any}, ) - @show size(data) - @show columns return DenseAxisArray(permutedims(data, (2, 3, 1)), columns[1], columns[2], 1:size(data)[1]) end @@ -390,16 +388,12 @@ function read_result( key::OptimizationContainerKey, index::Union{DecisionModelIndexType, EmulationModelIndexType}, ) - @show model_name if is_cached(store.cache, model_name, key, index) - @show "in cache" data = read_result(store.cache, model_name, key, index) columns = get_column_names(store, DecisionModelIndexType, model_name, key) else - @show "not in cache" data, columns = _read_result(store, model_name, key, index) end - @show columns return _make_denseaxisarray(data, columns) end @@ -531,7 +525,7 @@ function _read_result( dataset = _get_dm_dataset(store, model_name, key) dset = dataset.values row_index = (simulation_step - 1) * num_executions + execution_index - @show columns = get_column_names(key, dataset) + columns = get_column_names(key, dataset) # Uncomment for performance checking #TimerOutputs.@timeit RUN_SIMULATION_TIMER "Read dataset" begin @@ -709,10 +703,11 @@ function _deserialize_attributes!(store::HdfSimulationStore) empty!(get_dm_data(store)) for model in HDF5.read(HDF5.attributes(group)["problem_order"]) problem_group = store.file["simulation/decision_models/$model"] + horizon = HDF5.read(HDF5.attributes(problem_group)["horizon"]) model_name = Symbol(model) store.params.decision_models_params[model_name] = ModelStoreParams( HDF5.read(HDF5.attributes(problem_group)["num_executions"]), - HDF5.read(HDF5.attributes(problem_group)["horizon"]), + horizon, Dates.Millisecond(HDF5.read(HDF5.attributes(problem_group)["interval_ms"])), Dates.Millisecond(HDF5.read(HDF5.attributes(problem_group)["resolution_ms"])), HDF5.read(HDF5.attributes(problem_group)["base_power"]), @@ -727,7 +722,9 @@ function _deserialize_attributes!(store::HdfSimulationStore) column_dataset = group[_make_column_name(name)] resolution = get_resolution(get_decision_model_params(store, model_name)) - item = HDF5Dataset(dataset, column_dataset, resolution, initial_time) + dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) + n_dims = max(1, ndims(dataset) - 2) + item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) container_key = container_key_lookup[name] getfield(get_dm_data(store)[model_name], type)[container_key] = item add_output_cache!( @@ -745,6 +742,7 @@ function _deserialize_attributes!(store::HdfSimulationStore) end em_group = _get_emulation_model_path(store) + horizon = HDF5.read(HDF5.attributes(em_group)["horizon"]) model_name = Symbol(HDF5.read(HDF5.attributes(em_group)["name"])) resolution = Dates.Millisecond(HDF5.read(HDF5.attributes(em_group)["resolution_ms"])) store.params.emulation_model_params[model_name] = ModelStoreParams( @@ -761,7 +759,9 @@ function _deserialize_attributes!(store::HdfSimulationStore) if !endswith(name, "columns") dataset = group[name] column_dataset = group[_make_column_name(name)] - item = HDF5Dataset(dataset, column_dataset, resolution, initial_time) + dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) + n_dims = max(1, ndims(dataset) - 1) + item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) container_key = container_key_lookup[name] getfield(store.em_data, type)[container_key] = item add_output_cache!(store.cache, model_name, container_key, CacheFlushRule()) From 32c0759b15db51885447fb3949f1acc318d38e8e Mon Sep 17 00:00:00 2001 From: rodrigomha Date: Thu, 28 Sep 2023 16:19:54 -0700 Subject: [PATCH 17/90] remove subcomp set param val --- src/parameters/update_parameters.jl | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 088dfd1fca..efba16b68d 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -25,28 +25,6 @@ function _set_param_value!(param::JuMPFloatArray, value::Float64, name::String, return end -function _set_param_value!( - param::SparseAxisArray{Union{Nothing, JuMP.VariableRef}}, - value::Float64, - name::String, - subcomp::String, - t::Int, -) - fix_parameter_value(param[name, subcomp, t], value) - return -end - -function _set_param_value!( - param::SparseAxisArray{Float64}, - value::Float64, - name::String, - subcomp::String, - t::Int, -) - param[name, subcomp, t] = value - return -end - function _update_parameter_values!( parameter_array::AbstractArray{T}, attributes::TimeSeriesAttributes{U}, From 7263ecc3dc456b54f38e01b8b788c525babf9e58 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 17:20:28 -0600 Subject: [PATCH 18/90] formatter --- src/simulation/hdf_simulation_store.jl | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 7cbd461967..956a9c23fb 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -378,7 +378,12 @@ function _make_denseaxisarray( data::Array{Float64, 3}, columns::NTuple{2, <:Any}, ) - return DenseAxisArray(permutedims(data, (2, 3, 1)), columns[1], columns[2], 1:size(data)[1]) + return DenseAxisArray( + permutedims(data, (2, 3, 1)), + columns[1], + columns[2], + 1:size(data)[1], + ) end function read_result( @@ -724,7 +729,13 @@ function _deserialize_attributes!(store::HdfSimulationStore) get_resolution(get_decision_model_params(store, model_name)) dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 2) - item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) + item = HDF5Dataset{n_dims}( + dataset, + column_dataset, + dims, + resolution, + initial_time, + ) container_key = container_key_lookup[name] getfield(get_dm_data(store)[model_name], type)[container_key] = item add_output_cache!( @@ -761,7 +772,13 @@ function _deserialize_attributes!(store::HdfSimulationStore) column_dataset = group[_make_column_name(name)] dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 1) - item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) + item = HDF5Dataset{n_dims}( + dataset, + column_dataset, + dims, + resolution, + initial_time, + ) container_key = container_key_lookup[name] getfield(store.em_data, type)[container_key] = item add_output_cache!(store.cache, model_name, container_key, CacheFlushRule()) From c5ca8c212fc5f590feb259dc36792c08fb13a3e6 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 20:02:53 -0600 Subject: [PATCH 19/90] add missing method --- src/utils/dataframes_utils.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/dataframes_utils.jl b/src/utils/dataframes_utils.jl index 6db302134c..28de0efb63 100644 --- a/src/utils/dataframes_utils.jl +++ b/src/utils/dataframes_utils.jl @@ -7,7 +7,11 @@ Creates a DataFrame from a JuMP DenseAxisArray or SparseAxisArray. - `array`: JuMP DenseAxisArray or SparseAxisArray to convert - `key::OptimizationContainerKey`: """ -function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T} +function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T <: Number} + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) +end + +function to_dataframe(array::DenseAxisArray{T, 1}, key::OptimizationContainerKey) where {T <: Number} return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end From 404d165e13e789cfcd825abd70fc822a28ef4938 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 20:03:03 -0600 Subject: [PATCH 20/90] update tests --- test/test_utils.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_utils.jl b/test/test_utils.jl index 5d49df15cf..7e39118246 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -23,10 +23,12 @@ end @test two_df == test_df three = PSI.DenseAxisArray{Float64}(undef, [:a], 1:2, 1:3) - @test_throws ErrorException PSI.to_dataframe(three, mock_key) + fill!(three, 1.0) + @test_throws MethodError PSI.to_dataframe(three, mock_key) four = PSI.DenseAxisArray{Float64}(undef, [:a], 1:2, 1:3, 1:5) - @test_throws ErrorException PSI.to_dataframe(four, mock_key) + fill!(three, 1.0) + @test_throws MethodError PSI.to_dataframe(four, mock_key) sparse_num = JuMP.Containers.@container([i = 1:10, j = (i + 1):10, t = 1:24], 0.0 + i + j + t) From 8197b8c6ec5c74f6ebfaeb27243d38f51a61117d Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 20:39:24 -0600 Subject: [PATCH 21/90] formatter --- src/utils/dataframes_utils.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/utils/dataframes_utils.jl b/src/utils/dataframes_utils.jl index 28de0efb63..eb9607a909 100644 --- a/src/utils/dataframes_utils.jl +++ b/src/utils/dataframes_utils.jl @@ -7,11 +7,17 @@ Creates a DataFrame from a JuMP DenseAxisArray or SparseAxisArray. - `array`: JuMP DenseAxisArray or SparseAxisArray to convert - `key::OptimizationContainerKey`: """ -function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T <: Number} +function to_dataframe( + array::DenseAxisArray{T, 2}, + key::OptimizationContainerKey, +) where {T <: Number} return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end -function to_dataframe(array::DenseAxisArray{T, 1}, key::OptimizationContainerKey) where {T <: Number} +function to_dataframe( + array::DenseAxisArray{T, 1}, + key::OptimizationContainerKey, +) where {T <: Number} return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end From 22d3e725bffe7c7ca0ff4841aec4572ddaf045b0 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 21:56:59 -0600 Subject: [PATCH 22/90] add min_res --- src/simulation/simulation_state.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index 05b5c3d482..12aa0365e1 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -135,6 +135,7 @@ function _initialize_system_states!( decision_states = get_decision_states(sim_state) emulator_states = get_system_states(sim_state) emulation_container = get_optimization_container(emulation_model) + min_res = minimum([v.resolution for v in values(params)]) for field in fieldnames(DatasetContainer) field_containers = getfield(emulation_container, field) From ecf9a0a8862ff634a3f382030b1f9775a7e9956b Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 00:34:57 -0600 Subject: [PATCH 23/90] add code to store 3D in HDF --- src/core/dataset.jl | 31 ++++++++++--- src/core/definitions.jl | 1 + src/parameters/update_parameters.jl | 2 +- src/simulation/hdf_simulation_store.jl | 61 +++++++++++++++++++++----- src/simulation/simulation_state.jl | 3 +- 5 files changed, 79 insertions(+), 19 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 6228a811f1..1b3efe15f8 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -111,7 +111,10 @@ function make_system_state( return InMemoryDataset(NaN, timestamp, resolution, 0, 1, columns) end -function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) +function get_dataset_value( + s::T, + date::Dates.DateTime, +) where {T <: Union{InMemoryDataset{1}, InMemoryDataset{2}}} s_index = find_timestamp_index(s.timestamps, date) if isnothing(s_index) error("Request time stamp $date not in the state") @@ -119,17 +122,32 @@ function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) return s.values[:, s_index] end +function get_dataset_value(s::InMemoryDataset{3}, date::Dates.DateTime) + s_index = find_timestamp_index(s.timestamps, date) + if isnothing(s_index) + error("Request time stamp $date not in the state") + end + return s.values[:, :, s_index] +end + function get_column_names(k::OptimizationContainerKey, s::InMemoryDataset) return get_column_names(k, s.values) end -function get_last_recorded_value(s::InMemoryDataset) +function get_last_recorded_value(s::InMemoryDataset{2}) if get_last_recorded_row(s) == 0 error("The Dataset hasn't been written yet") end return s.values[:, get_last_recorded_row(s)] end +function get_last_recorded_value(s::InMemoryDataset{3}) + if get_last_recorded_row(s) == 0 + error("The Dataset hasn't been written yet") + end + return s.values[:, :, get_last_recorded_row(s)] +end + function get_end_of_step_timestamp(s::InMemoryDataset) return s.timestamps[s.end_of_step_index] end @@ -153,17 +171,20 @@ function get_value_timestamp(s::InMemoryDataset, date::Dates.DateTime) return s.timestamps[s_index] end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 2}, index::Int) +# These set_value! methods expect a single time_step value because they are used to update +#the state so the incoming vals will have one dimension less than the DataSet. The exception +# is for vals of Dimension 1 which are still stored in DataSets of dimension 2. +function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 2}, index::Int) s.values[:, index] = vals[:, index] return end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 1}, index::Int) +function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 1}, index::Int) s.values[:, index] = vals return end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 3}, index::Int) +function set_value!(s::InMemoryDataset{3}, vals::DenseAxisArray{Float64, 2}, index::Int) s.values[:, :, index] = vals return end diff --git a/src/core/definitions.jl b/src/core/definitions.jl index 1900cf773f..c365af7c19 100644 --- a/src/core/definitions.jl +++ b/src/core/definitions.jl @@ -24,6 +24,7 @@ const JuMPVariableMatrix = DenseAxisArray{ JuMP.Containers._AxisLookup{Tuple{Int64, Int64}}, }, } +const JuMPFloatMatrix = DenseAxisArray{Float64, 2} const JuMPFloatArray = DenseAxisArray{Float64} const JuMPVariableArray = DenseAxisArray{JuMP.VariableRef} diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index efba16b68d..67eafd8c03 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -560,7 +560,7 @@ end function _fix_parameter_value!( container::OptimizationContainer, - parameter_array::JuMPFloatArray, + parameter_array::DenseAxisArray{Float64, 2}, parameter_attributes::VariableValueAttributes, ) affected_variable_keys = parameter_attributes.affected_keys diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 956a9c23fb..2a8494eba2 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -592,11 +592,10 @@ function write_result!( ::Dates.DateTime, data::DenseAxisArray{Float64, 3, <:NTuple{3, Any}}, ) - @show "Here" - #= output_cache = get_output_cache(store.cache, model_name, key) cur_size = get_size(store.cache) - add_result!(output_cache, index, to_matrix(data), is_full(store.cache, cur_size)) + + add_result!(output_cache, index, data.data, is_full(store.cache, cur_size)) if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) discard = !should_keep_in_cache(output_cache) @@ -614,7 +613,6 @@ function write_result!( #end @debug "write_result" get_size(store.cache) encode_key_as_string(key) - =# return end @@ -627,10 +625,10 @@ function write_result!( key::OptimizationContainerKey, index::EmulationModelIndexType, simulation_time::Dates.DateTime, - data::Matrix{Float64}, + data::Array{Float64}, ) dataset = _get_em_dataset(store, key) - _write_dataset!(dataset.values, data, index:index) + _write_dataset!(dataset.values, data, index) set_last_recorded_row!(dataset, index) set_update_timestamp!(dataset, simulation_time) return @@ -642,7 +640,21 @@ function write_result!( key::OptimizationContainerKey, index::EmulationModelIndexType, simulation_time::Dates.DateTime, - data, + data::DenseAxisArray{Float64, 2}, +) + data_array = Array{Float64, 3}(undef, size(data)[1], size(data)[2], 1) + data_array[:, :, 1] = data + write_result!(store, model_name, key, index, simulation_time, data_array) + return +end + +function write_result!( + store::HdfSimulationStore, + model_name::Symbol, + key::OptimizationContainerKey, + index::EmulationModelIndexType, + simulation_time::Dates.DateTime, + data::DenseAxisArray{Float64, 1}, ) write_result!(store, model_name, key, index, simulation_time, to_matrix(data)) return @@ -984,7 +996,7 @@ function _read_length(::Type{OptimizerStats}, store::HdfSimulationStore) end function _write_dataset!( - dataset, + dataset::HDF5.Dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}, ::Val{3}, @@ -995,7 +1007,7 @@ function _write_dataset!( end function _write_dataset!( - dataset, + dataset::HDF5.Dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}, ::Val{2}, @@ -1005,13 +1017,38 @@ function _write_dataset!( return end -function _write_dataset!(dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}) - _write_dataset!(dataset, array, row_range, Val{ndims(dataset)}()) +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 3}, + row_range::UnitRange{Int64}, + ::Val{3}, +) + dataset[row_range, :, :] = array + @debug "wrote dataset" dataset row_range + return +end + +function _write_dataset!(dataset::HDF5.Dataset, array::Array{Float64}, index::Int) + _write_dataset!(dataset, array, index:index, Val{ndims(dataset)}()) return end -function _write_dataset!(dataset, array::Array{Float64, 3}, row_range::UnitRange{Int64}) +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 3}, + row_range::UnitRange{Int64}, +) dataset[:, :, row_range] = array @debug "wrote dataset" dataset row_range return end + +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 4}, + row_range::UnitRange{Int64}, +) + dataset[:, :, :, row_range] = array + @debug "wrote dataset" dataset row_range + return +end diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index 12aa0365e1..e628294e89 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -410,7 +410,8 @@ function update_system_state!( set_update_timestamp!(system_dataset, ts) # Keep coordination between fields. System state is an array of size 1 system_dataset.timestamps[1] = ts - set_dataset_values!(state, key, 1, get_dataset_value(decision_dataset, simulation_time)) + data_set_value = get_dataset_value(decision_dataset, simulation_time) + set_dataset_values!(state, key, 1, data_set_value) # This value shouldn't be other than one and after one execution is no-op. set_last_recorded_row!(system_dataset, 1) return From ba4e377d620c24ef04ab1675d47171224f81af1c Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 01:12:00 -0600 Subject: [PATCH 24/90] fixes to run --- src/simulation/hdf_simulation_store.jl | 2 +- src/simulation/optimization_output_cache.jl | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 2a8494eba2..e2b639f507 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -595,7 +595,7 @@ function write_result!( output_cache = get_output_cache(store.cache, model_name, key) cur_size = get_size(store.cache) - add_result!(output_cache, index, data.data, is_full(store.cache, cur_size)) + add_result!(output_cache, index, permutedims(data.data, (3, 1, 2)), is_full(store.cache, cur_size)) if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) discard = !should_keep_in_cache(output_cache) diff --git a/src/simulation/optimization_output_cache.jl b/src/simulation/optimization_output_cache.jl index da76708ff3..276c529273 100644 --- a/src/simulation/optimization_output_cache.jl +++ b/src/simulation/optimization_output_cache.jl @@ -52,7 +52,10 @@ end """ Add result to the cache. """ -function add_result!(cache::OptimizationOutputCache, timestamp, array, system_cache_is_full) +function add_result!(cache::OptimizationOutputCache, + timestamp::Dates.DateTime, + array::Array{Float64}, + system_cache_is_full::Bool) if cache.size_per_entry == 0 cache.size_per_entry = length(array) * sizeof(first(array)) end @@ -77,7 +80,11 @@ function add_result!(cache::OptimizationOutputCache, timestamp, array, system_ca return cache.size_per_entry end -function _add_result!(cache::OptimizationOutputCache, timestamp, data) +function _add_result!( + cache::OptimizationOutputCache, + timestamp::Dates.DateTime, + data::Array{Float64}, +) cache.data[timestamp] = data push!(cache.dirty_timestamps, timestamp) return From 5e83cfbbcfcc4cf21994b9c868bde338f438d20e Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 01:15:02 -0600 Subject: [PATCH 25/90] fix columns call --- src/simulation/simulation_state.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index e628294e89..b2f4a79d4c 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -148,27 +148,27 @@ function _initialize_system_states!( make_system_state( simulation_initial_time, min_res, - cols, + column_names, ), ) end end for key in get_dataset_keys(decision_states) + dm_cols = get_column_names(key, get_dataset(decision_states, key)) if has_dataset(emulator_states, key) - dm_cols = get_column_names(key, get_dataset(decision_states, key)) em_cols = get_column_names(key, get_dataset(emulator_states, key)) @assert_op dm_cols == em_cols continue end - cols = get_column_names(key, get_dataset(decision_states, key)) + set_dataset!( emulator_states, key, make_system_state( simulation_initial_time, min_res, - cols, + dm_cols, ), ) end From 8c4e109be56de7da21d46a6fb1eb9e177b2b5786 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 01:16:03 -0600 Subject: [PATCH 26/90] formatter --- src/simulation/hdf_simulation_store.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index e2b639f507..77ab196e26 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -595,7 +595,12 @@ function write_result!( output_cache = get_output_cache(store.cache, model_name, key) cur_size = get_size(store.cache) - add_result!(output_cache, index, permutedims(data.data, (3, 1, 2)), is_full(store.cache, cur_size)) + add_result!( + output_cache, + index, + permutedims(data.data, (3, 1, 2)), + is_full(store.cache, cur_size), + ) if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) discard = !should_keep_in_cache(output_cache) From 194d72abff397f5938faa3017d757f362f8ccbc3 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 15:28:43 -0600 Subject: [PATCH 27/90] fix passing of key --- src/simulation/realized_meta.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/simulation/realized_meta.jl b/src/simulation/realized_meta.jl index 4dbe971bc6..cb373cd05a 100644 --- a/src/simulation/realized_meta.jl +++ b/src/simulation/realized_meta.jl @@ -51,6 +51,7 @@ function _make_dataframe( results_by_time::ResultsByTime{Matrix{Float64}, 1}, num_rows::Int, meta::RealizedMeta, + key::OptimizationContainerKey, ) num_cols = length(columns[1]) matrix = Matrix{Float64}(undef, num_rows, num_cols) @@ -86,10 +87,11 @@ function get_realization( lk = ReentrantLock() num_rows = length(meta.realized_timestamps) start = time() - Threads.@threads for key in collect(keys(results)) + #Threads.@threads + for key in collect(keys(results)) results_by_time = results[key] columns = get_column_names(results_by_time) - df = _make_dataframe(columns, results_by_time, num_rows, meta) + df = _make_dataframe(columns, results_by_time, num_rows, meta, key) lock(lk) do realized_values[key] = df end From 9a06de3cdf6918006b703f769e285930f589fb9c Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 22:19:29 -0600 Subject: [PATCH 28/90] add logging --- src/simulation/hdf_simulation_store.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 77ab196e26..e994650cff 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -309,6 +309,7 @@ function initialize_problem_storage!( group = _get_group_or_create(emulation_group, string(type)) for (key, reqs) in getfield(em_problem_reqs, type) name = encode_key_as_string(key) + @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. @@ -699,6 +700,7 @@ function _compute_chunk_count(dims, dtype; max_chunk_bytes = DEFAULT_MAX_CHUNK_B end function _create_dataset(group, name, reqs) + @error reqs["dims"] dataset = HDF5.create_dataset( group, name, From 723ee7652752321ca3936ce6953049584716fc84 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 22:23:48 -0600 Subject: [PATCH 29/90] move logger --- src/simulation/hdf_simulation_store.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index e994650cff..c432972cca 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -257,6 +257,7 @@ function initialize_problem_storage!( for (key, reqs) in getfield(dm_problem_reqs[problem], type) !should_write_resulting_value(key) && continue name = encode_key_as_string(key) + @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. @@ -309,7 +310,6 @@ function initialize_problem_storage!( group = _get_group_or_create(emulation_group, string(type)) for (key, reqs) in getfield(em_problem_reqs, type) name = encode_key_as_string(key) - @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. From 761bad135daa8619e3aa43794007688e0bf2827a Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 2 Oct 2023 15:27:54 -0600 Subject: [PATCH 30/90] remove @error --- src/simulation/hdf_simulation_store.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index c432972cca..4794311282 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -257,7 +257,6 @@ function initialize_problem_storage!( for (key, reqs) in getfield(dm_problem_reqs[problem], type) !should_write_resulting_value(key) && continue name = encode_key_as_string(key) - @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. @@ -700,7 +699,6 @@ function _compute_chunk_count(dims, dtype; max_chunk_bytes = DEFAULT_MAX_CHUNK_B end function _create_dataset(group, name, reqs) - @error reqs["dims"] dataset = HDF5.create_dataset( group, name, @@ -1055,6 +1053,7 @@ function _write_dataset!( array::Array{Float64, 4}, row_range::UnitRange{Int64}, ) + @show dataset[:, :, :, row_range] = array @debug "wrote dataset" dataset row_range return From c7c9e01a9221315380a9b7192646d1a2380a9b47 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 2 Oct 2023 16:31:09 -0600 Subject: [PATCH 31/90] add logging --- src/simulation/hdf_simulation_store.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 4794311282..313759f01b 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -746,6 +746,8 @@ function _deserialize_attributes!(store::HdfSimulationStore) get_resolution(get_decision_model_params(store, model_name)) dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 2) + @error n_dims + @error dims item = HDF5Dataset{n_dims}( dataset, column_dataset, From fadf8f7a84ff15a70594b58a0a5560206b842270 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 2 Oct 2023 17:48:13 -0600 Subject: [PATCH 32/90] add method for results deserialize --- src/core/dataset.jl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 1b3efe15f8..af3c20368b 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -255,6 +255,28 @@ function HDF5Dataset{2}( ) end +function HDF5Dataset{2}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + dims::NTuple{5, Int}, + resolution::Dates.Period, + initial_time::Dates.DateTime, +) + col1 = column_dataset[1:dims[2]] + col2 = column_dataset[(dims[2] + 1):end] + HDF5Dataset{2}( + values, + column_dataset, + 1, + 0, + resolution, + initial_time, + UNSET_INI_TIME, + (col1, col2), + ) +end + + function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) s.column_names return s.column_names From 1cad58ccea435a6f67c21f76ddc57fbd175de990 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 19 Oct 2023 13:52:37 -0600 Subject: [PATCH 33/90] enable 3D dataframes --- src/core/dataset.jl | 2 - src/core/results_by_time.jl | 13 ++ .../decision_model_simulation_results.jl | 140 ++++++++++++------ src/simulation/hdf_simulation_store.jl | 20 ++- 4 files changed, 128 insertions(+), 47 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index af3c20368b..457f601296 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -276,9 +276,7 @@ function HDF5Dataset{2}( ) end - function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) - s.column_names return s.column_names end diff --git a/src/core/results_by_time.jl b/src/core/results_by_time.jl index 32d77ede00..f35267e0c3 100644 --- a/src/core/results_by_time.jl +++ b/src/core/results_by_time.jl @@ -89,6 +89,19 @@ function make_dataframe( return df end +function make_dataframe( + results::ResultsByTime{DenseAxisArray{Float64, 3}}, + timestamp::Dates.DateTime, +) + df = DataFrames.DataFrame() + array = results.data[timestamp] + for idx in Iterators.product(array.axes[1:2]...) + df[!, "$(idx)"] = array[idx..., :].data + end + # _add_timestamps!(df, results, timestamp, array) + return df +end + function make_dataframe(results::ResultsByTime{Matrix{Float64}}, timestamp::Dates.DateTime) array = results.data[timestamp] df = DataFrames.DataFrame(array, results.column_names) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 96cbd04289..8739bd1ca4 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -114,63 +114,115 @@ function get_forecast_horizon(res::SimulationProblemResults{DecisionModelSimulat end function _get_store_value( - ::Type{T}, res::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, timestamps, ::Nothing, -) where {T <: Union{Matrix{Float64}, DenseAxisArray{Float64, 2}}} +) simulation_store_path = joinpath(get_execution_path(res), "data_store") return open_store(HdfSimulationStore, simulation_store_path, "r") do store - _get_store_value(T, res, container_keys, timestamps, store) + _get_store_value(res, container_keys, timestamps, store) end end function _get_store_value( - ::Type{DenseAxisArray{Float64, 2}}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, timestamps, store::SimulationStore, ) - base_power = get_model_base_power(sim_results) - results_by_key = - Dict{OptimizationContainerKey, ResultsByTime{DenseAxisArray{Float64, 2}}}() + results_by_key = Dict{OptimizationContainerKey, ResultsByTime}() model_name = Symbol(get_model_name(sim_results)) + for ckey in container_keys + n_dims = get_number_of_dimensions(store, DecisionModelIndexType, model_name, ckey) + container_type = DenseAxisArray{Float64, n_dims + 1} + results_by_key[ckey] = _get_store_value(container_type, + sim_results, + ckey, + timestamps, store) + end + return results_by_key +end + +function _get_store_value( + ::Type{T}, + sim_results::SimulationProblemResults{DecisionModelSimulationResults}, + key::OptimizationContainerKey, + timestamps, + store::SimulationStore, +) where {T <: DenseAxisArray{Float64, 2}} resolution = get_resolution(sim_results) horizon = get_forecast_horizon(sim_results) + base_power = get_model_base_power(sim_results) + results_by_time = ResultsByTime( + key, + SortedDict{Dates.DateTime, T}(), + resolution, + get_column_names(store, DecisionModelIndexType, model_name, key), + ) + array_size::Union{Nothing, Tuple{Int, Int}} = nothing + for ts in timestamps + array = read_result(DenseAxisArray, store, model_name, key, ts) + if isnothing(array_size) + array_size = size(array) + elseif size(array) != array_size + error( + "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", + ) + end + if convert_result_to_natural_units(key) + array.data .*= base_power + end + if array_size[2] != horizon + @warn "$(encode_key_as_string(key)) has a different horizon than the " * + "problem specification. Can't assign timestamps to the resulting DataFrame." + results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) + end + results_by_time[ts] = array + end - for key in container_keys - results_by_time = ResultsByTime( - key, - SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}(), - resolution, - get_column_names(store, DecisionModelIndexType, model_name, key), - ) - array_size::Union{Nothing, Tuple{Int, Int}} = nothing - for ts in timestamps - array = read_result(DenseAxisArray, store, model_name, key, ts) - if isnothing(array_size) - array_size = size(array) - elseif size(array) != array_size - error( - "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", - ) - end - if convert_result_to_natural_units(key) - array.data .*= base_power - end - if array_size[2] != horizon - @warn "$(encode_key_as_string(key)) has a different horizon than the " * - "problem specification. Can't assign timestamps to the resulting DataFrame." - results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) - end - results_by_time[ts] = array + return results_by_time +end + +function _get_store_value( + ::Type{T}, + sim_results::SimulationProblemResults{DecisionModelSimulationResults}, + key::OptimizationContainerKey, + timestamps, + store::SimulationStore, +) where {T <: DenseAxisArray{Float64, 3}} + resolution = get_resolution(sim_results) + horizon = get_forecast_horizon(sim_results) + base_power = get_model_base_power(sim_results) + model_name = Symbol(get_model_name(sim_results)) + results_by_time = ResultsByTime( + key, + SortedDict{Dates.DateTime, T}(), + resolution, + get_column_names(store, DecisionModelIndexType, model_name, key), + ) + array_size::Union{Nothing, Tuple{Int, Int, Int}} = nothing + for ts in timestamps + array = read_result(DenseAxisArray, store, model_name, key, ts) + if isnothing(array_size) + array_size = size(array) + elseif size(array) != array_size + error( + "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", + ) end - results_by_key[key] = results_by_time + if convert_result_to_natural_units(key) + array.data .*= base_power + end + if array_size[3] != horizon + @warn "$(encode_key_as_string(key)) has a different horizon than the " * + "problem specification. Can't assign timestamps to the resulting DataFrame." + results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) + end + results_by_time[ts] = array end - return results_by_key + return results_by_time end function _get_store_value( @@ -186,6 +238,12 @@ function _get_store_value( resolution = get_resolution(sim_results) for key in container_keys + n_dims = get_number_of_dimensions(store, DecisionModelIndexType, model_name, key) + if n_dims != 1 + error( + "The number of dimensions $(n_dims) is not supported for $(encode_key_as_string(key))", + ) + end results_by_time = ResultsByTime{Matrix{Float64}, 1}( key, SortedDict{Dates.DateTime, Matrix{Float64}}(), @@ -233,13 +291,13 @@ function _process_timestamps( end function _read_results( - ::Type{T}, res::SimulationProblemResults{DecisionModelSimulationResults}, result_keys, timestamps::Vector{Dates.DateTime}, store::Union{Nothing, <:SimulationStore}, -) where {T <: Union{Matrix{Float64}, DenseAxisArray{Float64, 2}}} - isempty(result_keys) && return Dict{OptimizationContainerKey, ResultsByTime{T}}() +) + isempty(result_keys) && + return Dict{OptimizationContainerKey, ResultsByTime{DenseAxisArray{Float64, 2}}}() if store === nothing && res.store !== nothing # In this case we have an InMemorySimulationStore. @@ -253,7 +311,7 @@ function _read_results( vals = Dict(k => cached_results[k] for k in result_keys) else @debug "reading results from data store" - vals = _get_store_value(T, res, result_keys, timestamps, store) + vals = _get_store_value(res, result_keys, timestamps, store) end return vals end @@ -285,9 +343,7 @@ function read_variable( ) key = _deserialize_key(VariableKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) - return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], - ) + return make_dataframes(_read_results(res, [key], timestamps, store)[key]) end """ diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 313759f01b..fe235e6a79 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -459,6 +459,23 @@ function get_column_names( return get_column_names(key, dataset) end +function get_number_of_dimensions( + store::HdfSimulationStore, + i::Type{DecisionModelIndexType}, + model_name::Symbol, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + +function get_number_of_dimensions( + store::HdfSimulationStore, + i::Type{EmulationModelIndexType}, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + function get_emulation_model_dataset_size( store::HdfSimulationStore, key::OptimizationContainerKey, @@ -746,8 +763,6 @@ function _deserialize_attributes!(store::HdfSimulationStore) get_resolution(get_decision_model_params(store, model_name)) dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 2) - @error n_dims - @error dims item = HDF5Dataset{n_dims}( dataset, column_dataset, @@ -1055,7 +1070,6 @@ function _write_dataset!( array::Array{Float64, 4}, row_range::UnitRange{Int64}, ) - @show dataset[:, :, :, row_range] = array @debug "wrote dataset" dataset row_range return From 7da3309f750efc5d80324a3d9c3d6180a14c3214 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 19 Oct 2023 14:00:37 -0600 Subject: [PATCH 34/90] add missing methods --- .../decision_model_simulation_results.jl | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 8739bd1ca4..dbc8ef8bf2 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -125,6 +125,20 @@ function _get_store_value( end end +# TODO: Probably remove method later. It might be a duplicate. +function _get_store_value( + T::Type{Matrix{Float64}}, + res::SimulationProblemResults{DecisionModelSimulationResults}, + container_keys::Vector{<:OptimizationContainerKey}, + timestamps, + ::Nothing, +) + simulation_store_path = joinpath(get_execution_path(res), "data_store") + return open_store(HdfSimulationStore, simulation_store_path, "r") do store + _get_store_value(T, res, container_keys, timestamps, store) + end +end + function _get_store_value( sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, @@ -290,6 +304,17 @@ function _process_timestamps( return requested_range end +# Temporary workaround. To be removed +function _read_results( + T::Type{Matrix{Float64}}, + res::SimulationProblemResults{DecisionModelSimulationResults}, + result_keys, + timestamps::Vector{Dates.DateTime}, + store::Union{Nothing, <:SimulationStore}, +) + return _get_store_value(T, res, result_keys, timestamps, store) +end + function _read_results( res::SimulationProblemResults{DecisionModelSimulationResults}, result_keys, From 6f304e06a4b2fcdd736c0d0fb94f89e3a21cdd45 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 12:38:30 -0600 Subject: [PATCH 35/90] add comment to the code about the use of dims --- src/core/dataset.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 457f601296..940159f61d 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -241,7 +241,12 @@ function HDF5Dataset{2}( resolution::Dates.Period, initial_time::Dates.DateTime, ) + # The indexing is done in this way because we save all the names in an + # adjacent column entry in the HDF5 Datatset. The indexes for each column + # are known because we know how many elements are in each dimension. + # the names for the first column are store in the 1:first_column_number_of_elements. col1 = column_dataset[1:dims[2]] + # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names. col2 = column_dataset[(dims[2] + 1):end] HDF5Dataset{2}( values, From eb8de6f0e3b302bd4468a957673390ef9a939fd1 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 12:40:56 -0600 Subject: [PATCH 36/90] remove unnecessary assertions --- src/parameters/update_parameters.jl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 67eafd8c03..4d825049a4 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -155,10 +155,8 @@ function _update_parameter_values!( max_state_index = get_num_rows(state_data) if model_resolution < state_data.resolution t_step = 1 - elseif model_resolution >= state_data.resolution - t_step = model_resolution ÷ state_data.resolution else - @assert false + t_step = model_resolution ÷ state_data.resolution end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) @@ -200,10 +198,8 @@ function _update_parameter_values!( max_state_index = get_num_rows(state_data) if model_resolution < state_data.resolution t_step = 1 - elseif model_resolution >= state_data.resolution - t_step = model_resolution ÷ state_data.resolution else - @assert false + t_step = model_resolution ÷ state_data.resolution end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) @@ -245,10 +241,8 @@ function _update_parameter_values!( max_state_index = get_num_rows(state_data) if model_resolution < state_data.resolution t_step = 1 - elseif model_resolution >= state_data.resolution - t_step = model_resolution ÷ state_data.resolution else - @assert false + t_step = model_resolution ÷ state_data.resolution end state_data_index = find_timestamp_index(state_timestamps, current_time) From 00b2134bcbefb1e28ee6072f75a0364a8b4aba87 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 15:12:15 -0600 Subject: [PATCH 37/90] add missing variable in function --- src/simulation/decision_model_simulation_results.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index dbc8ef8bf2..98e8fabf0e 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -168,6 +168,7 @@ function _get_store_value( resolution = get_resolution(sim_results) horizon = get_forecast_horizon(sim_results) base_power = get_model_base_power(sim_results) + model_name = Symbol(get_model_name(sim_results)) results_by_time = ResultsByTime( key, SortedDict{Dates.DateTime, T}(), From a21d9bc7b143432bc8ae056079fc6e45cf9da6c5 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 16:38:23 -0600 Subject: [PATCH 38/90] fix tests --- .../decision_model_simulation_results.jl | 13 ++++--------- src/simulation/in_memory_simulation_store.jl | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 98e8fabf0e..94cba4295f 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -393,7 +393,7 @@ function read_dual( key = _deserialize_key(ConstraintKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -417,7 +417,7 @@ function read_parameter( key = _deserialize_key(ParameterKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -441,7 +441,7 @@ function read_aux_variable( key = _deserialize_key(AuxVarKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -465,7 +465,7 @@ function read_expression( key = _deserialize_key(ExpressionKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -565,7 +565,6 @@ function load_results!( merge!( get_cached_variables(res), _read_results( - DenseAxisArray{Float64, 2}, res, variable_keys, res.results_timestamps, @@ -575,7 +574,6 @@ function load_results!( merge!( get_cached_duals(res), _read_results( - DenseAxisArray{Float64, 2}, res, dual_keys, res.results_timestamps, @@ -585,7 +583,6 @@ function load_results!( merge!( get_cached_parameters(res), _read_results( - DenseAxisArray{Float64, 2}, res, parameter_keys, res.results_timestamps, @@ -595,7 +592,6 @@ function load_results!( merge!( get_cached_aux_variables(res), _read_results( - DenseAxisArray{Float64, 2}, res, aux_variable_keys, res.results_timestamps, @@ -605,7 +601,6 @@ function load_results!( merge!( get_cached_expressions(res), _read_results( - DenseAxisArray{Float64, 2}, res, expression_keys, res.results_timestamps, diff --git a/src/simulation/in_memory_simulation_store.jl b/src/simulation/in_memory_simulation_store.jl index 59cd94de1a..7182bffe98 100644 --- a/src/simulation/in_memory_simulation_store.jl +++ b/src/simulation/in_memory_simulation_store.jl @@ -17,6 +17,24 @@ function InMemorySimulationStore() ) end +function get_number_of_dimensions( + store::InMemorySimulationStore, + i::Type{EmulationModelIndexType}, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + +function get_number_of_dimensions( + store::InMemorySimulationStore, + i::Type{DecisionModelIndexType}, + model_name::Symbol, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + + function open_store( func::Function, ::Type{InMemorySimulationStore}, From c8b13887c5ce83f1f749f32ca97a0b94df176833 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 24 Oct 2023 11:11:47 -0600 Subject: [PATCH 39/90] add comment --- src/core/dataset.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 940159f61d..f263dfb711 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -267,7 +267,12 @@ function HDF5Dataset{2}( resolution::Dates.Period, initial_time::Dates.DateTime, ) + # The indexing is done in this way because we save all the names in an + # adjacent column entry in the HDF5 Datatset. The indexes for each column + # are known because we know how many elements are in each dimension. + # the names for the first column are store in the 1:first_column_number_of_elements. col1 = column_dataset[1:dims[2]] + # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names. col2 = column_dataset[(dims[2] + 1):end] HDF5Dataset{2}( values, From 3ebd3d3cf4419ef871acf509fcf79d38d0b74d3b Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 25 Oct 2023 14:27:07 -0600 Subject: [PATCH 40/90] add vector defs --- .../decision_model_simulation_results.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 94cba4295f..e3aca2ee3c 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -130,7 +130,7 @@ function _get_store_value( T::Type{Matrix{Float64}}, res::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, ::Nothing, ) simulation_store_path = joinpath(get_execution_path(res), "data_store") @@ -142,7 +142,7 @@ end function _get_store_value( sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) results_by_key = Dict{OptimizationContainerKey, ResultsByTime}() @@ -162,7 +162,7 @@ function _get_store_value( ::Type{T}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, key::OptimizationContainerKey, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) where {T <: DenseAxisArray{Float64, 2}} resolution = get_resolution(sim_results) @@ -203,7 +203,7 @@ function _get_store_value( ::Type{T}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, key::OptimizationContainerKey, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) where {T <: DenseAxisArray{Float64, 3}} resolution = get_resolution(sim_results) @@ -244,7 +244,7 @@ function _get_store_value( ::Type{Matrix{Float64}}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) base_power = get_model_base_power(sim_results) @@ -514,8 +514,8 @@ end function _are_results_cached( res::SimulationProblemResults{DecisionModelSimulationResults}, - output_keys, - timestamps, + output_keys::Vector{<:OptimizationContainerKey}, + timestamps::Vector{Dates.DateTime}, cached_keys, ) return isempty(setdiff(timestamps, res.results_timestamps)) && From 8cb2baef84039050fd878c8db6ac45e3c87ac30c Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 25 Oct 2023 16:27:11 -0600 Subject: [PATCH 41/90] fix read results interface --- src/simulation/decision_model_simulation_results.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index e3aca2ee3c..7f98a4276d 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -311,8 +311,15 @@ function _read_results( res::SimulationProblemResults{DecisionModelSimulationResults}, result_keys, timestamps::Vector{Dates.DateTime}, - store::Union{Nothing, <:SimulationStore}, + store::Nothing, ) + isempty(result_keys) && + return Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}() + + if res.store !== nothing + # In this case we have an InMemorySimulationStore. + store = res.store + end return _get_store_value(T, res, result_keys, timestamps, store) end From fce8884c122dfdda10cfdc1c8b3592d0b5493d59 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:18:40 -0700 Subject: [PATCH 42/90] remove not needed comments --- src/simulation/decision_model_simulation_results.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 7f98a4276d..961a21c201 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -125,7 +125,6 @@ function _get_store_value( end end -# TODO: Probably remove method later. It might be a duplicate. function _get_store_value( T::Type{Matrix{Float64}}, res::SimulationProblemResults{DecisionModelSimulationResults}, @@ -305,7 +304,6 @@ function _process_timestamps( return requested_range end -# Temporary workaround. To be removed function _read_results( T::Type{Matrix{Float64}}, res::SimulationProblemResults{DecisionModelSimulationResults}, From 03e2764a707b65bff74499b2700eb3833bc8e542 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:21:40 -0700 Subject: [PATCH 43/90] remove bad check --- src/simulation/hdf_simulation_store.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index fe235e6a79..09007a7b07 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -347,9 +347,6 @@ end log_cache_hit_percentages(x::HdfSimulationStore) = log_cache_hit_percentages(x.cache) function _make_dataframe(data::Matrix{Float64}, columns::Tuple{Vector{String}}) - if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] - data = reshape(data, length(data), 1) - end return DataFrames.DataFrame(data, columns[1]; copycols = false) end From 6039d279c0dce59b66185d439fd43ccf2865c11d Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:21:44 -0700 Subject: [PATCH 44/90] formatter --- src/simulation/decision_model_simulation_results.jl | 2 +- src/simulation/in_memory_simulation_store.jl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 961a21c201..f770ef3b12 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -312,7 +312,7 @@ function _read_results( store::Nothing, ) isempty(result_keys) && - return Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}() + return Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}() if res.store !== nothing # In this case we have an InMemorySimulationStore. diff --git a/src/simulation/in_memory_simulation_store.jl b/src/simulation/in_memory_simulation_store.jl index 7182bffe98..37beb6f846 100644 --- a/src/simulation/in_memory_simulation_store.jl +++ b/src/simulation/in_memory_simulation_store.jl @@ -34,7 +34,6 @@ function get_number_of_dimensions( return length(get_column_names(store, i, model_name, key)) end - function open_store( func::Function, ::Type{InMemorySimulationStore}, From d3c29eeeae75893f1e3b3a4f32923d42fcbbe50f Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:31:17 -0700 Subject: [PATCH 45/90] re-enable threads --- src/simulation/realized_meta.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simulation/realized_meta.jl b/src/simulation/realized_meta.jl index cb373cd05a..379a6e6c52 100644 --- a/src/simulation/realized_meta.jl +++ b/src/simulation/realized_meta.jl @@ -87,8 +87,7 @@ function get_realization( lk = ReentrantLock() num_rows = length(meta.realized_timestamps) start = time() - #Threads.@threads - for key in collect(keys(results)) + Threads.@threads for key in collect(keys(results)) results_by_time = results[key] columns = get_column_names(results_by_time) df = _make_dataframe(columns, results_by_time, num_rows, meta, key) From 3552fb097f62e9f6696c5543ae0d1fe30cdc923d Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 20 Sep 2023 11:32:28 -0600 Subject: [PATCH 46/90] add fix for resolution mismatches --- src/parameters/update_parameters.jl | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index ac021131bb..93871a3ce5 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -171,16 +171,15 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -211,16 +210,15 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix @@ -251,17 +249,16 @@ function _update_parameter_values!( current_time = get_current_time(model) state_values = get_dataset_values(state, get_attribute_key(attributes)) component_names, time = axes(parameter_array) - resolution = get_resolution(model) - + model_resolution = get_resolution(model) state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - + t_step = model_resolution ÷ state_data.resolution state_data_index = find_timestamp_index(state_timestamps, current_time) - sim_timestamps = range(current_time; step = resolution, length = time[end]) + sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time - timestamp_ix = min(max_state_index, state_data_index + 1) + timestamp_ix = min(max_state_index, state_data_index + t_step) @debug "parameter horizon is over the step" max_state_index > state_data_index + 1 if state_timestamps[timestamp_ix] <= sim_timestamps[t] state_data_index = timestamp_ix From 3b31244eba30a179a3f25967616bb140f0d57b82 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 20 Sep 2023 18:36:01 -0400 Subject: [PATCH 47/90] fix t_step calc --- src/operation/operation_model_interface.jl | 5 +++-- src/parameters/update_parameters.jl | 24 +++++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index e3511a754c..97c9faa313 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -87,10 +87,11 @@ function solve_impl!(model::OperationModel) if status != RunStatus.SUCCESSFUL settings = get_settings(model) model_name = get_name(model) + ts = get_current_timestamp(model) if !get_allow_fails(settings) - error("Solving model $(model_name) failed") + error("Solving model $(model_name) failed at $(ts)") else - @error "Solving model $(model_name) failed. Failure Allowed" + @error "Solving model $(model_name) failed at $(ts). Failure Allowed" end end return diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 93871a3ce5..a556d518b2 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -175,7 +175,13 @@ function _update_parameter_values!( state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - t_step = model_resolution ÷ state_data.resolution + if model_resolution < state_data.resolution + t_step = 1 + elseif model_resolution >= state_data.resolution + t_step = model_resolution ÷ state_data.resolution + else + @assert false + end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time @@ -214,7 +220,13 @@ function _update_parameter_values!( state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - t_step = model_resolution ÷ state_data.resolution + if model_resolution < state_data.resolution + t_step = 1 + elseif model_resolution >= state_data.resolution + t_step = model_resolution ÷ state_data.resolution + else + @assert false + end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) for t in time @@ -253,7 +265,13 @@ function _update_parameter_values!( state_data = get_dataset(state, get_attribute_key(attributes)) state_timestamps = state_data.timestamps max_state_index = get_num_rows(state_data) - t_step = model_resolution ÷ state_data.resolution + if model_resolution < state_data.resolution + t_step = 1 + elseif model_resolution >= state_data.resolution + t_step = model_resolution ÷ state_data.resolution + else + @assert false + end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) From be854a14854c9855dfc27db5bf32b6025ed551b1 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 16:19:33 -0600 Subject: [PATCH 48/90] allow 3D datasets --- src/core/dataset.jl | 58 +++++++++++++++----- src/core/optimizer_stats.jl | 2 +- src/core/results_by_time.jl | 25 ++++++--- src/operation/decision_model_store.jl | 4 +- src/operation/emulation_model_store.jl | 4 +- src/simulation/hdf_simulation_store.jl | 14 ++++- src/simulation/in_memory_simulation_store.jl | 2 +- src/simulation/simulation.jl | 11 ++-- src/simulation/simulation_state.jl | 28 ++++------ src/utils/dataframes_utils.jl | 6 +- src/utils/jump_utils.jl | 21 ++++--- 11 files changed, 112 insertions(+), 63 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 6340551e1f..be6e377783 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -20,9 +20,9 @@ end # Values field is accessed with dot syntax to avoid type instability -mutable struct InMemoryDataset <: AbstractDataset - "Data with dimensions (column names, row indexes)" - values::DenseAxisArray{Float64, 2} +mutable struct InMemoryDataset{N} <: AbstractDataset + "Data with dimensions (N column names, row indexes)" + values::DenseAxisArray{Float64, N} # We use Array here to allow for overwrites when updating the state timestamps::Vector{Dates.DateTime} # Resolution is needed because AbstractDataset might have just one row @@ -33,12 +33,12 @@ mutable struct InMemoryDataset <: AbstractDataset end function InMemoryDataset( - values::DenseAxisArray{Float64, 2}, + values::DenseAxisArray{Float64, N}, timestamps::Vector{Dates.DateTime}, resolution::Dates.Millisecond, end_of_step_index::Int, -) - return InMemoryDataset( +) where {N} + return InMemoryDataset{N}( values, timestamps, resolution, @@ -48,8 +48,8 @@ function InMemoryDataset( ) end -function InMemoryDataset(values::DenseAxisArray{Float64, 2}) - return InMemoryDataset( +function InMemoryDataset(values::DenseAxisArray{Float64, N}) where {N} + return InMemoryDataset{N}( values, Vector{Dates.DateTime}(), Dates.Second(0.0), @@ -59,14 +59,38 @@ function InMemoryDataset(values::DenseAxisArray{Float64, 2}) ) end -get_num_rows(s::InMemoryDataset) = size(s.values)[2] +function InMemoryDataset( + fill_val::Float64, + initial_time::Dates.DateTime, + resolution::Dates.Millisecond, + end_of_step_index::Int, + row_count::Int, + column_names::NTuple{N, <:Any}) where {N} + return InMemoryDataset( + fill!( + DenseAxisArray{Float64}(undef, column_names..., 1:row_count), + fill_val, + ), + collect( + range( + initial_time; + step = resolution, + length = row_count, + ), + ), + resolution, + end_of_step_index, + ) +end + +get_num_rows(s::InMemoryDataset{N}) where {N} = size(s.values)[N] function make_system_state( - values::DenseAxisArray{Float64, 2}, timestamp::Dates.DateTime, resolution::Dates.Millisecond, -) - return InMemoryDataset(values, [timestamp], resolution, 0, 1, UNSET_INI_TIME) + columns::NTuple{N, <:Any}, +) where {N} + return InMemoryDataset(NaN, timestamp, resolution, 0, 1, columns) end function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) @@ -77,8 +101,9 @@ function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) return s.values[:, s_index] end -get_column_names(s::InMemoryDataset) = axes(s.values)[1] -get_column_names(::OptimizationContainerKey, s::InMemoryDataset) = get_column_names(s) +function get_column_names(k::OptimizationContainerKey, s::InMemoryDataset) + return get_column_names(k, s.values) +end function get_last_recorded_value(s::InMemoryDataset) if get_last_recorded_row(s) == 0 @@ -120,6 +145,11 @@ function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 1}, index: return end +function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 3}, index::Int) + s.values[:, :, index] = vals + return +end + # HDF5Dataset does not account of overwrites in the data. Values are written sequentially. mutable struct HDF5Dataset <: AbstractDataset values::HDF5.Dataset diff --git a/src/core/optimizer_stats.jl b/src/core/optimizer_stats.jl index 6e2c98806a..19475381c6 100644 --- a/src/core/optimizer_stats.jl +++ b/src/core/optimizer_stats.jl @@ -100,5 +100,5 @@ function to_dict(stats::OptimizerStats) end function get_column_names(::Type{OptimizerStats}) - return collect(string.(fieldnames(OptimizerStats))) + return (collect(string.(fieldnames(OptimizerStats))),) end diff --git a/src/core/results_by_time.jl b/src/core/results_by_time.jl index 827b32d4c4..e71084234b 100644 --- a/src/core/results_by_time.jl +++ b/src/core/results_by_time.jl @@ -1,21 +1,26 @@ -mutable struct ResultsByTime{T} +mutable struct ResultsByTime{T, N} key::OptimizationContainerKey data::SortedDict{Dates.DateTime, T} resolution::Dates.Period - column_names::Vector{String} + column_names::NTuple{N, Vector{String}} end -function ResultsByTime(key, data, resolution, column_names) +function ResultsByTime( + key::OptimizationContainerKey, + data::SortedDict{Dates.DateTime, T}, + resolution::Dates.Period, + column_names, +) where {T} _check_column_consistency(data, column_names) ResultsByTime(key, data, resolution, column_names) end function _check_column_consistency( data::SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}, - cols::Vector{String}, + cols::Tuple{Vector{String}}, ) for val in values(data) - if axes(val)[1] != cols + if axes(val)[1] != cols[1] error("Mismatch in DenseAxisArray column names: $(axes(val)[1]) $cols") end end @@ -23,15 +28,19 @@ end function _check_column_consistency( data::SortedDict{Dates.DateTime, Matrix{Float64}}, - cols::Vector{String}, + cols::Tuple{Vector{String}}, ) for val in values(data) - if size(val)[2] != length(cols) - error("Mismatch in length of Matrix columns: $(size(val)[2]) $(length(cols))") + if size(val)[2] != length(cols[1]) + error( + "Mismatch in length of Matrix columns: $(size(val)[2]) $(length(cols[1]))", + ) end end end +# TODO: Implement consistency check for other sizes + # This struct behaves like a dict, delegating to its 'data' field. Base.length(res::ResultsByTime) = length(res.data) Base.iterate(res::ResultsByTime) = iterate(res.data) diff --git a/src/operation/decision_model_store.jl b/src/operation/decision_model_store.jl index bf10abcf42..b8c2591164 100644 --- a/src/operation/decision_model_store.jl +++ b/src/operation/decision_model_store.jl @@ -49,7 +49,7 @@ function initialize_storage!( for timestamp in range(initial_time; step = model_interval, length = num_of_executions) data[timestamp] = fill!( - DenseAxisArray{Float64}(undef, column_names, 1:time_steps_count), + DenseAxisArray{Float64}(undef, column_names..., 1:time_steps_count), NaN, ) end @@ -133,5 +133,5 @@ end function get_column_names(store::DecisionModelStore, key::OptimizationContainerKey) container = getfield(store, get_store_container_type(key)) - return axes(first(values(container[key])))[1] + return get_column_names(key, first(values(container[key]))) end diff --git a/src/operation/emulation_model_store.jl b/src/operation/emulation_model_store.jl index fe468b6483..deca8dbca6 100644 --- a/src/operation/emulation_model_store.jl +++ b/src/operation/emulation_model_store.jl @@ -77,7 +77,7 @@ function initialize_storage!( column_names = get_column_names(key, field_container) results_container[key] = InMemoryDataset( fill!( - DenseAxisArray{Float64}(undef, column_names, 1:num_of_executions), + DenseAxisArray{Float64}(undef, column_names..., 1:num_of_executions), NaN, ), ) @@ -139,7 +139,7 @@ end function get_column_names(store::EmulationModelStore, key::OptimizationContainerKey) container = get_data_field(store, get_store_container_type(key)) - return axes(container[key].values)[1] + return get_column_names(key, container[key].values) end function get_dataset_size(store::EmulationModelStore, key::OptimizationContainerKey) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 223407bea0..e0c45843ef 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -261,7 +261,12 @@ function initialize_problem_storage!( # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. col = _make_column_name(name) - HDF5.write_dataset(group, col, string.(reqs["columns"])) + if length(reqs["columns"]) == 1 + HDF5.write_dataset(group, col, string.(reqs["columns"][1])) + else + col_vals = vcat(reqs["columns"]...) + HDF5.write_dataset(group, col, string.(col_vals)) + end column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) datasets[key] = HDF5Dataset( @@ -305,7 +310,12 @@ function initialize_problem_storage!( # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. col = _make_column_name(name) - HDF5.write_dataset(group, col, string.(reqs["columns"])) + if length(reqs["columns"]) == 1 + HDF5.write_dataset(group, col, string.(reqs["columns"][1])) + else + col_vals = vcat(reqs["columns"]...) + HDF5.write_dataset(group, col, string.(col_vals)) + end column_dataset = group[col] datasets = getfield(store.em_data, type) datasets[key] = HDF5Dataset( diff --git a/src/simulation/in_memory_simulation_store.jl b/src/simulation/in_memory_simulation_store.jl index cff3ef5b60..59cd94de1a 100644 --- a/src/simulation/in_memory_simulation_store.jl +++ b/src/simulation/in_memory_simulation_store.jl @@ -143,7 +143,7 @@ function initialize_problem_storage!( container = get_data_field(get_em_data(store), type) container[key] = InMemoryDataset( fill!( - DenseAxisArray{Float64}(undef, reqs["columns"], 1:reqs["dims"][1]), + DenseAxisArray{Float64}(undef, reqs["columns"]..., 1:reqs["dims"][1]), NaN, ), ) diff --git a/src/simulation/simulation.jl b/src/simulation/simulation.jl index 0b2985a9ec..aa0a77754f 100644 --- a/src/simulation/simulation.jl +++ b/src/simulation/simulation.jl @@ -384,35 +384,36 @@ function _get_emulation_store_requirements(sim::Simulation) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.duals[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.duals[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_parameters_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.parameters[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.parameters[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_variables_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.variables[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.variables[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_aux_variables_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.aux_variables[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.aux_variables[key] = + Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end for (key, state_values) in get_expression_values(system_state) !should_write_resulting_value(key) && continue dims = sim_time ÷ get_data_resolution(state_values) cols = get_column_names(key, state_values) - reqs.expressions[key] = Dict("columns" => cols, "dims" => (dims, length(cols))) + reqs.expressions[key] = Dict("columns" => cols, "dims" => (dims, length.(cols)...)) end return reqs end diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index daaa02fcfa..b7e2932ef5 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -90,20 +90,12 @@ function _initialize_model_states!( column_names = get_column_names(key, value) if !haskey(field_states, key) || get_num_rows(field_states[key]) < value_counts field_states[key] = InMemoryDataset( - fill!( - DenseAxisArray{Float64}(undef, column_names, 1:value_counts), - NaN, - ), - collect( - range( - simulation_initial_time; - step = params[key].resolution, - length = value_counts, - ), - ), + NaN, + simulation_initial_time, params[key].resolution, Int(simulation_step / params[key].resolution), - ) + value_counts, + column_names) end end end @@ -125,9 +117,9 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, cols, 1:1), NaN), simulation_initial_time, min_res, + cols, ), ) end @@ -153,9 +145,9 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, column_names, 1:1), NaN), simulation_initial_time, - get_resolution(emulation_model), + min_res, + cols, ), ) end @@ -173,9 +165,9 @@ function _initialize_system_states!( emulator_states, key, make_system_state( - fill!(DenseAxisArray{Float64}(undef, cols, 1:1), NaN), simulation_initial_time, - get_resolution(emulation_model), + min_res, + cols, ), ) end @@ -212,7 +204,7 @@ function update_decision_state!( model_params::ModelStoreParams, ) state_data = get_decision_state_data(state, key) - column_names = get_column_names(state_data) + column_names = get_column_names(key, state_data)[1] model_resolution = get_resolution(model_params) state_resolution = get_data_resolution(state_data) resolution_ratio = model_resolution ÷ state_resolution diff --git a/src/utils/dataframes_utils.jl b/src/utils/dataframes_utils.jl index 4f78172b67..6db302134c 100644 --- a/src/utils/dataframes_utils.jl +++ b/src/utils/dataframes_utils.jl @@ -7,12 +7,12 @@ Creates a DataFrame from a JuMP DenseAxisArray or SparseAxisArray. - `array`: JuMP DenseAxisArray or SparseAxisArray to convert - `key::OptimizationContainerKey`: """ -function to_dataframe(array::DenseAxisArray, key::OptimizationContainerKey) - return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)) +function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T} + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end function to_dataframe(array::SparseAxisArray, key::OptimizationContainerKey) - return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)) + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end function to_matrix(df::DataFrames.DataFrame) diff --git a/src/utils/jump_utils.jl b/src/utils/jump_utils.jl index 5f1b2aa2ce..8d790e732c 100644 --- a/src/utils/jump_utils.jl +++ b/src/utils/jump_utils.jl @@ -61,15 +61,22 @@ end function get_column_names( key::OptimizationContainerKey, ::DenseAxisArray{T, 1, K}, -) where {T, K <: NTuple{1, Any}} +) where {T, K <: NTuple{1, Int}} return get_column_names(key) end function get_column_names( - ::OptimizationContainerKey, + k::OptimizationContainerKey, array::DenseAxisArray{T, 2, K}, ) where {T, K <: NTuple{2, Any}} - return string.(axes(array)[1]) + return (string.(axes(array)[1]),) +end + +function get_column_names( + k::OptimizationContainerKey, + array::DenseAxisArray{T, 3, K}, +) where {T, K <: NTuple{3, Any}} + return (string.(axes(array)[1]), string.(axes(array)[2])) end function _get_column_names(arr::SparseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} @@ -80,7 +87,7 @@ function get_column_names( ::OptimizationContainerKey, array::SparseAxisArray{T, N, K}, ) where {T, N, K <: NTuple{N, Any}} - return get_column_names(array) + return (get_column_names(array),) end function get_column_names(array::SparseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} @@ -198,12 +205,12 @@ function _calc_dimensions( if length(ax[2]) != horizon @debug "$(encode_key_as_string(key)) has length $(length(ax[1])). Different than horizon $horizon." end - dims = (length(ax[2]), length(columns), num_rows) + dims = (length(ax[2]), length(columns[1]), num_rows) elseif length(ax) == 3 if length(ax[3]) != horizon @debug "$(encode_key_as_string(key)) has length $(length(ax[1])). Different than horizon $horizon." end - dims = (length(ax[2]), length(ax[3]), length(columns), num_rows) + dims = (length(ax[3]), length(columns[1]), length(columns[2]), num_rows) else error("unsupported data size $(length(ax))") end @@ -218,7 +225,7 @@ function _calc_dimensions( horizon::Int, ) columns = get_column_names(key, array) - dims = (horizon, length(columns), num_rows) + dims = (horizon, length.(columns)..., num_rows) return Dict("columns" => columns, "dims" => dims) end From 5f9589c4ff1ea585fc8b6f4d56ec03747f333c04 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 16:40:36 -0600 Subject: [PATCH 49/90] fix get_columns --- src/utils/jump_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/jump_utils.jl b/src/utils/jump_utils.jl index 8d790e732c..c3aa86fe60 100644 --- a/src/utils/jump_utils.jl +++ b/src/utils/jump_utils.jl @@ -61,7 +61,7 @@ end function get_column_names( key::OptimizationContainerKey, ::DenseAxisArray{T, 1, K}, -) where {T, K <: NTuple{1, Int}} +) where {T, K <: NTuple{1, Any}} return get_column_names(key) end From 0febf078c3f3e9433e6b5c3649ca7361f576eb28 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 16:49:27 -0600 Subject: [PATCH 50/90] add method for 1 d data set --- src/core/dataset.jl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index be6e377783..199c5eb6ad 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -59,6 +59,24 @@ function InMemoryDataset(values::DenseAxisArray{Float64, N}) where {N} ) end +# Helper method for one dimensional cases +function InMemoryDataset( + fill_val::Float64, + initial_time::Dates.DateTime, + resolution::Dates.Millisecond, + end_of_step_index::Int, + row_count::Int, + column_names::Vector{String}) + return InMemoryDataset( + fill_val, + initial_time, + resolution, + end_of_step_index, + row_count, + (column_names,), + ) +end + function InMemoryDataset( fill_val::Float64, initial_time::Dates.DateTime, From 39e012edc151b63c94b05d0b78cde86501d30494 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 17:23:11 -0600 Subject: [PATCH 51/90] fix return --- src/utils/jump_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/jump_utils.jl b/src/utils/jump_utils.jl index c3aa86fe60..c3360c5da5 100644 --- a/src/utils/jump_utils.jl +++ b/src/utils/jump_utils.jl @@ -55,7 +55,7 @@ function to_matrix(::DenseAxisArray{T, N, K}) where {T, N, K <: NTuple{N, Any}} end function get_column_names(key::OptimizationContainerKey) - return [encode_key_as_string(key)] + return ([encode_key_as_string(key)],) end function get_column_names( From ddd6f270a3f9174fa4e37cc8c83104a10d5b4847 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 18:35:32 -0600 Subject: [PATCH 52/90] fix results processing --- src/core/dataset.jl | 12 +++- src/core/results_by_time.jl | 7 +++ .../decision_model_simulation_results.jl | 2 +- src/simulation/hdf_simulation_store.jl | 5 +- src/simulation/realized_meta.jl | 56 +++++++++++-------- src/simulation/simulation_state.jl | 6 +- 6 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 199c5eb6ad..9d996d3375 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -201,7 +201,17 @@ HDF5Dataset(values, column_dataset, resolution, initial_time) = column_dataset[:], ) -get_column_names(::OptimizationContainerKey, s::HDF5Dataset) = s.column_names +function _make_column_names(vals::Vector{String}) + return (vals,) +end + +function _make_column_names(vals::Matrix{String}) + return (vals[:, 1], vals[:, 2]) +end + +function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) + return _make_column_names(s.column_names) +end """ Return the timestamp from most recent data row updated in the dataset. This value may not be the same as the result from `get_update_timestamp` diff --git a/src/core/results_by_time.jl b/src/core/results_by_time.jl index e71084234b..32d77ede00 100644 --- a/src/core/results_by_time.jl +++ b/src/core/results_by_time.jl @@ -39,6 +39,13 @@ function _check_column_consistency( end end +function _check_column_consistency( + data::SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}, + cols::NTuple{N, Vector{String}}, +) where {N} + # TODO: +end + # TODO: Implement consistency check for other sizes # This struct behaves like a dict, delegating to its 'data' field. diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 38e6bb958c..96cbd04289 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -186,7 +186,7 @@ function _get_store_value( resolution = get_resolution(sim_results) for key in container_keys - results_by_time = ResultsByTime{Matrix{Float64}}( + results_by_time = ResultsByTime{Matrix{Float64}, 1}( key, SortedDict{Dates.DateTime, Matrix{Float64}}(), resolution, diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index e0c45843ef..6372ccb457 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -371,7 +371,7 @@ function read_result( else data, columns = _read_result(store, model_name, key, index) end - return DenseAxisArray(permutedims(data), columns, 1:size(data)[1]) + return DenseAxisArray(permutedims(data), columns..., 1:size(data)[1]) end function read_result( @@ -407,8 +407,7 @@ function read_results( data = dataset.values[index:(index + len - 1), :] end columns = get_column_names(key, dataset) - @assert_op size(data)[2] == length(columns) - return DenseAxisArray(permutedims(data), columns, 1:size(data)[1]) + return DenseAxisArray(permutedims(data), columns..., 1:size(data)[1]) end function get_column_names( diff --git a/src/simulation/realized_meta.jl b/src/simulation/realized_meta.jl index 3b8f9c34ea..4dbe971bc6 100644 --- a/src/simulation/realized_meta.jl +++ b/src/simulation/realized_meta.jl @@ -46,6 +46,38 @@ function RealizedMeta( ) end +function _make_dataframe( + columns::Tuple{Vector{String}}, + results_by_time::ResultsByTime{Matrix{Float64}, 1}, + num_rows::Int, + meta::RealizedMeta, +) + num_cols = length(columns[1]) + matrix = Matrix{Float64}(undef, num_rows, num_cols) + row_index = 1 + for (step, (_, array)) in enumerate(results_by_time) + first_id = step > 1 ? 1 : meta.start_offset + last_id = + step == meta.len ? meta.interval_len - meta.end_offset : meta.interval_len + if last_id - first_id > size(array, 1) + error( + "Variable $(encode_key_as_string(key)) has $(size(array, 1)) number of steps, that is different than the default problem horizon. \ + Can't calculate the realized variables. Use `read_variables` instead and write your own concatenation", + ) + end + row_end = row_index + last_id - first_id + matrix[row_index:row_end, :] = array[first_id:last_id, :] + row_index += last_id - first_id + 1 + end + df = DataFrames.DataFrame(matrix, collect(columns[1]); copycols = false) + DataFrames.insertcols!( + df, + 1, + :DateTime => meta.realized_timestamps, + ) + return df +end + function get_realization( results::Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}, meta::RealizedMeta, @@ -57,29 +89,7 @@ function get_realization( Threads.@threads for key in collect(keys(results)) results_by_time = results[key] columns = get_column_names(results_by_time) - num_cols = length(columns) - matrix = Matrix{Float64}(undef, num_rows, num_cols) - row_index = 1 - for (step, (_, array)) in enumerate(results_by_time) - first_id = step > 1 ? 1 : meta.start_offset - last_id = - step == meta.len ? meta.interval_len - meta.end_offset : meta.interval_len - if last_id - first_id > size(array, 1) - error( - "Variable $(encode_key_as_string(key)) has $(size(array, 1)) number of steps, that is different than the default problem horizon. \ - Can't calculate the realized variables. Use `read_variables` instead and write your own concatenation", - ) - end - row_end = row_index + last_id - first_id - matrix[row_index:row_end, :] = array[first_id:last_id, :] - row_index += last_id - first_id + 1 - end - df = DataFrames.DataFrame(matrix, collect(columns); copycols = false) - DataFrames.insertcols!( - df, - 1, - :DateTime => meta.realized_timestamps, - ) + df = _make_dataframe(columns, results_by_time, num_rows, meta) lock(lk) do realized_values[key] = df end diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index b7e2932ef5..05b5c3d482 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -199,7 +199,7 @@ end function update_decision_state!( state::SimulationState, key::OptimizationContainerKey, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) @@ -241,7 +241,7 @@ end function update_decision_state!( state::SimulationState, key::AuxVarKey{EnergyOutput, T}, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) where {T <: PSY.Component} @@ -283,7 +283,7 @@ end function update_decision_state!( state::SimulationState, key::AuxVarKey{S, T}, - store_data::DenseAxisArray{Float64}, + store_data::DenseAxisArray{Float64, 2}, simulation_time::Dates.DateTime, model_params::ModelStoreParams, ) where {T <: PSY.Component, S <: Union{TimeDurationOff, TimeDurationOn}} From 97c60e29823b150dd72e59e87e6f6926ae5db60e Mon Sep 17 00:00:00 2001 From: rodrigomha Date: Thu, 28 Sep 2023 11:42:06 -0700 Subject: [PATCH 53/90] update hdf debug for 3 dims Co-authored-by: jd-lara --- src/core/dataset.jl | 7 ++- src/simulation/hdf_simulation_store.jl | 68 ++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 9d996d3375..f0eee616bb 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -189,7 +189,8 @@ mutable struct HDF5Dataset <: AbstractDataset end end -HDF5Dataset(values, column_dataset, resolution, initial_time) = +function HDF5Dataset(values, column_dataset, resolution, initial_time) + @show column_dataset[:] HDF5Dataset( values, column_dataset, @@ -200,16 +201,20 @@ HDF5Dataset(values, column_dataset, resolution, initial_time) = UNSET_INI_TIME, column_dataset[:], ) +end function _make_column_names(vals::Vector{String}) + @show "col vector" return (vals,) end function _make_column_names(vals::Matrix{String}) + @show "col matrix" return (vals[:, 1], vals[:, 2]) end function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) + @show s.column_names return _make_column_names(s.column_names) end diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 6372ccb457..ba48ee7e5b 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -358,6 +358,22 @@ function read_result( return DataFrames.DataFrame(data, columns; copycols = false) end +function _make_denseaxisarray( + data::Matrix{Float64}, + columns::Tuple{Vector{String}}, +) + return DenseAxisArray(permutedims(data), columns[1], 1:size(data)[1]) +end + +function _make_denseaxisarray( + data::Array{Float64, 3}, + columns::NTuple{2, <:Any}, +) + @show data + @show columns + return DenseAxisArray(permutedims(data), columns[1], columns[2], 1:size(data)[1]) +end + function read_result( ::Type{DenseAxisArray}, store::HdfSimulationStore, @@ -365,13 +381,17 @@ function read_result( key::OptimizationContainerKey, index::Union{DecisionModelIndexType, EmulationModelIndexType}, ) + @show model_name if is_cached(store.cache, model_name, key, index) + @show "in cache" data = read_result(store.cache, model_name, key, index) columns = get_column_names(store, DecisionModelIndexType, model_name, key) else + @show "not in cache" data, columns = _read_result(store, model_name, key, index) end - return DenseAxisArray(permutedims(data), columns..., 1:size(data)[1]) + @show columns + return _make_denseaxisarray(data, columns) end function read_result( @@ -502,15 +522,15 @@ function _read_result( dataset = _get_dm_dataset(store, model_name, key) dset = dataset.values row_index = (simulation_step - 1) * num_executions + execution_index - columns = get_column_names(key, dataset) + @show columns = get_column_names(key, dataset) # Uncomment for performance checking #TimerOutputs.@timeit RUN_SIMULATION_TIMER "Read dataset" begin num_dims = ndims(dset) if num_dims == 3 data = dset[:, :, row_index] - #elseif num_dims == 4 - # data = dset[:, :, :, row_index] + elseif num_dims == 4 + data = dset[:, :, :, row_index] else error("unsupported dims: $num_dims") end @@ -519,6 +539,7 @@ function _read_result( return data, columns end + """ Write a decision model result for a timestamp to the store. """ @@ -553,6 +574,45 @@ function write_result!( return end +""" +Write a decision model result for a timestamp to the store. +""" +function write_result!( + store::HdfSimulationStore, + model_name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + ::Dates.DateTime, + data::DenseAxisArray{Float64, 3, <:NTuple{3, Any}}, +) + @show "Here" + #= + output_cache = get_output_cache(store.cache, model_name, key) + cur_size = get_size(store.cache) + add_result!(output_cache, index, to_matrix(data), is_full(store.cache, cur_size)) + + if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) + discard = !should_keep_in_cache(output_cache) + + # PERF: A potentially significant performance improvement would be to queue several + # flushes and submit them in parallel. + size_flushed = _flush_data!(output_cache, store, model_name, key, discard) + + @debug "flushed data" LOG_GROUP_SIMULATION_STORE key size_flushed discard cur_size + end + + # Disabled because this is currently a noop. + #if is_full(store.cache) + # _flush_data!(store.cache, store) + #end + + @debug "write_result" get_size(store.cache) encode_key_as_string(key) + =# + return +end + + + """ Write an emulation model result for an execution index value and the timestamp of the update """ From ed29e5ccb2f506e58ae68dc6e564d940f60f1fa6 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 27 Sep 2023 21:43:15 -0600 Subject: [PATCH 54/90] fix results testing --- src/simulation/hdf_simulation_store.jl | 23 ++++++++++++++++------- test/test_simulation_store.jl | 10 +++++----- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index ba48ee7e5b..b947623f61 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -340,6 +340,14 @@ end log_cache_hit_percentages(x::HdfSimulationStore) = log_cache_hit_percentages(x.cache) + +function _make_dataframe(data::Matrix{Float64}, columns::Tuple{Vector{String}}) + if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] + data = reshape(data, length(data), 1) + end + return DataFrames.DataFrame(data, columns[1]; copycols = false) +end + """ Return DataFrame, DenseAxisArray, or Array for a model result at a timestamp. """ @@ -351,11 +359,7 @@ function read_result( index::Union{DecisionModelIndexType, EmulationModelIndexType}, ) data, columns = _read_data_columns(store, model_name, key, index) - - if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] - data = reshape(data, length(data), 1) - end - return DataFrames.DataFrame(data, columns; copycols = false) + return _make_dataframe(data, columns) end function _make_denseaxisarray( @@ -925,11 +929,16 @@ function _read_data_columns( if is_cached(store.cache, model_name, key, index) data = read_result(store.cache, model_name, key, index) column_dataset = _get_dm_dataset(store, model_name, key).column_dataset - columns = column_dataset[:] + if ndims(column_dataset) == 1 + columns = (column_dataset[:],) + elseif ndims(column_dataset) == 2 + columns = (column_dataset[:, 1], column_dataset[:, 2]) + else + error("Datasets with $(ndims(column_dataset)) columns not supported") + end else data, columns = _read_result(store, model_name, key, index) end - return data, columns end diff --git a/test/test_simulation_store.jl b/test/test_simulation_store.jl index 5a29a38d35..bb7a27f4c1 100644 --- a/test/test_simulation_store.jl +++ b/test/test_simulation_store.jl @@ -39,7 +39,7 @@ function _initialize!(store, sim, variables, model_defs, cache_rules) for (key, array) in model_defs[model]["variables"] reqs.variables[key] = Dict( "columns" => model_defs[model]["names"], - "dims" => (horizon, length(model_defs[model]["names"]), num_rows), + "dims" => (horizon, length(model_defs[model]["names"][1]), num_rows), ) keep_in_cache = variables[key]["keep_in_cache"] add_rule!(cache_rules, model, key, keep_in_cache) @@ -146,8 +146,8 @@ function _verify_read_results(path, sim, variables, model_defs, seed) end end -function _verify_data(expected, store, model, name, time, columns) - expected_df = DataFrames.DataFrame(expected, columns) +function _verify_data(expected, store, model, name, time, columns::Tuple{Vector{Symbol}}) + expected_df = DataFrames.DataFrame(expected, columns[1]) df = read_result(DataFrames.DataFrame, store, model, name, time) @test expected_df == df end @@ -172,7 +172,7 @@ end :ED => Dict( "execution_count" => 24, "horizon" => 12, - "names" => [:dev1, :dev2, :dev3, :dev4, :dev5], + "names" => ([:dev1, :dev2, :dev3, :dev4, :dev5],), "variables" => Dict(x => ones(12, 5) for x in keys(variables)), "interval" => Dates.Hour(1), "resolution" => Dates.Hour(1), @@ -182,7 +182,7 @@ end :UC => Dict( "execution_count" => 1, "horizon" => 24, - "names" => [:dev1, :dev2, :dev3], + "names" => ([:dev1, :dev2, :dev3],), "variables" => Dict(x => ones(24, 3) for x in keys(variables)), "interval" => Dates.Hour(1), "resolution" => Dates.Hour(24), From bdff35f1c4f9e0e26b69704281761e00e8f96b4f Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:00:00 -0600 Subject: [PATCH 55/90] support 3D HDFD dataset --- src/core/dataset.jl | 61 +++++++++++++++++--------- src/simulation/hdf_simulation_store.jl | 13 +++--- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index f0eee616bb..6228a811f1 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -169,7 +169,7 @@ function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 3}, index: end # HDF5Dataset does not account of overwrites in the data. Values are written sequentially. -mutable struct HDF5Dataset <: AbstractDataset +mutable struct HDF5Dataset{N} <: AbstractDataset values::HDF5.Dataset column_dataset::HDF5.Dataset write_index::Int @@ -177,21 +177,31 @@ mutable struct HDF5Dataset <: AbstractDataset resolution::Dates.Millisecond initial_timestamp::Dates.DateTime update_timestamp::Dates.DateTime - column_names::Vector{String} + column_names::NTuple{N, Vector{String}} - function HDF5Dataset(values, column_dataset, write_index, last_recorded_row, resolution, + function HDF5Dataset{N}(values, + column_dataset, + write_index, + last_recorded_row, + resolution, initial_timestamp, - update_timestamp, column_names, - ) - new(values, column_dataset, write_index, last_recorded_row, resolution, + update_timestamp, + column_names::NTuple{N, Vector{String}}, + ) where {N} + new{N}(values, column_dataset, write_index, last_recorded_row, resolution, initial_timestamp, update_timestamp, column_names) end end -function HDF5Dataset(values, column_dataset, resolution, initial_time) - @show column_dataset[:] - HDF5Dataset( +function HDF5Dataset{1}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + ::Tuple, + resolution::Dates.Millisecond, + initial_time::Dates.DateTime, +) + HDF5Dataset{1}( values, column_dataset, 1, @@ -199,23 +209,34 @@ function HDF5Dataset(values, column_dataset, resolution, initial_time) resolution, initial_time, UNSET_INI_TIME, - column_dataset[:], + (column_dataset[:],), ) end -function _make_column_names(vals::Vector{String}) - @show "col vector" - return (vals,) -end - -function _make_column_names(vals::Matrix{String}) - @show "col matrix" - return (vals[:, 1], vals[:, 2]) +function HDF5Dataset{2}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + dims::NTuple{4, Int}, + resolution::Dates.Period, + initial_time::Dates.DateTime, +) + col1 = column_dataset[1:dims[2]] + col2 = column_dataset[(dims[2] + 1):end] + HDF5Dataset{2}( + values, + column_dataset, + 1, + 0, + resolution, + initial_time, + UNSET_INI_TIME, + (col1, col2), + ) end function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) - @show s.column_names - return _make_column_names(s.column_names) + s.column_names + return s.column_names end """ diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index b947623f61..7881831b6d 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -269,9 +269,12 @@ function initialize_problem_storage!( end column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) - datasets[key] = HDF5Dataset( + # First dim is Horizon, Last number of steps + n_dims = length(reqs["dims"]) - 2 + datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, + reqs["dims"], get_resolution(problem_params), initial_time, ) @@ -318,9 +321,11 @@ function initialize_problem_storage!( end column_dataset = group[col] datasets = getfield(store.em_data, type) - datasets[key] = HDF5Dataset( + n_dims = length(reqs["dims"]) - 1 + datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, + reqs["dims"], get_resolution(emulation_params), initial_time, ) @@ -340,7 +345,6 @@ end log_cache_hit_percentages(x::HdfSimulationStore) = log_cache_hit_percentages(x.cache) - function _make_dataframe(data::Matrix{Float64}, columns::Tuple{Vector{String}}) if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] data = reshape(data, length(data), 1) @@ -543,7 +547,6 @@ function _read_result( return data, columns end - """ Write a decision model result for a timestamp to the store. """ @@ -615,8 +618,6 @@ function write_result!( return end - - """ Write an emulation model result for an execution index value and the timestamp of the update """ From 4cfab7686376a2c7716c928dd28f1bc76981942a Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:13:33 -0600 Subject: [PATCH 56/90] fix dims count --- src/simulation/hdf_simulation_store.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 7881831b6d..bec4992144 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -321,7 +321,8 @@ function initialize_problem_storage!( end column_dataset = group[col] datasets = getfield(store.em_data, type) - n_dims = length(reqs["dims"]) - 1 + # First dim is Horizon, Last number of steps + n_dims = length(reqs["dims"]) - 2 datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, From 420c6d3017a776413c89dac910f9b0af9d973fc9 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:14:58 -0600 Subject: [PATCH 57/90] fix to dims --- src/simulation/hdf_simulation_store.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index bec4992144..d6d51c660e 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -270,7 +270,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) # First dim is Horizon, Last number of steps - n_dims = length(reqs["dims"]) - 2 + n_dims = min(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, @@ -322,7 +322,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(store.em_data, type) # First dim is Horizon, Last number of steps - n_dims = length(reqs["dims"]) - 2 + n_dims = min(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, From 34254fc48848393ce84739117a7cd3a76724e37b Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 15:28:32 -0600 Subject: [PATCH 58/90] use max --- src/simulation/hdf_simulation_store.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index d6d51c660e..3519f03f54 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -270,7 +270,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(get_dm_data(store)[problem], type) # First dim is Horizon, Last number of steps - n_dims = min(1, length(reqs["dims"]) - 2) + n_dims = max(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, @@ -322,7 +322,7 @@ function initialize_problem_storage!( column_dataset = group[col] datasets = getfield(store.em_data, type) # First dim is Horizon, Last number of steps - n_dims = min(1, length(reqs["dims"]) - 2) + n_dims = max(1, length(reqs["dims"]) - 2) datasets[key] = HDF5Dataset{n_dims}( dataset, column_dataset, From ea9d6399720fcce41909251dd03cd1350f60bd4d Mon Sep 17 00:00:00 2001 From: rodrigomha Date: Thu, 28 Sep 2023 14:38:10 -0700 Subject: [PATCH 59/90] force permute order --- src/simulation/hdf_simulation_store.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 3519f03f54..510f1ac395 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -378,9 +378,9 @@ function _make_denseaxisarray( data::Array{Float64, 3}, columns::NTuple{2, <:Any}, ) - @show data + @show size(data) @show columns - return DenseAxisArray(permutedims(data), columns[1], columns[2], 1:size(data)[1]) + return DenseAxisArray(permutedims(data, (2, 3, 1)), columns[1], columns[2], 1:size(data)[1]) end function read_result( From caa018f9e65d1e4b43e042654dcdbd8f7bd1dcf7 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 17:18:48 -0600 Subject: [PATCH 60/90] fix results --- src/simulation/hdf_simulation_store.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 510f1ac395..7cbd461967 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -378,8 +378,6 @@ function _make_denseaxisarray( data::Array{Float64, 3}, columns::NTuple{2, <:Any}, ) - @show size(data) - @show columns return DenseAxisArray(permutedims(data, (2, 3, 1)), columns[1], columns[2], 1:size(data)[1]) end @@ -390,16 +388,12 @@ function read_result( key::OptimizationContainerKey, index::Union{DecisionModelIndexType, EmulationModelIndexType}, ) - @show model_name if is_cached(store.cache, model_name, key, index) - @show "in cache" data = read_result(store.cache, model_name, key, index) columns = get_column_names(store, DecisionModelIndexType, model_name, key) else - @show "not in cache" data, columns = _read_result(store, model_name, key, index) end - @show columns return _make_denseaxisarray(data, columns) end @@ -531,7 +525,7 @@ function _read_result( dataset = _get_dm_dataset(store, model_name, key) dset = dataset.values row_index = (simulation_step - 1) * num_executions + execution_index - @show columns = get_column_names(key, dataset) + columns = get_column_names(key, dataset) # Uncomment for performance checking #TimerOutputs.@timeit RUN_SIMULATION_TIMER "Read dataset" begin @@ -709,10 +703,11 @@ function _deserialize_attributes!(store::HdfSimulationStore) empty!(get_dm_data(store)) for model in HDF5.read(HDF5.attributes(group)["problem_order"]) problem_group = store.file["simulation/decision_models/$model"] + horizon = HDF5.read(HDF5.attributes(problem_group)["horizon"]) model_name = Symbol(model) store.params.decision_models_params[model_name] = ModelStoreParams( HDF5.read(HDF5.attributes(problem_group)["num_executions"]), - HDF5.read(HDF5.attributes(problem_group)["horizon"]), + horizon, Dates.Millisecond(HDF5.read(HDF5.attributes(problem_group)["interval_ms"])), Dates.Millisecond(HDF5.read(HDF5.attributes(problem_group)["resolution_ms"])), HDF5.read(HDF5.attributes(problem_group)["base_power"]), @@ -727,7 +722,9 @@ function _deserialize_attributes!(store::HdfSimulationStore) column_dataset = group[_make_column_name(name)] resolution = get_resolution(get_decision_model_params(store, model_name)) - item = HDF5Dataset(dataset, column_dataset, resolution, initial_time) + dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) + n_dims = max(1, ndims(dataset) - 2) + item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) container_key = container_key_lookup[name] getfield(get_dm_data(store)[model_name], type)[container_key] = item add_output_cache!( @@ -745,6 +742,7 @@ function _deserialize_attributes!(store::HdfSimulationStore) end em_group = _get_emulation_model_path(store) + horizon = HDF5.read(HDF5.attributes(em_group)["horizon"]) model_name = Symbol(HDF5.read(HDF5.attributes(em_group)["name"])) resolution = Dates.Millisecond(HDF5.read(HDF5.attributes(em_group)["resolution_ms"])) store.params.emulation_model_params[model_name] = ModelStoreParams( @@ -761,7 +759,9 @@ function _deserialize_attributes!(store::HdfSimulationStore) if !endswith(name, "columns") dataset = group[name] column_dataset = group[_make_column_name(name)] - item = HDF5Dataset(dataset, column_dataset, resolution, initial_time) + dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) + n_dims = max(1, ndims(dataset) - 1) + item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) container_key = container_key_lookup[name] getfield(store.em_data, type)[container_key] = item add_output_cache!(store.cache, model_name, container_key, CacheFlushRule()) From b9f3f105b178f264bccdaaf212e6f1a1346f986f Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 17:20:28 -0600 Subject: [PATCH 61/90] formatter --- src/simulation/hdf_simulation_store.jl | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 7cbd461967..956a9c23fb 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -378,7 +378,12 @@ function _make_denseaxisarray( data::Array{Float64, 3}, columns::NTuple{2, <:Any}, ) - return DenseAxisArray(permutedims(data, (2, 3, 1)), columns[1], columns[2], 1:size(data)[1]) + return DenseAxisArray( + permutedims(data, (2, 3, 1)), + columns[1], + columns[2], + 1:size(data)[1], + ) end function read_result( @@ -724,7 +729,13 @@ function _deserialize_attributes!(store::HdfSimulationStore) get_resolution(get_decision_model_params(store, model_name)) dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 2) - item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) + item = HDF5Dataset{n_dims}( + dataset, + column_dataset, + dims, + resolution, + initial_time, + ) container_key = container_key_lookup[name] getfield(get_dm_data(store)[model_name], type)[container_key] = item add_output_cache!( @@ -761,7 +772,13 @@ function _deserialize_attributes!(store::HdfSimulationStore) column_dataset = group[_make_column_name(name)] dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 1) - item = HDF5Dataset{n_dims}(dataset, column_dataset, dims, resolution, initial_time) + item = HDF5Dataset{n_dims}( + dataset, + column_dataset, + dims, + resolution, + initial_time, + ) container_key = container_key_lookup[name] getfield(store.em_data, type)[container_key] = item add_output_cache!(store.cache, model_name, container_key, CacheFlushRule()) From 24ec5464c1c8e1303827a7ee1d7e75ae07d3eef9 Mon Sep 17 00:00:00 2001 From: rodrigomha Date: Thu, 28 Sep 2023 16:19:54 -0700 Subject: [PATCH 62/90] remove subcomp set param val --- src/parameters/update_parameters.jl | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index a556d518b2..6208ea8237 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -25,28 +25,6 @@ function _set_param_value!(param::JuMPFloatArray, value::Float64, name::String, return end -function _set_param_value!( - param::SparseAxisArray{Union{Nothing, JuMP.VariableRef}}, - value::Float64, - name::String, - subcomp::String, - t::Int, -) - fix_parameter_value(param[name, subcomp, t], value) - return -end - -function _set_param_value!( - param::SparseAxisArray{Float64}, - value::Float64, - name::String, - subcomp::String, - t::Int, -) - param[name, subcomp, t] = value - return -end - function _update_parameter_values!( parameter_array::AbstractArray{T}, attributes::TimeSeriesAttributes{U}, From f6e67a7865645214d4d5f370fabd366cbbe1c7da Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 20:02:53 -0600 Subject: [PATCH 63/90] add missing method --- src/utils/dataframes_utils.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/dataframes_utils.jl b/src/utils/dataframes_utils.jl index 6db302134c..28de0efb63 100644 --- a/src/utils/dataframes_utils.jl +++ b/src/utils/dataframes_utils.jl @@ -7,7 +7,11 @@ Creates a DataFrame from a JuMP DenseAxisArray or SparseAxisArray. - `array`: JuMP DenseAxisArray or SparseAxisArray to convert - `key::OptimizationContainerKey`: """ -function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T} +function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T <: Number} + return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) +end + +function to_dataframe(array::DenseAxisArray{T, 1}, key::OptimizationContainerKey) where {T <: Number} return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end From 1fa2f9f157d9aa0fbca7076285f80ada91369b9a Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 20:03:03 -0600 Subject: [PATCH 64/90] update tests --- test/test_utils.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_utils.jl b/test/test_utils.jl index 5d49df15cf..7e39118246 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -23,10 +23,12 @@ end @test two_df == test_df three = PSI.DenseAxisArray{Float64}(undef, [:a], 1:2, 1:3) - @test_throws ErrorException PSI.to_dataframe(three, mock_key) + fill!(three, 1.0) + @test_throws MethodError PSI.to_dataframe(three, mock_key) four = PSI.DenseAxisArray{Float64}(undef, [:a], 1:2, 1:3, 1:5) - @test_throws ErrorException PSI.to_dataframe(four, mock_key) + fill!(three, 1.0) + @test_throws MethodError PSI.to_dataframe(four, mock_key) sparse_num = JuMP.Containers.@container([i = 1:10, j = (i + 1):10, t = 1:24], 0.0 + i + j + t) From f2e57ce3c40dff0cc6b0dd8c179ccefac4efa666 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 20:39:24 -0600 Subject: [PATCH 65/90] formatter --- src/utils/dataframes_utils.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/utils/dataframes_utils.jl b/src/utils/dataframes_utils.jl index 28de0efb63..eb9607a909 100644 --- a/src/utils/dataframes_utils.jl +++ b/src/utils/dataframes_utils.jl @@ -7,11 +7,17 @@ Creates a DataFrame from a JuMP DenseAxisArray or SparseAxisArray. - `array`: JuMP DenseAxisArray or SparseAxisArray to convert - `key::OptimizationContainerKey`: """ -function to_dataframe(array::DenseAxisArray{T, 2}, key::OptimizationContainerKey) where {T <: Number} +function to_dataframe( + array::DenseAxisArray{T, 2}, + key::OptimizationContainerKey, +) where {T <: Number} return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end -function to_dataframe(array::DenseAxisArray{T, 1}, key::OptimizationContainerKey) where {T <: Number} +function to_dataframe( + array::DenseAxisArray{T, 1}, + key::OptimizationContainerKey, +) where {T <: Number} return DataFrames.DataFrame(to_matrix(array), get_column_names(key, array)[1]) end From 1691a945f0d111e3b23d054afb41b5312d834459 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 28 Sep 2023 21:56:59 -0600 Subject: [PATCH 66/90] add min_res --- src/simulation/simulation_state.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index 05b5c3d482..12aa0365e1 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -135,6 +135,7 @@ function _initialize_system_states!( decision_states = get_decision_states(sim_state) emulator_states = get_system_states(sim_state) emulation_container = get_optimization_container(emulation_model) + min_res = minimum([v.resolution for v in values(params)]) for field in fieldnames(DatasetContainer) field_containers = getfield(emulation_container, field) From 11d3777d223c8d88e06d934becdcd5f72346b23f Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 00:34:57 -0600 Subject: [PATCH 67/90] add code to store 3D in HDF --- src/core/dataset.jl | 31 ++++++++++--- src/core/definitions.jl | 1 + src/parameters/update_parameters.jl | 2 +- src/simulation/hdf_simulation_store.jl | 61 +++++++++++++++++++++----- src/simulation/simulation_state.jl | 3 +- 5 files changed, 79 insertions(+), 19 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 6228a811f1..1b3efe15f8 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -111,7 +111,10 @@ function make_system_state( return InMemoryDataset(NaN, timestamp, resolution, 0, 1, columns) end -function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) +function get_dataset_value( + s::T, + date::Dates.DateTime, +) where {T <: Union{InMemoryDataset{1}, InMemoryDataset{2}}} s_index = find_timestamp_index(s.timestamps, date) if isnothing(s_index) error("Request time stamp $date not in the state") @@ -119,17 +122,32 @@ function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime) return s.values[:, s_index] end +function get_dataset_value(s::InMemoryDataset{3}, date::Dates.DateTime) + s_index = find_timestamp_index(s.timestamps, date) + if isnothing(s_index) + error("Request time stamp $date not in the state") + end + return s.values[:, :, s_index] +end + function get_column_names(k::OptimizationContainerKey, s::InMemoryDataset) return get_column_names(k, s.values) end -function get_last_recorded_value(s::InMemoryDataset) +function get_last_recorded_value(s::InMemoryDataset{2}) if get_last_recorded_row(s) == 0 error("The Dataset hasn't been written yet") end return s.values[:, get_last_recorded_row(s)] end +function get_last_recorded_value(s::InMemoryDataset{3}) + if get_last_recorded_row(s) == 0 + error("The Dataset hasn't been written yet") + end + return s.values[:, :, get_last_recorded_row(s)] +end + function get_end_of_step_timestamp(s::InMemoryDataset) return s.timestamps[s.end_of_step_index] end @@ -153,17 +171,20 @@ function get_value_timestamp(s::InMemoryDataset, date::Dates.DateTime) return s.timestamps[s_index] end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 2}, index::Int) +# These set_value! methods expect a single time_step value because they are used to update +#the state so the incoming vals will have one dimension less than the DataSet. The exception +# is for vals of Dimension 1 which are still stored in DataSets of dimension 2. +function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 2}, index::Int) s.values[:, index] = vals[:, index] return end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 1}, index::Int) +function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 1}, index::Int) s.values[:, index] = vals return end -function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 3}, index::Int) +function set_value!(s::InMemoryDataset{3}, vals::DenseAxisArray{Float64, 2}, index::Int) s.values[:, :, index] = vals return end diff --git a/src/core/definitions.jl b/src/core/definitions.jl index 0097b123a2..b51e41e7db 100644 --- a/src/core/definitions.jl +++ b/src/core/definitions.jl @@ -24,6 +24,7 @@ const JuMPVariableMatrix = DenseAxisArray{ JuMP.Containers._AxisLookup{Tuple{Int64, Int64}}, }, } +const JuMPFloatMatrix = DenseAxisArray{Float64, 2} const JuMPFloatArray = DenseAxisArray{Float64} const JuMPVariableArray = DenseAxisArray{JuMP.VariableRef} diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 6208ea8237..0af903c683 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -478,7 +478,7 @@ end function _fix_parameter_value!( container::OptimizationContainer, - parameter_array::JuMPFloatArray, + parameter_array::DenseAxisArray{Float64, 2}, parameter_attributes::VariableValueAttributes, ) affected_variable_keys = parameter_attributes.affected_keys diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 956a9c23fb..2a8494eba2 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -592,11 +592,10 @@ function write_result!( ::Dates.DateTime, data::DenseAxisArray{Float64, 3, <:NTuple{3, Any}}, ) - @show "Here" - #= output_cache = get_output_cache(store.cache, model_name, key) cur_size = get_size(store.cache) - add_result!(output_cache, index, to_matrix(data), is_full(store.cache, cur_size)) + + add_result!(output_cache, index, data.data, is_full(store.cache, cur_size)) if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) discard = !should_keep_in_cache(output_cache) @@ -614,7 +613,6 @@ function write_result!( #end @debug "write_result" get_size(store.cache) encode_key_as_string(key) - =# return end @@ -627,10 +625,10 @@ function write_result!( key::OptimizationContainerKey, index::EmulationModelIndexType, simulation_time::Dates.DateTime, - data::Matrix{Float64}, + data::Array{Float64}, ) dataset = _get_em_dataset(store, key) - _write_dataset!(dataset.values, data, index:index) + _write_dataset!(dataset.values, data, index) set_last_recorded_row!(dataset, index) set_update_timestamp!(dataset, simulation_time) return @@ -642,7 +640,21 @@ function write_result!( key::OptimizationContainerKey, index::EmulationModelIndexType, simulation_time::Dates.DateTime, - data, + data::DenseAxisArray{Float64, 2}, +) + data_array = Array{Float64, 3}(undef, size(data)[1], size(data)[2], 1) + data_array[:, :, 1] = data + write_result!(store, model_name, key, index, simulation_time, data_array) + return +end + +function write_result!( + store::HdfSimulationStore, + model_name::Symbol, + key::OptimizationContainerKey, + index::EmulationModelIndexType, + simulation_time::Dates.DateTime, + data::DenseAxisArray{Float64, 1}, ) write_result!(store, model_name, key, index, simulation_time, to_matrix(data)) return @@ -984,7 +996,7 @@ function _read_length(::Type{OptimizerStats}, store::HdfSimulationStore) end function _write_dataset!( - dataset, + dataset::HDF5.Dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}, ::Val{3}, @@ -995,7 +1007,7 @@ function _write_dataset!( end function _write_dataset!( - dataset, + dataset::HDF5.Dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}, ::Val{2}, @@ -1005,13 +1017,38 @@ function _write_dataset!( return end -function _write_dataset!(dataset, array::Matrix{Float64}, row_range::UnitRange{Int64}) - _write_dataset!(dataset, array, row_range, Val{ndims(dataset)}()) +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 3}, + row_range::UnitRange{Int64}, + ::Val{3}, +) + dataset[row_range, :, :] = array + @debug "wrote dataset" dataset row_range + return +end + +function _write_dataset!(dataset::HDF5.Dataset, array::Array{Float64}, index::Int) + _write_dataset!(dataset, array, index:index, Val{ndims(dataset)}()) return end -function _write_dataset!(dataset, array::Array{Float64, 3}, row_range::UnitRange{Int64}) +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 3}, + row_range::UnitRange{Int64}, +) dataset[:, :, row_range] = array @debug "wrote dataset" dataset row_range return end + +function _write_dataset!( + dataset::HDF5.Dataset, + array::Array{Float64, 4}, + row_range::UnitRange{Int64}, +) + dataset[:, :, :, row_range] = array + @debug "wrote dataset" dataset row_range + return +end diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index 12aa0365e1..e628294e89 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -410,7 +410,8 @@ function update_system_state!( set_update_timestamp!(system_dataset, ts) # Keep coordination between fields. System state is an array of size 1 system_dataset.timestamps[1] = ts - set_dataset_values!(state, key, 1, get_dataset_value(decision_dataset, simulation_time)) + data_set_value = get_dataset_value(decision_dataset, simulation_time) + set_dataset_values!(state, key, 1, data_set_value) # This value shouldn't be other than one and after one execution is no-op. set_last_recorded_row!(system_dataset, 1) return From ea64d4f0082f02d316090d27d20825beaaafc891 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 01:12:00 -0600 Subject: [PATCH 68/90] fixes to run --- src/simulation/hdf_simulation_store.jl | 2 +- src/simulation/optimization_output_cache.jl | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 2a8494eba2..e2b639f507 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -595,7 +595,7 @@ function write_result!( output_cache = get_output_cache(store.cache, model_name, key) cur_size = get_size(store.cache) - add_result!(output_cache, index, data.data, is_full(store.cache, cur_size)) + add_result!(output_cache, index, permutedims(data.data, (3, 1, 2)), is_full(store.cache, cur_size)) if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) discard = !should_keep_in_cache(output_cache) diff --git a/src/simulation/optimization_output_cache.jl b/src/simulation/optimization_output_cache.jl index da76708ff3..276c529273 100644 --- a/src/simulation/optimization_output_cache.jl +++ b/src/simulation/optimization_output_cache.jl @@ -52,7 +52,10 @@ end """ Add result to the cache. """ -function add_result!(cache::OptimizationOutputCache, timestamp, array, system_cache_is_full) +function add_result!(cache::OptimizationOutputCache, + timestamp::Dates.DateTime, + array::Array{Float64}, + system_cache_is_full::Bool) if cache.size_per_entry == 0 cache.size_per_entry = length(array) * sizeof(first(array)) end @@ -77,7 +80,11 @@ function add_result!(cache::OptimizationOutputCache, timestamp, array, system_ca return cache.size_per_entry end -function _add_result!(cache::OptimizationOutputCache, timestamp, data) +function _add_result!( + cache::OptimizationOutputCache, + timestamp::Dates.DateTime, + data::Array{Float64}, +) cache.data[timestamp] = data push!(cache.dirty_timestamps, timestamp) return From 2eb8c3840f2a0c04f256d4924120e96bcd5a7269 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 01:15:02 -0600 Subject: [PATCH 69/90] fix columns call --- src/simulation/simulation_state.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/simulation/simulation_state.jl b/src/simulation/simulation_state.jl index e628294e89..b2f4a79d4c 100644 --- a/src/simulation/simulation_state.jl +++ b/src/simulation/simulation_state.jl @@ -148,27 +148,27 @@ function _initialize_system_states!( make_system_state( simulation_initial_time, min_res, - cols, + column_names, ), ) end end for key in get_dataset_keys(decision_states) + dm_cols = get_column_names(key, get_dataset(decision_states, key)) if has_dataset(emulator_states, key) - dm_cols = get_column_names(key, get_dataset(decision_states, key)) em_cols = get_column_names(key, get_dataset(emulator_states, key)) @assert_op dm_cols == em_cols continue end - cols = get_column_names(key, get_dataset(decision_states, key)) + set_dataset!( emulator_states, key, make_system_state( simulation_initial_time, min_res, - cols, + dm_cols, ), ) end From d3fe841bfee0a5734286d8b72dc14d646215e135 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 01:16:03 -0600 Subject: [PATCH 70/90] formatter --- src/simulation/hdf_simulation_store.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index e2b639f507..77ab196e26 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -595,7 +595,12 @@ function write_result!( output_cache = get_output_cache(store.cache, model_name, key) cur_size = get_size(store.cache) - add_result!(output_cache, index, permutedims(data.data, (3, 1, 2)), is_full(store.cache, cur_size)) + add_result!( + output_cache, + index, + permutedims(data.data, (3, 1, 2)), + is_full(store.cache, cur_size), + ) if get_dirty_size(output_cache) >= get_min_flush_size(store.cache) discard = !should_keep_in_cache(output_cache) From 456e717367cd7bd0e08462f3a28e959ac102ec17 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 15:28:43 -0600 Subject: [PATCH 71/90] fix passing of key --- src/simulation/realized_meta.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/simulation/realized_meta.jl b/src/simulation/realized_meta.jl index 4dbe971bc6..cb373cd05a 100644 --- a/src/simulation/realized_meta.jl +++ b/src/simulation/realized_meta.jl @@ -51,6 +51,7 @@ function _make_dataframe( results_by_time::ResultsByTime{Matrix{Float64}, 1}, num_rows::Int, meta::RealizedMeta, + key::OptimizationContainerKey, ) num_cols = length(columns[1]) matrix = Matrix{Float64}(undef, num_rows, num_cols) @@ -86,10 +87,11 @@ function get_realization( lk = ReentrantLock() num_rows = length(meta.realized_timestamps) start = time() - Threads.@threads for key in collect(keys(results)) + #Threads.@threads + for key in collect(keys(results)) results_by_time = results[key] columns = get_column_names(results_by_time) - df = _make_dataframe(columns, results_by_time, num_rows, meta) + df = _make_dataframe(columns, results_by_time, num_rows, meta, key) lock(lk) do realized_values[key] = df end From 01d42b735186cee41f2e1b8ab13d797b9ba4546e Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 22:19:29 -0600 Subject: [PATCH 72/90] add logging --- src/simulation/hdf_simulation_store.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 77ab196e26..e994650cff 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -309,6 +309,7 @@ function initialize_problem_storage!( group = _get_group_or_create(emulation_group, string(type)) for (key, reqs) in getfield(em_problem_reqs, type) name = encode_key_as_string(key) + @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. @@ -699,6 +700,7 @@ function _compute_chunk_count(dims, dtype; max_chunk_bytes = DEFAULT_MAX_CHUNK_B end function _create_dataset(group, name, reqs) + @error reqs["dims"] dataset = HDF5.create_dataset( group, name, From 49b736694b47c53b90601c4896cf9f7387d02494 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Fri, 29 Sep 2023 22:23:48 -0600 Subject: [PATCH 73/90] move logger --- src/simulation/hdf_simulation_store.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index e994650cff..c432972cca 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -257,6 +257,7 @@ function initialize_problem_storage!( for (key, reqs) in getfield(dm_problem_reqs[problem], type) !should_write_resulting_value(key) && continue name = encode_key_as_string(key) + @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. @@ -309,7 +310,6 @@ function initialize_problem_storage!( group = _get_group_or_create(emulation_group, string(type)) for (key, reqs) in getfield(em_problem_reqs, type) name = encode_key_as_string(key) - @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. From 739350ac090a7a42346a3a107a11ac982f6fee9d Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 2 Oct 2023 15:27:54 -0600 Subject: [PATCH 74/90] remove @error --- src/simulation/hdf_simulation_store.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index c432972cca..4794311282 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -257,7 +257,6 @@ function initialize_problem_storage!( for (key, reqs) in getfield(dm_problem_reqs[problem], type) !should_write_resulting_value(key) && continue name = encode_key_as_string(key) - @error name dataset = _create_dataset(group, name, reqs) # Columns can't be stored in attributes because they might be larger than # the max size of 64 KiB. @@ -700,7 +699,6 @@ function _compute_chunk_count(dims, dtype; max_chunk_bytes = DEFAULT_MAX_CHUNK_B end function _create_dataset(group, name, reqs) - @error reqs["dims"] dataset = HDF5.create_dataset( group, name, @@ -1055,6 +1053,7 @@ function _write_dataset!( array::Array{Float64, 4}, row_range::UnitRange{Int64}, ) + @show dataset[:, :, :, row_range] = array @debug "wrote dataset" dataset row_range return From 18a6362a7cb062d2ac2400eb43e638d8566712b5 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 2 Oct 2023 16:31:09 -0600 Subject: [PATCH 75/90] add logging --- src/simulation/hdf_simulation_store.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 4794311282..313759f01b 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -746,6 +746,8 @@ function _deserialize_attributes!(store::HdfSimulationStore) get_resolution(get_decision_model_params(store, model_name)) dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 2) + @error n_dims + @error dims item = HDF5Dataset{n_dims}( dataset, column_dataset, From ef5a6f91bc87bf360025875d9102fa40b45b9dc0 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 2 Oct 2023 17:48:13 -0600 Subject: [PATCH 76/90] add method for results deserialize --- src/core/dataset.jl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 1b3efe15f8..af3c20368b 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -255,6 +255,28 @@ function HDF5Dataset{2}( ) end +function HDF5Dataset{2}( + values::HDF5.Dataset, + column_dataset::HDF5.Dataset, + dims::NTuple{5, Int}, + resolution::Dates.Period, + initial_time::Dates.DateTime, +) + col1 = column_dataset[1:dims[2]] + col2 = column_dataset[(dims[2] + 1):end] + HDF5Dataset{2}( + values, + column_dataset, + 1, + 0, + resolution, + initial_time, + UNSET_INI_TIME, + (col1, col2), + ) +end + + function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) s.column_names return s.column_names From a6926adafa57ab97c835a6739d5ac829316d29a6 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 19 Oct 2023 13:52:37 -0600 Subject: [PATCH 77/90] enable 3D dataframes --- src/core/dataset.jl | 2 - src/core/results_by_time.jl | 13 ++ .../decision_model_simulation_results.jl | 140 ++++++++++++------ src/simulation/hdf_simulation_store.jl | 20 ++- 4 files changed, 128 insertions(+), 47 deletions(-) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index af3c20368b..457f601296 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -276,9 +276,7 @@ function HDF5Dataset{2}( ) end - function get_column_names(::OptimizationContainerKey, s::HDF5Dataset) - s.column_names return s.column_names end diff --git a/src/core/results_by_time.jl b/src/core/results_by_time.jl index 32d77ede00..f35267e0c3 100644 --- a/src/core/results_by_time.jl +++ b/src/core/results_by_time.jl @@ -89,6 +89,19 @@ function make_dataframe( return df end +function make_dataframe( + results::ResultsByTime{DenseAxisArray{Float64, 3}}, + timestamp::Dates.DateTime, +) + df = DataFrames.DataFrame() + array = results.data[timestamp] + for idx in Iterators.product(array.axes[1:2]...) + df[!, "$(idx)"] = array[idx..., :].data + end + # _add_timestamps!(df, results, timestamp, array) + return df +end + function make_dataframe(results::ResultsByTime{Matrix{Float64}}, timestamp::Dates.DateTime) array = results.data[timestamp] df = DataFrames.DataFrame(array, results.column_names) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 96cbd04289..8739bd1ca4 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -114,63 +114,115 @@ function get_forecast_horizon(res::SimulationProblemResults{DecisionModelSimulat end function _get_store_value( - ::Type{T}, res::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, timestamps, ::Nothing, -) where {T <: Union{Matrix{Float64}, DenseAxisArray{Float64, 2}}} +) simulation_store_path = joinpath(get_execution_path(res), "data_store") return open_store(HdfSimulationStore, simulation_store_path, "r") do store - _get_store_value(T, res, container_keys, timestamps, store) + _get_store_value(res, container_keys, timestamps, store) end end function _get_store_value( - ::Type{DenseAxisArray{Float64, 2}}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, timestamps, store::SimulationStore, ) - base_power = get_model_base_power(sim_results) - results_by_key = - Dict{OptimizationContainerKey, ResultsByTime{DenseAxisArray{Float64, 2}}}() + results_by_key = Dict{OptimizationContainerKey, ResultsByTime}() model_name = Symbol(get_model_name(sim_results)) + for ckey in container_keys + n_dims = get_number_of_dimensions(store, DecisionModelIndexType, model_name, ckey) + container_type = DenseAxisArray{Float64, n_dims + 1} + results_by_key[ckey] = _get_store_value(container_type, + sim_results, + ckey, + timestamps, store) + end + return results_by_key +end + +function _get_store_value( + ::Type{T}, + sim_results::SimulationProblemResults{DecisionModelSimulationResults}, + key::OptimizationContainerKey, + timestamps, + store::SimulationStore, +) where {T <: DenseAxisArray{Float64, 2}} resolution = get_resolution(sim_results) horizon = get_forecast_horizon(sim_results) + base_power = get_model_base_power(sim_results) + results_by_time = ResultsByTime( + key, + SortedDict{Dates.DateTime, T}(), + resolution, + get_column_names(store, DecisionModelIndexType, model_name, key), + ) + array_size::Union{Nothing, Tuple{Int, Int}} = nothing + for ts in timestamps + array = read_result(DenseAxisArray, store, model_name, key, ts) + if isnothing(array_size) + array_size = size(array) + elseif size(array) != array_size + error( + "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", + ) + end + if convert_result_to_natural_units(key) + array.data .*= base_power + end + if array_size[2] != horizon + @warn "$(encode_key_as_string(key)) has a different horizon than the " * + "problem specification. Can't assign timestamps to the resulting DataFrame." + results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) + end + results_by_time[ts] = array + end - for key in container_keys - results_by_time = ResultsByTime( - key, - SortedDict{Dates.DateTime, DenseAxisArray{Float64, 2}}(), - resolution, - get_column_names(store, DecisionModelIndexType, model_name, key), - ) - array_size::Union{Nothing, Tuple{Int, Int}} = nothing - for ts in timestamps - array = read_result(DenseAxisArray, store, model_name, key, ts) - if isnothing(array_size) - array_size = size(array) - elseif size(array) != array_size - error( - "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", - ) - end - if convert_result_to_natural_units(key) - array.data .*= base_power - end - if array_size[2] != horizon - @warn "$(encode_key_as_string(key)) has a different horizon than the " * - "problem specification. Can't assign timestamps to the resulting DataFrame." - results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) - end - results_by_time[ts] = array + return results_by_time +end + +function _get_store_value( + ::Type{T}, + sim_results::SimulationProblemResults{DecisionModelSimulationResults}, + key::OptimizationContainerKey, + timestamps, + store::SimulationStore, +) where {T <: DenseAxisArray{Float64, 3}} + resolution = get_resolution(sim_results) + horizon = get_forecast_horizon(sim_results) + base_power = get_model_base_power(sim_results) + model_name = Symbol(get_model_name(sim_results)) + results_by_time = ResultsByTime( + key, + SortedDict{Dates.DateTime, T}(), + resolution, + get_column_names(store, DecisionModelIndexType, model_name, key), + ) + array_size::Union{Nothing, Tuple{Int, Int, Int}} = nothing + for ts in timestamps + array = read_result(DenseAxisArray, store, model_name, key, ts) + if isnothing(array_size) + array_size = size(array) + elseif size(array) != array_size + error( + "Arrays for $(encode_key_as_string(key)) at different timestamps have different sizes", + ) end - results_by_key[key] = results_by_time + if convert_result_to_natural_units(key) + array.data .*= base_power + end + if array_size[3] != horizon + @warn "$(encode_key_as_string(key)) has a different horizon than the " * + "problem specification. Can't assign timestamps to the resulting DataFrame." + results_by_time.resolution = Dates.Period(Dates.Millisecond(0)) + end + results_by_time[ts] = array end - return results_by_key + return results_by_time end function _get_store_value( @@ -186,6 +238,12 @@ function _get_store_value( resolution = get_resolution(sim_results) for key in container_keys + n_dims = get_number_of_dimensions(store, DecisionModelIndexType, model_name, key) + if n_dims != 1 + error( + "The number of dimensions $(n_dims) is not supported for $(encode_key_as_string(key))", + ) + end results_by_time = ResultsByTime{Matrix{Float64}, 1}( key, SortedDict{Dates.DateTime, Matrix{Float64}}(), @@ -233,13 +291,13 @@ function _process_timestamps( end function _read_results( - ::Type{T}, res::SimulationProblemResults{DecisionModelSimulationResults}, result_keys, timestamps::Vector{Dates.DateTime}, store::Union{Nothing, <:SimulationStore}, -) where {T <: Union{Matrix{Float64}, DenseAxisArray{Float64, 2}}} - isempty(result_keys) && return Dict{OptimizationContainerKey, ResultsByTime{T}}() +) + isempty(result_keys) && + return Dict{OptimizationContainerKey, ResultsByTime{DenseAxisArray{Float64, 2}}}() if store === nothing && res.store !== nothing # In this case we have an InMemorySimulationStore. @@ -253,7 +311,7 @@ function _read_results( vals = Dict(k => cached_results[k] for k in result_keys) else @debug "reading results from data store" - vals = _get_store_value(T, res, result_keys, timestamps, store) + vals = _get_store_value(res, result_keys, timestamps, store) end return vals end @@ -285,9 +343,7 @@ function read_variable( ) key = _deserialize_key(VariableKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) - return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], - ) + return make_dataframes(_read_results(res, [key], timestamps, store)[key]) end """ diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index 313759f01b..fe235e6a79 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -459,6 +459,23 @@ function get_column_names( return get_column_names(key, dataset) end +function get_number_of_dimensions( + store::HdfSimulationStore, + i::Type{DecisionModelIndexType}, + model_name::Symbol, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + +function get_number_of_dimensions( + store::HdfSimulationStore, + i::Type{EmulationModelIndexType}, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + function get_emulation_model_dataset_size( store::HdfSimulationStore, key::OptimizationContainerKey, @@ -746,8 +763,6 @@ function _deserialize_attributes!(store::HdfSimulationStore) get_resolution(get_decision_model_params(store, model_name)) dims = (horizon, size(dataset)[2:end]..., size(dataset)[1]) n_dims = max(1, ndims(dataset) - 2) - @error n_dims - @error dims item = HDF5Dataset{n_dims}( dataset, column_dataset, @@ -1055,7 +1070,6 @@ function _write_dataset!( array::Array{Float64, 4}, row_range::UnitRange{Int64}, ) - @show dataset[:, :, :, row_range] = array @debug "wrote dataset" dataset row_range return From 616cc356b9f46b2a566b3e9beaeca1101d9f7310 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Thu, 19 Oct 2023 14:00:37 -0600 Subject: [PATCH 78/90] add missing methods --- .../decision_model_simulation_results.jl | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 8739bd1ca4..dbc8ef8bf2 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -125,6 +125,20 @@ function _get_store_value( end end +# TODO: Probably remove method later. It might be a duplicate. +function _get_store_value( + T::Type{Matrix{Float64}}, + res::SimulationProblemResults{DecisionModelSimulationResults}, + container_keys::Vector{<:OptimizationContainerKey}, + timestamps, + ::Nothing, +) + simulation_store_path = joinpath(get_execution_path(res), "data_store") + return open_store(HdfSimulationStore, simulation_store_path, "r") do store + _get_store_value(T, res, container_keys, timestamps, store) + end +end + function _get_store_value( sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, @@ -290,6 +304,17 @@ function _process_timestamps( return requested_range end +# Temporary workaround. To be removed +function _read_results( + T::Type{Matrix{Float64}}, + res::SimulationProblemResults{DecisionModelSimulationResults}, + result_keys, + timestamps::Vector{Dates.DateTime}, + store::Union{Nothing, <:SimulationStore}, +) + return _get_store_value(T, res, result_keys, timestamps, store) +end + function _read_results( res::SimulationProblemResults{DecisionModelSimulationResults}, result_keys, From 28cce32be174ee8f8d3eaa1b48e46ab45e6e047a Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 12:38:30 -0600 Subject: [PATCH 79/90] add comment to the code about the use of dims --- src/core/dataset.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 457f601296..940159f61d 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -241,7 +241,12 @@ function HDF5Dataset{2}( resolution::Dates.Period, initial_time::Dates.DateTime, ) + # The indexing is done in this way because we save all the names in an + # adjacent column entry in the HDF5 Datatset. The indexes for each column + # are known because we know how many elements are in each dimension. + # the names for the first column are store in the 1:first_column_number_of_elements. col1 = column_dataset[1:dims[2]] + # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names. col2 = column_dataset[(dims[2] + 1):end] HDF5Dataset{2}( values, From cbbfe612411246417025182c831b70be67439f14 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 12:40:56 -0600 Subject: [PATCH 80/90] remove unnecessary assertions --- src/parameters/update_parameters.jl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/parameters/update_parameters.jl b/src/parameters/update_parameters.jl index 0af903c683..bc81e0259c 100644 --- a/src/parameters/update_parameters.jl +++ b/src/parameters/update_parameters.jl @@ -155,10 +155,8 @@ function _update_parameter_values!( max_state_index = get_num_rows(state_data) if model_resolution < state_data.resolution t_step = 1 - elseif model_resolution >= state_data.resolution - t_step = model_resolution ÷ state_data.resolution else - @assert false + t_step = model_resolution ÷ state_data.resolution end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) @@ -200,10 +198,8 @@ function _update_parameter_values!( max_state_index = get_num_rows(state_data) if model_resolution < state_data.resolution t_step = 1 - elseif model_resolution >= state_data.resolution - t_step = model_resolution ÷ state_data.resolution else - @assert false + t_step = model_resolution ÷ state_data.resolution end state_data_index = find_timestamp_index(state_timestamps, current_time) sim_timestamps = range(current_time; step = model_resolution, length = time[end]) @@ -245,10 +241,8 @@ function _update_parameter_values!( max_state_index = get_num_rows(state_data) if model_resolution < state_data.resolution t_step = 1 - elseif model_resolution >= state_data.resolution - t_step = model_resolution ÷ state_data.resolution else - @assert false + t_step = model_resolution ÷ state_data.resolution end state_data_index = find_timestamp_index(state_timestamps, current_time) From 3302cc1fefef7c3e1a95135adebef490b4abe9b8 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 15:12:15 -0600 Subject: [PATCH 81/90] add missing variable in function --- src/simulation/decision_model_simulation_results.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index dbc8ef8bf2..98e8fabf0e 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -168,6 +168,7 @@ function _get_store_value( resolution = get_resolution(sim_results) horizon = get_forecast_horizon(sim_results) base_power = get_model_base_power(sim_results) + model_name = Symbol(get_model_name(sim_results)) results_by_time = ResultsByTime( key, SortedDict{Dates.DateTime, T}(), From 439f7b212bb6517627b0aaf621dc7ee9c96c8ee3 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 23 Oct 2023 16:38:23 -0600 Subject: [PATCH 82/90] fix tests --- .../decision_model_simulation_results.jl | 13 ++++--------- src/simulation/in_memory_simulation_store.jl | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 98e8fabf0e..94cba4295f 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -393,7 +393,7 @@ function read_dual( key = _deserialize_key(ConstraintKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -417,7 +417,7 @@ function read_parameter( key = _deserialize_key(ParameterKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -441,7 +441,7 @@ function read_aux_variable( key = _deserialize_key(AuxVarKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -465,7 +465,7 @@ function read_expression( key = _deserialize_key(ExpressionKey, res, args...) timestamps = _process_timestamps(res, initial_time, count) return make_dataframes( - _read_results(DenseAxisArray{Float64, 2}, res, [key], timestamps, store)[key], + _read_results(res, [key], timestamps, store)[key], ) end @@ -565,7 +565,6 @@ function load_results!( merge!( get_cached_variables(res), _read_results( - DenseAxisArray{Float64, 2}, res, variable_keys, res.results_timestamps, @@ -575,7 +574,6 @@ function load_results!( merge!( get_cached_duals(res), _read_results( - DenseAxisArray{Float64, 2}, res, dual_keys, res.results_timestamps, @@ -585,7 +583,6 @@ function load_results!( merge!( get_cached_parameters(res), _read_results( - DenseAxisArray{Float64, 2}, res, parameter_keys, res.results_timestamps, @@ -595,7 +592,6 @@ function load_results!( merge!( get_cached_aux_variables(res), _read_results( - DenseAxisArray{Float64, 2}, res, aux_variable_keys, res.results_timestamps, @@ -605,7 +601,6 @@ function load_results!( merge!( get_cached_expressions(res), _read_results( - DenseAxisArray{Float64, 2}, res, expression_keys, res.results_timestamps, diff --git a/src/simulation/in_memory_simulation_store.jl b/src/simulation/in_memory_simulation_store.jl index 59cd94de1a..7182bffe98 100644 --- a/src/simulation/in_memory_simulation_store.jl +++ b/src/simulation/in_memory_simulation_store.jl @@ -17,6 +17,24 @@ function InMemorySimulationStore() ) end +function get_number_of_dimensions( + store::InMemorySimulationStore, + i::Type{EmulationModelIndexType}, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + +function get_number_of_dimensions( + store::InMemorySimulationStore, + i::Type{DecisionModelIndexType}, + model_name::Symbol, + key::OptimizationContainerKey, +) + return length(get_column_names(store, i, model_name, key)) +end + + function open_store( func::Function, ::Type{InMemorySimulationStore}, From b41fda7b34bc7f506a8c0b06aa115f15082dc6d5 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 24 Oct 2023 11:11:47 -0600 Subject: [PATCH 83/90] add comment --- src/core/dataset.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/dataset.jl b/src/core/dataset.jl index 940159f61d..f263dfb711 100644 --- a/src/core/dataset.jl +++ b/src/core/dataset.jl @@ -267,7 +267,12 @@ function HDF5Dataset{2}( resolution::Dates.Period, initial_time::Dates.DateTime, ) + # The indexing is done in this way because we save all the names in an + # adjacent column entry in the HDF5 Datatset. The indexes for each column + # are known because we know how many elements are in each dimension. + # the names for the first column are store in the 1:first_column_number_of_elements. col1 = column_dataset[1:dims[2]] + # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names. col2 = column_dataset[(dims[2] + 1):end] HDF5Dataset{2}( values, From 84bb449a63e60c531c1fbe304eb70fa1b52b9e28 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 25 Oct 2023 14:27:07 -0600 Subject: [PATCH 84/90] add vector defs --- .../decision_model_simulation_results.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 94cba4295f..e3aca2ee3c 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -130,7 +130,7 @@ function _get_store_value( T::Type{Matrix{Float64}}, res::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, ::Nothing, ) simulation_store_path = joinpath(get_execution_path(res), "data_store") @@ -142,7 +142,7 @@ end function _get_store_value( sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) results_by_key = Dict{OptimizationContainerKey, ResultsByTime}() @@ -162,7 +162,7 @@ function _get_store_value( ::Type{T}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, key::OptimizationContainerKey, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) where {T <: DenseAxisArray{Float64, 2}} resolution = get_resolution(sim_results) @@ -203,7 +203,7 @@ function _get_store_value( ::Type{T}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, key::OptimizationContainerKey, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) where {T <: DenseAxisArray{Float64, 3}} resolution = get_resolution(sim_results) @@ -244,7 +244,7 @@ function _get_store_value( ::Type{Matrix{Float64}}, sim_results::SimulationProblemResults{DecisionModelSimulationResults}, container_keys::Vector{<:OptimizationContainerKey}, - timestamps, + timestamps::Vector{Dates.DateTime}, store::SimulationStore, ) base_power = get_model_base_power(sim_results) @@ -514,8 +514,8 @@ end function _are_results_cached( res::SimulationProblemResults{DecisionModelSimulationResults}, - output_keys, - timestamps, + output_keys::Vector{<:OptimizationContainerKey}, + timestamps::Vector{Dates.DateTime}, cached_keys, ) return isempty(setdiff(timestamps, res.results_timestamps)) && From dc6c53a3b831f101872d776f17da20a1cc874eab Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Wed, 25 Oct 2023 16:27:11 -0600 Subject: [PATCH 85/90] fix read results interface --- src/simulation/decision_model_simulation_results.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index e3aca2ee3c..7f98a4276d 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -311,8 +311,15 @@ function _read_results( res::SimulationProblemResults{DecisionModelSimulationResults}, result_keys, timestamps::Vector{Dates.DateTime}, - store::Union{Nothing, <:SimulationStore}, + store::Nothing, ) + isempty(result_keys) && + return Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}() + + if res.store !== nothing + # In this case we have an InMemorySimulationStore. + store = res.store + end return _get_store_value(T, res, result_keys, timestamps, store) end From ede614775614016762191eafb41bdcc8de8a7bba Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:18:40 -0700 Subject: [PATCH 86/90] remove not needed comments --- src/simulation/decision_model_simulation_results.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 7f98a4276d..961a21c201 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -125,7 +125,6 @@ function _get_store_value( end end -# TODO: Probably remove method later. It might be a duplicate. function _get_store_value( T::Type{Matrix{Float64}}, res::SimulationProblemResults{DecisionModelSimulationResults}, @@ -305,7 +304,6 @@ function _process_timestamps( return requested_range end -# Temporary workaround. To be removed function _read_results( T::Type{Matrix{Float64}}, res::SimulationProblemResults{DecisionModelSimulationResults}, From 73178c3754d521a083341ad20fa7acf055faa619 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:21:40 -0700 Subject: [PATCH 87/90] remove bad check --- src/simulation/hdf_simulation_store.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/simulation/hdf_simulation_store.jl b/src/simulation/hdf_simulation_store.jl index fe235e6a79..09007a7b07 100644 --- a/src/simulation/hdf_simulation_store.jl +++ b/src/simulation/hdf_simulation_store.jl @@ -347,9 +347,6 @@ end log_cache_hit_percentages(x::HdfSimulationStore) = log_cache_hit_percentages(x.cache) function _make_dataframe(data::Matrix{Float64}, columns::Tuple{Vector{String}}) - if (ndims(data) < 2 || size(data)[1] == 1) && size(data)[2] != size(columns)[1] - data = reshape(data, length(data), 1) - end return DataFrames.DataFrame(data, columns[1]; copycols = false) end From 31e1afcf2e3c476c437273cf113b29625584a723 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:21:44 -0700 Subject: [PATCH 88/90] formatter --- src/simulation/decision_model_simulation_results.jl | 2 +- src/simulation/in_memory_simulation_store.jl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simulation/decision_model_simulation_results.jl b/src/simulation/decision_model_simulation_results.jl index 961a21c201..f770ef3b12 100644 --- a/src/simulation/decision_model_simulation_results.jl +++ b/src/simulation/decision_model_simulation_results.jl @@ -312,7 +312,7 @@ function _read_results( store::Nothing, ) isempty(result_keys) && - return Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}() + return Dict{OptimizationContainerKey, ResultsByTime{Matrix{Float64}}}() if res.store !== nothing # In this case we have an InMemorySimulationStore. diff --git a/src/simulation/in_memory_simulation_store.jl b/src/simulation/in_memory_simulation_store.jl index 7182bffe98..37beb6f846 100644 --- a/src/simulation/in_memory_simulation_store.jl +++ b/src/simulation/in_memory_simulation_store.jl @@ -34,7 +34,6 @@ function get_number_of_dimensions( return length(get_column_names(store, i, model_name, key)) end - function open_store( func::Function, ::Type{InMemorySimulationStore}, From 901ec14d52905a5595bc99f5249ce3b6a8ff6260 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:31:17 -0700 Subject: [PATCH 89/90] re-enable threads --- src/simulation/realized_meta.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simulation/realized_meta.jl b/src/simulation/realized_meta.jl index cb373cd05a..379a6e6c52 100644 --- a/src/simulation/realized_meta.jl +++ b/src/simulation/realized_meta.jl @@ -87,8 +87,7 @@ function get_realization( lk = ReentrantLock() num_rows = length(meta.realized_timestamps) start = time() - #Threads.@threads - for key in collect(keys(results)) + Threads.@threads for key in collect(keys(results)) results_by_time = results[key] columns = get_column_names(results_by_time) df = _make_dataframe(columns, results_by_time, num_rows, meta, key) From ffd6238e3d87cdc6faac69ea369b167ec8f61dc9 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Tue, 21 Nov 2023 14:57:26 -0700 Subject: [PATCH 90/90] fix pretty table issue --- Project.toml | 2 +- src/utils/printing.jl | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index a41f96c0bc..ab805d1d1f 100644 --- a/Project.toml +++ b/Project.toml @@ -48,7 +48,7 @@ MathOptInterface = "1" PowerModels = "~0.19" PowerNetworkMatrices = "^0.9" PowerSystems = "^3" -PrettyTables = "^1.3, 2" +PrettyTables = "2" ProgressMeter = "^1.5" SHA = "0.7" Serialization = "1" diff --git a/src/utils/printing.jl b/src/utils/printing.jl index 47abf8cce6..8cdb139e29 100644 --- a/src/utils/printing.jl +++ b/src/utils/printing.jl @@ -84,7 +84,7 @@ function _show_method( PrettyTables.pretty_table( io, table; - noheader = true, + show_header = false, backend = Val(backend), title = "Duals", alignment = :l, @@ -176,7 +176,7 @@ function _show_method(io::IO, template::ProblemTemplate, backend::Symbol; kwargs io, table; backend = Val(backend), - noheader = true, + show_header = false, title = "Network Model", alignment = :l, kwargs..., @@ -338,7 +338,7 @@ function _show_method(io::IO, sequence::SimulationSequence, backend::Symbol; kwa io, table; backend = Val(backend), - noheader = true, + show_header = false, title = "Simulation Sequence", alignment = :l, kwargs..., @@ -432,7 +432,7 @@ function _show_method(io::IO, sim::Simulation, backend::Symbol; kwargs...) io, table; backend = Val(backend), - noheader = true, + show_header = false, title = "Simulation", alignment = :l, kwargs..., @@ -479,7 +479,7 @@ function _show_method(io::IO, results::SimulationResults, backend::Symbol; kwarg PrettyTables.pretty_table( io, table; - noheader = true, + show_header = false, backend = Val(backend), title = "Emulator Results", alignment = :l, @@ -534,7 +534,7 @@ function _show_method( PrettyTables.pretty_table( io, val; - noheader = true, + show_header = false, backend = Val(backend), title = "$name Problem $k Results", alignment = :l,