Jd/parameter update fixes

Project.toml

@@ -1,7 +1,7 @@
 name = "PowerSimulations"
 uuid = "e690365d-45e2-57bb-ac84-44ba829e73c4"
 authors = ["Jose Daniel Lara", "Clayton Barrows", "Daniel Thom", "Dheepak Krishnamurthy", "Sourabh Dalvi"]
-version = "0.23.3"
+version = "0.24.1"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -19,7 +19,6 @@ JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PowerModels = "c36e90e8-916a-50a6-bd94-075b64ef4655"
 PowerNetworkMatrices = "bed98974-b02a-5e2f-9fe0-a103f5c450dd"
 PowerSystems = "bcd98974-b02a-5e2f-9ee0-a103f5c450dd"
@@ -34,18 +33,25 @@ TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
 CSV = "~0.10"
 DataFrames = "1"
 DataStructures = "~0.18"
+Dates = "1"
+Distributed = "1"
 DocStringExtensions = "~v0.9"
 HDF5 = "~0.17"
 InfrastructureSystems = "^1.21"
+InteractiveUtils = "1"
 JSON = "0.21"
 JSON3 = "1"
 JuMP = "1"
+LinearAlgebra = "1"
+Logging = "1"
 MathOptInterface = "1"
 PowerModels = "~0.19"
 PowerNetworkMatrices = "^0.9"
 PowerSystems = "^3"
-PrettyTables = "^1.3, 2"
+PrettyTables = "2"
 ProgressMeter = "^1.5"
+SHA = "0.7"
+Serialization = "1"
 TimeSeries = "~0.23"
 TimerOutputs = "~0.5"
 julia = "^1.6"

docs/src/formulation_library/README.md

@@ -5,12 +5,12 @@ Formulation documentation should *roughly* follow the template established by Re
 ## Auto generated items
 
 - Valid DeviceModel table: just change the device category in the filter function
-- Time Series Parameters: just change the device category and formulation in the `get_defualt_time_series_names` method call
+- Time Series Parameters: just change the device category and formulation in the `get_default_time_series_names` method call
 
 
 ## Linked items
 
 - Formulations in the Valid DeviceModel table must have a docstring in src/core/formulations.jl
 - The Formulation in the @docs block must have a docstring in src/core/formulations.jl
 - The Variables must have docstrings in src/core/variables.jl 
-- The Time Series Paraemters must have docstrings in src/core/paramters.jl
+- The Time Series Parameters must have docstrings in src/core/parameters.jl

docs/src/modeler_guide/debugging_infeasible_models.md

@@ -1,6 +1,6 @@
 # Debugging infeasible models
 
-Getting infeasible solutions to models is a common ocurrence in operations simulations, there are multiple reasons why this can happen.
+Getting infeasible solutions to models is a common occurrence in operations simulations, there are multiple reasons why this can happen.
 `PowerSimulations.jl` has several tools to help debug this situation.
 
 ## Adding slacks to the model

docs/src/quick_start_guide.md

@@ -1,6 +1,6 @@
 # Quick Start Guide
 
-* **Julia:** If this is your first time using Julia visit our [Introduction to Julia](https://nrel-Sienna.github.io/SIIP-Tutorial/fundamentals/introduction-to-julia/) and the official[Getting started with Julia](https://julialang.org/learning/).
+* **Julia:** If this is your first time using Julia visit our [Introduction to Julia](https://nrel-Sienna.github.io/SIIP-Tutorial/fundamentals/introduction-to-julia/) and the official [Getting started with Julia](https://julialang.org/learning/).
 * **Package Installation:** If you want to install packages check the [Package Manager](https://pkgdocs.julialang.org/v1/environments/) instructions, or you can refer to the [PowerSimulations installation instructions](@ref Installation).
 * **PowerSystems:** [PowerSystems.jl](https://github.com/nrel-Sienna/PowerSystems.jl) manages the data and is a fundamental dependency of PowerSimulations.jl. Check the [PowerSystems.jl Basics Tutorial](https://nrel-sienna.github.io/PowerSystems.jl/stable/tutorials/basics/) and [PowerSystems.jl documentation](https://nrel-Sienna.github.io/PowerSystems.jl/stable/) to understand how the inputs to the models are organized.
 * **Dataset Library:** If you don't have a data set to start using `PowerSimulations.jl` check the test systems provided in [`PowerSystemCaseBuilder.jl`](https://nrel-sienna.github.io/PowerSystems.jl/stable/tutorials/powersystembuilder/)

docs/src/tutorials/adding_new_problem_model.md

@@ -36,7 +36,7 @@ It is possible to define a Custom Decision Problem that gives the user full cont
 struct MyCustomDecisionProblem <: PSI.DecisionProblem end
 ```
 
-Alternatevely, it is possible to define a Custom Decision Problem subtyping from `DefaultDecisionProblem` which imposes more requirements and structure onto the developer but employs more checks and validations in the process. Be aware that this route will decrease the flexibility of what can be done inside the custom model.
+Alternatively, it is possible to define a Custom Decision Problem subtyping from `DefaultDecisionProblem` which imposes more requirements and structure onto the developer but employs more checks and validations in the process. Be aware that this route will decrease the flexibility of what can be done inside the custom model.
 
 ```julia
 struct MyCustomDecisionProblem <: PSI.DefaultDecisionProblem end

src/PowerSimulations.jl

@@ -77,9 +77,7 @@ export ReserveLimitedRegulation
 export UpperBoundFeedforward
 export LowerBoundFeedforward
 export SemiContinuousFeedforward
-export EnergyLimitFeedforward
 export FixValueFeedforward
-export EnergyTargetFeedforward
 
 # InitialConditions chrons
 export InterProblemChronology

src/core/dataset.jl

@@ -20,9 +20,9 @@
 
 # Values field is accessed with dot syntax to avoid type instability
 
-mutable struct InMemoryDataset <: AbstractDataset
-    "Data with dimensions (column names, row indexes)"
-    values::DenseAxisArray{Float64, 2}
+mutable struct InMemoryDataset{N} <: AbstractDataset
+    "Data with dimensions (N column names, row indexes)"
+    values::DenseAxisArray{Float64, N}
     # We use Array here to allow for overwrites when updating the state
     timestamps::Vector{Dates.DateTime}
     # Resolution is needed because AbstractDataset might have just one row
@@ -33,12 +33,12 @@
 end
 
 function InMemoryDataset(
-    values::DenseAxisArray{Float64, 2},
+    values::DenseAxisArray{Float64, N},
     timestamps::Vector{Dates.DateTime},
     resolution::Dates.Millisecond,
     end_of_step_index::Int,
-)
-    return InMemoryDataset(
+) where {N}
+    return InMemoryDataset{N}(
         values,
         timestamps,
         resolution,
@@ -48,8 +48,8 @@
     )
 end
 
-function InMemoryDataset(values::DenseAxisArray{Float64, 2})
-    return InMemoryDataset(
+function InMemoryDataset(values::DenseAxisArray{Float64, N}) where {N}
+    return InMemoryDataset{N}(
         values,
         Vector{Dates.DateTime}(),
         Dates.Second(0.0),
@@ -59,34 +59,95 @@
     )
 end
 
-get_num_rows(s::InMemoryDataset) = size(s.values)[2]
+# Helper method for one dimensional cases
+function InMemoryDataset(
+    fill_val::Float64,
+    initial_time::Dates.DateTime,
+    resolution::Dates.Millisecond,
+    end_of_step_index::Int,
+    row_count::Int,
+    column_names::Vector{String})
+    return InMemoryDataset(
+        fill_val,
+        initial_time,
+        resolution,
+        end_of_step_index,
+        row_count,
+        (column_names,),
+    )
+end
+
+function InMemoryDataset(
+    fill_val::Float64,
+    initial_time::Dates.DateTime,
+    resolution::Dates.Millisecond,
+    end_of_step_index::Int,
+    row_count::Int,
+    column_names::NTuple{N, <:Any}) where {N}
+    return InMemoryDataset(
+        fill!(
+            DenseAxisArray{Float64}(undef, column_names..., 1:row_count),
+            fill_val,
+        ),
+        collect(
+            range(
+                initial_time;
+                step = resolution,
+                length = row_count,
+            ),
+        ),
+        resolution,
+        end_of_step_index,
+    )
+end
+
+get_num_rows(s::InMemoryDataset{N}) where {N} = size(s.values)[N]
 
 function make_system_state(
-    values::DenseAxisArray{Float64, 2},
     timestamp::Dates.DateTime,
     resolution::Dates.Millisecond,
-)
-    return InMemoryDataset(values, [timestamp], resolution, 0, 1, UNSET_INI_TIME)
+    columns::NTuple{N, <:Any},
+) where {N}
+    return InMemoryDataset(NaN, timestamp, resolution, 0, 1, columns)
 end
 
-function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime)
+function get_dataset_value(
+    s::T,
+    date::Dates.DateTime,
+) where {T <: Union{InMemoryDataset{1}, InMemoryDataset{2}}}
     s_index = find_timestamp_index(s.timestamps, date)
     if isnothing(s_index)
         error("Request time stamp $date not in the state")
     end
     return s.values[:, s_index]
 end
 
-get_column_names(s::InMemoryDataset) = axes(s.values)[1]
-get_column_names(::OptimizationContainerKey, s::InMemoryDataset) = get_column_names(s)
+function get_dataset_value(s::InMemoryDataset{3}, date::Dates.DateTime)
+    s_index = find_timestamp_index(s.timestamps, date)
+    if isnothing(s_index)
+        error("Request time stamp $date not in the state")
+    end
+    return s.values[:, :, s_index]
+end
+
+function get_column_names(k::OptimizationContainerKey, s::InMemoryDataset)
+    return get_column_names(k, s.values)
+end
 
-function get_last_recorded_value(s::InMemoryDataset)
+function get_last_recorded_value(s::InMemoryDataset{2})
     if get_last_recorded_row(s) == 0
         error("The Dataset hasn't been written yet")
     end
     return s.values[:, get_last_recorded_row(s)]
 end
 
+function get_last_recorded_value(s::InMemoryDataset{3})
+    if get_last_recorded_row(s) == 0
+        error("The Dataset hasn't been written yet")
+    end
+    return s.values[:, :, get_last_recorded_row(s)]
+end
+
 function get_end_of_step_timestamp(s::InMemoryDataset)
     return s.timestamps[s.end_of_step_index]
 end
@@ -110,50 +171,124 @@
     return s.timestamps[s_index]
 end
 
-function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 2}, index::Int)
+# These set_value! methods expect a single time_step value because they are used to update
+#the state so the incoming vals will have one dimension less than the DataSet. The exception
+# is for vals of Dimension 1 which are still stored in DataSets of dimension 2.
+function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 2}, index::Int)
     s.values[:, index] = vals[:, index]
     return
 end
 
-function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 1}, index::Int)
+function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 1}, index::Int)
     s.values[:, index] = vals
     return
 end
 
+function set_value!(s::InMemoryDataset{3}, vals::DenseAxisArray{Float64, 2}, index::Int)
+    s.values[:, :, index] = vals
+    return
+end
+
 # HDF5Dataset does not account of overwrites in the data. Values are written sequentially.
-mutable struct HDF5Dataset <: AbstractDataset
+mutable struct HDF5Dataset{N} <: AbstractDataset
     values::HDF5.Dataset
     column_dataset::HDF5.Dataset
     write_index::Int
     last_recorded_row::Int
     resolution::Dates.Millisecond
     initial_timestamp::Dates.DateTime
     update_timestamp::Dates.DateTime
-    column_names::Vector{String}
+    column_names::NTuple{N, Vector{String}}
 
-    function HDF5Dataset(values, column_dataset, write_index, last_recorded_row, resolution,
+    function HDF5Dataset{N}(values,
+        column_dataset,
+        write_index,
+        last_recorded_row,
+        resolution,
         initial_timestamp,
-        update_timestamp, column_names,
-    )
-        new(values, column_dataset, write_index, last_recorded_row, resolution,
+        update_timestamp,
+        column_names::NTuple{N, Vector{String}},
+    ) where {N}
+        new{N}(values, column_dataset, write_index, last_recorded_row, resolution,
             initial_timestamp,
             update_timestamp, column_names)
     end
 end
 
-HDF5Dataset(values, column_dataset, resolution, initial_time) =
-    HDF5Dataset(
+function HDF5Dataset{1}(
+    values::HDF5.Dataset,
+    column_dataset::HDF5.Dataset,
+    ::Tuple,
+    resolution::Dates.Millisecond,
+    initial_time::Dates.DateTime,
+)
+    HDF5Dataset{1}(
         values,
         column_dataset,
         1,
         0,
         resolution,
         initial_time,
         UNSET_INI_TIME,
-        column_dataset[:],
+        (column_dataset[:],),
     )
+end
 
-get_column_names(::OptimizationContainerKey, s::HDF5Dataset) = s.column_names
+function HDF5Dataset{2}(
+    values::HDF5.Dataset,
+    column_dataset::HDF5.Dataset,
+    dims::NTuple{4, Int},
+    resolution::Dates.Period,
+    initial_time::Dates.DateTime,
+)
+    # The indexing is done in this way because we save all the names in an
+    # adjacent column entry in the HDF5 Datatset. The indexes for each column
+    # are known because we know how many elements are in each dimension.
+    # the names for the first column are store in the 1:first_column_number_of_elements.
+    col1 = column_dataset[1:dims[2]]
+    # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names.
+    col2 = column_dataset[(dims[2] + 1):end]
+    HDF5Dataset{2}(
+        values,
+        column_dataset,
+        1,
+        0,
+        resolution,
+        initial_time,
+        UNSET_INI_TIME,
+        (col1, col2),
+    )
+end
+
+function HDF5Dataset{2}(
+    values::HDF5.Dataset,
+    column_dataset::HDF5.Dataset,
+    dims::NTuple{5, Int},
+    resolution::Dates.Period,
+    initial_time::Dates.DateTime,
+)
+    # The indexing is done in this way because we save all the names in an
+    # adjacent column entry in the HDF5 Datatset. The indexes for each column
+    # are known because we know how many elements are in each dimension.
+    # the names for the first column are store in the 1:first_column_number_of_elements.
+    col1 = column_dataset[1:dims[2]]
+    # the names for the second column are store in the first_column_number_of elements + 1:end of the column with the names.
+    col2 = column_dataset[(dims[2] + 1):end]
+    HDF5Dataset{2}(
+        values,
+        column_dataset,
+        1,
+        0,
+        resolution,
+        initial_time,
+        UNSET_INI_TIME,
+        (col1, col2),
+    )
+end
+
+function get_column_names(::OptimizationContainerKey, s::HDF5Dataset)
+    return s.column_names
+end
 
 """
 Return the timestamp from most recent data row updated in the dataset. This value may not be the same as the result from `get_update_timestamp`

src/core/definitions.jl

@@ -24,6 +24,7 @@ const JuMPVariableMatrix = DenseAxisArray{
         JuMP.Containers._AxisLookup{Tuple{Int64, Int64}},
     },
 }
+const JuMPFloatMatrix = DenseAxisArray{Float64, 2}
 const JuMPFloatArray = DenseAxisArray{Float64}
 const JuMPVariableArray = DenseAxisArray{JuMP.VariableRef}
 
@@ -46,7 +47,7 @@ const MAX_START_STAGES = 3
 const OBJECTIVE_FUNCTION_POSITIVE = 1.0
 const OBJECTIVE_FUNCTION_NEGATIVE = -1.0
 const INITIALIZATION_PROBLEM_HORIZON = 3
-# The DEFAULT_RESERVE_COST value is used to avoid degeneracy of the solutions, reseve cost isn't provided.
+# The DEFAULT_RESERVE_COST value is used to avoid degeneracy of the solutions, reserve cost isn't provided.
 const DEFAULT_RESERVE_COST = 1.0
 const KiB = 1024
 const MiB = KiB * KiB