From 55d1455f7a6d52d86ae4bfffcc59d8951edfc877 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 2 Apr 2024 08:30:13 -0600 Subject: [PATCH 01/19] feat(time-series): Re-design time series management - Store time series metadata in a SQLite database instead of per-component dictionaries. This allows system-wide SQL queries instead of looping across component dictionaries. - Consolidate management of time series in TimeSeriesManager instead of individual time series storage implementations. - Support addition of user-defined features to time series arrays. - Add support for different time series resolutions. --- Project.toml | 20 +- src/InfrastructureSystems.jl | 24 +- src/abstract_time_series.jl | 12 + src/component.jl | 245 ++--- src/components.jl | 14 +- src/containers.jl | 33 +- src/descriptors/structs.json | 24 + src/deterministic_metadata.jl | 3 +- src/generated/DeterministicMetadata.jl | 16 +- src/generated/ProbabilisticMetadata.jl | 16 +- src/generated/ScenariosMetadata.jl | 16 +- src/generated/SingleTimeSeriesMetadata.jl | 16 +- src/generated/includes.jl | 2 + src/hdf5_time_series_storage.jl | 437 ++------- src/in_memory_time_series_storage.jl | 125 +-- src/probabilistic.jl | 3 +- src/scenarios.jl | 3 +- src/serialization.jl | 12 - src/single_time_series.jl | 3 +- src/supplemental_attribute.jl | 60 +- src/supplemental_attributes.jl | 22 +- src/system_data.jl | 481 ++++------ src/time_series_cache.jl | 6 +- src/time_series_container.jl | 126 +-- src/time_series_formats.jl | 6 +- src/time_series_interface.jl | 523 +++++------ src/time_series_manager.jl | 289 ++++++ src/time_series_metadata_store.jl | 1012 +++++++++++++++++++++ src/time_series_parameters.jl | 254 ------ src/time_series_storage.jl | 9 +- src/time_series_structs.jl | 82 ++ src/utils/print.jl | 90 +- src/utils/test.jl | 2 +- src/utils/utils.jl | 4 +- test/test_components.jl | 24 +- test/test_printing.jl | 2 +- test/test_serialization.jl | 38 +- test/test_supplemental_attributes.jl | 14 +- test/test_system_data.jl | 120 ++- test/test_time_series.jl | 517 ++++++----- test/test_time_series_cache.jl | 4 +- test/test_time_series_storage.jl | 74 +- test/test_utils.jl | 3 +- 43 files changed, 2645 insertions(+), 2141 deletions(-) create mode 100644 src/time_series_manager.jl create mode 100644 src/time_series_metadata_store.jl create mode 100644 src/time_series_structs.jl diff --git a/Project.toml b/Project.toml index 3e5649d34..bdc8012ef 100644 --- a/Project.toml +++ b/Project.toml @@ -18,8 +18,11 @@ Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -29,21 +32,24 @@ YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" CSV = "0.9, 0.10" DataFrames = "~1.6" DataStructures = "~0.18" +Dates = "1" DocStringExtensions = "0.8, 0.9" H5Zblosc = "0.1" HDF5 = "0.17" +InteractiveUtils = "1" JSON3 = "^1.11" +Logging = "1" Mustache = "1" +Pkg = "1" PrettyTables = "^1.3, 2" +Random = "1" +SHA = "0.7" +SQLite = "^1.6" StructTypes = "^1.9" +TOML = "1" +Tables = "^1.11" TerminalLoggers = "~0.1" TimeSeries = "0.23, 0.24" +UUIDs = "1" YAML = "~0.4" julia = "^1.6" -Dates = "1" -InteractiveUtils = "1" -Logging = "1" -Pkg = "1" -Random = "1" -TOML = "1" -UUIDs = "1" diff --git a/src/InfrastructureSystems.jl b/src/InfrastructureSystems.jl index 5c0c18371..f45288c0a 100644 --- a/src/InfrastructureSystems.jl +++ b/src/InfrastructureSystems.jl @@ -4,17 +4,21 @@ module InfrastructureSystems import CSV import DataFrames +import DataFrames: DataFrame import Dates import JSON3 import Logging import Random import Pkg import PrettyTables +import SHA import StructTypes import TerminalLoggers: TerminalLogger, ProgressLevel import TimeSeries import TOML using DataStructures: OrderedDict, SortedDict +import SQLite +import Tables using DocStringExtensions @@ -115,17 +119,11 @@ include("time_series_storage.jl") include("abstract_time_series.jl") include("forecasts.jl") include("static_time_series.jl") -include("time_series_container.jl") -include("time_series_parser.jl") include("containers.jl") include("component_uuids.jl") -include("supplemental_attribute.jl") -include("supplemental_attributes_container.jl") -include("supplemental_attributes.jl") -include("components.jl") -include("iterators.jl") include("geographic_supplemental_attribute.jl") include("generated/includes.jl") +include("time_series_parser.jl") include("single_time_series.jl") include("deterministic_single_time_series.jl") include("deterministic.jl") @@ -134,16 +132,24 @@ include("scenarios.jl") include("deterministic_metadata.jl") include("hdf5_time_series_storage.jl") include("in_memory_time_series_storage.jl") +include("time_series_structs.jl") include("time_series_formats.jl") +include("time_series_metadata_store.jl") +include("time_series_manager.jl") +include("time_series_interface.jl") +include("time_series_container.jl") include("time_series_cache.jl") -include("time_series_parameters.jl") include("time_series_utils.jl") +include("supplemental_attribute.jl") +include("supplemental_attributes_container.jl") +include("supplemental_attributes.jl") +include("components.jl") +include("iterators.jl") include("component.jl") include("results.jl") include("serialization.jl") include("system_data.jl") include("subsystems.jl") -include("time_series_interface.jl") include("validation.jl") include("utils/print.jl") include("utils/test.jl") diff --git a/src/abstract_time_series.jl b/src/abstract_time_series.jl index e1955a55a..675e47d5e 100644 --- a/src/abstract_time_series.jl +++ b/src/abstract_time_series.jl @@ -26,3 +26,15 @@ abstract type TimeSeriesData <: InfrastructureSystemsComponent end # - get_resolution # - make_time_array # - eltype_data + +abstract type AbstractTimeSeriesParameters <: InfrastructureSystemsType end + +struct StaticTimeSeriesParameters <: AbstractTimeSeriesParameters end + +Base.@kwdef struct ForecastParameters <: AbstractTimeSeriesParameters + horizon::Int + initial_timestamp::Dates.DateTime + interval::Dates.Period + count::Int + resolution::Dates.Period +end diff --git a/src/component.jl b/src/component.jl index 8a48d5f82..01367b44a 100644 --- a/src/component.jl +++ b/src/component.jl @@ -2,11 +2,8 @@ This function must be called when a component is removed from a system. """ function prepare_for_removal!(component::InfrastructureSystemsComponent) - # TimeSeriesContainer can only be part of a component when that component is part of a - # system. - clear_time_series_storage!(component) - set_time_series_storage!(component, nothing) clear_time_series!(component) + set_time_series_manager!(component, nothing) @debug "cleared all time series data from" _group = LOG_GROUP_SYSTEM get_name(component) return end @@ -21,37 +18,31 @@ Call `collect` on the result to get an array. # Arguments - - `component::InfrastructureSystemsComponent`: component from which to get time_series + - `owner::InfrastructureSystemsComponent`: component or attribute from which to get time_series - `filter_func = nothing`: Only return time_series for which this returns true. - `type = nothing`: Only return time_series with this type. - `name = nothing`: Only return time_series matching this value. """ function get_time_series_multiple( - component::InfrastructureSystemsComponent, + owner::TimeSeriesOwners, filter_func = nothing; type = nothing, - start_time = nothing, name = nothing, ) - container = get_time_series_container(component) - storage = _get_time_series_storage(component) + throw_if_does_not_support_time_series(owner) + mgr = get_time_series_manager(owner) + # This is true when the component is not part of a system. + isnothing(mgr) && return () + storage = get_time_series_storage(owner) Channel() do channel - for key in keys(container.data) - ts_metadata = container.data[key] - ts_type = time_series_metadata_to_data(ts_metadata) - if !isnothing(type) && !(ts_type <: type) - continue - end - if !isnothing(name) && key.name != name - continue - end + for metadata in list_metadata(mgr, owner; time_series_type = type, name = name) ts = deserialize_time_series( - ts_type, + isnothing(type) ? time_series_metadata_to_data(metadata) : type, storage, - ts_metadata, - UnitRange(1, length(ts_metadata)), - UnitRange(1, get_count(ts_metadata)), + metadata, + UnitRange(1, length(metadata)), + UnitRange(1, get_count(metadata)), ) if !isnothing(filter_func) && !filter_func(ts) continue @@ -61,56 +52,6 @@ function get_time_series_multiple( end end -""" -Returns an iterator of TimeSeriesMetadata instances attached to the component. -""" -function get_time_series_multiple( - ::Type{TimeSeriesMetadata}, - component::InfrastructureSystemsComponent, -) - container = get_time_series_container(component) - Channel() do channel - for key in keys(container.data) - put!(channel, container.data[key]) - end - end -end - -function get_time_series_with_metadata_multiple( - component::InfrastructureSystemsComponent, - filter_func = nothing; - type = nothing, - start_time = nothing, - name = nothing, -) - container = get_time_series_container(component) - storage = _get_time_series_storage(component) - - Channel() do channel - for key in keys(container.data) - ts_metadata = container.data[key] - ts_type = time_series_metadata_to_data(ts_metadata) - if !isnothing(type) && !(ts_type <: type) - continue - end - if !isnothing(name) && key.name != name - continue - end - ts = deserialize_time_series( - ts_type, - storage, - ts_metadata, - UnitRange(1, length(ts_metadata)), - UnitRange(1, get_count(ts_metadata)), - ) - if !isnothing(filter_func) && !filter_func(ts) - continue - end - put!(channel, (ts, ts_metadata)) - end - end -end - """ Transform all instances of SingleTimeSeries to DeterministicSingleTimeSeries. Do nothing if the component does not contain any instances. @@ -122,40 +63,38 @@ Return true if a transformation occurs. function transform_single_time_series_internal!( component::InfrastructureSystemsComponent, ::Type{T}, - params::TimeSeriesParameters, + horizon::Int, + interval::Dates.Period, ) where {T <: DeterministicSingleTimeSeries} - container = get_time_series_container(component) + mgr = get_time_series_manager(component) metadata_to_add = [] - for ts_metadata in values(container.data) - if ts_metadata isa SingleTimeSeriesMetadata - resolution = get_resolution(ts_metadata) - _params = _get_single_time_series_transformed_parameters( - ts_metadata, - T, - params.forecast_params.horizon, - params.forecast_params.interval, - ) - check_params_compatibility(params, _params) - new_metadata = DeterministicMetadata(; - name = get_name(ts_metadata), - resolution = params.resolution, - initial_timestamp = params.forecast_params.initial_timestamp, - interval = params.forecast_params.interval, - count = params.forecast_params.count, - time_series_uuid = get_time_series_uuid(ts_metadata), - horizon = params.forecast_params.horizon, - time_series_type = DeterministicSingleTimeSeries, - scaling_factor_multiplier = get_scaling_factor_multiplier(ts_metadata), - internal = get_internal(ts_metadata), - ) - push!(metadata_to_add, new_metadata) - end + for metadata in list_metadata(mgr, component; time_series_type = SingleTimeSeries) + params = _get_single_time_series_transformed_parameters( + metadata, + T, + horizon, + interval, + ) + check_params_compatibility(mgr.metadata_store, params) + new_metadata = DeterministicMetadata(; + name = get_name(metadata), + resolution = get_resolution(metadata), + initial_timestamp = params.initial_timestamp, + interval = params.interval, + count = params.count, + time_series_uuid = get_time_series_uuid(metadata), + horizon = params.horizon, + time_series_type = DeterministicSingleTimeSeries, + scaling_factor_multiplier = get_scaling_factor_multiplier(metadata), + internal = get_internal(metadata), + ) + push!(metadata_to_add, new_metadata) end isempty(metadata_to_add) && return false for new_metadata in metadata_to_add - add_time_series!(container, new_metadata) + add_metadata!(mgr.metadata_store, component, new_metadata) @debug "Added $new_metadata." _group = LOG_GROUP_TIME_SERIES end @@ -168,29 +107,27 @@ function get_single_time_series_transformed_parameters( horizon::Int, interval::Dates.Period, ) where {T <: Forecast} - container = get_time_series_container(component) - for (key, ts_metadata) in container.data - if ts_metadata isa SingleTimeSeriesMetadata - return _get_single_time_series_transformed_parameters( - ts_metadata, - T, - horizon, - interval, - ) - end + mgr = get_time_series_manager(component) + for metadata in list_metadata(mgr, component; time_series_type = SingleTimeSeries) + return _get_single_time_series_transformed_parameters( + metadata, + T, + horizon, + interval, + ) end return end function _get_single_time_series_transformed_parameters( - ts_metadata::SingleTimeSeriesMetadata, + metadata::SingleTimeSeriesMetadata, ::Type{T}, horizon::Int, interval::Dates.Period, ) where {T <: Forecast} - resolution = get_resolution(ts_metadata) - len = length(ts_metadata) + resolution = get_resolution(metadata) + len = length(metadata) if len < horizon throw( ConflictingInputsError("existing length=$len is shorter than horizon=$horizon"), @@ -209,87 +146,23 @@ function _get_single_time_series_transformed_parameters( ) end - initial_timestamp = get_initial_timestamp(ts_metadata) - return TimeSeriesParameters(initial_timestamp, resolution, len, horizon, interval) -end - -function clear_time_series_storage!(component::InfrastructureSystemsComponent) - storage = _get_time_series_storage(component) - if !isnothing(storage) - # In the case of Deterministic and DeterministicSingleTimeSeries the UUIDs - # can be shared. - uuids = Set{Base.UUID}() - for (uuid, name) in get_time_series_uuids(component) - if !(uuid in uuids) - remove_time_series!(storage, uuid, get_uuid(component), name) - push!(uuids, uuid) - end - end - end - return -end - -function set_time_series_storage!( - component::InfrastructureSystemsComponent, - storage::Union{Nothing, TimeSeriesStorage}, -) - container = get_time_series_container(component) - if !isnothing(container) - set_time_series_storage!(container, storage) - end - return -end - -function _get_time_series_storage(component::InfrastructureSystemsComponent) - container = get_time_series_container(component) - if isnothing(container) - return nothing - end - - return container.time_series_storage -end - -function get_time_series_by_key( - key::TimeSeriesKey, - component::InfrastructureSystemsComponent; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Nothing, Int} = nothing, - count::Union{Nothing, Int} = nothing, -) - container = get_time_series_container(component) - ts_metadata = container.data[key] - ts_type = time_series_metadata_to_data(ts_metadata) - return get_time_series( - ts_type, - component, - key.name; - start_time = start_time, - len = len, + initial_timestamp = get_initial_timestamp(metadata) + count = get_forecast_window_count(initial_timestamp, interval, resolution, len, horizon) + return ForecastParameters(; + initial_timestamp = initial_timestamp, count = count, + horizon = horizon, + interval = interval, + resolution = resolution, ) end function assign_new_uuid_internal!(component::InfrastructureSystemsComponent) old_uuid = get_uuid(component) new_uuid = make_uuid() - if has_time_series(component) - container = get_time_series_container(component) - # There may be duplicates because of transform operations. - changed_uuids = Set{Tuple{Base.UUID, String}}() - for (key, ts_metadata) in container.data - ts_uuid = get_time_series_uuid(ts_metadata) - changed_uuid = (ts_uuid, key.name) - if !in(changed_uuid, changed_uuids) - replace_component_uuid!( - container.time_series_storage, - ts_uuid, - old_uuid, - new_uuid, - key.name, - ) - push!(changed_uuids, changed_uuid) - end - end + mgr = get_time_series_manager(component) + if !isnothing(mgr) + replace_component_uuid!(mgr.metadata_store, old_uuid, new_uuid) end set_uuid!(get_internal(component), new_uuid) diff --git a/src/components.jl b/src/components.jl index 66ed52bdd..0af8a5ce7 100644 --- a/src/components.jl +++ b/src/components.jl @@ -2,21 +2,21 @@ const ComponentsByType = Dict{DataType, Dict{String, <:InfrastructureSystemsComp struct Components <: InfrastructureSystemsContainer data::ComponentsByType - time_series_storage::TimeSeriesStorage + time_series_manager::TimeSeriesManager validation_descriptors::Vector end get_member_string(::Components) = "components" function Components( - time_series_storage::TimeSeriesStorage, + time_series_manager::TimeSeriesManager, validation_descriptors = nothing, ) if isnothing(validation_descriptors) validation_descriptors = Vector() end - return Components(ComponentsByType(), time_series_storage, validation_descriptors) + return Components(ComponentsByType(), time_series_manager, validation_descriptors) end function _add_component!( @@ -42,7 +42,7 @@ function _add_component!( throw(ArgumentError("cannot add a component with time_series: $component")) end - set_time_series_storage!(component, components.time_series_storage) + set_time_series_manager!(component, components.time_series_manager) components.data[T][component_name] = component return end @@ -309,10 +309,6 @@ function iterate_components(components::Components) iterate_container(components) end -function iterate_components_with_time_series(components::Components) - iterate_container_with_time_series(components) -end - function get_num_components(components::Components) return get_num_members(components) end @@ -366,7 +362,7 @@ function compare_values( for name in fieldnames(Components) name in exclude && continue # This gets validated in SystemData. - name == :time_series_storage && continue + name == :time_series_manager && continue val_x = getfield(x, name) val_y = getfield(y, name) if !compare_values(val_x, val_y; compare_uuids = compare_uuids, exclude = exclude) diff --git a/src/containers.jl b/src/containers.jl index c3705ae7b..19b991abf 100644 --- a/src/containers.jl +++ b/src/containers.jl @@ -11,38 +11,9 @@ end Iterates over all data in the container. """ function iterate_container(container::InfrastructureSystemsContainer) - Channel() do channel - for m_dict in values(container.data) - for member in values(m_dict) - put!(channel, member) - end - end - end -end - -function iterate_container_with_time_series(container::InfrastructureSystemsContainer) - Channel() do channel - for m_dict in values(container.data) - for member in values(m_dict) - if has_time_series(member) - put!(channel, member) - end - end - end - end + return (y for x in values(container.data) for y in values(x)) end function get_num_members(container::InfrastructureSystemsContainer) - count = 0 - for members in values(container.data) - count += length(members) - end - return count -end - -function clear_time_series!(container::InfrastructureSystemsContainer) - for member in iterate_components_with_time_series(container) - clear_time_series!(member) - end - return + return mapreduce(length, +, values(container.data); init = 0) end diff --git a/src/descriptors/structs.json b/src/descriptors/structs.json index d09fb6819..0816406e7 100644 --- a/src/descriptors/structs.json +++ b/src/descriptors/structs.json @@ -49,6 +49,12 @@ "default": "nothing", "comment": "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." }, + { + "name": "features", + "data_type": "Dict{String, <:Any}", + "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "default": "Dict{String, Any}()" + }, { "name": "internal", "data_type": "InfrastructureSystemsInternal", @@ -106,6 +112,12 @@ "default": "nothing", "comment": "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." }, + { + "name": "features", + "data_type": "Dict{String, <:Any}", + "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "default": "Dict{String, Any}()" + }, { "name": "internal", "data_type": "InfrastructureSystemsInternal", @@ -163,6 +175,12 @@ "default": "nothing", "comment": "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." }, + { + "name": "features", + "data_type": "Dict{String, <:Any}", + "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "default": "Dict{String, Any}()" + }, { "name": "internal", "data_type": "InfrastructureSystemsInternal", @@ -205,6 +223,12 @@ "default": "nothing", "comment": "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." }, + { + "name": "features", + "data_type": "Dict{String, <:Any}", + "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "default": "Dict{String, Any}()" + }, { "name": "internal", "data_type": "InfrastructureSystemsInternal", diff --git a/src/deterministic_metadata.jl b/src/deterministic_metadata.jl index 66d4273b4..5342342b4 100644 --- a/src/deterministic_metadata.jl +++ b/src/deterministic_metadata.jl @@ -1,4 +1,4 @@ -function DeterministicMetadata(ts::AbstractDeterministic) +function DeterministicMetadata(ts::AbstractDeterministic; features...) return DeterministicMetadata( get_name(ts), get_resolution(ts), @@ -9,6 +9,7 @@ function DeterministicMetadata(ts::AbstractDeterministic) get_horizon(ts), typeof(ts), get_scaling_factor_multiplier(ts), + Dict{String, Any}(string(k) => v for (k, v) in features), ) end diff --git a/src/generated/DeterministicMetadata.jl b/src/generated/DeterministicMetadata.jl index 316c079f3..432f34244 100644 --- a/src/generated/DeterministicMetadata.jl +++ b/src/generated/DeterministicMetadata.jl @@ -15,6 +15,7 @@ This file is auto-generated. Do not edit. horizon::Int time_series_type::Type{<:AbstractDeterministic} scaling_factor_multiplier::Union{Nothing, Function} + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end @@ -30,6 +31,7 @@ A deterministic forecast for a particular data field in a Component. - `horizon::Int`: length of this time series - `time_series_type::Type{<:AbstractDeterministic}`: Type of the time series data associated with this metadata. - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. +- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. - `internal::InfrastructureSystemsInternal` """ mutable struct DeterministicMetadata <: ForecastMetadata @@ -50,15 +52,17 @@ mutable struct DeterministicMetadata <: ForecastMetadata time_series_type::Type{<:AbstractDeterministic} "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} + "User-defined tags that describe the relationship between a component and a time series array." + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end -function DeterministicMetadata(name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier=nothing, ) - DeterministicMetadata(name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier, InfrastructureSystemsInternal(), ) +function DeterministicMetadata(name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), ) + DeterministicMetadata(name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier, features, InfrastructureSystemsInternal(), ) end -function DeterministicMetadata(; name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier=nothing, internal=InfrastructureSystemsInternal(), ) - DeterministicMetadata(name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier, internal, ) +function DeterministicMetadata(; name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), internal=InfrastructureSystemsInternal(), ) + DeterministicMetadata(name, resolution, initial_timestamp, interval, count, time_series_uuid, horizon, time_series_type, scaling_factor_multiplier, features, internal, ) end """Get [`DeterministicMetadata`](@ref) `name`.""" @@ -79,6 +83,8 @@ get_horizon(value::DeterministicMetadata) = value.horizon get_time_series_type(value::DeterministicMetadata) = value.time_series_type """Get [`DeterministicMetadata`](@ref) `scaling_factor_multiplier`.""" get_scaling_factor_multiplier(value::DeterministicMetadata) = value.scaling_factor_multiplier +"""Get [`DeterministicMetadata`](@ref) `features`.""" +get_features(value::DeterministicMetadata) = value.features """Get [`DeterministicMetadata`](@ref) `internal`.""" get_internal(value::DeterministicMetadata) = value.internal @@ -100,5 +106,7 @@ set_horizon!(value::DeterministicMetadata, val) = value.horizon = val set_time_series_type!(value::DeterministicMetadata, val) = value.time_series_type = val """Set [`DeterministicMetadata`](@ref) `scaling_factor_multiplier`.""" set_scaling_factor_multiplier!(value::DeterministicMetadata, val) = value.scaling_factor_multiplier = val +"""Set [`DeterministicMetadata`](@ref) `features`.""" +set_features!(value::DeterministicMetadata, val) = value.features = val """Set [`DeterministicMetadata`](@ref) `internal`.""" set_internal!(value::DeterministicMetadata, val) = value.internal = val diff --git a/src/generated/ProbabilisticMetadata.jl b/src/generated/ProbabilisticMetadata.jl index 6b956cbaf..167055e32 100644 --- a/src/generated/ProbabilisticMetadata.jl +++ b/src/generated/ProbabilisticMetadata.jl @@ -15,6 +15,7 @@ This file is auto-generated. Do not edit. time_series_uuid::UUIDs.UUID horizon::Int scaling_factor_multiplier::Union{Nothing, Function} + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end @@ -30,6 +31,7 @@ A Probabilistic forecast for a particular data field in a Component. - `time_series_uuid::UUIDs.UUID`: reference to time series data - `horizon::Int`: length of this time series - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. +- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. - `internal::InfrastructureSystemsInternal` """ mutable struct ProbabilisticMetadata <: ForecastMetadata @@ -50,15 +52,17 @@ mutable struct ProbabilisticMetadata <: ForecastMetadata horizon::Int "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} + "User-defined tags that describe the relationship between a component and a time series array." + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end -function ProbabilisticMetadata(name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier=nothing, ) - ProbabilisticMetadata(name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier, InfrastructureSystemsInternal(), ) +function ProbabilisticMetadata(name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), ) + ProbabilisticMetadata(name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier, features, InfrastructureSystemsInternal(), ) end -function ProbabilisticMetadata(; name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier=nothing, internal=InfrastructureSystemsInternal(), ) - ProbabilisticMetadata(name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier, internal, ) +function ProbabilisticMetadata(; name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), internal=InfrastructureSystemsInternal(), ) + ProbabilisticMetadata(name, initial_timestamp, resolution, interval, count, percentiles, time_series_uuid, horizon, scaling_factor_multiplier, features, internal, ) end """Get [`ProbabilisticMetadata`](@ref) `name`.""" @@ -79,6 +83,8 @@ get_time_series_uuid(value::ProbabilisticMetadata) = value.time_series_uuid get_horizon(value::ProbabilisticMetadata) = value.horizon """Get [`ProbabilisticMetadata`](@ref) `scaling_factor_multiplier`.""" get_scaling_factor_multiplier(value::ProbabilisticMetadata) = value.scaling_factor_multiplier +"""Get [`ProbabilisticMetadata`](@ref) `features`.""" +get_features(value::ProbabilisticMetadata) = value.features """Get [`ProbabilisticMetadata`](@ref) `internal`.""" get_internal(value::ProbabilisticMetadata) = value.internal @@ -100,5 +106,7 @@ set_time_series_uuid!(value::ProbabilisticMetadata, val) = value.time_series_uui set_horizon!(value::ProbabilisticMetadata, val) = value.horizon = val """Set [`ProbabilisticMetadata`](@ref) `scaling_factor_multiplier`.""" set_scaling_factor_multiplier!(value::ProbabilisticMetadata, val) = value.scaling_factor_multiplier = val +"""Set [`ProbabilisticMetadata`](@ref) `features`.""" +set_features!(value::ProbabilisticMetadata, val) = value.features = val """Set [`ProbabilisticMetadata`](@ref) `internal`.""" set_internal!(value::ProbabilisticMetadata, val) = value.internal = val diff --git a/src/generated/ScenariosMetadata.jl b/src/generated/ScenariosMetadata.jl index 14c8b8d7b..d1efe20c5 100644 --- a/src/generated/ScenariosMetadata.jl +++ b/src/generated/ScenariosMetadata.jl @@ -15,6 +15,7 @@ This file is auto-generated. Do not edit. time_series_uuid::UUIDs.UUID horizon::Int scaling_factor_multiplier::Union{Nothing, Function} + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end @@ -30,6 +31,7 @@ A Discrete Scenario Based time series for a particular data field in a Component - `time_series_uuid::UUIDs.UUID`: reference to time series data - `horizon::Int`: length of this time series - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. +- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. - `internal::InfrastructureSystemsInternal` """ mutable struct ScenariosMetadata <: ForecastMetadata @@ -50,15 +52,17 @@ mutable struct ScenariosMetadata <: ForecastMetadata horizon::Int "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} + "User-defined tags that describe the relationship between a component and a time series array." + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end -function ScenariosMetadata(name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier=nothing, ) - ScenariosMetadata(name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier, InfrastructureSystemsInternal(), ) +function ScenariosMetadata(name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), ) + ScenariosMetadata(name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier, features, InfrastructureSystemsInternal(), ) end -function ScenariosMetadata(; name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier=nothing, internal=InfrastructureSystemsInternal(), ) - ScenariosMetadata(name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier, internal, ) +function ScenariosMetadata(; name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), internal=InfrastructureSystemsInternal(), ) + ScenariosMetadata(name, resolution, initial_timestamp, interval, scenario_count, count, time_series_uuid, horizon, scaling_factor_multiplier, features, internal, ) end """Get [`ScenariosMetadata`](@ref) `name`.""" @@ -79,6 +83,8 @@ get_time_series_uuid(value::ScenariosMetadata) = value.time_series_uuid get_horizon(value::ScenariosMetadata) = value.horizon """Get [`ScenariosMetadata`](@ref) `scaling_factor_multiplier`.""" get_scaling_factor_multiplier(value::ScenariosMetadata) = value.scaling_factor_multiplier +"""Get [`ScenariosMetadata`](@ref) `features`.""" +get_features(value::ScenariosMetadata) = value.features """Get [`ScenariosMetadata`](@ref) `internal`.""" get_internal(value::ScenariosMetadata) = value.internal @@ -100,5 +106,7 @@ set_time_series_uuid!(value::ScenariosMetadata, val) = value.time_series_uuid = set_horizon!(value::ScenariosMetadata, val) = value.horizon = val """Set [`ScenariosMetadata`](@ref) `scaling_factor_multiplier`.""" set_scaling_factor_multiplier!(value::ScenariosMetadata, val) = value.scaling_factor_multiplier = val +"""Set [`ScenariosMetadata`](@ref) `features`.""" +set_features!(value::ScenariosMetadata, val) = value.features = val """Set [`ScenariosMetadata`](@ref) `internal`.""" set_internal!(value::ScenariosMetadata, val) = value.internal = val diff --git a/src/generated/SingleTimeSeriesMetadata.jl b/src/generated/SingleTimeSeriesMetadata.jl index 4505c967d..30e76a12c 100644 --- a/src/generated/SingleTimeSeriesMetadata.jl +++ b/src/generated/SingleTimeSeriesMetadata.jl @@ -12,6 +12,7 @@ This file is auto-generated. Do not edit. time_series_uuid::UUIDs.UUID length::Int scaling_factor_multiplier::Union{Nothing, Function} + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end @@ -24,6 +25,7 @@ A TimeSeries Data object in contigous form. - `time_series_uuid::UUIDs.UUID`: reference to time series data - `length::Int`: length of this time series - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. +- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. - `internal::InfrastructureSystemsInternal` """ mutable struct SingleTimeSeriesMetadata <: StaticTimeSeriesMetadata @@ -38,15 +40,17 @@ mutable struct SingleTimeSeriesMetadata <: StaticTimeSeriesMetadata length::Int "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} + "User-defined tags that describe the relationship between a component and a time series array." + features::Dict{String, <:Any} internal::InfrastructureSystemsInternal end -function SingleTimeSeriesMetadata(name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier=nothing, ) - SingleTimeSeriesMetadata(name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier, InfrastructureSystemsInternal(), ) +function SingleTimeSeriesMetadata(name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), ) + SingleTimeSeriesMetadata(name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier, features, InfrastructureSystemsInternal(), ) end -function SingleTimeSeriesMetadata(; name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier=nothing, internal=InfrastructureSystemsInternal(), ) - SingleTimeSeriesMetadata(name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier, internal, ) +function SingleTimeSeriesMetadata(; name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier=nothing, features=Dict{String, Any}(), internal=InfrastructureSystemsInternal(), ) + SingleTimeSeriesMetadata(name, resolution, initial_timestamp, time_series_uuid, length, scaling_factor_multiplier, features, internal, ) end """Get [`SingleTimeSeriesMetadata`](@ref) `name`.""" @@ -61,6 +65,8 @@ get_time_series_uuid(value::SingleTimeSeriesMetadata) = value.time_series_uuid get_length(value::SingleTimeSeriesMetadata) = value.length """Get [`SingleTimeSeriesMetadata`](@ref) `scaling_factor_multiplier`.""" get_scaling_factor_multiplier(value::SingleTimeSeriesMetadata) = value.scaling_factor_multiplier +"""Get [`SingleTimeSeriesMetadata`](@ref) `features`.""" +get_features(value::SingleTimeSeriesMetadata) = value.features """Get [`SingleTimeSeriesMetadata`](@ref) `internal`.""" get_internal(value::SingleTimeSeriesMetadata) = value.internal @@ -76,6 +82,8 @@ set_time_series_uuid!(value::SingleTimeSeriesMetadata, val) = value.time_series_ set_length!(value::SingleTimeSeriesMetadata, val) = value.length = val """Set [`SingleTimeSeriesMetadata`](@ref) `scaling_factor_multiplier`.""" set_scaling_factor_multiplier!(value::SingleTimeSeriesMetadata, val) = value.scaling_factor_multiplier = val +"""Set [`SingleTimeSeriesMetadata`](@ref) `features`.""" +set_features!(value::SingleTimeSeriesMetadata, val) = value.features = val """Set [`SingleTimeSeriesMetadata`](@ref) `internal`.""" set_internal!(value::SingleTimeSeriesMetadata, val) = value.internal = val diff --git a/src/generated/includes.jl b/src/generated/includes.jl index 0941655cb..97cb0e0c5 100644 --- a/src/generated/includes.jl +++ b/src/generated/includes.jl @@ -4,6 +4,7 @@ include("ScenariosMetadata.jl") include("SingleTimeSeriesMetadata.jl") export get_count +export get_features export get_horizon export get_initial_timestamp export get_interval @@ -16,6 +17,7 @@ export get_scenario_count export get_time_series_type export get_time_series_uuid export set_count! +export set_features! export set_horizon! export set_initial_timestamp! export set_interval! diff --git a/src/hdf5_time_series_storage.jl b/src/hdf5_time_series_storage.jl index aaf1ab7a5..efdfe6d3d 100644 --- a/src/hdf5_time_series_storage.jl +++ b/src/hdf5_time_series_storage.jl @@ -3,9 +3,9 @@ import HDF5 import H5Zblosc const HDF5_TS_ROOT_PATH = "time_series" -const TIME_SERIES_DATA_FORMAT_VERSION = "1.0.1" +const HDF5_TS_METADATA_ROOT_PATH = "time_series_metadata" +const TIME_SERIES_DATA_FORMAT_VERSION = "2.0.0" const TIME_SERIES_VERSION_KEY = "data_format_version" -const COMPONENT_REFERENCES_KEY = "component_references" """ Stores all time series data in an HDF5 file. @@ -15,7 +15,6 @@ no more references to the storage object. """ mutable struct Hdf5TimeSeriesStorage <: TimeSeriesStorage file_path::String - read_only::Bool compression::CompressionSettings file::Union{Nothing, HDF5.File} end @@ -38,14 +37,11 @@ Constructs Hdf5TimeSeriesStorage. directory. If it is not set, use the environment variable SIENNA_TIME_SERIES_DIRECTORY. If that is not set, use tempdir(). This should be set if the time series data is larger than the tmp filesystem can hold. - - `read_only = false`: If true, don't allow changes to the file. Allows simultaneous read - access. """ function Hdf5TimeSeriesStorage( create_file::Bool; filename = nothing, directory = nothing, - read_only = false, compression = CompressionSettings(), ) if create_file @@ -55,13 +51,13 @@ function Hdf5TimeSeriesStorage( close(io) end - storage = Hdf5TimeSeriesStorage(filename, read_only, compression, nothing) + storage = Hdf5TimeSeriesStorage(filename, compression, nothing) _make_file(storage) else - storage = Hdf5TimeSeriesStorage(filename, read_only, compression, nothing) + storage = Hdf5TimeSeriesStorage(filename, compression, nothing) end - @debug "Constructed new Hdf5TimeSeriesStorage" _group = LOG_GROUP_TIME_SERIES storage.file_path read_only compression + @debug "Constructed new Hdf5TimeSeriesStorage" _group = LOG_GROUP_TIME_SERIES storage.file_path compression return storage end @@ -105,12 +101,8 @@ function from_file( copy_h5_file(filename, file_path) end - storage = Hdf5TimeSeriesStorage(false; filename = file_path, read_only = read_only) + storage = Hdf5TimeSeriesStorage(false; filename = file_path) if !read_only - version = read_data_format_version(storage) - if version == "1.0.0" - _convert_from_1_0_0!(storage) - end _deserialize_compression_settings!(storage) end @@ -169,8 +161,9 @@ undergoing a deepcopy. function copy_to_new_file!(storage::Hdf5TimeSeriesStorage, directory = nothing) directory = _get_time_series_parent_dir(directory) - # If we ever choose to keep the HDF5 file open then this will break. - # Any open buffers will need to be flushed. + if !isnothing(storage.file) && isopen(storage.file) + error("This operation is not allowed when the HDF5 file handle is open.") + end filename, io = mktemp(directory) close(io) copy_h5_file(get_file_path(storage), filename) @@ -186,6 +179,13 @@ function copy_h5_file(src::AbstractString, dst::AbstractString) HDF5.h5open(dst, "w") do fw HDF5.h5open(src, "r") do fr HDF5.copy_object(fr[HDF5_TS_ROOT_PATH], fw, HDF5_TS_ROOT_PATH) + if HDF5_TS_METADATA_ROOT_PATH in keys(fr) + HDF5.copy_object( + fr[HDF5_TS_METADATA_ROOT_PATH], + fw, + HDF5_TS_METADATA_ROOT_PATH, + ) + end end end @@ -208,52 +208,37 @@ end function _read_data_format_version(storage::Hdf5TimeSeriesStorage, file::HDF5.File) root = _get_root(storage, file) - if !haskey(HDF5.attributes(root), TIME_SERIES_VERSION_KEY) - return "1.0.0" - end return HDF5.read(HDF5.attributes(root)[TIME_SERIES_VERSION_KEY]) end function serialize_time_series!( storage::Hdf5TimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, ts::TimeSeriesData, ) - check_read_only(storage) - _serialize_time_series!(storage, component_uuid, name, ts, storage.file) + _serialize_time_series!(storage, ts, storage.file) return end function _serialize_time_series!( storage::Hdf5TimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, ts::TimeSeriesData, ::Nothing, ) HDF5.h5open(storage.file_path, "r+") do file - _serialize_time_series!(storage, component_uuid, name, ts, file) + _serialize_time_series!(storage, ts, file) end return end function _serialize_time_series!( storage::Hdf5TimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, ts::TimeSeriesData, file::HDF5.File, ) root = _get_root(storage, file) uuid = string(get_uuid(ts)) - component_name = make_component_name(component_uuid, name) if !haskey(root, uuid) group = HDF5.create_group(root, uuid) - # Create a group to store component references as attributes. - # Use this instead of this time series' group or the dataset so that - # the only attributes are component references. - component_refs = HDF5.create_group(group, COMPONENT_REFERENCES_KEY) data = get_array_for_hdf(ts) settings = storage.compression if settings.enabled @@ -271,14 +256,9 @@ function _serialize_time_series!( else group["data"] = data end - _write_time_series_attributes!(storage, ts, group) - @debug "Create new time series entry." _group = LOG_GROUP_TIME_SERIES uuid component_uuid name - else - component_refs = root[uuid][COMPONENT_REFERENCES_KEY] - @debug "Add reference to existing time series entry." _group = - LOG_GROUP_TIME_SERIES uuid component_uuid name + _write_time_series_attributes!(ts, group) + @debug "Create new time series entry." _group = LOG_GROUP_TIME_SERIES uuid end - HDF5.attributes(component_refs)[component_name] = true return end @@ -293,54 +273,22 @@ function get_data_type(ts::TimeSeriesData) end function _write_time_series_attributes!( - storage::Hdf5TimeSeriesStorage, - ts::T, - path, -) where {T <: StaticTimeSeries} - return _write_time_series_attributes_common!(storage, ts, path) -end - -function _write_time_series_attributes!( - storage::Hdf5TimeSeriesStorage, ts::T, path, -) where {T <: Forecast} - _write_time_series_attributes_common!(storage, ts, path) - interval = get_interval(ts) - HDF5.attributes(path)["interval"] = time_period_conversion(interval).value - return -end - -function _write_time_series_attributes_common!(storage::Hdf5TimeSeriesStorage, ts, path) - initial_timestamp = Dates.datetime2epochms(get_initial_timestamp(ts)) - resolution = get_resolution(ts) +) where {T <: TimeSeriesData} data_type = get_data_type(ts) HDF5.attributes(path)["module"] = string(parentmodule(typeof(ts))) HDF5.attributes(path)["type"] = string(nameof(typeof(ts))) - HDF5.attributes(path)["initial_timestamp"] = initial_timestamp - HDF5.attributes(path)["resolution"] = time_period_conversion(resolution).value HDF5.attributes(path)["data_type"] = data_type return end -function _read_time_series_attributes( - storage::Hdf5TimeSeriesStorage, - path, - rows, - ::Type{T}, -) where {T <: StaticTimeSeries} - return _read_time_series_attributes_common(storage, path, rows) -end - -function _read_time_series_attributes( - storage::Hdf5TimeSeriesStorage, - path, - rows, - ::Type{T}, -) where {T <: Forecast} - data = _read_time_series_attributes_common(storage, path, rows) - data["interval"] = Dates.Millisecond(HDF5.read(HDF5.attributes(path)["interval"])) - return data +function _read_time_series_attributes(path) + return Dict( + "type" => _read_time_series_type(path), + "dataset_size" => size(path["data"]), + "data_type" => _TYPE_DICT[HDF5.read(HDF5.attributes(path)["data_type"])], + ) end # TODO I suspect this could be designed better using reflection even without the security risks of eval discussed above @@ -355,87 +303,21 @@ const _TYPE_DICT = Dict( ) _TYPE_DICT["CONSTANT"] = CONSTANT -function _read_time_series_attributes_common(storage::Hdf5TimeSeriesStorage, path, rows) - initial_timestamp = - Dates.epochms2datetime(HDF5.read(HDF5.attributes(path)["initial_timestamp"])) - resolution = Dates.Millisecond(HDF5.read(HDF5.attributes(path)["resolution"])) - data_type = _TYPE_DICT[HDF5.read(HDF5.attributes(path)["data_type"])] - return Dict( - "type" => _read_time_series_type(path), - "initial_timestamp" => initial_timestamp, - "resolution" => resolution, - "dataset_size" => size(path["data"]), - "start_time" => initial_timestamp + resolution * (rows.start - 1), - "data_type" => data_type, - ) -end - function _read_time_series_type(path) module_str = HDF5.read(HDF5.attributes(path)["module"]) type_str = HDF5.read(HDF5.attributes(path)["type"]) return get_type_from_strings(module_str, type_str) end -function add_time_series_reference!( - storage::Hdf5TimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, - ts_uuid::UUIDs.UUID, -) - _add_time_series_reference!(storage, component_uuid, name, ts_uuid, storage.file) -end - -function _add_time_series_reference!( - storage::Hdf5TimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, - ts_uuid::UUIDs.UUID, - ::Nothing, -) - HDF5.h5open(storage.file_path, "r+") do file - _add_time_series_reference!(storage, component_uuid, name, ts_uuid, file) - end -end - -function _add_time_series_reference!( - storage::Hdf5TimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, - ts_uuid::UUIDs.UUID, - file::HDF5.File, -) - check_read_only(storage) - uuid = string(ts_uuid) - component_name = make_component_name(component_uuid, name) - root = _get_root(storage, file) - path = root[uuid][COMPONENT_REFERENCES_KEY] - - # It's possible that this is overly restrictive, but as of now there is not a good - # reason for a caller to add a reference multiple times. This should be a bug. - @assert !haskey(HDF5.attributes(path), component_name) "There is already a reference to $component_name for time series $ts_uuid" - - HDF5.attributes(path)[component_name] = true - @debug "Add reference to existing time series entry." _group = LOG_GROUP_TIME_SERIES uuid component_uuid name - return -end - # TODO: This needs to change if we want to directly convert Hdf5TimeSeriesStorage to # InMemoryTimeSeriesStorage, which is currently not supported at System deserialization. function iterate_time_series(storage::Hdf5TimeSeriesStorage) Channel() do channel HDF5.h5open(storage.file_path, "r") do file root = _get_root(storage, file) - for uuid_group in root - data = HDF5.read(uuid_group["data"]) - attributes = Dict() - for name in keys(HDF5.attributes(uuid_group)) - attributes[name] = HDF5.read(HDF5.attributes(uuid_group)[name]) - end - refs = uuid_group[COMPONENT_REFERENCES_KEY] - for ref in keys(HDF5.attributes(refs)) - component, name = deserialize_component_name(ref) - put!(channel, (component, name, data, attributes)) - end + for uuid in keys(root) + data = HDF5.read(root[uuid]["data"]) + put!(channel, (Base.UUID(uuid), data)) end end end @@ -456,164 +338,145 @@ function _make_rows_columns(dataset, ::Type{T}) where T <: Forecast end =# -function remove_time_series!( - storage::Hdf5TimeSeriesStorage, - uuid::UUIDs.UUID, - component_uuid::UUIDs.UUID, - name::AbstractString, -) - check_read_only(storage) - _remove_time_series!(storage, uuid, component_uuid, name, storage.file) +function remove_time_series!(storage::Hdf5TimeSeriesStorage, uuid::UUIDs.UUID) + _remove_time_series!(storage, uuid, storage.file) end function _remove_time_series!( storage::Hdf5TimeSeriesStorage, uuid::UUIDs.UUID, - component_uuid::UUIDs.UUID, - name::AbstractString, ::Nothing, ) HDF5.h5open(storage.file_path, "r+") do file - _remove_time_series!(storage, uuid, component_uuid, name, file) + _remove_time_series!(storage, uuid, file) end end function _remove_time_series!( storage::Hdf5TimeSeriesStorage, uuid::UUIDs.UUID, - component_uuid::UUIDs.UUID, - name::AbstractString, file::HDF5.File, ) root = _get_root(storage, file) path = _get_time_series_path(root, uuid) - components = path[COMPONENT_REFERENCES_KEY] - HDF5.delete_attribute(components, make_component_name(component_uuid, name)) - if isempty(keys(HDF5.attributes(components))) - @debug "$path has no more references; delete it." _group = LOG_GROUP_TIME_SERIES - HDF5.delete_object(path) - end + HDF5.delete_object(path) return end function deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ) where {T <: StaticTimeSeries} - _deserialize_time_series(T, storage, ts_metadata, rows, columns, storage.file) + _deserialize_time_series(T, storage, metadata, rows, columns, storage.file) end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ::Nothing, ) where {T <: StaticTimeSeries} return HDF5.h5open(storage.file_path, "r") do file - _deserialize_time_series(T, storage, ts_metadata, rows, columns, file) + _deserialize_time_series(T, storage, metadata, rows, columns, file) end end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, file::HDF5.File, ) where {T <: StaticTimeSeries} # Note that all range checks must occur at a higher level. root = _get_root(storage, file) - uuid = get_time_series_uuid(ts_metadata) + uuid = get_time_series_uuid(metadata) path = _get_time_series_path(root, uuid) - attributes = _read_time_series_attributes(storage, path, rows, T) - @assert_op attributes["type"] == T + attributes = _read_time_series_attributes(path) @debug "deserializing a StaticTimeSeries" _group = LOG_GROUP_TIME_SERIES T data_type = attributes["data_type"] data = get_hdf_array(path["data"], data_type, rows) - return T( - ts_metadata, - TimeSeries.TimeArray( - range( - attributes["start_time"]; - length = length(rows), - step = attributes["resolution"], - ), - data, - ), + resolution = get_resolution(metadata) + start_time = get_initial_timestamp(metadata) + resolution * (rows.start - 1) + timestamps = range( + start_time; + length = length(rows), + step = resolution, ) + return T(metadata, TimeSeries.TimeArray(timestamps, data)) end function deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ) where {T <: AbstractDeterministic} # Note that all range checks must occur at a higher level. - _deserialize_time_series(T, storage, ts_metadata, rows, columns, storage.file) + _deserialize_time_series(T, storage, metadata, rows, columns, storage.file) end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ::Nothing, ) where {T <: AbstractDeterministic} return HDF5.h5open(storage.file_path, "r") do file - _deserialize_time_series(T, storage, ts_metadata, rows, columns, file) + _deserialize_time_series(T, storage, metadata, rows, columns, file) end end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, file::HDF5.File, ) where {T <: AbstractDeterministic} root = _get_root(storage, file) - uuid = get_time_series_uuid(ts_metadata) + uuid = get_time_series_uuid(metadata) path = _get_time_series_path(root, uuid) actual_type = _read_time_series_type(path) - if actual_type == SingleTimeSeries + if actual_type === SingleTimeSeries last_index = size(path["data"])[1] return deserialize_deterministic_from_single_time_series( storage, - ts_metadata, + metadata, rows, columns, last_index, ) end - attributes = _read_time_series_attributes(storage, path, rows, T) @assert actual_type <: T "actual_type = $actual_type T = $T" @debug "deserializing a Forecast" _group = LOG_GROUP_TIME_SERIES T - data_type = attributes["data_type"] - data = get_hdf_array(path["data"], data_type, attributes, rows, columns) - return actual_type(ts_metadata, data) + attributes = _read_time_series_attributes(path) + data = get_hdf_array(path["data"], attributes["data_type"], metadata, rows, columns) + return actual_type(metadata, data) end function get_hdf_array( dataset, ::Type{<:CONSTANT}, - attributes::Dict{String, Any}, + metadata::ForecastMetadata, rows::UnitRange{Int}, columns::UnitRange{Int}, ) data = SortedDict{Dates.DateTime, Vector{Float64}}() - initial_timestamp = attributes["start_time"] - interval = attributes["interval"] + resolution = get_resolution(metadata) + initial_timestamp = get_initial_timestamp(metadata) + resolution * (rows.start - 1) + interval = get_interval(metadata) start_time = initial_timestamp + interval * (columns.start - 1) if length(columns) == 1 data[start_time] = dataset[rows, columns.start] @@ -630,11 +493,11 @@ end function get_hdf_array( dataset, ::Type{LinearFunctionData}, - attributes::Dict{String, Any}, + metadata::TimeSeriesMetadata, rows::UnitRange{Int}, columns::UnitRange{Int}, ) - data = get_hdf_array(dataset, CONSTANT, attributes, rows, columns) + data = get_hdf_array(dataset, CONSTANT, metadata, rows, columns) return SortedDict{Dates.DateTime, Vector{LinearFunctionData}}( k => LinearFunctionData.(v) for (k, v) in data ) @@ -645,13 +508,14 @@ _quadratic_from_tuple((a, b)::Tuple{Float64, Float64}) = QuadraticFunctionData(a function get_hdf_array( dataset, type::Type{QuadraticFunctionData}, - attributes::Dict{String, Any}, + metadata::ForecastMetadata, rows::UnitRange{Int}, columns::UnitRange{Int}, ) data = SortedDict{Dates.DateTime, Vector{QuadraticFunctionData}}() - initial_timestamp = attributes["start_time"] - interval = attributes["interval"] + resolution = get_resolution(metadata) + initial_timestamp = get_initial_timestamp(metadata) + resolution * (rows.start - 1) + interval = get_interval(metadata) start_time = initial_timestamp + interval * (columns.start - 1) if length(columns) == 1 data[start_time] = retransform_hdf_array(dataset[rows, columns.start, :], type) @@ -668,13 +532,14 @@ end function get_hdf_array( dataset, type::Type{PiecewiseLinearPointData}, - attributes::Dict{String, Any}, + metadata::ForecastMetadata, rows::UnitRange{Int}, columns::UnitRange{Int}, ) data = SortedDict{Dates.DateTime, Vector{PiecewiseLinearPointData}}() - initial_timestamp = attributes["start_time"] - interval = attributes["interval"] + resolution = get_resolution(metadata) + initial_timestamp = get_initial_timestamp(metadata) + resolution * (rows.start - 1) + interval = get_interval(metadata) start_time = initial_timestamp + interval * (columns.start - 1) if length(columns) == 1 data[start_time] = retransform_hdf_array(dataset[rows, columns.start, :, :], type) @@ -780,46 +645,45 @@ end function deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ) where {T <: Probabilistic} - _deserialize_time_series(T, storage, ts_metadata, rows, columns, storage.file) + _deserialize_time_series(T, storage, metadata, rows, columns, storage.file) end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ::Nothing, ) where {T <: Probabilistic} return HDF5.h5open(storage.file_path, "r") do file - _deserialize_time_series(T, storage, ts_metadata, rows, columns, file) + _deserialize_time_series(T, storage, metadata, rows, columns, file) end end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, file::HDF5.File, ) where {T <: Probabilistic} # Note that all range checks must occur at a higher level. - total_percentiles = length(get_percentiles(ts_metadata)) + total_percentiles = length(get_percentiles(metadata)) root = _get_root(storage, file) - uuid = get_time_series_uuid(ts_metadata) + uuid = get_time_series_uuid(metadata) path = _get_time_series_path(root, uuid) - attributes = _read_time_series_attributes(storage, path, rows, T) - @assert_op attributes["type"] == T + attributes = _read_time_series_attributes(path) @assert_op length(attributes["dataset_size"]) == 3 @debug "deserializing a Forecast" _group = LOG_GROUP_TIME_SERIES T data = SortedDict{Dates.DateTime, Matrix{attributes["data_type"]}}() - initial_timestamp = attributes["start_time"] - interval = attributes["interval"] + initial_timestamp = get_initial_timestamp(metadata) + interval = get_interval(metadata) start_time = initial_timestamp + interval * (first(columns) - 1) if length(columns) == 1 data[start_time] = @@ -830,59 +694,59 @@ function _deserialize_time_series( [3, 2, 1], ) for (i, it) in enumerate( - range(start_time; length = length(columns), step = attributes["interval"]), + range(start_time; length = length(columns), step = interval), ) data[it] = @view data_read[i, 1:length(rows), 1:total_percentiles] end end - return T(ts_metadata, data) + return T(metadata, data) end function deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ) where {T <: Scenarios} - _deserialize_time_series(T, storage, ts_metadata, rows, columns, storage.file) + _deserialize_time_series(T, storage, metadata, rows, columns, storage.file) end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, ::Nothing, ) where {T <: Scenarios} return HDF5.h5open(storage.file_path, "r") do file - _deserialize_time_series(T, storage, ts_metadata, rows, columns, file) + _deserialize_time_series(T, storage, metadata, rows, columns, file) end end function _deserialize_time_series( ::Type{T}, storage::Hdf5TimeSeriesStorage, - ts_metadata::TimeSeriesMetadata, + metadata::TimeSeriesMetadata, rows::UnitRange, columns::UnitRange, file::HDF5.File, ) where {T <: Scenarios} # Note that all range checks must occur at a higher level. - total_scenarios = get_scenario_count(ts_metadata) + total_scenarios = get_scenario_count(metadata) root = _get_root(storage, file) - uuid = get_time_series_uuid(ts_metadata) + uuid = get_time_series_uuid(metadata) path = _get_time_series_path(root, uuid) - attributes = _read_time_series_attributes(storage, path, rows, T) + attributes = _read_time_series_attributes(path) @assert_op attributes["type"] == T @assert_op length(attributes["dataset_size"]) == 3 @debug "deserializing a Forecast" _group = LOG_GROUP_TIME_SERIES T data = SortedDict{Dates.DateTime, Matrix{attributes["data_type"]}}() - initial_timestamp = attributes["start_time"] - interval = attributes["interval"] + initial_timestamp = get_initial_timestamp(metadata) + interval = get_interval(metadata) start_time = initial_timestamp + interval * (first(columns) - 1) if length(columns) == 1 data[start_time] = @@ -891,17 +755,16 @@ function _deserialize_time_series( data_read = PermutedDimsArray(path["data"][1:total_scenarios, rows, columns], [3, 2, 1]) for (i, it) in enumerate( - range(start_time; length = length(columns), step = attributes["interval"]), + range(start_time; length = length(columns), step = interval), ) data[it] = @view data_read[i, 1:length(rows), 1:total_scenarios] end end - return T(ts_metadata, data) + return T(metadata, data) end function clear_time_series!(storage::Hdf5TimeSeriesStorage) - check_read_only(storage) # Re-create the file. HDF5 will not actually free up the deleted space until h5repack # is run on the file. _make_file(storage) @@ -920,65 +783,6 @@ end _get_num_time_series(storage::Hdf5TimeSeriesStorage, file::HDF5.File) = length(_get_root(storage, file)) -function replace_component_uuid!( - storage::Hdf5TimeSeriesStorage, - ts_uuid, - old_component_uuid, - new_component_uuid, - name, -) - check_read_only(storage) - _replace_component_uuid!( - storage, - ts_uuid, - old_component_uuid, - new_component_uuid, - name, - storage.file, - ) -end - -function _replace_component_uuid!( - storage::Hdf5TimeSeriesStorage, - ts_uuid, - old_component_uuid, - new_component_uuid, - name, - ::Nothing, -) - HDF5.h5open(storage.file_path, "r+") do file - _replace_component_uuid!( - storage, - ts_uuid, - old_component_uuid, - new_component_uuid, - name, - file, - ) - end -end - -function _replace_component_uuid!( - storage::Hdf5TimeSeriesStorage, - ts_uuid, - old_component_uuid, - new_component_uuid, - name, - file::HDF5.File, -) - root = _get_root(storage, file) - path = _get_time_series_path(root, ts_uuid) - components = path[COMPONENT_REFERENCES_KEY] - HDF5.delete_attribute(components, make_component_name(old_component_uuid, name)) - new_component_name = make_component_name(new_component_uuid, name) - if haskey(HDF5.attributes(components), new_component_name) - error("BUG! $new_component_name is already stored in time series $ts_uuid") - end - - HDF5.attributes(components)[new_component_name] = true - return -end - _make_file(storage::Hdf5TimeSeriesStorage) = _make_file(storage, storage.file) function _make_file(storage::Hdf5TimeSeriesStorage, ::Nothing) @@ -1034,14 +838,6 @@ function _get_time_series_path(root::HDF5.Group, uuid::UUIDs.UUID) return root[uuid_str] end -function check_read_only(storage::Hdf5TimeSeriesStorage) - if storage.read_only - error("Operation not permitted; this time series file is read-only") - end -end - -is_read_only(storage::Hdf5TimeSeriesStorage) = storage.read_only - function compare_values( x::Hdf5TimeSeriesStorage, y::Hdf5TimeSeriesStorage; @@ -1061,51 +857,16 @@ function compare_values( return true end - for ((uuid_x, name_x, data_x, attrs_x), (uuid_y, name_y, data_y, attrs_y)) in - zip(item_x, item_y) + for ((uuid_x, data_x), (uuid_y, data_y)) in zip(item_x, item_y) if uuid_x != uuid_y - @error "component UUIDs don't match" uuid_x uuid_y - return false - end - if name_x != name_y - @error "names don't match" name_x name_y + @error "UUIDs doesn't match" uuid_x uuid_y return false end if data_x != data_y @error "data doesn't match" data_x data_y return false end - if sort!(collect(keys(attrs_x))) != sort!(collect(keys(attrs_y))) - @error "attr keys don't match" attrs_x attrs_y - end - if collect(values(attrs_x)) != collect(values(attrs_y)) - @error "attr values don't match" attrs_x attrs_y - end end return true end - -function _convert_from_1_0_0!(storage::Hdf5TimeSeriesStorage) - # 1.0.0 version did not support compression. - # 1.0.0 stored component name/UUID pairs in a dataset. - # That wasn't efficient if a user added many shared references. - HDF5.h5open(storage.file_path, "r+") do file - root = _get_root(storage, file) - for uuid_group in root - components = HDF5.create_group(uuid_group, COMPONENT_REFERENCES_KEY) - component_names = uuid_group["components"][:] - for name in component_names - HDF5.attributes(components)[name] = true - end - HDF5.delete_object(uuid_group["components"]) - end - - HDF5.attributes(root)[TIME_SERIES_VERSION_KEY] = TIME_SERIES_DATA_FORMAT_VERSION - compression = CompressionSettings() - _serialize_compression_settings(storage, root) - return - end - - @debug "Converted file from 1.0.0 format" _group = LOG_GROUP_TIME_SERIES -end diff --git a/src/in_memory_time_series_storage.jl b/src/in_memory_time_series_storage.jl index e115f6af4..0906d52f9 100644 --- a/src/in_memory_time_series_storage.jl +++ b/src/in_memory_time_series_storage.jl @@ -1,26 +1,12 @@ - -const _ComponentnameReferences = Set{Tuple{UUIDs.UUID, String}} - -struct _TimeSeriesRecord - component_names::_ComponentnameReferences - ts::TimeSeriesData -end - -function _TimeSeriesRecord(component_uuid, name, ts) - record = _TimeSeriesRecord(_ComponentnameReferences(), ts) - push!(record.component_names, (component_uuid, name)) - return record -end - """ Stores all time series data in memory. """ struct InMemoryTimeSeriesStorage <: TimeSeriesStorage - data::Dict{UUIDs.UUID, _TimeSeriesRecord} + data::Dict{UUIDs.UUID, <:TimeSeriesData} end function InMemoryTimeSeriesStorage() - storage = InMemoryTimeSeriesStorage(Dict{UUIDs.UUID, _TimeSeriesRecord}()) + storage = InMemoryTimeSeriesStorage(Dict{UUIDs.UUID, TimeSeriesData}()) @debug "Created InMemoryTimeSeriesStorage" _group = LOG_GROUP_TIME_SERIES return storage end @@ -30,8 +16,8 @@ Constructs InMemoryTimeSeriesStorage from an instance of Hdf5TimeSeriesStorage. """ function InMemoryTimeSeriesStorage(hdf5_storage::Hdf5TimeSeriesStorage) storage = InMemoryTimeSeriesStorage() - for (component, name, time_series) in iterate_time_series(hdf5_storage) - serialize_time_series!(storage, component, name, time_series) + for (_, time_series) in iterate_time_series(hdf5_storage) + serialize_time_series!(storage, time_series) end return storage @@ -49,71 +35,32 @@ end Base.isempty(storage::InMemoryTimeSeriesStorage) = isempty(storage.data) -check_read_only(::InMemoryTimeSeriesStorage) = nothing - get_compression_settings(::InMemoryTimeSeriesStorage) = CompressionSettings(; enabled = false) -is_read_only(storage::InMemoryTimeSeriesStorage) = false - function serialize_time_series!( storage::InMemoryTimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, ts::TimeSeriesData, ) uuid = get_uuid(ts) - if !haskey(storage.data, uuid) - @debug "Create new time series entry." _group = LOG_GROUP_TIME_SERIES uuid component_uuid name - storage.data[uuid] = _TimeSeriesRecord(component_uuid, name, ts) - else - add_time_series_reference!(storage, component_uuid, name, uuid) + if haskey(storage.data, uuid) + throw(ArgumentError("Time series UUID = $uuid is already stored")) end - return -end - -function add_time_series_reference!( - storage::InMemoryTimeSeriesStorage, - component_uuid::UUIDs.UUID, - name::AbstractString, - ts_uuid::UUIDs.UUID, -) - @debug "Add reference to existing time series entry." _group = LOG_GROUP_TIME_SERIES ts_uuid component_uuid name - record = storage.data[ts_uuid] - key = (component_uuid, name) - - # It's possible that this is overly restrictive, but as of now there is not a good - # reason for a caller to add a reference multiple times. This should be a bug. - @assert !in(key, record.component_names) "There is already a reference to $key for time series $ts_uuid" - - push!(record.component_names, key) + storage.data[uuid] = ts + @debug "Create new time series entry." _group = LOG_GROUP_TIME_SERIES uuid return end function remove_time_series!( storage::InMemoryTimeSeriesStorage, uuid::UUIDs.UUID, - component_uuid::UUIDs.UUID, - name::AbstractString, ) if !haskey(storage.data, uuid) throw(ArgumentError("$uuid is not stored")) end - record = storage.data[uuid] - component_name = (component_uuid, name) - if !(component_name in record.component_names) - throw(ArgumentError("$component_name wasn't stored for $uuid")) - end - - pop!(record.component_names, component_name) - @debug "Removed $component_name from $uuid." _group = LOG_GROUP_TIME_SERIES - - if isempty(record.component_names) - @debug "$uuid has no more references; delete it." _group = LOG_GROUP_TIME_SERIES - pop!(storage.data, uuid) - end + pop!(storage.data, uuid) end function deserialize_time_series( @@ -128,7 +75,7 @@ function deserialize_time_series( throw(ArgumentError("$uuid is not stored")) end - ts_data = get_data(storage.data[uuid].ts) + ts_data = get_data(storage.data[uuid]) total_rows = length(ts_metadata) if rows.start == 1 && length(rows) == total_rows # No memory allocation @@ -151,7 +98,7 @@ function deserialize_time_series( throw(ArgumentError("$uuid is not stored")) end - ts = storage.data[uuid].ts + ts = storage.data[uuid] ts_data = get_data(ts) if ts isa SingleTimeSeries return deserialize_deterministic_from_single_time_series( @@ -199,41 +146,11 @@ function get_num_time_series(storage::InMemoryTimeSeriesStorage) return length(storage.data) end -function replace_component_uuid!( - storage::InMemoryTimeSeriesStorage, - ts_uuid, - old_component_uuid, - new_component_uuid, - name, -) - if !haskey(storage.data, ts_uuid) - throw(ArgumentError("$ts_uuid is not stored")) - end - - record = storage.data[ts_uuid] - component_name = (old_component_uuid, name) - if !(component_name in record.component_names) - throw(ArgumentError("$component_name wasn't stored for $ts_uuid")) - end - - pop!(record.component_names, component_name) - new_component_name = (new_component_uuid, name) - if new_component_name in record.component_names - error("BUG! $new_component_name is already stored in time series $ts_uuid") - end - push!(record.component_names, new_component_name) - - @debug "Replaced $component_name with $new_component_name for $ts_uuid." _group = - LOG_GROUP_TIME_SERIES -end - function convert_to_hdf5(storage::InMemoryTimeSeriesStorage, filename::AbstractString) create_file = true hdf5_storage = Hdf5TimeSeriesStorage(create_file; filename = filename) - for record in values(storage.data) - for pair in record.component_names - serialize_time_series!(hdf5_storage, pair[1], pair[2], record.ts) - end + for ts in values(storage.data) + serialize_time_series!(hdf5_storage, ts) end end @@ -251,18 +168,14 @@ function compare_values( end for key in keys_x - record_x = x.data[key] - record_y = y.data[key] - if compare_uuids && record_x.component_names != record_y.component_names - @error "component_names don't match" record_x.component_names record_y.component_names - return false - end - if TimeSeries.timestamp(record_x.ts.data) != TimeSeries.timestamp(record_y.ts.data) - @error "timestamps don't match" record_x record_y + ts_x = x.data[key] + ts_y = y.data[key] + if TimeSeries.timestamp(get_data(ts_x)) != TimeSeries.timestamp(get_data(ts_y)) + @error "timestamps don't match" ts_x ts_y return false end - if TimeSeries.values(record_x.ts.data) != TimeSeries.values(record_y.ts.data) - @error "values don't match" record_x record_y + if TimeSeries.values(get_data(ts_x)) != TimeSeries.values(get_data(ts_y)) + @error "values don't match" ts_x ts_y return false end end diff --git a/src/probabilistic.jl b/src/probabilistic.jl index e2cc7b2c8..5c22b881d 100644 --- a/src/probabilistic.jl +++ b/src/probabilistic.jl @@ -204,7 +204,7 @@ function Probabilistic( ) end -function ProbabilisticMetadata(time_series::Probabilistic) +function ProbabilisticMetadata(time_series::Probabilistic; features...) return ProbabilisticMetadata( get_name(time_series), get_initial_timestamp(time_series), @@ -215,6 +215,7 @@ function ProbabilisticMetadata(time_series::Probabilistic) get_uuid(time_series), get_horizon(time_series), get_scaling_factor_multiplier(time_series), + Dict{String, Any}(string(k) => v for (k, v) in features), ) end diff --git a/src/scenarios.jl b/src/scenarios.jl index 46e563aa6..f510ca8a9 100644 --- a/src/scenarios.jl +++ b/src/scenarios.jl @@ -170,7 +170,7 @@ function Scenarios(info::TimeSeriesParsedInfo) ) end -function ScenariosMetadata(time_series::Scenarios) +function ScenariosMetadata(time_series::Scenarios; features...) return ScenariosMetadata( get_name(time_series), get_resolution(time_series), @@ -181,6 +181,7 @@ function ScenariosMetadata(time_series::Scenarios) get_uuid(time_series), get_horizon(time_series), get_scaling_factor_multiplier(time_series), + Dict{String, Any}(string(k) => v for (k, v) in features), ) end diff --git a/src/serialization.jl b/src/serialization.jl index 73914ab96..9895d696a 100644 --- a/src/serialization.jl +++ b/src/serialization.jl @@ -159,18 +159,6 @@ function deserialize(::Type{T}, data::Dict) where {T <: InfrastructureSystemsTyp return deserialize_struct(T, data) end -function deserialize_struct(::Type{TimeSeriesKey}, data::Dict) - vals = Dict{Symbol, Any}() - for field_name in fieldnames(TimeSeriesKey) - val = data[string(field_name)] - if field_name == :time_series_type - val = getfield(InfrastructureSystems, Symbol(strip_module_name(val))) - end - vals[field_name] = val - end - return TimeSeriesKey(; vals...) -end - function deserialize_to_dict(::Type{T}, data::Dict) where {T} # Note: mostly duplicated in src/deterministic_metadata.jl vals = Dict{Symbol, Any}() diff --git a/src/single_time_series.jl b/src/single_time_series.jl index cfd8a0029..437b85b90 100644 --- a/src/single_time_series.jl +++ b/src/single_time_series.jl @@ -194,7 +194,7 @@ function SingleTimeSeries(ts_metadata::SingleTimeSeriesMetadata, data::TimeSerie ) end -function SingleTimeSeriesMetadata(ts::SingleTimeSeries) +function SingleTimeSeriesMetadata(ts::SingleTimeSeries; features...) return SingleTimeSeriesMetadata( get_name(ts), get_resolution(ts), @@ -202,6 +202,7 @@ function SingleTimeSeriesMetadata(ts::SingleTimeSeries) get_uuid(ts), length(ts), get_scaling_factor_multiplier(ts), + Dict{String, Any}(string(k) => v for (k, v) in features), ) end diff --git a/src/supplemental_attribute.jl b/src/supplemental_attribute.jl index a9dd3aa70..e5fe03813 100644 --- a/src/supplemental_attribute.jl +++ b/src/supplemental_attribute.jl @@ -31,40 +31,6 @@ function is_attached_to_component(attribute::SupplementalAttribute) return !isempty(get_component_uuids(attribute)) end -""" -Return true if the attribute has time series data. -""" -function has_time_series(attribute::SupplementalAttribute) - container = get_time_series_container(attribute) - return !isnothing(container) && !isempty(container) -end - -function clear_time_series_storage!(attribute::SupplementalAttribute) - storage = _get_time_series_storage(attribute) - if !isnothing(storage) - # In the case of Deterministic and DeterministicSingleTimeSeries the UUIDs - # can be shared. - uuids = Set{Base.UUID}() - for (uuid, name) in get_time_series_uuids(attribute) - if !(uuid in uuids) - remove_time_series!(storage, uuid, get_uuid(attribute), name) - push!(uuids, uuid) - end - end - end -end - -function set_time_series_storage!( - attribute::SupplementalAttribute, - storage::Union{Nothing, TimeSeriesStorage}, -) - container = get_time_series_container(attribute) - if !isnothing(container) - set_time_series_storage!(container, storage) - end - return -end - """ This function must be called when an attribute is removed from a system. """ @@ -79,32 +45,8 @@ function prepare_for_removal!( ) end - # TimeSeriesContainer can only be part of a component when that component is part of a - # system. - clear_time_series_storage!(attribute) - set_time_series_storage!(attribute, nothing) clear_time_series!(attribute) + set_time_series_manager!(attribute, nothing) @debug "cleared all time series data from" _group = LOG_GROUP_SYSTEM get_uuid(attribute) return end - -function _get_time_series_storage(attribute::SupplementalAttribute) - container = get_time_series_container(attribute) - if isnothing(container) - return nothing - end - - return container.time_series_storage -end - -function clear_time_series!( - attribute::T, -) where {T <: SupplementalAttribute} - container = get_time_series_container(attribute) - if !isnothing(container) - clear_time_series!(container) - @debug "Cleared time_series in attribute type $T, $(get_uuid(attribute))." _group = - LOG_GROUP_TIME_SERIES - end - return -end diff --git a/src/supplemental_attributes.jl b/src/supplemental_attributes.jl index fb14f9db3..eea8e8183 100644 --- a/src/supplemental_attributes.jl +++ b/src/supplemental_attributes.jl @@ -1,12 +1,12 @@ struct SupplementalAttributes <: InfrastructureSystemsContainer data::SupplementalAttributesContainer - time_series_storage::TimeSeriesStorage + time_series_manager::TimeSeriesManager end get_member_string(::SupplementalAttributes) = "supplemental attributes" -function SupplementalAttributes(time_series_storage::TimeSeriesStorage) - return SupplementalAttributes(SupplementalAttributesContainer(), time_series_storage) +function SupplementalAttributes(time_series_manager::TimeSeriesManager) + return SupplementalAttributes(SupplementalAttributesContainer(), time_series_manager) end function add_supplemental_attribute!( @@ -65,9 +65,9 @@ function _add_supplemental_attribute!( ) end - set_time_series_storage!( + set_time_series_manager!( supplemental_attribute, - supplemental_attributes.time_series_storage, + supplemental_attributes.time_series_manager, ) supplemental_attributes.data[T][supplemental_attribute_uuid] = supplemental_attribute return @@ -88,12 +88,6 @@ function iterate_supplemental_attributes(supplemental_attributes::SupplementalAt iterate_container(supplemental_attributes) end -function iterate_supplemental_attributes_with_time_series( - supplemental_attributes::SupplementalAttributes, -) - iterate_container_with_time_series(supplemental_attributes) -end - """ Returns the total number of stored supplemental_attributes """ @@ -126,7 +120,7 @@ function remove_supplemental_attribute!( if isempty(supplemental_attributes.data[T]) pop!(supplemental_attributes.data, T) end - clear_time_series_storage!(supplemental_attribute) + set_time_series_manager!(supplemental_attribute, nothing) return end @@ -191,7 +185,7 @@ end function deserialize( ::Type{SupplementalAttributes}, data::Vector, - time_series_storage::TimeSeriesStorage, + time_series_manager::TimeSeriesManager, ) attributes = SupplementalAttributesByType() for attr_dict in data @@ -211,6 +205,6 @@ function deserialize( return SupplementalAttributes( SupplementalAttributesContainer(attributes), - time_series_storage, + time_series_manager, ) end diff --git a/src/system_data.jl b/src/system_data.jl index 5d350d0e6..91a5e9bd4 100644 --- a/src/system_data.jl +++ b/src/system_data.jl @@ -11,10 +11,7 @@ const SERIALIZATION_METADATA_KEY = "__serialization_metadata__" are not exposed in the standard library calls like [`get_components`](@ref). Examples are components in a subsystem." masked_components::Components - time_series_params::TimeSeriesParameters validation_descriptors::Vector - time_series_storage::TimeSeriesStorage - time_series_storage_file::Union{Nothing, String} internal::InfrastructureSystemsInternal end @@ -28,8 +25,7 @@ mutable struct SystemData <: InfrastructureSystemsType "User-defined subystems. Components can be regular or masked." subsystems::Dict{String, Set{Base.UUID}} attributes::SupplementalAttributes - time_series_params::TimeSeriesParameters - time_series_storage::TimeSeriesStorage + time_series_manager::TimeSeriesManager validation_descriptors::Vector internal::InfrastructureSystemsInternal end @@ -60,49 +56,42 @@ function SystemData(; read_validation_descriptor(validation_descriptor_file) end - if time_series_directory === nothing && haskey(ENV, TIME_SERIES_DIRECTORY_ENV_VAR) - time_series_directory = ENV[TIME_SERIES_DIRECTORY_ENV_VAR] - end - - ts_storage = make_time_series_storage(; + time_series_manager = TimeSeriesManager(; in_memory = time_series_in_memory, directory = time_series_directory, compression = compression, ) - components = Components(ts_storage, validation_descriptors) - attributes = SupplementalAttributes(ts_storage) - masked_components = Components(ts_storage, validation_descriptors) + components = Components(time_series_manager, validation_descriptors) + attributes = SupplementalAttributes(time_series_manager) + masked_components = Components(time_series_manager, validation_descriptors) return SystemData( components, masked_components, Dict{Base.UUID, InfrastructureSystemsComponent}(), Dict{String, Set{Base.UUID}}(), attributes, - TimeSeriesParameters(), - ts_storage, + time_series_manager, validation_descriptors, InfrastructureSystemsInternal(), ) end function SystemData( - time_series_params, validation_descriptors, - time_series_storage, + time_series_manager, subsystems, attributes, internal, ) - components = Components(time_series_storage, validation_descriptors) - masked_components = Components(time_series_storage, validation_descriptors) + components = Components(time_series_manager, validation_descriptors) + masked_components = Components(time_series_manager, validation_descriptors) return SystemData( components, masked_components, Dict{Base.UUID, InfrastructureSystemsComponent}(), subsystems, attributes, - time_series_params, - time_series_storage, + time_series_manager, validation_descriptors, internal, ) @@ -115,7 +104,13 @@ function open_time_series_store!( args...; kwargs..., ) - open_store!(func, data.time_series_storage, mode, args...; kwargs...) + open_store!( + func, + data.time_series_manager.data_store, + mode, + args...; + kwargs..., + ) end """ @@ -166,62 +161,31 @@ function add_time_series_from_file_metadata!( end """ -Add time series data to a component. - -# Arguments - - - `data::SystemData`: SystemData - - `component::InfrastructureSystemsComponent`: will store the time series reference - - `time_series::TimeSeriesData`: Any object of subtype TimeSeriesData - -Throws ArgumentError if the component is not stored in the system. -""" -function add_time_series!( - data::SystemData, - component::InfrastructureSystemsComponent, - time_series::TimeSeriesData; - skip_if_present = false, -) - metadata_type = time_series_data_to_metadata(typeof(time_series)) - ts_metadata = metadata_type(time_series) - _validate_component(data, component) - attach_time_series_and_serialize!( - data, - component, - ts_metadata, - time_series; - skip_if_present = skip_if_present, - ) - return -end - -""" -Add time series data to an attribute. +Add time series data to a component or supplemental attribute. # Arguments - `data::SystemData`: SystemData - - `attribute::SupplementalAttribute`: will store the time series reference + - `owner::InfrastructureSystemsComponent`: will store the time series reference - `time_series::TimeSeriesData`: Any object of subtype TimeSeriesData -Throws ArgumentError if the attribute is not stored in the system. +Throws ArgumentError if the owner is not stored in the system. """ function add_time_series!( data::SystemData, - attribute::SupplementalAttribute, + owner::TimeSeriesOwners, time_series::TimeSeriesData; skip_if_present = false, + features..., ) - metadata_type = time_series_data_to_metadata(typeof(time_series)) - ts_metadata = metadata_type(time_series) - attach_time_series_and_serialize!( - data, - attribute, - ts_metadata, + _validate(data, owner) + add_time_series!( + data.time_series_manager, + owner, time_series; skip_if_present = skip_if_present, + features..., ) - return end """ @@ -238,11 +202,19 @@ individually with the same data because in this case, only one time series array Throws ArgumentError if a component is not stored in the system. """ -function add_time_series!(data::SystemData, components, time_series::TimeSeriesData) - metadata_type = time_series_data_to_metadata(typeof(time_series)) - ts_metadata = metadata_type(time_series) +function add_time_series!( + data::SystemData, + components, + time_series::TimeSeriesData; + features..., +) for component in components - attach_time_series_and_serialize!(data, component, ts_metadata, time_series) + add_time_series!( + data, + component, + time_series; + features..., + ) end end @@ -266,16 +238,10 @@ function remove_time_series!( data::SystemData, ::Type{T}, component::InfrastructureSystemsComponent, - name::String, + name::String; + features..., ) where {T <: TimeSeriesData} - type = time_series_data_to_metadata(T) - time_series = get_time_series_metadata(type, component, name) - uuid = get_time_series_uuid(time_series) - if remove_time_series_metadata!(component, type, name) - remove_time_series!(data.time_series_storage, uuid, get_uuid(component), name) - end - - return + return remove_time_series!(data.time_series_manager, T, component, name; features...) end function remove_time_series!( @@ -283,46 +249,30 @@ function remove_time_series!( component::InfrastructureSystemsComponent, ts_metadata::TimeSeriesMetadata, ) - uuid = get_time_series_uuid(ts_metadata) - name = get_name(ts_metadata) - if remove_time_series_metadata!(component, typeof(ts_metadata), name) - remove_time_series!(data.time_series_storage, uuid, get_uuid(component), name) - end - - return + return remove_time_series!(data.time_series_manager, component, ts_metadata) end """ -Return a time series from TimeSeriesFileMetadata. +Removes all time series of a particular type from a System. # Arguments - - `cache::TimeSeriesParsingCache`: cached data - - `ts_file_metadata::TimeSeriesFileMetadata`: metadata - - `resolution::{Nothing, Dates.Period}`: skip any time_series that don't match this resolution + - `data::SystemData`: system + - `type::Type{<:TimeSeriesData}`: Type of time series objects to remove. """ -function make_time_series!( - cache::TimeSeriesParsingCache, - ts_file_metadata::TimeSeriesFileMetadata, -) - info = add_time_series_info!(cache, ts_file_metadata) - return ts_file_metadata.time_series_type(info) -end - -function add_time_series_info!( - cache::TimeSeriesParsingCache, - metadata::TimeSeriesFileMetadata, -) - time_series = _add_time_series_info!(cache, metadata) - info = TimeSeriesParsedInfo(metadata, time_series) - @debug "Added TimeSeriesParsedInfo" _group = LOG_GROUP_TIME_SERIES metadata - return info +function remove_time_series!(data::SystemData, ::Type{T}) where {T <: TimeSeriesData} + _throw_if_read_only(data.time_series_manager) + for component in iterate_components_with_time_series(data; time_series_type = T) + for ts_metadata in list_time_series_metadata(component; time_series_type = T) + remove_time_series!(data, component, ts_metadata) + end + end end """ -Checks that the component exists in data and the UUID's match. +Checks that the component exists in data and is the same object. """ -function _validate_component( +function _validate( data::SystemData, component::T, ) where {T <: InfrastructureSystemsComponent} @@ -335,13 +285,23 @@ function _validate_component( end end - user_uuid = get_uuid(component) - ps_uuid = get_uuid(comp) - if user_uuid != ps_uuid + if component !== comp throw( ArgumentError( - "comp UUID doesn't match, perhaps it was copied?; " * - "$T name=$(get_name(component)) user=$user_uuid system=$ps_uuid", + "$(summary(component)) does not match the stored component of the same " * + "type and name. Was it copied?", + ), + ) + end +end + +function _validate(data::SystemData, attribute::SupplementalAttribute) + _attribute = get_supplemental_attribute(data, get_uuid(attribute)) + if attribute !== _attribute + throw( + ArgumentError( + "$(summary(attribute)) does not match the stored attribute of the same " * + "type and name. Was it copied?", ), ) end @@ -363,12 +323,6 @@ function compare_values( end val_x = getfield(x, name) val_y = getfield(y, name) - if name == :time_series_storage && typeof(val_x) != typeof(val_y) - @warn "Cannot compare $(typeof(val_x)) and $(typeof(val_y))" - # TODO 1.0: workaround for not being able to convert Hdf5TimeSeriesStorage to - # InMemoryTimeSeriesStorage - continue - end if !compare_values(val_x, val_y; compare_uuids = compare_uuids, exclude = exclude) @error "SystemData field = $name does not match" getfield(x, name) getfield( y, @@ -422,7 +376,7 @@ function mask_component!( remove_time_series = false, ) remove_component!(data.components, component; remove_time_series = remove_time_series) - set_time_series_storage!(component, nothing) + set_time_series_manager!(component, nothing) return add_masked_component!( data, component; @@ -431,50 +385,34 @@ function mask_component!( ) end -function clear_time_series!(data::SystemData) - clear_time_series!(data.time_series_storage) - clear_time_series!(data.components) - reset_info!(data.time_series_params) - return -end - -function iterate_components_with_time_series(data::SystemData) - return Iterators.Flatten(( - iterate_components_with_time_series(data.components), - iterate_components_with_time_series(data.masked_components), - )) -end +clear_time_series!(data::SystemData) = clear_time_series!(data.time_series_manager) -function iterate_supplemental_attributes_with_time_series(data::SystemData) - iterate_supplemental_attributes_with_time_series(data.attributes) +function iterate_components_with_time_series( + data::SystemData; + time_series_type::Union{Nothing, Type{<:TimeSeriesData}} = nothing, +) + return ( + get_component(data, x) for + x in list_owner_uuids_with_time_series( + data.time_series_manager.metadata_store, + InfrastructureSystemsComponent; + time_series_type = time_series_type, + ) + ) end -""" -Removes all time series of a particular type from a System. - -# Arguments - - - `data::SystemData`: system - - `type::Type{<:TimeSeriesData}`: Type of time series objects to remove. -""" -function remove_time_series!(data::SystemData, ::Type{T}) where {T <: TimeSeriesData} - for component in iterate_components_with_time_series(data) - for ts_metadata in list_time_series_metadata(component) - if time_series_metadata_to_data(ts_metadata) <: T - remove_time_series!(data, component, ts_metadata) - end - end - end - counts = get_time_series_counts(data) - if counts.forecast_count == 0 - if counts.static_time_series_count == 0 - reset_info!(data.time_series_params) - else - reset_info!(data.time_series_params.forecast_params) - end - end - - return +function iterate_supplemental_attributes_with_time_series( + data::SystemData, + time_series_type::Union{Nothing, Type{<:TimeSeriesData}} = nothing, +) + return ( + get_supplemental_attribute(data, x) for + x in list_owner_uuids_with_time_series( + data.time_series_manager.metadata_store, + SupplementalAttribute; + time_series_type = time_series_type, + ) + ) end """ @@ -499,7 +437,7 @@ function get_time_series_multiple( name = nothing, ) Channel() do channel - for component in iterate_components_with_time_series(data) + for component in iterate_components_with_time_series(data; time_series_type = type) for time_series in get_time_series_multiple(component, filter_func; type = type, name = name) put!(channel, time_series) @@ -508,105 +446,8 @@ function get_time_series_multiple( end end -# These are guaranteed to be consistent already. -check_time_series_consistency(::SystemData, ::Type{<:Forecast}) = nothing - -function check_time_series_consistency(data::SystemData, ::Type{SingleTimeSeries}) - first_initial_timestamp = Dates.now() - first_len = 0 - found_first = false - for component in iterate_components_with_time_series(data) - container = get_time_series_container(component) - for key in keys(container.data) - ts_metadata = container.data[key] - if time_series_metadata_to_data(ts_metadata) === SingleTimeSeries - initial_timestamp = get_initial_timestamp(ts_metadata) - len = get_length(ts_metadata) - if !found_first - first_initial_timestamp = get_initial_timestamp(ts_metadata) - first_len = get_length(ts_metadata) - found_first = true - else - if initial_timestamp != first_initial_timestamp - throw( - InvalidValue( - "SingleTimeSeries initial timestamps are inconsistent; $initial_timestamp != $first_initial_timestamp", - ), - ) - end - if len != first_len - throw( - InvalidValue( - "SingleTimeSeries lengths are inconsistent; $len != $first_len", - ), - ) - end - end - end - end - end - - return first_initial_timestamp, first_len -end - -""" -Provides counts of time series including attachments to components and supplemental -attributes. -""" -struct TimeSeriesCounts - components_with_time_series::Int - supplemental_attributes_with_time_series::Int - static_time_series_count::Int - forecast_count::Int -end - -""" -Build an instance of TimeSeriesCounts by scanning the system. -""" -function get_time_series_counts(data::SystemData) - component_count = 0 - attribute_count = 0 - static_time_series_count = 0 - forecast_count = 0 - # Note that the same time series UUID can exist in in multiple types, such as with - # SingleTimeSeries and DeterministicSingleTimeSeries. - uuids_by_type = Dict{DataType, Set{Base.UUID}}() - - function update_time_series_counts(object) - for metadata in iterate_time_series_metadata(get_time_series_container(object)) - ts_type = time_series_metadata_to_data(metadata) - if !haskey(uuids_by_type, ts_type) - uuids_by_type[ts_type] = Set{Base.UUID}() - end - uuid = get_time_series_uuid(metadata) - if !in(uuid, uuids_by_type[ts_type]) - if ts_type <: StaticTimeSeries - static_time_series_count += 1 - elseif ts_type <: Forecast - forecast_count += 1 - end - push!(uuids_by_type[ts_type], uuid) - end - end - end - - for component in iterate_components_with_time_series(data) - component_count += 1 - update_time_series_counts(component) - end - - for attr in iterate_supplemental_attributes_with_time_series(data) - attribute_count += 1 - update_time_series_counts(attr) - end - - return TimeSeriesCounts( - component_count, - attribute_count, - static_time_series_count, - forecast_count, - ) -end +check_time_series_consistency(data::SystemData, ts_type) = + check_consistency(data.time_series_manager.metadata_store, ts_type) """ Transform all instances of SingleTimeSeries to DeterministicSingleTimeSeries. @@ -620,29 +461,37 @@ function transform_single_time_series!( horizon::Int, interval::Dates.Period, ) where {T <: DeterministicSingleTimeSeries} + resolutions = list_time_series_resolutions(data; time_series_type = SingleTimeSeries) + if length(resolutions) > 1 + # TODO: This needs to support an alternate method where horizon is expressed as a + # Period (horizon * resolution) + error( + "transform_single_time_series! is not yet supported when there is more than " * + "one resolution: $resolutions", + ) + end + remove_time_series!(data, DeterministicSingleTimeSeries) - params = nothing - set_params = false - for component in iterate_components_with_time_series(data) - if params === nothing + for (i, uuid) in enumerate( + list_owner_uuids_with_time_series( + data.time_series_manager.metadata_store, + InfrastructureSystemsComponent; + time_series_type = SingleTimeSeries, + ), + ) + component = get_component(data, uuid) + if i == 1 params = get_single_time_series_transformed_parameters( component, T, horizon, interval, ) - if params === nothing - # This component doesn't have SingleTimeSeries. - continue - end # This will throw if there is another forecast type with conflicting parameters. - check_params_compatibility(data.time_series_params, params) + check_params_compatibility(data.time_series_manager.metadata_store, params) end - if transform_single_time_series_internal!(component, T, params) && !set_params - set_parameters!(data.time_series_params, params) - set_params = true - end + transform_single_time_series_internal!(component, T, horizon, interval) end end @@ -724,7 +573,6 @@ function to_dict(data::SystemData) :masked_components, :subsystems, :attributes, - :time_series_params, :internal, ) serialized_data[string(field)] = serialize(getfield(data, field)) @@ -744,17 +592,19 @@ function serialize(data::SystemData) directory = metadata["serialization_directory"] base = metadata["basename"] - if isempty(data.time_series_storage) + if isempty(data.time_series_manager.data_store) json_data["time_series_compression_enabled"] = - get_compression_settings(data.time_series_storage).enabled + get_compression_settings(data.time_series_manager.data_store).enabled json_data["time_series_in_memory"] = - data.time_series_storage isa InMemoryTimeSeriesStorage + data.time_series_manager.data_store isa InMemoryTimeSeriesStorage else time_series_base_name = _get_secondary_basename(base, TIME_SERIES_STORAGE_FILE) time_series_storage_file = joinpath(directory, time_series_base_name) - serialize(data.time_series_storage, time_series_storage_file) + serialize(data.time_series_manager.data_store, time_series_storage_file) + to_h5_file(data.time_series_manager.metadata_store, time_series_storage_file) json_data["time_series_storage_file"] = time_series_base_name - json_data["time_series_storage_type"] = string(typeof(data.time_series_storage)) + json_data["time_series_storage_type"] = + string(typeof(data.time_series_manager.data_store)) end end @@ -769,9 +619,9 @@ function deserialize( time_series_read_only = false, time_series_directory = nothing, validation_descriptor_file = nothing, + kwargs..., ) @debug "deserialize" raw _group = LOG_GROUP_SERIALIZATION - time_series_params = deserialize(TimeSeriesParameters, raw["time_series_params"]) if haskey(raw, "time_series_storage_file") if !isfile(raw["time_series_storage_file"]) @@ -786,8 +636,13 @@ function deserialize( time_series_storage = from_file( Hdf5TimeSeriesStorage, raw["time_series_storage_file"]; - read_only = time_series_read_only, directory = time_series_directory, + read_only = time_series_read_only, + ) + time_series_metadata_store = from_h5_file( + TimeSeriesMetadataStore, + time_series_storage.file_path, + time_series_directory, ) else time_series_storage = make_time_series_storage(; @@ -796,10 +651,16 @@ function deserialize( ), directory = time_series_directory, ) + time_series_metadata_store = nothing end + time_series_manager = TimeSeriesManager(; + data_store = time_series_storage, + read_only = time_series_read_only, + metadata_store = time_series_metadata_store, + ) subsystems = Dict(k => Set(Base.UUID.(v)) for (k, v) in raw["subsystems"]) - attributes = deserialize(SupplementalAttributes, raw["attributes"], time_series_storage) + attributes = deserialize(SupplementalAttributes, raw["attributes"], time_series_manager) internal = deserialize(InfrastructureSystemsInternal, raw["internal"]) validation_descriptors = if isnothing(validation_descriptor_file) [] @@ -808,9 +669,8 @@ function deserialize( end @debug "deserialize" _group = LOG_GROUP_SERIALIZATION time_series_storage internal sys = SystemData( - time_series_params, validation_descriptors, - time_series_storage, + time_series_manager, subsystems, attributes, internal, @@ -972,17 +832,30 @@ function get_masked_component(data::SystemData, uuid::Base.UUID) end get_forecast_initial_times(data::SystemData) = - get_forecast_initial_times(data.time_series_params) -get_forecast_total_period(data::SystemData) = - get_forecast_total_period(data.time_series_params) + get_forecast_initial_times(data.time_series_manager.metadata_store) get_forecast_window_count(data::SystemData) = - get_forecast_window_count(data.time_series_params) -get_forecast_horizon(data::SystemData) = get_forecast_horizon(data.time_series_params) + get_forecast_window_count(data.time_series_manager.metadata_store) +get_forecast_horizon(data::SystemData) = + get_forecast_horizon(data.time_series_manager.metadata_store) get_forecast_initial_timestamp(data::SystemData) = - get_forecast_initial_timestamp(data.time_series_params) -get_forecast_interval(data::SystemData) = get_forecast_interval(data.time_series_params) -get_time_series_resolution(data::SystemData) = - get_time_series_resolution(data.time_series_params) + get_forecast_initial_timestamp(data.time_series_manager.metadata_store) +get_forecast_interval(data::SystemData) = + get_forecast_interval(data.time_series_manager.metadata_store) + +list_time_series_resolutions( + data::SystemData; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, +) = list_time_series_resolutions( + data.time_series_manager.metadata_store; + time_series_type = time_series_type, +) + +# TODO: do we need this? The old way of calculating this required a single resolution. +# function get_forecast_total_period(data::SystemData) +# params = get_forecast_parameters(data.time_series_manager.metadata_store) +# isnothing(params) && return Dates.Second(0) +# return get_total_period(params.initial_timestamp, params.count, params.interval, params.horizon) +# end clear_components!(data::SystemData) = clear_components!(data.components) @@ -995,7 +868,7 @@ end check_component(data::SystemData, component) = check_component(data.components, component) get_compression_settings(data::SystemData) = - get_compression_settings(data.time_series_storage) + get_compression_settings(data.time_series_manager.data_store) set_name!(data::SystemData, component, name) = set_name!(data.components, component, name) @@ -1010,22 +883,14 @@ function get_component_counts_by_type(data::SystemData) ] end -function get_time_series_counts_by_type(data::SystemData) - counts = Dict{String, Int}() - for component in iterate_components_with_time_series(data) - for (type, count) in get_num_time_series_by_type(component) - if haskey(counts, type) - counts[type] += count - else - counts[type] = count - end - end - end - - return [ - OrderedDict("type" => x, "count" => counts[x]) for x in sort(collect(keys(counts))) - ] -end +get_num_time_series(data::SystemData) = + get_num_time_series(data.time_series_manager.metadata_store) +get_time_series_counts(data::SystemData) = + get_time_series_counts(data.time_series_manager.metadata_store) +get_time_series_counts_by_type(data::SystemData) = + get_time_series_counts_by_type(data.time_series_manager.metadata_store) +get_time_series_summary_table(data::SystemData) = + get_time_series_summary_table(data.time_series_manager.metadata_store) _get_system_basename(system_file) = splitext(basename(system_file))[1] _get_secondary_basename(system_basename, name) = system_basename * "_" * name diff --git a/src/time_series_cache.jl b/src/time_series_cache.jl index 384aaafbc..edee30f9b 100644 --- a/src/time_series_cache.jl +++ b/src/time_series_cache.jl @@ -228,8 +228,7 @@ function ForecastCache( cache_size_bytes = TIME_SERIES_CACHE_SIZE_BYTES, ignore_scaling_factors = false, ) where {T <: Forecast} - metadata_type = time_series_data_to_metadata(T) - ts_metadata = get_time_series_metadata(metadata_type, component, name) + ts_metadata = get_time_series_metadata(T, component, name) initial_timestamp = get_initial_timestamp(ts_metadata) if start_time === nothing start_time = initial_timestamp @@ -345,8 +344,7 @@ function StaticTimeSeriesCache( start_time::Union{Nothing, Dates.DateTime} = nothing, ignore_scaling_factors = false, ) where {T <: StaticTimeSeries} - metadata_type = time_series_data_to_metadata(T) - ts_metadata = get_time_series_metadata(metadata_type, component, name) + ts_metadata = get_time_series_metadata(T, component, name) initial_timestamp = get_initial_timestamp(ts_metadata) if start_time === nothing start_time = initial_timestamp diff --git a/src/time_series_container.jl b/src/time_series_container.jl index f0e297bab..2c631326f 100644 --- a/src/time_series_container.jl +++ b/src/time_series_container.jl @@ -1,136 +1,34 @@ -struct TimeSeriesKey <: InfrastructureSystemsType - time_series_type::Type{<:TimeSeriesMetadata} - name::String -end - -function TimeSeriesKey(; time_series_type::Type{<:TimeSeriesMetadata}, name::String) - return TimeSeriesKey(time_series_type, name) -end - -function TimeSeriesKey(data::TimeSeriesData) - metadata_type = time_series_data_to_metadata(typeof(data)) - return TimeSeriesKey(metadata_type, get_name(data)) -end - -const TimeSeriesByType = Dict{TimeSeriesKey, TimeSeriesMetadata} - """ Time series container for a component. """ mutable struct TimeSeriesContainer - data::TimeSeriesByType - time_series_storage::Union{Nothing, TimeSeriesStorage} + manager::Union{Nothing, TimeSeriesManager} end function TimeSeriesContainer() - return TimeSeriesContainer(TimeSeriesByType(), nothing) + return TimeSeriesContainer(nothing) end -Base.length(container::TimeSeriesContainer) = length(container.data) -Base.isempty(container::TimeSeriesContainer) = isempty(container.data) +getproperty(::TimeSeriesContainer, x) = nothing + +get_time_series_manager(x::TimeSeriesContainer) = x.manager -function set_time_series_storage!( +function set_time_series_manager!( container::TimeSeriesContainer, - storage::Union{Nothing, TimeSeriesStorage}, + time_series_manager::Union{Nothing, TimeSeriesManager}, ) - if !isnothing(container.time_series_storage) && !isnothing(storage) + if !isnothing(container.manager) && !isnothing(time_series_manager) throw( ArgumentError( - "The time_series_storage reference is already set. Is this component being " * + "The time_series_manager reference is already set. Is this component being " * "added to multiple systems?", ), ) end - container.time_series_storage = storage - return -end - -function add_time_series!( - container::TimeSeriesContainer, - ts_metadata::T; - skip_if_present = false, -) where {T <: TimeSeriesMetadata} - key = TimeSeriesKey(T, get_name(ts_metadata)) - if haskey(container.data, key) - if skip_if_present - @warn "time_series $key is already present, skipping overwrite" - else - throw(ArgumentError("time_series $key is already stored")) - end - else - container.data[key] = ts_metadata - end -end - -function remove_time_series!( - container::TimeSeriesContainer, - ::Type{T}, - name::AbstractString, -) where {T <: TimeSeriesMetadata} - key = TimeSeriesKey(T, name) - if !haskey(container.data, key) - throw(ArgumentError("time_series $key is not stored")) - end - - pop!(container.data, key) - return -end - -function clear_time_series!(container::TimeSeriesContainer) - empty!(container.data) + container.manager = time_series_manager return end -function get_time_series_metadata( - ::Type{T}, - container::TimeSeriesContainer, - name::AbstractString, -) where {T <: TimeSeriesMetadata} - key = TimeSeriesKey(T, name) - if !haskey(container.data, key) - throw(ArgumentError("time_series $key is not stored")) - end - - return container.data[key] -end - -function get_time_series_names( - ::Type{T}, - container::TimeSeriesContainer, -) where {T <: TimeSeriesMetadata} - names = Set{String}() - for key in keys(container.data) - if key.time_series_type <: T - push!(names, key.name) - end - end - - return Vector{String}(collect(names)) -end - -function has_time_series_internal( - container::TimeSeriesContainer, - ::Type{T}, - name::AbstractString, -) where {T <: TimeSeriesMetadata} - return haskey(container.data, TimeSeriesKey(T, name)) -end - -function serialize(container::TimeSeriesContainer) - # Store a flat array of time series. Deserialization can unwind it. - return serialize_struct.(values(container.data)) -end - -function deserialize(::Type{TimeSeriesContainer}, data::Vector) - container = TimeSeriesContainer() - for ts_dict in data - type = get_type_from_serialization_metadata(get_serialization_metadata(ts_dict)) - time_series = deserialize(type, ts_dict) - add_time_series!(container, time_series) - end - - return container -end - -iterate_time_series_metadata(container::TimeSeriesContainer) = values(container.data) +serialize(::TimeSeriesContainer) = Dict() +deserialize(::Type{TimeSeriesContainer}, ::Dict) = TimeSeriesContainer() diff --git a/src/time_series_formats.jl b/src/time_series_formats.jl index e90dedd46..69ba078f8 100644 --- a/src/time_series_formats.jl +++ b/src/time_series_formats.jl @@ -207,7 +207,7 @@ function read_time_series( ) where {T <: Union{TimeSeriesFormatPeriodAsColumn, TimeSeriesFormatDateTimeAsColumn}} first_timestamp = get_timestamp(T, file, 1) value_columns = get_value_columns(T, file) - vals = [(string(x) => getproperty(file, x)) for x in value_columns] + vals = [(string(x) => Base.getproperty(file, x)) for x in value_columns] series_length = length(vals[1][2]) return RawTimeSeries(first_timestamp, Dict(vals...), series_length) end @@ -229,7 +229,7 @@ function read_time_series( vals = Vector{Float64}() for i in 1:length(file) for period in period_cols_as_symbols - val = getproperty(file, period)[i] + val = Base.getproperty(file, period)[i] push!(vals, val) end end @@ -253,7 +253,7 @@ function read_time_series( ) where {T <: TimeSeriesFormatComponentsAsColumnsNoTime} first_timestamp = get(kwargs, :start_datetime, Dates.DateTime(Dates.today())) value_columns = get_value_columns(T, file) - vals = [(string(x) => getproperty(file, x)) for x in value_columns] + vals = [(string(x) => Base.getproperty(file, x)) for x in value_columns] series_length = length(vals[1][2]) return RawTimeSeries(first_timestamp, Dict(vals...), series_length) end diff --git a/src/time_series_interface.jl b/src/time_series_interface.jl index e1a1faeef..b3d5126ad 100644 --- a/src/time_series_interface.jl +++ b/src/time_series_interface.jl @@ -1,137 +1,31 @@ -const SupportedTimeSeriesTypes = - Union{InfrastructureSystemsComponent, SupplementalAttribute} - -function add_time_series!( - component::T, - time_series::TimeSeriesMetadata; - skip_if_present = false, -) where {T <: SupportedTimeSeriesTypes} - component_id = get_uuid(component) - container = get_time_series_container(component) - if isnothing(container) - throw(ArgumentError("type $T does not support storing time series")) - end - - add_time_series!(container, time_series; skip_if_present = skip_if_present) - @debug "Added $time_series to $(typeof(component)) $(component_id) " * - "num_time_series=$(length(get_time_series_container(component).data))." _group = - LOG_GROUP_TIME_SERIES -end - """ -Removes the metadata for a time_series. -If this returns true then the caller must also remove the actual time series data. +Return the TimeSeriesManager or nothing if the component/attribute does not support time +series. """ -function remove_time_series_metadata!( - component::SupportedTimeSeriesTypes, - ::Type{T}, - name::AbstractString, -) where {T <: TimeSeriesMetadata} - container = get_time_series_container(component) - remove_time_series!(container, T, name) - @debug "Removed time_series from $(get_name(component)): $name." _group = - LOG_GROUP_TIME_SERIES - if T <: DeterministicMetadata && - has_time_series_internal(container, SingleTimeSeriesMetadata, name) - return false - elseif T <: SingleTimeSeriesMetadata && - has_time_series_internal(container, DeterministicMetadata, name) - return false - end - - return true +function get_time_series_manager(owner::TimeSeriesOwners) + container = get_time_series_container(owner) + isnothing(container) && return nothing + return get_time_series_manager(container) end -function clear_time_series!(component::SupportedTimeSeriesTypes) - container = get_time_series_container(component) +function set_time_series_manager!( + owner::TimeSeriesOwners, + time_series_manager::Union{Nothing, TimeSeriesManager}, +) + container = get_time_series_container(owner) if !isnothing(container) - clear_time_series!(container) - @debug "Cleared time_series in $(get_name(component))." _group = - LOG_GROUP_TIME_SERIES + set_time_series_manager!(container, time_series_manager) end return end -function _get_columns(start_time, count, ts_metadata::ForecastMetadata) - offset = start_time - get_initial_timestamp(ts_metadata) - interval = time_period_conversion(get_interval(ts_metadata)) - window_count = get_count(ts_metadata) - if window_count > 1 - index = Int(offset / interval) + 1 - else - index = 1 - end - if count === nothing - count = window_count - index + 1 +function get_time_series_storage(owner::TimeSeriesOwners) + mgr = get_time_series_manager(owner) + if isnothing(mgr) + return nothing end - if index + count - 1 > get_count(ts_metadata) - throw( - ArgumentError( - "The requested start_time $start_time and count $count are invalid", - ), - ) - end - return UnitRange(index, index + count - 1) -end - -_get_columns(start_time, count, ts_metadata::StaticTimeSeriesMetadata) = UnitRange(1, 1) - -function _get_rows(start_time, len, ts_metadata::StaticTimeSeriesMetadata) - index = - Int( - (start_time - get_initial_timestamp(ts_metadata)) / get_resolution(ts_metadata), - ) + 1 - if len === nothing - len = length(ts_metadata) - index + 1 - end - if index + len - 1 > length(ts_metadata) - throw( - ArgumentError( - "The requested index=$index len=$len exceeds the range $(length(ts_metadata))", - ), - ) - end - - return UnitRange(index, index + len - 1) -end - -function _get_rows(start_time, len, ts_metadata::ForecastMetadata) - if len === nothing - len = get_horizon(ts_metadata) - end - - return UnitRange(1, len) -end - -function _check_start_time(start_time, ts_metadata::TimeSeriesMetadata) - if start_time === nothing - return get_initial_timestamp(ts_metadata) - end - - time_diff = start_time - get_initial_timestamp(ts_metadata) - if time_diff < Dates.Second(0) - throw( - ArgumentError( - "start_time=$start_time is earlier than $(get_initial_timestamp(ts_metadata))", - ), - ) - end - - if typeof(ts_metadata) <: ForecastMetadata - window_count = get_count(ts_metadata) - interval = get_interval(ts_metadata) - if window_count > 1 && - Dates.Millisecond(time_diff) % Dates.Millisecond(interval) != Dates.Second(0) - throw( - ArgumentError( - "start_time=$start_time is not on a multiple of interval=$interval", - ), - ) - end - end - - return start_time + return mgr.data_store end """ @@ -140,7 +34,7 @@ Return a time series corresponding to the given parameters. # Arguments - `::Type{T}`: Concrete subtype of TimeSeriesData to return - - `component::SupportedTimeSeriesTypes`: Component containing the time series + - `owner::TimeSeriesOwners`: Component or attribute containing the time series - `name::AbstractString`: name of time series - `start_time::Union{Nothing, Dates.DateTime} = nothing`: If nothing, use the `initial_timestamp` of the time series. If T is a subtype of Forecast then `start_time` @@ -152,64 +46,62 @@ Return a time series corresponding to the given parameters. """ function get_time_series( ::Type{T}, - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, name::AbstractString; start_time::Union{Nothing, Dates.DateTime} = nothing, len::Union{Nothing, Int} = nothing, count::Union{Nothing, Int} = nothing, + features..., ) where {T <: TimeSeriesData} - if !has_time_series(component) - throw(ArgumentError("no forecasts are stored in $component")) - end - - metadata_type = time_series_data_to_metadata(T) - ts_metadata = get_time_series_metadata(metadata_type, component, name) + ts_metadata = get_time_series_metadata(T, owner, name; features...) start_time = _check_start_time(start_time, ts_metadata) rows = _get_rows(start_time, len, ts_metadata) columns = _get_columns(start_time, count, ts_metadata) - storage = _get_time_series_storage(component) + storage = get_time_series_storage(owner) return deserialize_time_series(T, storage, ts_metadata, rows, columns) end function get_time_series_uuid( ::Type{T}, - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, name::AbstractString, ) where {T <: TimeSeriesData} metadata_type = time_series_data_to_metadata(T) - metadata = get_time_series_metadata(metadata_type, component, name) + metadata = get_time_series_metadata(metadata_type, owner, name) return get_time_series_uuid(metadata) end function get_time_series_metadata( ::Type{T}, - component::SupportedTimeSeriesTypes, - name::AbstractString, -) where {T <: TimeSeriesMetadata} - return get_time_series_metadata(T, get_time_series_container(component), name) + owner::TimeSeriesOwners, + name::AbstractString; + features..., +) where {T <: TimeSeriesData} + mgr = get_time_series_manager(owner) + return get_metadata(mgr, owner, T, name; features...) end """ Return a TimeSeries.TimeArray from storage for the given time series parameters. If the data are scaling factors then the stored scaling_factor_multiplier will be called on -the component and applied to the data unless ignore_scaling_factors is true. +the owner and applied to the data unless ignore_scaling_factors is true. """ function get_time_series_array( ::Type{T}, - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, name::AbstractString; start_time::Union{Nothing, Dates.DateTime} = nothing, len::Union{Nothing, Int} = nothing, ignore_scaling_factors = false, ) where {T <: TimeSeriesData} - ts = get_time_series(T, component, name; start_time = start_time, len = len, count = 1) + ts = get_time_series(T, owner, name; start_time = start_time, len = len, count = 1) if start_time === nothing start_time = get_initial_timestamp(ts) end return get_time_series_array( - component, + owner, ts, start_time; len = len, @@ -221,30 +113,30 @@ end Return a TimeSeries.TimeArray for one forecast window from a cached Forecast instance. If the data are scaling factors then the stored scaling_factor_multiplier will be called on -the component and applied to the data unless ignore_scaling_factors is true. +the owner and applied to the data unless ignore_scaling_factors is true. See also [`ForecastCache`](@ref). """ function get_time_series_array( - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, forecast::Forecast, start_time::Dates.DateTime; len = nothing, ignore_scaling_factors = false, ) - return _make_time_array(component, forecast, start_time, len, ignore_scaling_factors) + return _make_time_array(owner, forecast, start_time, len, ignore_scaling_factors) end """ Return a TimeSeries.TimeArray from a cached StaticTimeSeries instance. If the data are scaling factors then the stored scaling_factor_multiplier will be called on -the component and applied to the data unless ignore_scaling_factors is true. +the owner and applied to the data unless ignore_scaling_factors is true. See also [`StaticTimeSeriesCache`](@ref). """ function get_time_series_array( - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, time_series::StaticTimeSeries, start_time::Union{Nothing, Dates.DateTime} = nothing; len::Union{Nothing, Int} = nothing, @@ -258,7 +150,7 @@ function get_time_series_array( len = length(time_series) end - return _make_time_array(component, time_series, start_time, len, ignore_scaling_factors) + return _make_time_array(owner, time_series, start_time, len, ignore_scaling_factors) end """ @@ -266,13 +158,13 @@ Return a vector of timestamps from storage for the given time series parameters. """ function get_time_series_timestamps( ::Type{T}, - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, name::AbstractString; start_time::Union{Nothing, Dates.DateTime} = nothing, len::Union{Nothing, Int} = nothing, ) where {T <: TimeSeriesData} return TimeSeries.timestamp( - get_time_series_array(T, component, name; start_time = start_time, len = len), + get_time_series_array(T, owner, name; start_time = start_time, len = len), ) end @@ -280,13 +172,13 @@ end Return a vector of timestamps from a cached Forecast instance. """ function get_time_series_timestamps( - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, forecast::Forecast, start_time::Union{Nothing, Dates.DateTime} = nothing; len::Union{Nothing, Int} = nothing, ) return TimeSeries.timestamp( - get_time_series_array(component, forecast, start_time; len = len), + get_time_series_array(owner, forecast, start_time; len = len), ) end @@ -294,13 +186,13 @@ end Return a vector of timestamps from a cached StaticTimeSeries instance. """ function get_time_series_timestamps( - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, time_series::StaticTimeSeries, start_time::Union{Nothing, Dates.DateTime} = nothing; len::Union{Nothing, Int} = nothing, ) return TimeSeries.timestamp( - get_time_series_array(component, time_series, start_time; len = len), + get_time_series_array(owner, time_series, start_time; len = len), ) end @@ -312,7 +204,7 @@ that accepts a cached TimeSeriesData instance. """ function get_time_series_values( ::Type{T}, - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, name::AbstractString; start_time::Union{Nothing, Dates.DateTime} = nothing, len::Union{Nothing, Int} = nothing, @@ -321,7 +213,7 @@ function get_time_series_values( return TimeSeries.values( get_time_series_array( T, - component, + owner, name; start_time = start_time, len = len, @@ -334,7 +226,7 @@ end Return an Array of values for one forecast window from a cached Forecast instance. """ function get_time_series_values( - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, forecast::Forecast, start_time::Dates.DateTime; len::Union{Nothing, Int} = nothing, @@ -342,7 +234,7 @@ function get_time_series_values( ) return TimeSeries.values( get_time_series_array( - component, + owner, forecast, start_time; len = len, @@ -356,7 +248,7 @@ Return an Array of values from a cached StaticTimeSeries instance for the reques series parameters. """ function get_time_series_values( - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, time_series::StaticTimeSeries, start_time::Union{Nothing, Dates.DateTime} = nothing; len::Union{Nothing, Int} = nothing, @@ -364,7 +256,7 @@ function get_time_series_values( ) return TimeSeries.values( get_time_series_array( - component, + owner, time_series, start_time; len = len, @@ -373,7 +265,7 @@ function get_time_series_values( ) end -function _make_time_array(component, time_series, start_time, len, ignore_scaling_factors) +function _make_time_array(owner, time_series, start_time, len, ignore_scaling_factors) ta = make_time_array(time_series, start_time; len = len) if ignore_scaling_factors return ta @@ -384,50 +276,50 @@ function _make_time_array(component, time_series, start_time, len, ignore_scalin return ta end - return ta .* multiplier(component) + return ta .* multiplier(owner) end """ -Return true if the component has time series data. +Return true if the component or supplemental attribute has time series data. """ -function has_time_series(component::SupportedTimeSeriesTypes) - container = get_time_series_container(component) - return !isnothing(container) && !isempty(container) +function has_time_series(owner::TimeSeriesOwners) + return has_time_series(get_time_series_manager(owner), owner) end """ -Return true if the component has time series data of type T. +Return true if the component or supplemental attribute has time series data of type T. """ function has_time_series( - component::SupportedTimeSeriesTypes, + val::TimeSeriesOwners, ::Type{T}, ) where {T <: TimeSeriesData} - container = get_time_series_container(component) - if container === nothing - return false - end + mgr = get_time_series_manager(val) + isnothing(mgr) && return false + return has_time_series(mgr.metadata_store, val, T) +end - for key in keys(container.data) - if isabstracttype(T) - if is_time_series_sub_type(key.time_series_type, T) - return true - end - elseif time_series_data_to_metadata(T) <: key.time_series_type - return true - end - end +function has_time_series( + val::TimeSeriesOwners, + ::Type{T}, + name::AbstractString; + features..., +) where {T <: TimeSeriesData} + mgr = get_time_series_manager(val) + isnothing(mgr) && return false + return has_time_series(mgr.metadata_store, val, T, name; features...) +end - return false +""" +Return true if the component or supplemental attribute supports time series data. +""" +function supports_time_series(owner::TimeSeriesOwners) + return !isnothing(get_time_series_container(owner)) end -function has_time_series( - component::SupportedTimeSeriesTypes, - type::Type{<:TimeSeriesMetadata}, - name::AbstractString, -) - container = get_time_series_container(component) - container === nothing && return false - return has_time_series_internal(container, type, name) +function throw_if_does_not_support_time_series(owner::TimeSeriesOwners) + if !supports_time_series(owner) + throw(ArgumentError("$(summary(owner)) does not support time series")) + end end """ @@ -436,8 +328,8 @@ references. # Arguments - - `dst::SupportedTimeSeriesTypes`: Destination component - - `src::SupportedTimeSeriesTypes`: Source component + - `dst::TimeSeriesOwners`: Destination owner + - `src::TimeSeriesOwners`: Source owner - `name_mapping::Dict = nothing`: Optionally map src names to different dst names. If provided and src has a time_series with a name not present in name_mapping, that time_series will not copied. If name_mapping is nothing then all time_series will be @@ -449,27 +341,25 @@ references. src's multipliers. """ function copy_time_series!( - dst::SupportedTimeSeriesTypes, - src::SupportedTimeSeriesTypes; + dst::TimeSeriesOwners, + src::TimeSeriesOwners; name_mapping::Union{Nothing, Dict{Tuple{String, String}, String}} = nothing, scaling_factor_multiplier_mapping::Union{Nothing, Dict{String, String}} = nothing, ) - storage = _get_time_series_storage(dst) + storage = get_time_series_storage(dst) if isnothing(storage) throw( ArgumentError( - "Component does not have time series storage. " * + "$(summary(dst)) does not have time series storage. " * "It may not be attached to the system.", ), ) end - # There may be time series that share time series arrays as a result of - # transform_single_time_series! being called. - # Don't add these references to the storage more than once. - refs = Set{Tuple{String, Base.UUID}}() + mgr = get_time_series_manager(dst) + @assert !isnothing(mgr) - for ts_metadata in get_time_series_multiple(TimeSeriesMetadata, src) + for ts_metadata in list_time_series_metadata(src) name = get_name(ts_metadata) new_name = name if !isnothing(name_mapping) @@ -494,46 +384,45 @@ function copy_time_series!( assign_new_uuid_internal!(new_time_series) set_name!(new_time_series, new_name) set_scaling_factor_multiplier!(new_time_series, new_multiplier) - add_time_series!(dst, new_time_series) - ts_uuid = get_time_series_uuid(new_time_series) - ref = (new_name, ts_uuid) - if !in(ref, refs) - add_time_series_reference!(storage, get_uuid(dst), new_name, ts_uuid) - push!(refs, ref) - end + add_metadata!(mgr.metadata_store, dst, new_time_series) end end -function get_time_series_keys(component::SupportedTimeSeriesTypes) - return keys(get_time_series_container(component).data) -end - -function list_time_series_metadata(component::SupportedTimeSeriesTypes) - return collect(values(get_time_series_container(component).data)) +function list_time_series_info(owner::TimeSeriesOwners) + mgr = get_time_series_manager(owner) + isnothing(mgr) && return [] + return list_time_series_info(mgr.metadata_store, owner) end -function get_time_series_names( - ::Type{T}, - component::SupportedTimeSeriesTypes, -) where {T <: TimeSeriesData} - return get_time_series_names( - time_series_data_to_metadata(T), - get_time_series_container(component), +function list_time_series_metadata( + owner::TimeSeriesOwners; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, + name::Union{String, Nothing} = nothing, + features..., +) + mgr = get_time_series_manager(owner) + isnothing(mgr) && return [] + return list_metadata( + mgr, + owner; + time_series_type = time_series_type, + name = name, + features..., ) end -function get_num_time_series(component::SupportedTimeSeriesTypes) - container = get_time_series_container(component) - if isnothing(container) +function get_num_time_series(owner::TimeSeriesOwners) + mgr = get_time_series_manager(owner) + if isnothing(mgr) return (0, 0) end static_ts_count = 0 forecast_count = 0 - for key in keys(container.data) - if key.time_series_type <: StaticTimeSeriesMetadata + for metadata in list_metadata(mgr.metadata_store, owner) + if metadata isa StaticTimeSeriesMetadata static_ts_count += 1 - elseif key.time_series_type <: ForecastMetadata + elseif metadata isa ForecastMetadata forecast_count += 1 else error("panic") @@ -543,14 +432,14 @@ function get_num_time_series(component::SupportedTimeSeriesTypes) return (static_ts_count, forecast_count) end -function get_num_time_series_by_type(component::SupportedTimeSeriesTypes) +function get_num_time_series_by_type(owner::TimeSeriesOwners) counts = Dict{String, Int}() - container = get_time_series_container(component) - if isnothing(container) + mgr = get_time_series_manager(owner) + if isnothing(mgr) return counts end - for metadata in values(container.data) + for metadata in list_metadata(mgr.metadata_store, owner) type = string(nameof(time_series_metadata_to_data(metadata))) if haskey(counts, type) counts[type] += 1 @@ -563,44 +452,160 @@ function get_num_time_series_by_type(component::SupportedTimeSeriesTypes) end function get_time_series( - component::SupportedTimeSeriesTypes, + owner::TimeSeriesOwners, time_series::TimeSeriesData, ) - storage = _get_time_series_storage(component) + storage = get_time_series_storage(owner) return get_time_series(storage, get_time_series_uuid(time_series)) end -function get_time_series_uuids(component::SupportedTimeSeriesTypes) - container = get_time_series_container(component) +function get_time_series_uuids(owner::TimeSeriesOwners) + mgr = get_time_series_manager(owner) + if isnothing(mgr) + return [] + end return [ - (get_time_series_uuid(container.data[key]), key.name) for - key in get_time_series_keys(component) + (get_time_series_uuid(x), get_name(x)) for + x in list_metadata(mgr.metadata_store, owner) ] end -function attach_time_series_and_serialize!( - data::SystemData, - component::SupportedTimeSeriesTypes, - ts_metadata::T, - ts::TimeSeriesData; - skip_if_present = false, -) where {T <: TimeSeriesMetadata} - check_add_time_series(data.time_series_params, ts) - check_read_only(data.time_series_storage) - if has_time_series(component, T, get_name(ts)) - skip_if_present && return - throw(ArgumentError("time_series $(typeof(ts)) $(get_name(ts)) is already stored")) +function clear_time_series!(owner::TimeSeriesOwners) + mgr = get_time_series_manager(owner) + if !isnothing(mgr) + clear_time_series!(mgr, owner) end - - serialize_time_series!( - data.time_series_storage, - get_uuid(component), - get_name(ts_metadata), - ts, - ) - add_time_series!(component, ts_metadata; skip_if_present = skip_if_present) - # Order is important. Set this last in case exceptions are thrown at previous steps. - set_parameters!(data.time_series_params, ts) return end + +""" +Return a time series from TimeSeriesFileMetadata. + +# Arguments + + - `cache::TimeSeriesParsingCache`: cached data + - `ts_file_metadata::TimeSeriesFileMetadata`: metadata + - `resolution::{Nothing, Dates.Period}`: skip any time_series that don't match this resolution +""" +function make_time_series!( + cache::TimeSeriesParsingCache, + ts_file_metadata::TimeSeriesFileMetadata, +) + info = add_time_series_info!(cache, ts_file_metadata) + return ts_file_metadata.time_series_type(info) +end + +function add_time_series_info!( + cache::TimeSeriesParsingCache, + metadata::TimeSeriesFileMetadata, +) + time_series = _add_time_series_info!(cache, metadata) + info = TimeSeriesParsedInfo(metadata, time_series) + @debug "Added TimeSeriesParsedInfo" _group = LOG_GROUP_TIME_SERIES metadata + return info +end + +function _get_columns(start_time, count, ts_metadata::ForecastMetadata) + offset = start_time - get_initial_timestamp(ts_metadata) + interval = time_period_conversion(get_interval(ts_metadata)) + window_count = get_count(ts_metadata) + if window_count > 1 + index = Int(offset / interval) + 1 + else + index = 1 + end + if count === nothing + count = window_count - index + 1 + end + + if index + count - 1 > get_count(ts_metadata) + throw( + ArgumentError( + "The requested start_time $start_time and count $count are invalid", + ), + ) + end + return UnitRange(index, index + count - 1) +end + +_get_columns(start_time, count, ts_metadata::StaticTimeSeriesMetadata) = UnitRange(1, 1) + +function _get_rows(start_time, len, ts_metadata::StaticTimeSeriesMetadata) + index = + Int( + (start_time - get_initial_timestamp(ts_metadata)) / get_resolution(ts_metadata), + ) + 1 + if len === nothing + len = length(ts_metadata) - index + 1 + end + if index + len - 1 > length(ts_metadata) + throw( + ArgumentError( + "The requested index=$index len=$len exceeds the range $(length(ts_metadata))", + ), + ) + end + + return UnitRange(index, index + len - 1) +end + +function _get_rows(start_time, len, ts_metadata::ForecastMetadata) + if len === nothing + len = get_horizon(ts_metadata) + end + + return UnitRange(1, len) +end + +function _check_start_time(start_time, ts_metadata::TimeSeriesMetadata) + if start_time === nothing + return get_initial_timestamp(ts_metadata) + end + + time_diff = start_time - get_initial_timestamp(ts_metadata) + if time_diff < Dates.Second(0) + throw( + ArgumentError( + "start_time=$start_time is earlier than $(get_initial_timestamp(ts_metadata))", + ), + ) + end + + if typeof(ts_metadata) <: ForecastMetadata + window_count = get_count(ts_metadata) + interval = get_interval(ts_metadata) + if window_count > 1 && + Dates.Millisecond(time_diff) % Dates.Millisecond(interval) != Dates.Second(0) + throw( + ArgumentError( + "start_time=$start_time is not on a multiple of interval=$interval", + ), + ) + end + end + + return start_time +end + +function get_forecast_window_count(initial_timestamp, interval, resolution, len, horizon) + if interval == Dates.Second(0) + count = 1 + else + last_timestamp = initial_timestamp + resolution * (len - 1) + last_initial_time = last_timestamp - resolution * (horizon - 1) + + # Reduce last_initial_time to the nearest interval if necessary. + diff = + Dates.Millisecond(last_initial_time - initial_timestamp) % + Dates.Millisecond(interval) + if diff != Dates.Millisecond(0) + last_initial_time -= diff + end + count = + Dates.Millisecond(last_initial_time - initial_timestamp) / + Dates.Millisecond(interval) + 1 + end + + return count +end diff --git a/src/time_series_manager.jl b/src/time_series_manager.jl new file mode 100644 index 000000000..5f1447e78 --- /dev/null +++ b/src/time_series_manager.jl @@ -0,0 +1,289 @@ +mutable struct TimeSeriesManager <: InfrastructureSystemsType + data_store::TimeSeriesStorage + metadata_store::TimeSeriesMetadataStore + read_only::Bool +end + +function TimeSeriesManager(; + data_store = nothing, + metadata_store = nothing, + in_memory = false, + read_only = false, + directory = nothing, + compression = CompressionSettings(), +) + if isnothing(directory) && haskey(ENV, TIME_SERIES_DIRECTORY_ENV_VAR) + directory = ENV[TIME_SERIES_DIRECTORY_ENV_VAR] + end + + if isnothing(metadata_store) + filename, io = mktemp(isnothing(directory) ? tempdir() : directory) + close(io) + metadata_store = TimeSeriesMetadataStore(filename) + end + + if isnothing(data_store) + data_store = make_time_series_storage(; + in_memory = in_memory, + directory = directory, + compression = compression, + ) + end + return TimeSeriesManager(data_store, metadata_store, read_only) +end + +function add_metadata!( + mgr::TimeSeriesManager, + component::TimeSeriesOwners, + metadata::TimeSeriesMetadata; + skip_if_present = false, + features..., +) + _throw_if_read_only(mgr) + add_metadata!( + mgr.metadata_store, + metadata, + component; + skip_if_present = skip_if_present, + ) + @debug "Added $(summary(metadata)) to $(summary(component)) " _group = + LOG_GROUP_TIME_SERIES + return +end + +function add_time_series!( + mgr::TimeSeriesManager, + owner::TimeSeriesOwners, + time_series::TimeSeriesData; + skip_if_present = false, + features..., +) + throw_if_does_not_support_time_series(owner) + _check_time_series_params(mgr, time_series) + metadata_type = time_series_data_to_metadata(typeof(time_series)) + metadata = metadata_type(time_series; features...) + data_exists = has_time_series(mgr.metadata_store, get_uuid(time_series)) + metadata_exists = has_metadata(mgr.metadata_store, owner, metadata) + + if metadata_exists && !skip_if_present + throw(ArgumentError("$(summary(metadata)) is already stored")) + end + + if !data_exists + serialize_time_series!(mgr.data_store, time_series) + end + + # Order matters. Don't add metadata unless serialize works. + if !metadata_exists + add_metadata!( + mgr.metadata_store, + owner, + metadata; + ) + end + return +end + +function clear_time_series!(mgr::TimeSeriesManager) + _throw_if_read_only(mgr) + clear_metadata!(mgr.metadata_store) + clear_time_series!(mgr.data_store) +end + +function clear_time_series!(mgr::TimeSeriesManager, component::TimeSeriesOwners) + _throw_if_read_only(mgr) + for metadata in list_metadata(mgr.metadata_store, component) + remove_time_series!(mgr, component, metadata) + end + @debug "Cleared time_series in $(summary(component))." _group = + LOG_GROUP_TIME_SERIES + return +end + +has_time_series(mgr::TimeSeriesManager, component::TimeSeriesOwners) = + has_time_series(mgr.metadata_store, component) +has_time_series(::Nothing, component::TimeSeriesOwners) = false + +function Base.deepcopy_internal(mgr::TimeSeriesManager, dict::IdDict) + if haskey(dict, mgr) + return dict[mgr] + end + data_store = deepcopy(mgr.data_store) + if mgr.data_store isa Hdf5TimeSeriesStorage + copy_to_new_file!(data_store, dirname(mgr.data_store.file_path)) + end + + close_temporarily!(mgr.metadata_store) do + # TODO: Change this implementation when SQLite.jl supports backup. + # https://github.com/JuliaDatabases/SQLite.jl/issues/210 + new_db_file, io = mktemp() + close(io) + cp(mgr.metadata_store.db.file, new_db_file; force = true) + metadata_store = from_file(TimeSeriesMetadataStore, new_db_file) + new_mgr = TimeSeriesManager(data_store, metadata_store, mgr.read_only) + dict[mgr] = new_mgr + dict[mgr.data_store] = new_mgr.data_store + dict[mgr.metadata_store] = new_mgr.metadata_store + return new_mgr + end +end + +get_metadata( + mgr::TimeSeriesManager, + component::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, + name::String; + features..., +) = get_metadata( + mgr.metadata_store, + component, + time_series_type, + name; + features..., +) + +list_metadata( + mgr::TimeSeriesManager, + component::TimeSeriesOwners; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, + name::Union{String, Nothing} = nothing, + features..., +) = list_metadata( + mgr.metadata_store, + component; + time_series_type = time_series_type, + name = name, + features..., +) + +""" +Remove the time series data for a component. +""" +function remove_time_series!( + mgr::TimeSeriesManager, + time_series_type::Type{<:TimeSeriesData}, + component::TimeSeriesOwners, + name::String; + features..., +) + _throw_if_read_only(mgr) + uuids = list_matching_time_series_uuids( + mgr.metadata_store; + time_series_type = time_series_type, + name = name, + features..., + ) + remove_metadata!( + mgr.metadata_store, + component; + time_series_type = time_series_type, + name = name, + features..., + ) + + @debug "Removed time_series metadata in $(summary(component))." _group = + LOG_GROUP_TIME_SERIES component time_series_type name features + + for uuid in uuids + _remove_data_if_no_more_references(mgr, uuid) + end + + return +end + +function remove_time_series!( + mgr::TimeSeriesManager, + component::InfrastructureSystemsComponent, + metadata::TimeSeriesMetadata, +) + _throw_if_read_only(mgr) + remove_metadata!(mgr.metadata_store, component, metadata) + @debug "Removed time_series metadata in $(summary(component)) $(summary(metadata))." _group = + LOG_GROUP_TIME_SERIES + _remove_data_if_no_more_references(mgr, get_time_series_uuid(metadata)) + return +end + +function _check_time_series_params(mgr::TimeSeriesManager, ts::StaticTimeSeries) + check_params_compatibility(mgr.metadata_store, StaticTimeSeriesParameters()) + data = get_data(ts) + if length(data) < 2 + throw(ArgumentError("data array length must be at least 2: $(length(data))")) + end + if length(data) != length(ts) + throw(ConflictingInputsError("length mismatch: $(length(data)) $(length(ts))")) + end + + timestamps = TimeSeries.timestamp(data) + difft = timestamps[2] - timestamps[1] + if difft != get_resolution(ts) + throw(ConflictingInputsError("resolution mismatch: $difft $(get_resolution(ts))")) + end + return +end + +function _check_time_series_params(mgr::TimeSeriesManager, ts::Forecast) + check_params_compatibility( + mgr.metadata_store, + ForecastParameters(; + horizon = get_horizon(ts), + initial_timestamp = get_initial_timestamp(ts), + interval = get_interval(ts), + count = get_count(ts), + resolution = get_resolution(ts), + ), + ) + horizon = get_horizon(ts) + if horizon < 2 + throw(ArgumentError("horizon must be at least 2: $horizon")) + end + for window in iterate_windows(ts) + if size(window)[1] != horizon + throw(ConflictingInputsError("length mismatch: $(size(window)[1]) $horizon")) + end + end +end + +function _remove_data_if_no_more_references(mgr::TimeSeriesManager, uuid::Base.UUID) + if !has_time_series(mgr.metadata_store, uuid) + remove_time_series!(mgr.data_store, uuid) + @debug "Removed time_series data $uuid." _group = LOG_GROUP_TIME_SERIES + end + + return +end + +function _throw_if_read_only(mgr::TimeSeriesManager) + if mgr.read_only + throw(ArgumentError("Time series operation is not allowed in read-only mode.")) + end +end + +function compare_values( + x::TimeSeriesManager, + y::TimeSeriesManager; + compare_uuids = false, + exclude = Set{Symbol}(), +) + match = true + for name in fieldnames(TimeSeriesManager) + val_x = getfield(x, name) + val_y = getfield(y, name) + if name == :data_store && typeof(val_x) != typeof(val_y) + @warn "Cannot compare $(typeof(val_x)) and $(typeof(val_y))" + # TODO 1.0: workaround for not being able to convert Hdf5TimeSeriesStorage to + # InMemoryTimeSeriesStorage + continue + end + + if !compare_values(val_x, val_y; compare_uuids = compare_uuids, exclude = exclude) + @error "TimeSeriesManager field = $name does not match" getfield(x, name) getfield( + y, + name, + ) + match = false + end + end + + return match +end diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl new file mode 100644 index 000000000..f2a371423 --- /dev/null +++ b/src/time_series_metadata_store.jl @@ -0,0 +1,1012 @@ +const METADATA_TABLE_NAME = "time_series_metadata" +const DB_FILENAME = "time_series_metadata.db" + +mutable struct TimeSeriesMetadataStore + db::SQLite.DB +end + +function TimeSeriesMetadataStore(filename::AbstractString) + # An ideal solution would be to create an in-memory database and then perform a SQLite + # backup to a file whenever the user serializes the system. However, SQLite.jl does + # not support that feature yet: https://github.com/JuliaDatabases/SQLite.jl/issues/210 + store = TimeSeriesMetadataStore(SQLite.DB(filename)) + _create_metadata_table!(store) + _create_indexes!(store) + @debug "Initializedd new time series metadata table" _group = LOG_GROUP_TIME_SERIES + return store +end + +function from_file(::Type{TimeSeriesMetadataStore}, filename::AbstractString) + store = TimeSeriesMetadataStore(SQLite.DB(filename)) + @debug "Loaded time series metadata from file" _group = LOG_GROUP_TIME_SERIES filename + return store +end + +function from_h5_file(::Type{TimeSeriesMetadataStore}, src::AbstractString, directory) + data = HDF5.h5open(src, "r") do file + file[HDF5_TS_METADATA_ROOT_PATH][:] + end + + filename, io = mktemp(isnothing(directory) ? tempdir() : directory) + write(io, data) + close(io) + return from_file(TimeSeriesMetadataStore, filename) +end + +function _create_metadata_table!(store::TimeSeriesMetadataStore) + # TODO: SQLite createtable!() doesn't provide a way to create a primary key. + # https://github.com/JuliaDatabases/SQLite.jl/issues/286 + # We can use that function if they ever add the feature. + schema = [ + "id INTEGER PRIMARY KEY", + "time_series_uuid TEXT NOT NULL", + "time_series_type TEXT NOT NULL", + "time_series_category TEXT NOT NULL", + "initial_timestamp TEXT NOT NULL", + "resolution_ms INTEGER NOT NULL", + "horizon INTEGER", + "horizon_time_ms INTEGER", + "interval_ms INTEGER", + "window_count INTEGER", + "length INTEGER", + "name TEXT NOT NULL", + "owner_uuid TEXT NOT NULL", + "owner_type TEXT NOT NULL", + "owner_category TEXT NOT NULL", + "features TEXT NOT NULL", + "metadata JSON NOT NULL", + ] + schema_text = join(schema, ",") + _execute(store, "CREATE TABLE $(METADATA_TABLE_NAME)($(schema_text))") + @debug "Created time series metadata table" schema _group = LOG_GROUP_TIME_SERIES + return +end + +function _create_indexes!(store::TimeSeriesMetadataStore) + # Index strategy: + # 1. Optimize for these user queries with indexes: + # 1a. all time series attached to one component/attribute + # 1b. time series for one component/attribute + name + type + # 1c. time series for one component/attribute with all features + # 2. Optimize for checks at system.add_time_series. Use all fields and features. + # 3. Optimize for returning all metadata for a time series UUID. + SQLite.createindex!(store.db, METADATA_TABLE_NAME, "by_id", "id"; unique = true) + SQLite.createindex!(store.db, METADATA_TABLE_NAME, "by_c", "owner_uuid"; unique = false) + SQLite.createindex!( + store.db, + METADATA_TABLE_NAME, + "by_c_n_tst", + ["owner_uuid", "name", "time_series_type"]; + unique = false, + ) + SQLite.createindex!( + store.db, + METADATA_TABLE_NAME, + "by_c_n_tst_features", + ["owner_uuid", "name", "time_series_type", "features"]; + unique = false, + ) + SQLite.createindex!( + store.db, + METADATA_TABLE_NAME, + "by_ts_uuid", + "time_series_uuid"; + unique = false, + ) + return +end + +""" +Add metadata to the store. +""" +function add_metadata!( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + metadata::TimeSeriesMetadata; +) + if has_metadata(store, owner, metadata) + throw(ArgumentError("time_series $(summary(metadata)) is already stored")) + end + + check_params_compatibility(store, metadata) + owner_category = _get_owner_category(owner) + ts_type = time_series_metadata_to_data(metadata) + ts_category = _get_time_series_category(ts_type) + features = _make_features_string(metadata.features) + vals = _create_row( + metadata, + owner, + owner_category, + string(nameof(ts_type)), + ts_category, + features, + ) + params = chop(repeat("?,", length(vals))) + SQLite.DBInterface.execute( + store.db, + "INSERT INTO $METADATA_TABLE_NAME VALUES($params)", + vals, + ) + @debug "Added metadata = $metadata to $(summary(owner))" _group = + LOG_GROUP_TIME_SERIES + return +end + +""" +Clear all time series metadata from the store. +""" +function clear_metadata!(store::TimeSeriesMetadataStore) + _execute(store, "DELETE FROM $METADATA_TABLE_NAME") +end + +function backup(store::TimeSeriesMetadataStore, filename::String) + # This is an unfortunate implementation. SQLite supports backup but SQLite.jl does not. + # https://github.com/JuliaDatabases/SQLite.jl/issues/210 + # When they address the limitation, search the IS repo for this github issue number + # to fix all locations. + was_open = isopen(store.db) + if was_open + close(store.db) + end + + cp(store.db.file, filename) + @debug "Backed up time series metadata" _group = LOG_GROUP_TIME_SERIES filename + + if was_open + store.db = SQLite.DB(store.db.file) + end + + return +end + +function check_params_compatibility( + store::TimeSeriesMetadataStore, + metadata::ForecastMetadata, +) + params = ForecastParameters(; + count = get_count(metadata), + horizon = get_horizon(metadata), + initial_timestamp = get_initial_timestamp(metadata), + interval = get_interval(metadata), + resolution = get_resolution(metadata), + ) + check_params_compatibility(store, params) + return +end + +check_params_compatibility( + store::TimeSeriesMetadataStore, + metadata::StaticTimeSeriesMetadata, +) = nothing +check_params_compatibility( + store::TimeSeriesMetadataStore, + params::StaticTimeSeriesParameters, +) = nothing + +function check_params_compatibility( + store::TimeSeriesMetadataStore, + params::ForecastParameters, +) + store_params = get_forecast_parameters(store) + isnothing(store_params) && return + + if params.count != store_params.count + throw( + ConflictingInputsError( + "forecast count $(params.count) does not match system count $(store_params.count)", + ), + ) + end + + if params.initial_timestamp != store_params.initial_timestamp + throw( + ConflictingInputsError( + "forecast initial_timestamp $(params.initial_timestamp) does not match system " * + "initial_timestamp $(store_params.initial_timestamp)", + ), + ) + end + + horizon_as_time = params.horizon * params.resolution + store_horizon_as_time = store_params.horizon * store_params.resolution + if horizon_as_time != store_horizon_as_time + throw( + ConflictingInputsError( + "forecast horizon $(horizon_as_time) " * + "does not match system horizon $(store_horizon_as_time)", + ), + ) + end +end + +# These are guaranteed to be consistent already. +check_consistency(::TimeSeriesMetadataStore, ::Type{<:Forecast}) = nothing + +""" +Throw InvalidValue if the SingleTimeSeries arrays have different initial times or lengths. +Return the initial timestamp and length as a tuple. +""" +function check_consistency(store::TimeSeriesMetadataStore, ::Type{<:StaticTimeSeries}) + query = """ + SELECT + DISTINCT initial_timestamp + ,length + FROM $METADATA_TABLE_NAME + WHERE time_series_type = 'SingleTimeSeries' + """ + table = Tables.rowtable(_execute(store, query)) + len = length(table) + if len == 0 + return Dates.DateTime(Dates.Minute(0)), 0 + elseif len > 1 + throw( + InvalidValue( + "There are more than one sets of SingleTimeSeries initial times and lengths: $table", + ), + ) + end + + row = table[1] + return Dates.DateTime(row.initial_timestamp), row.length +end + +function close_temporarily!(func::Function, store::TimeSeriesMetadataStore) + try + close(store.db) + func() + finally + store.db = SQLite.DB(store.db.file) + end +end + +function get_forecast_initial_times(store::TimeSeriesMetadataStore) + params = get_forecast_parameters(store) + isnothing(params) && return [] + return get_initial_times(params.initial_timestamp, params.count, params.interval) +end + +function get_forecast_parameters(store::TimeSeriesMetadataStore) + query = """ + SELECT + horizon + ,initial_timestamp + ,interval_ms + ,resolution_ms + ,window_count + FROM $METADATA_TABLE_NAME + WHERE horizon IS NOT NULL + LIMIT 1 + """ + table = Tables.rowtable(_execute(store, query)) + isempty(table) && return nothing + row = table[1] + return ForecastParameters(; + horizon = row.horizon, + initial_timestamp = Dates.DateTime(row.initial_timestamp), + interval = Dates.Millisecond(row.interval_ms), + count = row.window_count, + resolution = Dates.Millisecond(row.resolution_ms), + ) +end + +function get_forecast_window_count(store::TimeSeriesMetadataStore) + query = """ + SELECT + window_count + FROM $METADATA_TABLE_NAME + WHERE window_count IS NOT NULL + LIMIT 1 + """ + table = Tables.rowtable(_execute(store, query)) + return isempty(table) ? 0 : table[1].window_count +end + +function get_forecast_horizon(store::TimeSeriesMetadataStore) + query = """ + SELECT + horizon + FROM $METADATA_TABLE_NAME + WHERE horizon IS NOT NULL + LIMIT 1 + """ + table = Tables.rowtable(_execute(store, query)) + return isempty(table) ? 0 : table[1].horizon +end + +function get_forecast_initial_timestamp(store::TimeSeriesMetadataStore) + query = """ + SELECT + initial_timestamp + FROM $METADATA_TABLE_NAME + WHERE horizon IS NOT NULL + LIMIT 1 + """ + table = Tables.rowtable(_execute(store, query)) + return if isempty(table) + Dates.DateTime(Dates.Minute(0)) + else + Dates.DateTime(table[1].initial_timestamp) + end +end + +function get_forecast_interval(store::TimeSeriesMetadataStore) + query = """ + SELECT + interval_ms + FROM $METADATA_TABLE_NAME + WHERE interval_ms IS NOT NULL + LIMIT 1 + """ + table = Tables.rowtable(_execute(store, query)) + return if isempty(table) + Dates.Period(Dates.Millisecond(0)) + else + Dates.Millisecond(table[1].interval_ms) + end +end + +""" +Return the metadata matching the inputs. Throw an exception if there is more than one +matching input. +""" +function get_metadata( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, + name::String; + features..., +) + metadata = _try_get_time_series_metadata_by_full_params( + store, + owner, + time_series_type, + name; + features..., + ) + !isnothing(metadata) && return metadata + + metadata_items = list_metadata( + store, + owner; + time_series_type = time_series_type, + name = name, + features..., + ) + len = length(metadata_items) + if len == 0 + if time_series_type === Deterministic + # This is a hack to account for the fact that we allow non-standard behavior + # with DeterministicSingleTimeSeries. + try + return get_metadata( + store, + owner, + DeterministicSingleTimeSeries, + name; + features..., + ) + catch _ + throw( + ArgumentError( + "No matching metadata is stored. " * + "Tried $time_series_type and DeterministicSingleTimeSeries.", + ), + ) + end + end + throw(ArgumentError("No matching metadata is stored.")) + elseif len > 1 + throw(ArgumentError("Found more than one matching metadata: $len")) + end + + return metadata_items[1] +end + +""" +Return the number of unique time series arrays. +""" +function get_num_time_series(store::TimeSeriesMetadataStore) + return Tables.rowtable( + _execute( + store, + "SELECT COUNT(DISTINCT time_series_uuid) AS count FROM $METADATA_TABLE_NAME", + ), + )[1].count +end + +""" +Return an instance of TimeSeriesCounts. +""" +function get_time_series_counts(store::TimeSeriesMetadataStore) + query_components = """ + SELECT + COUNT(DISTINCT owner_uuid) AS count + FROM $METADATA_TABLE_NAME + WHERE owner_category = 'Component' + """ + query_attributes = """ + SELECT + COUNT(DISTINCT owner_uuid) AS count + FROM $METADATA_TABLE_NAME + WHERE owner_category = 'SupplementalAttribute' + """ + query_sts = """ + SELECT + COUNT(DISTINCT time_series_uuid) AS count + FROM $METADATA_TABLE_NAME + WHERE interval_ms IS NULL + """ + query_forecasts = """ + SELECT + COUNT(DISTINCT time_series_uuid) AS count + FROM $METADATA_TABLE_NAME + WHERE interval_ms IS NOT NULL + """ + + count_components = _execute_count(store, query_components) + count_attributes = _execute_count(store, query_attributes) + count_sts = _execute_count(store, query_sts) + count_forecasts = _execute_count(store, query_forecasts) + + return TimeSeriesCounts(; + components_with_time_series = count_components, + supplemental_attributes_with_time_series = count_attributes, + static_time_series_count = count_sts, + forecast_count = count_forecasts, + ) +end + +""" +Return a Vector of OrderedDict of stored time series counts by type. +""" +function get_time_series_counts_by_type(store::TimeSeriesMetadataStore) + query = """ + SELECT + time_series_type + ,count(*) AS count + FROM $METADATA_TABLE_NAME + GROUP BY + time_series_type + ORDER BY + time_series_type + """ + table = Tables.rowtable(_execute(store, query)) + return [ + OrderedDict("type" => x.time_series_type, "count" => x.count) for x in table + ] +end + +""" +Return a DataFrame with the number of time series by type for components and supplemental +attributes. +""" +function get_time_series_summary_table(store::TimeSeriesMetadataStore) + query = """ + SELECT + owner_type + ,owner_category + ,time_series_type + ,time_series_category + ,initial_timestamp + ,resolution_ms + ,count(*) AS count + FROM $METADATA_TABLE_NAME + GROUP BY + owner_type + ,owner_category + ,time_series_type + ,initial_timestamp + ,resolution_ms + ORDER BY + owner_category + ,owner_type + ,time_series_type + ,initial_timestamp + ,resolution_ms + """ + return DataFrame(_execute(store, query)) +end + +""" +Return True if there is time series metadata matching the inputs. +""" +function has_metadata( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + metadata::TimeSeriesMetadata, +) + features = Dict(Symbol(k) => v for (k, v) in get_features(metadata)) + return has_metadata( + store, + owner, + time_series_metadata_to_data(metadata), + get_name(metadata); + features..., + ) +end + +function has_metadata( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, + name::String; + features..., +) + if _try_has_time_series_metadata_by_full_params( + store, + owner, + time_series_type, + name; + features..., + ) + return true + end + + where_clause = _make_where_clause( + owner; + time_series_type = time_series_type, + name = name, + features..., + ) + query = "SELECT COUNT(*) AS count FROM $METADATA_TABLE_NAME WHERE $where_clause" + return _execute_count(store, query) > 0 +end + +""" +Return True if there is time series matching the UUID. +""" +function has_time_series(store::TimeSeriesMetadataStore, time_series_uuid::Base.UUID) + where_clause = "time_series_uuid = '$time_series_uuid'" + return _has_time_series(store, where_clause) +end + +function has_time_series( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, +) + where_clause = _make_owner_where_clause(owner) + return _has_time_series(store, where_clause) +end + +function has_time_series( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, +) + where_clause = _make_where_clause(owner; time_series_type = time_series_type) + return _has_time_series(store, where_clause) +end + +has_time_series( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, + name::String; + features..., +) = has_metadata(store, owner, time_series_type, name, features...) + +""" +Return a sorted Vector of distinct resolutions for all time series of the given type +(or all types). +""" +function list_time_series_resolutions( + store::TimeSeriesMetadataStore; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, +) + where_clause = if isnothing(time_series_type) + "" + else + "WHERE time_series_type = '$(nameof(time_series_type))'" + end + query = """ + SELECT + DISTINCT resolution_ms + FROM $METADATA_TABLE_NAME $where_clause + ORDER BY resolution_ms + """ + return Dates.Millisecond.(Tables.columntable(_execute(store, query)).resolution_ms) +end + +""" +Return the time series UUIDs that match the inputs. +""" +function list_matching_time_series_uuids( + store::TimeSeriesMetadataStore; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, + name::Union{String, Nothing} = nothing, + features..., +) + where_clause = _make_where_clause( + nothing; + time_series_type = time_series_type, + name = name, + features..., + ) + query = "SELECT DISTINCT time_series_uuid FROM $METADATA_TABLE_NAME WHERE $where_clause" + table = Tables.columntable(_execute(store, query)) + return Base.UUID.(table.time_series_uuid) +end + +function list_metadata( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, + name::Union{String, Nothing} = nothing, + features..., +) + where_clause = _make_where_clause( + owner; + time_series_type = time_series_type, + name = name, + features..., + ) + query = "SELECT metadata FROM $METADATA_TABLE_NAME WHERE $where_clause" + table = Tables.rowtable(_execute(store, query)) + return [_deserialize_metadata(x.metadata) for x in table] +end + +function list_owner_uuids_with_time_series( + store::TimeSeriesMetadataStore, + owner_type::Type{<:TimeSeriesOwners}; + time_series_type::Union{Nothing, Type{<:TimeSeriesData}} = nothing, +) + category = _get_owner_category(owner_type) + vals = ["owner_category = '$category'"] + if !isnothing(time_series_type) + push!(vals, "time_series_type = '$(nameof(time_series_type))'") + end + + where_clause = join(vals, " AND ") + query = """ + SELECT + DISTINCT owner_uuid + FROM $METADATA_TABLE_NAME + WHERE $where_clause + """ + return Base.UUID.(Tables.columntable(_execute(store, query)).owner_uuid) +end + +""" +Return information about each time series array attached to the owner. +This information can be used to call get_time_series. +""" +function list_time_series_info(store::TimeSeriesMetadataStore, owner::TimeSeriesOwners) + return [make_time_series_info(x) for x in list_metadata(store, owner)] +end + +""" +Remove the matching metadata from the store. +""" +function remove_metadata!( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + metadata::TimeSeriesMetadata, +) + where_clause = _make_where_clause(owner, metadata) + num_deleted = _remove_metadata!(store, where_clause) + if num_deleted != 1 + error("Bug: unexpected number of deletions: $num_deleted. Should have been 1.") + end +end + +function remove_metadata!( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, + name::Union{String, Nothing} = nothing, + features..., +) + where_clause = _make_where_clause( + owner; + time_series_type = time_series_type, + name = name, + require_full_feature_match = false, # TODO: needs more consideration + features..., + ) + num_deleted = _remove_metadata!(store, where_clause) + if num_deleted == 0 + if time_series_type === Deterministic + # This is a hack to account for the fact that we allow non-standard behavior + # with DeterministicSingleTimeSeries. + remove_metadata!( + store, + owner; + time_series_type = DeterministicSingleTimeSeries, + name = name, + features..., + ) + else + @warn "No time series metadata was deleted." + end + end +end + +function replace_component_uuid!( + store::TimeSeriesMetadataStore, + old_uuid::Base.UUID, + new_uuid::Base.UUID, +) + query = """ + UPDATE $METADATA_TABLE_NAME + SET owner_uuid = '$new_uuid' + WHERE owner_uuid = '$old_uuid' + """ + _execute(store, query) + return +end + +""" +Run a query and return the results in a DataFrame. +""" +function sql(store::TimeSeriesMetadataStore, query::String) + """Run a SQL query on the time series metadata table.""" + return DataFrames.DataFrame(_execute(store, query)) +end + +function to_h5_file(store::TimeSeriesMetadataStore, dst::String) + metadata_path, io = mktemp() + close(io) + rm(metadata_path) + backup(store, metadata_path) + + data = open(metadata_path, "r") do io + read(io) + end + + HDF5.h5open(dst, "r+") do file + if HDF5_TS_METADATA_ROOT_PATH in keys(file) + HDF5.delete_object(file, HDF5_TS_METADATA_ROOT_PATH) + end + file[HDF5_TS_METADATA_ROOT_PATH] = data + end + + return +end + +function _create_row( + metadata::ForecastMetadata, + owner, + owner_category, + ts_type, + ts_category, + features, +) + return ( + missing, # auto-assigned by sqlite + string(get_time_series_uuid(metadata)), + ts_type, + ts_category, + string(get_initial_timestamp(metadata)), + Dates.Millisecond(get_resolution(metadata)).value, + get_horizon(metadata), + Dates.Millisecond(get_horizon(metadata) * get_resolution(metadata)).value, + Dates.Millisecond(get_interval(metadata)).value, + get_count(metadata), + missing, + get_name(metadata), + string(get_uuid(owner)), + string(nameof(typeof(owner))), + owner_category, + features, + JSON3.write(serialize(metadata)), + ) +end + +function _create_row( + metadata::StaticTimeSeriesMetadata, + owner, + owner_category, + ts_type, + ts_category, + features, +) + return ( + missing, # auto-assigned by sqlite + string(get_time_series_uuid(metadata)), + ts_type, + ts_category, + string(get_initial_timestamp(metadata)), + Dates.Millisecond(get_resolution(metadata)).value, + missing, + missing, + missing, + missing, + get_length(metadata), + get_name(metadata), + string(get_uuid(owner)), + string(nameof(typeof(owner))), + owner_category, + features, + JSON3.write(serialize(metadata)), + ) +end + +function _execute(store::TimeSeriesMetadataStore, query::AbstractString) + @debug "Run SQL" query _group = LOG_GROUP_TIME_SERIES + res = SQLite.DBInterface.execute(store.db, query) + return res +end + +function _execute_count(store::TimeSeriesMetadataStore, query::AbstractString) + for row in Tables.rows(_execute(store, query)) + return row.count + end + + error("Bug: unexpectedly did not receive any rows") +end + +function _has_time_series(store::TimeSeriesMetadataStore, where_clause::String) + query = "SELECT COUNT(*) AS count FROM $METADATA_TABLE_NAME WHERE $where_clause" + return _execute_count(store, query) > 0 +end + +function _remove_metadata!( + store::TimeSeriesMetadataStore, + where_clause::AbstractString, +) + _execute(store, "DELETE FROM $METADATA_TABLE_NAME WHERE $where_clause") + table = Tables.rowtable(_execute(store, "SELECT CHANGES() AS changes")) + @assert_op length(table) == 1 + @debug "Deleted $(table[1].changes) rows from the time series metadata table" _group = + LOG_GROUP_TIME_SERIES + return table[1].changes +end + +function _try_get_time_series_metadata_by_full_params( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, + name::String; + features..., +) + rows = _try_time_series_metadata_by_full_params( + store, + owner, + time_series_type, + name, + "metadata"; + features..., + ) + len = length(rows) + if len == 0 + return nothing + elseif len == 1 + return _deserialize_metadata(rows[1].metadata) + else + throw(ArgumentError("Found more than one matching time series: $len")) + end +end + +function _try_has_time_series_metadata_by_full_params( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, + name::String; + features..., +) + row = _try_time_series_metadata_by_full_params( + store, + owner, + time_series_type, + name, + "id"; + features..., + ) + return !isempty(row) +end + +function _try_time_series_metadata_by_full_params( + store::TimeSeriesMetadataStore, + owner::TimeSeriesOwners, + time_series_type::Type{<:TimeSeriesData}, + name::String, + column::String; + features..., +) + where_clause = _make_where_clause( + owner; + time_series_type = time_series_type, + name = name, + require_full_feature_match = true, + features..., + ) + query = "SELECT $column FROM $METADATA_TABLE_NAME WHERE $where_clause" + return Tables.rowtable(_execute(store, query)) +end + +function compare_values( + x::TimeSeriesMetadataStore, + y::TimeSeriesMetadataStore; + compare_uuids = false, + exclude = Set{Symbol}(), +) + # Note that we can't compare missing values. + owner_uuid = compare_uuids ? ", owner_uuid" : "" + query = """ + SELECT id, metadata, time_series_uuid $owner_uuid + FROM $METADATA_TABLE_NAME ORDER BY id + """ + table_x = Tables.rowtable(_execute(x, query)) + table_y = Tables.rowtable(_execute(y, query)) + return table_x == table_y +end + +### Non-TimeSeriesMetadataStore functions ### + +_convert_ts_type_to_string(ts_type::Type{<:TimeSeriesData}) = string(nameof(ts_type)) + +function _deserialize_metadata(text::String) + val = JSON3.read(text, Dict) + return deserialize(get_type_from_serialization_data(val), val) +end + +_get_owner_category( + ::Union{InfrastructureSystemsComponent, Type{<:InfrastructureSystemsComponent}}, +) = "Component" +_get_owner_category(::Union{SupplementalAttribute, Type{<:SupplementalAttribute}}) = + "SupplementalAttribute" +_get_time_series_category(::Type{<:Forecast}) = "Forecast" +_get_time_series_category(::Type{<:StaticTimeSeries}) = "Forecast" + +function _make_feature_filter(; features...) + data = _make_sorted_feature_array(; features...) + return join((["metadata->>'\$.features.$k' = '$v'" for (k, v) in data]), "AND ") +end + +function _make_features_string(features::Dict{String, <:Any}) + key_names = sort!(collect(keys(features))) + data = [Dict(k => features[k]) for k in key_names] + return JSON3.write(data) +end + +function _make_features_string(; features...) + key_names = sort!(collect(string.(keys(features)))) + data = [Dict(k => features[Symbol(k)]) for (k) in key_names] + return JSON3.write(data) +end + +_make_owner_where_clause(owner::TimeSeriesOwners) = + "owner_uuid = '$(get_uuid(owner))'" + +function _make_sorted_feature_array(; features...) + key_names = sort!(collect(string.(keys(features)))) + return [(key, features[Symbol(key)]) for key in key_names] +end + +function _make_where_clause( + owner::Union{TimeSeriesOwners, Nothing}; + time_series_type::Union{Type{<:TimeSeriesData}, Nothing} = nothing, + name::Union{String, Nothing} = nothing, + require_full_feature_match = false, + features..., +) + vals = String[] + if !isnothing(owner) + push!(vals, _make_owner_where_clause(owner)) + end + if !isnothing(name) + push!(vals, "name = '$name'") + end + if !isnothing(time_series_type) + push!(vals, "time_series_type = '$(_convert_ts_type_to_string(time_series_type))'") + end + if !isempty(features) + if require_full_feature_match + val = "features = '$(_make_features_string(; features...))'" + else + val = "$(_make_feature_filter(; features...))" + end + push!(vals, val) + end + + return "(" * join(vals, " AND ") * ")" +end + +function _make_where_clause(owner::TimeSeriesOwners, metadata::TimeSeriesMetadata) + return _make_where_clause( + owner; + time_series_type = time_series_metadata_to_data(metadata), + name = get_name(metadata), + get_features(metadata)..., + ) +end diff --git a/src/time_series_parameters.jl b/src/time_series_parameters.jl index 2bf6e79da..8b1378917 100644 --- a/src/time_series_parameters.jl +++ b/src/time_series_parameters.jl @@ -1,255 +1 @@ -const UNINITIALIZED_DATETIME = Dates.DateTime(Dates.Minute(0)) -const UNINITIALIZED_LENGTH = 0 -const UNINITIALIZED_PERIOD = Dates.Period(Dates.Minute(0)) -mutable struct ForecastParameters <: InfrastructureSystemsType - horizon::Int - initial_timestamp::Dates.DateTime - interval::Dates.Period - count::Int -end - -function ForecastParameters(; - horizon = UNINITIALIZED_LENGTH, - initial_timestamp = UNINITIALIZED_DATETIME, - interval = UNINITIALIZED_PERIOD, - count = UNINITIALIZED_LENGTH, -) - return ForecastParameters(horizon, initial_timestamp, interval, count) -end - -function check_params_compatibility(params::ForecastParameters, other::ForecastParameters) - _is_uninitialized(params) && return true - - if other.count != params.count - throw( - ConflictingInputsError( - "forecast count $(other.count) does not match system count $(params.count)", - ), - ) - end - - if other.horizon != params.horizon - throw( - ConflictingInputsError( - "forecast horizon $(other.horizon) does not match system horizon $(params.horizon)", - ), - ) - end - - if other.initial_timestamp != params.initial_timestamp - throw( - ConflictingInputsError( - "forecast initial_timestamp $(other.initial_timestamp) does not match system " * - "initial_timestamp $(params.initial_timestamp)", - ), - ) - end - - return -end - -function _is_uninitialized(params::ForecastParameters) - return params.horizon == UNINITIALIZED_LENGTH && - params.initial_timestamp == UNINITIALIZED_DATETIME && - params.interval == UNINITIALIZED_PERIOD && - params.count == UNINITIALIZED_LENGTH -end - -function reset_info!(params::ForecastParameters) - params.horizon = UNINITIALIZED_LENGTH - params.initial_timestamp = UNINITIALIZED_DATETIME - params.interval = UNINITIALIZED_PERIOD - params.count = UNINITIALIZED_LENGTH - return -end - -function get_forecast_initial_times(params::ForecastParameters) - return get_initial_times(params.initial_timestamp, params.count, params.interval) -end - -function set_parameters!(params::ForecastParameters, other::ForecastParameters) - params.horizon = other.horizon - params.initial_timestamp = other.initial_timestamp - params.interval = other.interval - params.count = other.count - return -end - -function set_parameters!(params::ForecastParameters, forecast::Forecast) - set_parameters!(params, TimeSeriesParameters(forecast).forecast_params) - return -end - -mutable struct TimeSeriesParameters <: InfrastructureSystemsType - resolution::Dates.Period - forecast_params::ForecastParameters -end - -function TimeSeriesParameters(; - resolution = UNINITIALIZED_PERIOD, - forecast_params = ForecastParameters(), -) - return TimeSeriesParameters(resolution, forecast_params) -end - -function TimeSeriesParameters(ts::StaticTimeSeries) - return TimeSeriesParameters(; resolution = get_resolution(ts)) -end - -function TimeSeriesParameters(ts::Forecast) - forecast_params = ForecastParameters(; - count = get_count(ts), - horizon = get_horizon(ts), - initial_timestamp = get_initial_timestamp(ts), - interval = get_interval(ts), - ) - return TimeSeriesParameters(get_resolution(ts), forecast_params) -end - -function TimeSeriesParameters( - initial_timestamp::Dates.DateTime, - resolution::Dates.Period, - len::Int, - horizon::Int, - interval::Dates.Period, -) - if interval == Dates.Second(0) - count = 1 - else - last_timestamp = initial_timestamp + resolution * (len - 1) - last_initial_time = last_timestamp - resolution * (horizon - 1) - - # Reduce last_initial_time to the nearest interval if necessary. - diff = - Dates.Millisecond(last_initial_time - initial_timestamp) % - Dates.Millisecond(interval) - if diff != Dates.Millisecond(0) - last_initial_time -= diff - end - count = - Dates.Millisecond(last_initial_time - initial_timestamp) / - Dates.Millisecond(interval) + 1 - end - fparams = ForecastParameters(; - horizon = horizon, - initial_timestamp = initial_timestamp, - interval = interval, - count = count, - ) - return TimeSeriesParameters(resolution, fparams) -end - -function reset_info!(params::TimeSeriesParameters) - params.resolution = UNINITIALIZED_PERIOD - reset_info!(params.forecast_params) - @info "Reset system time series parameters." -end - -function _is_uninitialized(params::TimeSeriesParameters) - return params.resolution == UNINITIALIZED_PERIOD -end - -""" -Return true if `params` match `other` or if one of them is uninitialized. -""" -function check_params_compatibility( - params::TimeSeriesParameters, - other::TimeSeriesParameters, -) - _is_uninitialized(params) && return true - - if other.resolution != params.resolution - throw( - ConflictingInputsError( - "time series resolution $(other.resolution) does not match system " * - "resolution $(params.resolution)", - ), - ) - end - - # `other` might be for a static time series and not have forecast params - if !_is_uninitialized(other.forecast_params) - check_params_compatibility(params.forecast_params, other.forecast_params) - end -end - -function check_add_time_series(params::TimeSeriesParameters, ts::TimeSeriesData) - check_params_compatibility(params, TimeSeriesParameters(ts)) - _check_time_series_lengths(ts) - return -end - -function set_parameters!(params::TimeSeriesParameters, ts::StaticTimeSeries) - params.resolution = get_resolution(ts) - return -end - -function set_parameters!(params::TimeSeriesParameters, forecast::Forecast) - params.resolution = get_resolution(forecast) - set_parameters!(params.forecast_params, forecast) - return -end - -function set_parameters!(params::TimeSeriesParameters, other::TimeSeriesParameters) - if _is_uninitialized(params) - # This is the first time series added. - params.resolution = other.resolution - end - - if _is_uninitialized(params.forecast_params) - set_parameters!(params.forecast_params, other.forecast_params) - end - - return -end - -function _check_time_series_lengths(ts::StaticTimeSeries) - data = get_data(ts) - if length(data) < 2 - throw(ArgumentError("data array length must be at least 2: $(length(data))")) - end - if length(data) != length(ts) - throw(ConflictingInputsError("length mismatch: $(length(data)) $(length(ts))")) - end - - timestamps = TimeSeries.timestamp(data) - difft = timestamps[2] - timestamps[1] - if difft != get_resolution(ts) - throw(ConflictingInputsError("resolution mismatch: $difft $(get_resolution(ts))")) - end - return -end - -function _check_time_series_lengths(ts::Forecast) - horizon = get_horizon(ts) - if horizon < 2 - throw(ArgumentError("horizon must be at least 2: $horizon")) - end - for window in iterate_windows(ts) - if size(window)[1] != horizon - throw(ConflictingInputsError("length mismatch: $(size(window)[1]) $horizon")) - end - end -end - -get_forecast_window_count(params::TimeSeriesParameters) = params.forecast_params.count -get_forecast_initial_times(params::TimeSeriesParameters) = - get_forecast_initial_times(params.forecast_params) -get_forecast_horizon(params::TimeSeriesParameters) = params.forecast_params.horizon -get_forecast_initial_timestamp(params::TimeSeriesParameters) = - params.forecast_params.initial_timestamp -get_forecast_interval(params::TimeSeriesParameters) = params.forecast_params.interval -get_time_series_resolution(params::TimeSeriesParameters) = params.resolution - -function get_forecast_total_period(p::TimeSeriesParameters) - f = p.forecast_params - _is_uninitialized(f) && return Dates.Second(0) - return get_total_period( - f.initial_timestamp, - f.count, - f.interval, - f.horizon, - p.resolution, - ) -end diff --git a/src/time_series_storage.jl b/src/time_series_storage.jl index c00b3a48c..46823486f 100644 --- a/src/time_series_storage.jl +++ b/src/time_series_storage.jl @@ -4,7 +4,6 @@ Abstract type for time series storage implementations. All subtypes must implement: - - add_time_series_reference! - check_read_only - clear_time_series! - deserialize_time_series @@ -80,13 +79,7 @@ end function serialize(storage::TimeSeriesStorage, file_path::AbstractString) if storage isa Hdf5TimeSeriesStorage if abspath(get_file_path(storage)) == abspath(file_path) - if !is_read_only(storage) - error("Attempting to overwrite identical time series file") - end - - @debug "Skip time series serialization because the paths are identical" _group = - LOG_GROUP_TIME_SERIES - return + error("Attempting to overwrite identical time series file") end copy_h5_file(get_file_path(storage), file_path) diff --git a/src/time_series_structs.jl b/src/time_series_structs.jl new file mode 100644 index 000000000..cf05d2ced --- /dev/null +++ b/src/time_series_structs.jl @@ -0,0 +1,82 @@ +const TimeSeriesOwners = Union{InfrastructureSystemsComponent, SupplementalAttribute} + +Base.@kwdef struct StaticTimeSeriesInfo <: InfrastructureSystemsType + type::DataType + name::String + initial_timestamp::Dates.DateTime + resolution::Dates.Period + length::Int + features::Dict{String, Any} +end + +function make_time_series_info(metadata::StaticTimeSeriesMetadata) + return StaticTimeSeriesInfo(; + type = time_series_metadata_to_data(metadata), + name = get_name(metadata), + initial_timestamp = get_initial_timestamp(metadata), + resolution = get_resolution(metadata), + length = get_length(metadata), + features = get_features(metadata), + ) +end + +Base.@kwdef struct ForecastInfo <: InfrastructureSystemsType + type::DataType + name::String + initial_timestamp::Dates.DateTime + resolution::Dates.Period + horizon::Int + interval::Dates.Period + count::Int + features::Dict{String, Any} +end + +function make_time_series_info(metadata::ForecastMetadata) + return ForecastInfo(; + type = time_series_metadata_to_data(metadata), + name = get_name(metadata), + initial_timestamp = get_initial_timestamp(metadata), + resolution = get_resolution(metadata), + horizon = get_horizon(metadata), + interval = get_interval(metadata), + count = get_count(metadata), + features = get_features(metadata), + ) +end + +""" +Provides counts of time series including attachments to components and supplemental +attributes. +""" +Base.@kwdef struct TimeSeriesCounts + components_with_time_series::Int + supplemental_attributes_with_time_series::Int + static_time_series_count::Int + forecast_count::Int +end + +# TODO: This is now only used in PSY. Consider moving. +struct TimeSeriesKey <: InfrastructureSystemsType + time_series_type::Type{<:TimeSeriesData} + name::String +end + +function TimeSeriesKey(; time_series_type::Type{<:TimeSeriesData}, name::String) + return TimeSeriesKey(time_series_type, name) +end + +function TimeSeriesKey(data::TimeSeriesData) + return TimeSeriesKey(typeof(data), get_name(data)) +end + +function deserialize_struct(::Type{TimeSeriesKey}, data::Dict) + vals = Dict{Symbol, Any}() + for field_name in fieldnames(TimeSeriesKey) + val = data[string(field_name)] + if field_name == :time_series_type + val = getfield(InfrastructureSystems, Symbol(strip_module_name(val))) + end + vals[field_name] = val + end + return TimeSeriesKey(; vals...) +end diff --git a/src/utils/print.jl b/src/utils/print.jl index 714c42122..1800f33d8 100644 --- a/src/utils/print.jl +++ b/src/utils/print.jl @@ -46,29 +46,18 @@ function Base.show(io::IO, ::MIME"text/html", container::InfrastructureSystemsCo end end -function Base.summary(container::TimeSeriesContainer) - return "$(typeof(container)): $(length(container))" -end - -function Base.show(io::IO, ::MIME"text/plain", container::TimeSeriesContainer) - println(io, summary(container)) - for key in keys(container.data) - println(io, "$(key.time_series_type): name=$(key.name)") - end -end - function Base.summary(time_series::TimeSeriesData) - return "$(typeof(time_series)) time_series ($(length(time_series)))" + return "$(typeof(time_series)).$(get_name(time_series))" end function Base.summary(time_series::TimeSeriesMetadata) - return "$(typeof(time_series)) time_series" + return "$(typeof(time_series)).$(get_name(time_series))" end function Base.show(io::IO, data::SystemData) show(io, data.components) println(io, "\n") - return show(io, data.time_series_params) + show_time_series_data(io, data) end function Base.show(io::IO, ::MIME"text/plain", data::SystemData) @@ -77,7 +66,6 @@ function Base.show(io::IO, ::MIME"text/plain", data::SystemData) show(io, MIME"text/plain"(), data.attributes) println(io, "\n") show_time_series_data(io, data; backend = Val(:auto)) - show(io, data.time_series_params) end function Base.show(io::IO, ::MIME"text/html", data::SystemData) @@ -86,43 +74,13 @@ function Base.show(io::IO, ::MIME"text/html", data::SystemData) show(io, MIME"text/html"(), data.attributes) println(io, "\n") show_time_series_data(io, data; backend = Val(:html), standalone = false) - show(io, data.time_series_params) end function show_time_series_data(io::IO, data::SystemData; kwargs...) - counts = get_time_series_counts(data) - if counts.static_time_series_count == 0 && counts.forecast_count == 0 - return - end - - res = get_time_series_resolution(data) - res = res <= Dates.Minute(1) ? Dates.Second(res) : Dates.Minute(res) - - header = ["Property", "Value"] - table = [ - "Components with time series data" string(counts.components_with_time_series) - "Supplemental attributes with time series data" string(counts.supplemental_attributes_with_time_series) - "Unique StaticTimeSeries" string(counts.static_time_series_count) - "Unique Forecasts" string(counts.forecast_count) - "Resolution" string(res) - ] - - if counts.forecast_count > 0 - initial_times = get_forecast_initial_times(data) - table2 = [ - "First initial time" string(first(initial_times)) - "Last initial time" string(last(initial_times)) - "Horizon" string(get_forecast_horizon(data)) - "Interval" string(Dates.Minute(get_forecast_interval(data))) - "Forecast window count" string(get_forecast_window_count(data)) - ] - table = vcat(table, table2) - end - + table = get_time_series_summary_table(data) PrettyTables.pretty_table( io, table; - header = header, title = "Time Series Summary", alignment = :l, kwargs..., @@ -133,7 +91,7 @@ end function Base.summary(ist::InfrastructureSystemsComponent) # All InfrastructureSystemsComponent subtypes are supposed to implement get_name. # Some don't. They need to override this function. - return "$(get_name(ist)) ($(typeof(ist)))" + return "$(typeof(ist)).$(get_name(ist))" end function Base.show(io::IO, ::MIME"text/plain", system_units::SystemUnitsSettings) @@ -148,9 +106,9 @@ function Base.show(io::IO, ::MIME"text/plain", ist::InfrastructureSystemsCompone print(io, summary(ist), ":") for name in fieldnames(typeof(ist)) obj = getfield(ist, name) - if obj isa InfrastructureSystemsInternal + if obj isa InfrastructureSystemsInternal || obj isa TimeSeriesContainer continue - elseif obj isa TimeSeriesContainer || obj isa InfrastructureSystemsType + elseif obj isa InfrastructureSystemsType val = summary(getfield(ist, name)) elseif obj isa Vector{<:InfrastructureSystemsComponent} val = summary(getfield(ist, name)) @@ -272,7 +230,7 @@ function show_components( data[i, 1] = get_name(component) j = 2 if has_available - data[i, 2] = getproperty(component, :available) + data[i, 2] = Base.getproperty(component, :available) j += 1 end @@ -287,13 +245,14 @@ function show_components( parent = parentmodule(component_type) # This logic enables application of system units in PowerSystems through # its getter functions. - val = getproperty(component, column) - if val isa TimeSeriesContainer || - val isa InfrastructureSystemsType || - val isa Vector{<:InfrastructureSystemsComponent} + val = Base.getproperty(component, column) + if val isa TimeSeriesContainer + continue + elseif val isa InfrastructureSystemsType || + val isa Vector{<:InfrastructureSystemsComponent} val = summary(val) elseif hasproperty(parent, getter_name) - getter_func = getproperty(parent, getter_name) + getter_func = Base.getproperty(parent, getter_name) val = getter_func(component) end data[i, j] = val @@ -313,6 +272,27 @@ function show_components( return end +function show_time_series(owner::TimeSeriesOwners) + data_by_type = Dict{Any, Vector{OrderedDict{String, Any}}}() + for info in list_time_series_info(owner) + if !haskey(data_by_type, info.type) + data_by_type[info.type] = Vector{OrderedDict{String, Any}}() + end + data = OrderedDict{String, Any}() + for field in fieldnames(typeof(info)) + if field == :type + data[string(field)] = string(nameof(Base.getproperty(info, field))) + else + data[string(field)] = Base.getproperty(info, field) + end + end + push!(data_by_type[info.type], data) + end + for rows in values(data_by_type) + PrettyTables.pretty_table(DataFrame(rows)) + end +end + ## This function takes in a time period or date period and returns a compound period function convert_compound_period(period::Union{Dates.TimePeriod, Dates.DatePeriod}) period = time_period_conversion(period) diff --git a/src/utils/test.jl b/src/utils/test.jl index 52eeb69ce..b496dad27 100644 --- a/src/utils/test.jl +++ b/src/utils/test.jl @@ -73,7 +73,7 @@ function deserialize(::Type{TestComponent}, data::Dict) data["name"], data["val"], data["val2"], - deserialize(TimeSeriesContainer, data["time_series_container"]), + TimeSeriesContainer(), data["supplemental_attributes_container"], deserialize(InfrastructureSystemsInternal, data["internal"]), ) diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 7279064f9..2af048354 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -132,8 +132,8 @@ function compare_values( else for field_name in fields field_name in exclude && continue - if (T <: TimeSeriesContainer || T <: SupplementalAttributes) && - field_name == :time_series_storage + if (T <: TimeSeriesContainer && field_name == :manager) || + (T <: SupplementalAttributes && field_name == :time_series_manager) # This gets validated at SystemData. Don't repeat for each component. continue end diff --git a/test/test_components.jl b/test/test_components.jl index 84e4087f0..4bb52777d 100644 --- a/test/test_components.jl +++ b/test/test_components.jl @@ -1,6 +1,6 @@ @testset "Test add_component" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) @@ -15,13 +15,13 @@ val::Int end - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = BadComponent("component1", 5) @test_throws MethodError IS.add_component!(container, component) end @testset "Test clear_components" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) @@ -34,7 +34,7 @@ end end @testset "Test remove_component" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) @@ -47,7 +47,7 @@ end end @testset "Test remove_component by name" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) name = "component1" component = IS.TestComponent(name, 5) @@ -75,7 +75,7 @@ end end @testset "Test get_components" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) # empty components = IS.get_components(IS.TestComponent, container) @@ -118,7 +118,7 @@ end end @testset "Test get_component" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) @@ -138,12 +138,12 @@ end end @testset "Test empty get_component" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) @test isempty(collect(IS.get_components(IS.TestComponent, container))) end @testset "Test get_components_by_name" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) @@ -159,7 +159,7 @@ end end @testset "Test iterate_components" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) @@ -171,7 +171,7 @@ end end @testset "Test components serialization" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) data = IS.serialize(container) @@ -181,7 +181,7 @@ end end @testset "Summarize components" begin - container = IS.Components(IS.InMemoryTimeSeriesStorage()) + container = IS.Components(IS.TimeSeriesManager(; in_memory = true)) component = IS.TestComponent("component1", 5) IS.add_component!(container, component) summary(devnull, container) diff --git a/test/test_printing.jl b/test/test_printing.jl index 4fc53c8ef..bd456f291 100644 --- a/test/test_printing.jl +++ b/test/test_printing.jl @@ -4,7 +4,7 @@ show(io, "text/plain", sys) text = String(take!(io)) @test occursin("TestComponent", text) - @test occursin("Time Series Summary", text) + @test occursin("time_series_type", text) end @testset "Test show_component_tables" begin diff --git a/test/test_serialization.jl b/test/test_serialization.jl index 3ba3d36c5..c6273f11c 100644 --- a/test/test_serialization.jl +++ b/test/test_serialization.jl @@ -1,31 +1,23 @@ function validate_serialization(sys::IS.SystemData; time_series_read_only = false) - #path, io = mktemp() - # For some reason files aren't getting deleted when written to /tmp. Using current dir. - filename = "test_system_serialization.json" - @info "Serializing to $filename" - - try - if isfile(filename) - rm(filename) - end - IS.prepare_for_serialization_to_file!(sys, filename; force = true) - data = IS.serialize(sys) - open(filename, "w") do io - return JSON3.write(io, data) - end - catch - rm(filename) - rethrow() + directory = mktempdir() + filename = joinpath(directory, "test_system_serialization.json") + IS.prepare_for_serialization_to_file!(sys, filename; force = true) + data = IS.serialize(sys) + open(filename, "w") do io + JSON3.write(io, data) end # Make sure the code supports the files changing directories. - test_dir = mktempdir() - path = mv(filename, joinpath(test_dir, filename)) + test_dir = mktempdir(directory) + path = mv(filename, joinpath(test_dir, basename(filename))) - @test haskey(data, "time_series_storage_file") == !isempty(sys.time_series_storage) - t_file = splitext(basename(path))[1] * "_" * IS.TIME_SERIES_STORAGE_FILE + @test haskey(data, "time_series_storage_file") == + !isempty(sys.time_series_manager.data_store) + t_file = + joinpath(directory, splitext(basename(path))[1] * "_" * IS.TIME_SERIES_STORAGE_FILE) if haskey(data, "time_series_storage_file") - mv(t_file, joinpath(test_dir, t_file)) + dst_file = joinpath(test_dir, basename(t_file)) + mv(t_file, dst_file) else @test !isfile(t_file) end @@ -74,7 +66,7 @@ end @testset "Test JSON serialization of with read-only time series" begin sys = create_system_data_shared_time_series(; time_series_in_memory = false) - sys2, result = validate_serialization(sys; time_series_read_only = true) + sys2, result = validate_serialization(sys) @test result end diff --git a/test/test_supplemental_attributes.jl b/test/test_supplemental_attributes.jl index e94f8843b..e10f8149f 100644 --- a/test/test_supplemental_attributes.jl +++ b/test/test_supplemental_attributes.jl @@ -1,5 +1,5 @@ @testset "Test add_supplemental_attribute" begin - container = IS.SupplementalAttributes(IS.InMemoryTimeSeriesStorage()) + container = IS.SupplementalAttributes(IS.TimeSeriesManager(; in_memory = true)) geo_supplemental_attribute = IS.GeographicInfo() component = IS.TestComponent("component1", 5) IS.add_supplemental_attribute!(container, component, geo_supplemental_attribute) @@ -12,7 +12,7 @@ geo_supplemental_attribute, ) - container = IS.SupplementalAttributes(IS.InMemoryTimeSeriesStorage()) + container = IS.SupplementalAttributes(IS.TimeSeriesManager(; in_memory = true)) geo_supplemental_attribute = IS.GeographicInfo() @test_throws ArgumentError IS._add_supplemental_attribute!( container, @@ -21,7 +21,7 @@ end @testset "Test clear_supplemental_attributes" begin - container = IS.SupplementalAttributes(IS.InMemoryTimeSeriesStorage()) + container = IS.SupplementalAttributes(IS.TimeSeriesManager(; in_memory = true)) geo_supplemental_attribute = IS.GeographicInfo() component = IS.TestComponent("component1", 5) IS.add_supplemental_attribute!(container, component, geo_supplemental_attribute) @@ -35,7 +35,7 @@ end end @testset "Test remove_supplemental_attribute" begin - container = IS.SupplementalAttributes(IS.InMemoryTimeSeriesStorage()) + container = IS.SupplementalAttributes(IS.TimeSeriesManager(; in_memory = true)) geo_supplemental_attribute = IS.GeographicInfo() component = IS.TestComponent("component1", 5) IS.add_supplemental_attribute!(container, component, geo_supplemental_attribute) @@ -65,7 +65,7 @@ end end @testset "Test iterate_SupplementalAttributes" begin - container = IS.SupplementalAttributes(IS.InMemoryTimeSeriesStorage()) + container = IS.SupplementalAttributes(IS.TimeSeriesManager(; in_memory = true)) geo_supplemental_attribute = IS.GeographicInfo() component = IS.TestComponent("component1", 5) IS.add_supplemental_attribute!(container, component, geo_supplemental_attribute) @@ -78,7 +78,7 @@ end end @testset "Summarize SupplementalAttributes" begin - container = IS.SupplementalAttributes(IS.InMemoryTimeSeriesStorage()) + container = IS.SupplementalAttributes(IS.TimeSeriesManager(; in_memory = true)) geo_supplemental_attribute = IS.GeographicInfo() component = IS.TestComponent("component1", 5) IS.add_supplemental_attribute!(container, component, geo_supplemental_attribute) @@ -86,7 +86,7 @@ end end @testset "Test supplemental_attributes serialization" begin - container = IS.SupplementalAttributes(IS.InMemoryTimeSeriesStorage()) + container = IS.SupplementalAttributes(IS.TimeSeriesManager(; in_memory = true)) geo_supplemental_attribute = IS.GeographicInfo() component = IS.TestComponent("component1", 5) IS.add_supplemental_attribute!(container, component, geo_supplemental_attribute) diff --git a/test/test_system_data.jl b/test/test_system_data.jl index 1c5849e0b..e7710cdc8 100644 --- a/test/test_system_data.jl +++ b/test/test_system_data.jl @@ -11,8 +11,12 @@ components = IS.get_components(IS.TestComponent, data) @test length(components) == 1 - @test length(IS.get_components(x -> (IS.get_val(x) != 5), IS.TestComponent, data)) == 0 - @test length(IS.get_components(x -> (IS.get_val(x) == 5), IS.TestComponent, data)) == 1 + @test length( + IS.get_components(x -> (IS.get_val(x) != 5), IS.TestComponent, data), + ) == 0 + @test length( + IS.get_components(x -> (IS.get_val(x) == 5), IS.TestComponent, data), + ) == 1 i = 0 for component in IS.iterate_components(data) @@ -32,7 +36,8 @@ @test isempty(data.component_uuids) IS.add_component!(data, component) - components = IS.get_components_by_name(IS.InfrastructureSystemsComponent, data, name) + components = + IS.get_components_by_name(IS.InfrastructureSystemsComponent, data, name) @test length(components) == 1 @test components[1].name == name @@ -65,7 +70,10 @@ end data = IS.SystemData() initial_time = Dates.DateTime("2020-09-01") resolution = Dates.Hour(1) - ta = TimeSeries.TimeArray(range(initial_time; length = 24, step = resolution), ones(24)) + ta = TimeSeries.TimeArray( + range(initial_time; length = 24, step = resolution), + ones(24), + ) ts = IS.SingleTimeSeries(; data = ta, name = "test") for i in 1:3 @@ -89,11 +97,14 @@ end data, "component_2", ) == [component] - @test IS.get_time_series(IS.SingleTimeSeries, component, "test") isa IS.SingleTimeSeries + @test IS.get_time_series(IS.SingleTimeSeries, component, "test") isa + IS.SingleTimeSeries @test IS.is_attached(component, data.masked_components) # This needs to return time series for masked components. - @test length(collect(IS.get_time_series_multiple(data; type = IS.SingleTimeSeries))) == + @test length( + collect(IS.get_time_series_multiple(data; type = IS.SingleTimeSeries)), + ) == 3 IS.remove_masked_component!( @@ -152,9 +163,12 @@ end @testset "Test compression settings" begin none = IS.CompressionSettings(; enabled = false) @test IS.get_compression_settings(IS.SystemData()) == none - @test IS.get_compression_settings(IS.SystemData(; time_series_in_memory = true)) == none - settings = IS.CompressionSettings(; enabled = true, type = IS.CompressionTypes.DEFLATE) - @test IS.get_compression_settings(IS.SystemData(; compression = settings)) == settings + @test IS.get_compression_settings(IS.SystemData(; time_series_in_memory = true)) == + none + settings = + IS.CompressionSettings(; enabled = true, type = IS.CompressionTypes.DEFLATE) + @test IS.get_compression_settings(IS.SystemData(; compression = settings)) == + settings end @testset "Test single time series consistency" begin @@ -175,7 +189,8 @@ end IS.add_time_series!(data, component, ts) end - returned_it, returned_len = IS.check_time_series_consistency(data, IS.SingleTimeSeries) + returned_it, returned_len = + IS.check_time_series_consistency(data, IS.SingleTimeSeries) @test returned_it == initial_time @test returned_len == len end @@ -196,7 +211,10 @@ end IS.add_time_series!(data, component, ts) end - @test_throws IS.InvalidValue IS.check_time_series_consistency(data, IS.SingleTimeSeries) + @test_throws IS.InvalidValue IS.check_time_series_consistency( + data, + IS.SingleTimeSeries, + ) end @testset "Test single time series length inconsistency" begin @@ -218,7 +236,10 @@ end IS.add_time_series!(data, component, ts) end - @test_throws IS.InvalidValue IS.check_time_series_consistency(data, IS.SingleTimeSeries) + @test_throws IS.InvalidValue IS.check_time_series_consistency( + data, + IS.SingleTimeSeries, + ) end @testset "Test check_components" begin @@ -243,7 +264,10 @@ end data = IS.SystemData() initial_time = Dates.DateTime("2020-09-01") resolution = Dates.Hour(1) - ta = TimeSeries.TimeArray(range(initial_time; length = 24, step = resolution), ones(24)) + ta = TimeSeries.TimeArray( + range(initial_time; length = 24, step = resolution), + ones(24), + ) ts = IS.SingleTimeSeries(; data = ta, name = "test") for i in 1:5 @@ -268,7 +292,10 @@ end data = IS.SystemData() initial_time = Dates.DateTime("2020-09-01") resolution = Dates.Hour(1) - ta = TimeSeries.TimeArray(range(initial_time; length = 24, step = resolution), ones(24)) + ta = TimeSeries.TimeArray( + range(initial_time; length = 24, step = resolution), + ones(24), + ) ts = IS.SingleTimeSeries(; data = ta, name = "test") for i in 1:5 @@ -336,8 +363,12 @@ end # Test all permutations of abstract vs concrete, system vs component, filter vs not. @test length(IS.get_supplemental_attributes(IS.SupplementalAttribute, data)) == 3 - @test length(IS.get_supplemental_attributes(IS.SupplementalAttribute, component1)) == 2 - @test length(IS.get_supplemental_attributes(IS.SupplementalAttribute, component2)) == 2 + @test length( + IS.get_supplemental_attributes(IS.SupplementalAttribute, component1), + ) == 2 + @test length( + IS.get_supplemental_attributes(IS.SupplementalAttribute, component2), + ) == 2 @test length( IS.get_supplemental_attributes( x -> x isa IS.TestSupplemental, @@ -506,3 +537,60 @@ end @test ts.data == ta end end + +@testset "Test list_time_series_resolutions" begin + sys = IS.SystemData() + initial_time = Dates.DateTime("2020-09-01") + resolution1 = Dates.Minute(5) + resolution2 = Dates.Hour(1) + len = 24 + timestamps1 = range(initial_time; length = len, step = resolution1) + timestamps2 = range(initial_time; length = len, step = resolution2) + array1 = TimeSeries.TimeArray(timestamps1, rand(len)) + array2 = TimeSeries.TimeArray(timestamps2, rand(len)) + name = "component" + component = IS.TestComponent(name, 3) + IS.add_component!(sys, component) + ts1 = IS.SingleTimeSeries(; data = array1, name = "test1") + ts2 = IS.SingleTimeSeries(; data = array2, name = "test2") + IS.add_time_series!(sys, component, ts1) + IS.add_time_series!(sys, component, ts2) + + other_time = initial_time + resolution2 + horizon = 24 + data = SortedDict(initial_time => rand(horizon), other_time => rand(horizon)) + + forecast = IS.Deterministic(; data = data, name = "test3", resolution = resolution2) + IS.add_time_series!(sys, component, forecast) + @test IS.list_time_series_resolutions(sys) == + [Dates.Minute(5), Dates.Hour(1)] + @test IS.list_time_series_resolutions( + sys; + time_series_type = IS.SingleTimeSeries, + ) == [Dates.Minute(5), Dates.Hour(1)] + @test IS.list_time_series_resolutions( + sys; + time_series_type = IS.Deterministic, + ) == [Dates.Hour(1)] +end + +@testset "Test deepcopy of system" begin + for in_memory in (false, true) + sys = IS.SystemData(; time_series_in_memory = in_memory) + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + len = 24 + timestamps = range(initial_time; length = len, step = resolution) + array = TimeSeries.TimeArray(timestamps, rand(len)) + ts_name = "test" + name = "component" + component = IS.TestComponent(name, 3) + IS.add_component!(sys, component) + ts = IS.SingleTimeSeries(; data = array, name = ts_name) + IS.add_time_series!(sys, component, ts) + sys2 = deepcopy(sys) + component2 = IS.get_component(IS.TestComponent, sys2, name) + ts2 = IS.get_time_series(IS.SingleTimeSeries, component2, ts_name) + @test ts2.data == array + end +end diff --git a/test/test_time_series.jl b/test/test_time_series.jl index b4e7b3265..768cd69af 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -14,7 +14,8 @@ forecast = IS.Deterministic(; data = data, name = name, resolution = resolution) IS.add_time_series!(sys, component, forecast) - var1 = IS.get_time_series(IS.Deterministic, component, name; start_time = initial_time) + var1 = + IS.get_time_series(IS.Deterministic, component, name; start_time = initial_time) @test length(var1) == 2 @test IS.get_horizon(var1) == horizon @test IS.get_initial_timestamp(var1) == initial_time @@ -28,7 +29,8 @@ ) @test length(var2) == 2 - var3 = IS.get_time_series(IS.Deterministic, component, name; start_time = other_time) + var3 = + IS.get_time_series(IS.Deterministic, component, name; start_time = other_time) @test length(var2) == 2 # Throws errors @test_throws ArgumentError IS.get_time_series( @@ -124,7 +126,7 @@ end @test_throws ArgumentError IS.Deterministic(name, data_ts_two_cols) end -@testset "Test add Deterministic Cost Timeseries " begin +@testset "Test add Deterministic Cost Timeseries" begin initial_time = Dates.DateTime("2020-09-01") resolution = Dates.Hour(1) other_time = initial_time + resolution @@ -210,7 +212,12 @@ end @test IS.has_time_series(component) @test IS.get_initial_timestamp(forecast) == initial_time forecast_retrieved = - IS.get_time_series(IS.Probabilistic, component, "test"; start_time = initial_time) + IS.get_time_series( + IS.Probabilistic, + component, + "test"; + start_time = initial_time, + ) @test IS.get_initial_timestamp(forecast_retrieved) == initial_time data_ts = Dict( @@ -311,7 +318,8 @@ end range(initial_time; length = 365, step = resolution), ones(365), ) - data = IS.SingleTimeSeries(; data = data, name = "test_c") + ts_name = "test_c" + data = IS.SingleTimeSeries(; data = data, name = ts_name) IS.add_time_series!(sys, component, data) _test_add_single_time_series_helper(component, initial_time) @@ -332,13 +340,151 @@ end len = 12, ) - # Conflicting resolution + # As of PSY 4.0, multiple resolutions are supported. data = TimeSeries.TimeArray( range(initial_time; length = 365, step = Dates.Minute(5)), ones(365), ) data = IS.SingleTimeSeries(; data = data, name = "test_d") - @test_throws IS.ConflictingInputsError IS.add_time_series!(sys, component, data) + IS.add_time_series!(sys, component, data) +end + +@testset "Test add SingleTimeSeries with features" begin + sys = IS.SystemData() + name = "Component1" + component = IS.TestComponent(name, 5) + IS.add_component!(sys, component) + + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + + data = TimeSeries.TimeArray( + range(initial_time; length = 365, step = resolution), + rand(365), + ) + ts_name = "test_c" + data = IS.SingleTimeSeries(; data = data, name = ts_name) + IS.add_time_series!(sys, component, data; scenario = "low", model_year = "2030") + IS.add_time_series!(sys, component, data; scenario = "high", model_year = "2030") + IS.add_time_series!(sys, component, data; scenario = "low", model_year = "2035") + IS.add_time_series!(sys, component, data; scenario = "high", model_year = "2035") + + @test_throws ArgumentError IS.get_time_series( + IS.SingleTimeSeries, + component, + ts_name, + ) + @test_throws ArgumentError IS.get_time_series( + IS.SingleTimeSeries, + component, + ts_name, + scenario = "low", + ) + @test IS.get_time_series( + IS.SingleTimeSeries, + component, + ts_name; + scenario = "low", + model_year = "2035", + ) isa IS.SingleTimeSeries +end + +@testset "Test add Deterministic with features" begin + sys = IS.SystemData() + name = "Component1" + component = IS.TestComponent(name, 5) + IS.add_component!(sys, component) + + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + + other_time = initial_time + resolution + ts_name = "test" + horizon = 24 + data = SortedDict(initial_time => rand(horizon), other_time => rand(horizon)) + + forecast = IS.Deterministic(; data = data, name = ts_name, resolution = resolution) + IS.add_time_series!(sys, component, forecast; scenario = "low", model_year = "2030") + IS.add_time_series!( + sys, + component, + forecast; + scenario = "high", + model_year = "2030", + ) + IS.add_time_series!(sys, component, forecast; scenario = "low", model_year = "2035") + IS.add_time_series!( + sys, + component, + forecast; + scenario = "high", + model_year = "2035", + ) + + @test_throws ArgumentError IS.get_time_series( + IS.Deterministic, + component, + ts_name, + ) + @test_throws ArgumentError IS.get_time_series( + IS.Deterministic, + component, + ts_name, + scenario = "low", + ) + @test IS.get_time_series( + IS.Deterministic, + component, + ts_name; + scenario = "low", + model_year = "2035", + ) isa IS.Deterministic + @test length(IS.list_time_series_metadata(component)) == 4 + @test length( + IS.list_time_series_metadata(component; time_series_type = IS.Deterministic), + ) == 4 + @test length( + IS.list_time_series_metadata( + component; + time_series_type = IS.Deterministic, + name = ts_name, + ), + ) == 4 + @test length( + IS.list_time_series_metadata( + component; + time_series_type = IS.Deterministic, + name = ts_name, + scenario = "low", + ), + ) == 2 + @test length( + IS.list_time_series_metadata( + component; + time_series_type = IS.Deterministic, + name = ts_name, + scenario = "low", + model_year = "2035", + ), + ) == 1 + @test IS.list_time_series_metadata( + component; + time_series_type = IS.Deterministic, + name = ts_name, + scenario = "low", + model_year = "2035", + )[1].features["model_year"] == "2035" + + IS.remove_time_series!(sys, IS.Deterministic, component, ts_name; scenario = "low") + @test length( + IS.list_time_series_metadata(component; time_series_type = IS.Deterministic), + ) == 2 + for metadata in + IS.list_time_series_metadata(component; time_series_type = IS.Deterministic) + @test metadata.features["scenario"] == "high" + end + IS.remove_time_series!(sys, IS.Deterministic, component, ts_name) + @test isempty(IS.list_time_series_metadata(component)) end @testset "Test Deterministic with a wrapped SingleTimeSeries" begin @@ -364,7 +510,11 @@ end fdata[dates[i]] = ones(horizon) end bystander = - IS.Deterministic(; data = fdata, name = "bystander", resolution = resolution) + IS.Deterministic(; + data = fdata, + name = "bystander", + resolution = resolution, + ) IS.add_time_series!(sys, component, bystander) counts = IS.get_time_series_counts(sys) @@ -418,7 +568,8 @@ end last_it_index = length(dates) - 1 - Int(Dates.Minute(interval) / resolution) @test last_initial_time == dates[last_it_index] last_val_index = last_it_index + horizon - 1 - @test TimeSeries.values(windows[exp_length]) == data[last_it_index:last_val_index] + @test TimeSeries.values(windows[exp_length]) == + data[last_it_index:last_val_index] # Do the same thing but pass Deterministic instead. forecast = IS.get_time_series(IS.Deterministic, component, name) @@ -430,14 +581,14 @@ end # Verify that get_time_series_multiple works with these types. forecasts = collect(IS.get_time_series_multiple(sys)) @test length(forecasts) == 3 - forecasts = - collect(IS.get_time_series_multiple(sys; type = IS.AbstractDeterministic)) - @test length(forecasts) == 2 forecasts = collect(IS.get_time_series_multiple(sys; type = IS.Deterministic)) @test length(forecasts) == 1 forecasts = collect( - IS.get_time_series_multiple(sys; type = IS.DeterministicSingleTimeSeries), + IS.get_time_series_multiple( + sys; + type = IS.DeterministicSingleTimeSeries, + ), ) @test length(forecasts) == 1 @test forecasts[1] isa IS.DeterministicSingleTimeSeries @@ -612,7 +763,11 @@ end resolution = Dates.Hour(1) horizon = 24 dates = collect( - range(Dates.DateTime("2020-01-01T00:00:00"); length = horizon, step = resolution), + range( + Dates.DateTime("2020-01-01T00:00:00"); + length = horizon, + step = resolution, + ), ) data = collect(1:horizon) ta = TimeSeries.TimeArray(dates, data, [IS.get_name(component)]) @@ -642,7 +797,11 @@ end resolution = Dates.Hour(1) horizon = 24 dates = collect( - range(Dates.DateTime("2020-01-01T00:00:00"); length = horizon, step = resolution), + range( + Dates.DateTime("2020-01-01T00:00:00"); + length = horizon, + step = resolution, + ), ) data = collect(1:horizon) ta = TimeSeries.TimeArray(dates, data, [IS.get_name(component)]) @@ -703,32 +862,32 @@ function _test_add_single_time_series_type(test_value, type_name) ) data = IS.SingleTimeSeries(; data = data_series, name = "test_c") IS.add_time_series!(sys, component, data) - ts = IS.get_time_series(IS.SingleTimeSeries, component, "test_c";) - @test IS.get_data_type(ts) == type_name - @test reshape(TimeSeries.values(IS.get_data(ts)), 365) == TimeSeries.values(data_series) - _test_add_single_time_series_helper(component, initial_time) -end - -@testset "Test add SingleTimeSeries with LinearFunctionData Cost" begin - _test_add_single_time_series_type( - repeat([IS.LinearFunctionData(3.14)], 365), - "LinearFunctionData", - ) -end - -@testset "Test add SingleTimeSeries with QuadraticFunctionData Cost" begin - _test_add_single_time_series_type( - repeat([IS.QuadraticFunctionData(999.0, 1.0, 0.0)], 365), - "QuadraticFunctionData", - ) -end - -@testset "Test add SingleTimeSeries with PiecewiseLinearPointData Cost" begin - _test_add_single_time_series_type( - repeat([IS.PiecewiseLinearPointData(repeat([(999.0, 1.0)], 5))], 365), - "PiecewiseLinearPointData", - ) -end + #ts = IS.get_time_series(IS.SingleTimeSeries, component, "test_c";) + #@test IS.get_data_type(ts) == type_name + #@test reshape(TimeSeries.values(IS.get_data(ts)), 365) == TimeSeries.values(data_series) + #_test_add_single_time_series_helper(component, initial_time) +end + +#@testset "Test add SingleTimeSeries with LinearFunctionData Cost" begin +# _test_add_single_time_series_type( +# repeat([IS.LinearFunctionData(3.14)], 365), +# "LinearFunctionData", +# ) +#end +# +#@testset "Test add SingleTimeSeries with QuadraticFunctionData Cost" begin +# _test_add_single_time_series_type( +# repeat([IS.QuadraticFunctionData(999.0, 1.0, 0.0)], 365), +# "QuadraticFunctionData", +# ) +#end +# +#@testset "Test add SingleTimeSeries with PiecewiseLinearPointData Cost" begin +# _test_add_single_time_series_type( +# repeat([IS.PiecewiseLinearPointData(repeat([(999.0, 1.0)], 5))], 365), +# "PiecewiseLinearPointData", +# ) +#end @testset "Test read_time_series_file_metadata" begin file = joinpath(FORECASTS_DIR, "ComponentsAsColumnsNoTime.json") @@ -749,7 +908,11 @@ end @test !IS.has_time_series(component) file = joinpath(FORECASTS_DIR, "ComponentsAsColumnsNoTime.json") - IS.add_time_series_from_file_metadata!(data, IS.InfrastructureSystemsComponent, file) + IS.add_time_series_from_file_metadata!( + data, + IS.InfrastructureSystemsComponent, + file, + ) @test IS.has_time_series(component) all_time_series = get_all_time_series(data) @@ -764,13 +927,13 @@ end start_time = IS.get_initial_timestamp(time_series), ) @test length(time_series) == length(time_series2) - @test IS.get_initial_timestamp(time_series) == IS.get_initial_timestamp(time_series2) + @test IS.get_initial_timestamp(time_series) == + IS.get_initial_timestamp(time_series2) it = IS.get_initial_timestamp(time_series) all_time_series = get_all_time_series(data) @test length(collect(all_time_series)) == 1 - @test IS.get_time_series_resolution(data) == IS.get_resolution(time_series) data = IS.SystemData() name = "Component1" @@ -778,7 +941,11 @@ end IS.add_component!(data, component) @test !IS.has_time_series(component) file = joinpath(FORECASTS_DIR, "ForecastPointers.json") - IS.add_time_series_from_file_metadata!(data, IS.InfrastructureSystemsComponent, file) + IS.add_time_series_from_file_metadata!( + data, + IS.InfrastructureSystemsComponent, + file, + ) @test IS.has_time_series(component) sys = IS.SystemData() @@ -862,7 +1029,7 @@ end end end - ts_storage = sys.time_series_storage + ts_storage = sys.time_series_manager.data_store @test ts_storage isa IS.Hdf5TimeSeriesStorage @test IS.get_num_time_series(ts_storage) == 1 end @@ -901,8 +1068,11 @@ end @test length(collect(IS.get_time_series_multiple(component))) == 2 @test length(collect(IS.get_time_series_multiple(sys))) == 2 - @test length(collect(IS.get_time_series_multiple(sys; type = IS.SingleTimeSeries))) == 2 - @test length(collect(IS.get_time_series_multiple(sys; type = IS.Probabilistic))) == 0 + @test length( + collect(IS.get_time_series_multiple(sys; type = IS.SingleTimeSeries)), + ) == 2 + @test length(collect(IS.get_time_series_multiple(sys; type = IS.Probabilistic))) == + 0 time_series = collect(IS.get_time_series_multiple(sys)) @test length(time_series) == 2 @@ -910,72 +1080,9 @@ end @test length(collect(IS.get_time_series_multiple(sys; name = "val"))) == 1 @test length(collect(IS.get_time_series_multiple(sys; name = "bad_name"))) == 0 - filter_func = x -> TimeSeries.values(IS.get_data(x))[12] == 12 - @test length(collect(IS.get_time_series_multiple(sys, filter_func; name = "val2"))) == 0 -end - -@testset "Test get_time_series_with_metadata_multiple" begin - sys = IS.SystemData() - name = "Component1" - component_val = 5 - component = IS.TestComponent(name, component_val) - IS.add_component!(sys, component) - initial_time1 = Dates.DateTime("2020-01-01T00:00:00") - initial_time2 = Dates.DateTime("2020-01-02T00:00:00") - - dates1 = collect(initial_time1:Dates.Hour(1):Dates.DateTime("2020-01-01T23:00:00")) - dates2 = collect(initial_time2:Dates.Hour(1):Dates.DateTime("2020-01-02T23:00:00")) - data1 = collect(1:24) - data2 = collect(25:48) - ta1 = TimeSeries.TimeArray(dates1, data1, [IS.get_name(component)]) - ta2 = TimeSeries.TimeArray(dates2, data2, [IS.get_name(component)]) - time_series1 = - IS.SingleTimeSeries(; - name = "val", - data = ta1, - scaling_factor_multiplier = IS.get_val, - ) - time_series2 = - IS.SingleTimeSeries(; - name = "val2", - data = ta2, - scaling_factor_multiplier = IS.get_val, - ) - IS.add_time_series!(sys, component, time_series1) - IS.add_time_series!(sys, component, time_series2) - - @test length(collect(IS.get_time_series_with_metadata_multiple(component))) == 2 - - @test length( - collect( - IS.get_time_series_with_metadata_multiple( - component; - type = IS.SingleTimeSeries, - ), - ), - ) == 2 - @test length( - collect( - IS.get_time_series_with_metadata_multiple(component; type = IS.Probabilistic), - ), - ) == 0 - - @test length( - collect(IS.get_time_series_with_metadata_multiple(component; name = "val")), - ) == 1 - @test length( - collect(IS.get_time_series_with_metadata_multiple(component; name = "bad_name")), - ) == 0 - filter_func = x -> TimeSeries.values(IS.get_data(x))[12] == 12 @test length( - collect( - IS.get_time_series_with_metadata_multiple( - component, - filter_func; - name = "val2", - ), - ), + collect(IS.get_time_series_multiple(sys, filter_func; name = "val2")), ) == 0 end @@ -1005,10 +1112,15 @@ end @test length(get_all_time_series(data)) == 1 time_series = time_series[1] - IS.remove_time_series!(data, typeof(time_series), component, IS.get_name(time_series)) + IS.remove_time_series!( + data, + typeof(time_series), + component, + IS.get_name(time_series), + ) @test length(get_all_time_series(data)) == 0 - @test IS.get_num_time_series(data.time_series_storage) == 0 + @test IS.get_num_time_series(data) == 0 end @testset "Test clear_time_series" begin @@ -1027,13 +1139,14 @@ end all_time_series = collect(IS.get_time_series_multiple(data)) @test length(all_time_series) == 1 time_series = all_time_series[1] - @test IS.get_num_time_series(data.time_series_storage) == 1 + @test IS.get_num_time_series(data) == 1 IS.remove_component!(data, component) @test length(collect(IS.get_time_series_multiple(component))) == 0 - @test length(collect(IS.get_components(IS.InfrastructureSystemsComponent, data))) == 0 + @test length(collect(IS.get_components(IS.InfrastructureSystemsComponent, data))) == + 0 @test length(get_all_time_series(data)) == 0 - @test IS.get_num_time_series(data.time_series_storage) == 0 + @test IS.get_num_time_series(data) == 0 end @testset "Test get_time_series_array" begin @@ -1181,7 +1294,11 @@ end @test time_series isa IS.SingleTimeSeries @test IS.get_initial_timestamp(time_series) == initial_time2 @test IS.get_name(time_series) == name2b - @test_throws ArgumentError IS.get_time_series(IS.SingleTimeSeries, component2, name2a) + @test_throws ArgumentError IS.get_time_series( + IS.SingleTimeSeries, + component2, + name2a, + ) end @testset "Test copy time_series with transformed time series" begin @@ -1238,11 +1355,6 @@ end @test_throws ArgumentError IS.add_component!(sys1, component) end -@testset "Summarize time_series" begin - data = create_system_data(; with_time_series = true) - summary(devnull, data.time_series_params) -end - @testset "Test time_series forwarding methods" begin data = create_system_data(; with_time_series = true) time_series = get_all_time_series(data)[1] @@ -1425,7 +1537,6 @@ end sts = IS.SingleTimeSeries(; data = sts_data, name = "test_sts") IS.add_time_series!(sys, component, sts) - @test IS.get_time_series_resolution(sys) == resolution @test IS.get_forecast_window_count(sys) == 2 @test IS.get_forecast_horizon(sys) == horizon @test IS.get_forecast_initial_timestamp(sys) == initial_time @@ -1433,9 +1544,6 @@ end @test IS.get_forecast_initial_times(sys) == [initial_time, second_time] @test collect(IS.get_initial_times(forecast)) == collect(IS.get_forecast_initial_times(sys)) - @test Dates.Hour(IS.get_forecast_total_period(sys)) == - Dates.Hour(second_time - initial_time) + Dates.Hour(resolution * horizon) - @test IS.get_forecast_total_period(sys) == IS.get_total_period(forecast) end # TODO something like this could be much more widespread to reduce code duplication @@ -1470,7 +1578,8 @@ function _test_get_time_series_option_type(test_data, in_memory, extended) @test IS.get_count(f2) == length(test_data) @test IS.get_initial_timestamp(f2) == default_time_params.initial_times[1] for (i, window) in enumerate(IS.iterate_windows(f2)) - @test TimeSeries.values(window) == test_data[default_time_params.initial_times[i]] + @test TimeSeries.values(window) == + test_data[default_time_params.initial_times[i]] end if extended @@ -1706,7 +1815,12 @@ end start_time = initial_timestamp + interval # Verify all permutations with defaults. ta2 = - IS.get_time_series_array(IS.Deterministic, component, name; start_time = start_time) + IS.get_time_series_array( + IS.Deterministic, + component, + name; + start_time = start_time, + ) @test ta2 isa TimeSeries.TimeArray @test TimeSeries.timestamp(ta2) == @@ -1798,7 +1912,12 @@ end @test IS.has_time_series(component) @test IS.get_initial_timestamp(forecast) == initial_time forecast_retrieved = - IS.get_time_series(IS.Probabilistic, component, "test"; start_time = initial_time) + IS.get_time_series( + IS.Probabilistic, + component, + "test"; + start_time = initial_time, + ) @test IS.get_initial_timestamp(forecast_retrieved) == initial_time t = IS.get_time_series_array( IS.Probabilistic, @@ -1843,7 +1962,12 @@ end forecast_retrieved = IS.get_time_series(IS.Scenarios, component, "test"; start_time = initial_time) @test IS.get_initial_timestamp(forecast_retrieved) == initial_time - t = IS.get_time_series_array(IS.Scenarios, component, "test"; start_time = initial_time) + t = IS.get_time_series_array( + IS.Scenarios, + component, + "test"; + start_time = initial_time, + ) @test size(t) == (24, 99) @test TimeSeries.values(t) == data1 @@ -1856,7 +1980,8 @@ end ) @test size(t) == (12, 99) @test TimeSeries.values(t) == data1[1:12, :] - t_other = IS.get_time_series(IS.Scenarios, component, "test"; start_time = other_time) + t_other = + IS.get_time_series(IS.Scenarios, component, "test"; start_time = other_time) @test collect(keys(IS.get_data(t_other)))[1] == other_time end @@ -1895,20 +2020,16 @@ end forecast = IS.Deterministic(; data = data, name = name, resolution = resolution) @test_throws IS.ConflictingInputsError IS.add_time_series!(sys, component, forecast) - # Conflicting resolution + # As of PSY 4.0, different resolutions are allowed. resolution2 = Dates.Minute(5) name = "test2" data = SortedDict(initial_time => ones(horizon), second_time => ones(horizon)) - forecast = IS.Deterministic(; data = data, name = name, resolution = resolution2) - @test_throws IS.ConflictingInputsError IS.add_time_series!(sys, component, forecast) + forecast = IS.Deterministic(; data = data, name = name, resolution = resolution) + IS.add_time_series!(sys, component, forecast) # Conflicting horizon - name = "test2" - horizon2 = 23 - data = SortedDict(initial_time => ones(horizon2), second_time => ones(horizon2)) - - forecast = IS.Deterministic(; data = data, name = name, resolution = resolution) + forecast = IS.Deterministic(; data = data, name = name, resolution = resolution2) @test_throws IS.ConflictingInputsError IS.add_time_series!(sys, component, forecast) # Conflicting count @@ -1924,56 +2045,6 @@ end @test_throws IS.ConflictingInputsError IS.add_time_series!(sys, component, forecast) end -@testset "Test get_time_series_by_key and TimeSeriesKey constructor" begin - sys = IS.SystemData() - name = "Component1" - component = IS.TestComponent(name, 5) - IS.add_component!(sys, component) - - initial_time = Dates.DateTime("2020-09-01") - resolution = Dates.Hour(1) - - other_time = initial_time + resolution - name = "test" - horizon = 24 - data = SortedDict(initial_time => ones(horizon), other_time => ones(horizon)) - - forecast = IS.Deterministic(; data = data, name = name, resolution = resolution) - IS.add_time_series!(sys, component, forecast) - key = IS.TimeSeriesKey(forecast) - @test key == IS.TimeSeriesKey(IS.DeterministicMetadata, name) - @test key == - IS.TimeSeriesKey(; time_series_type = IS.DeterministicMetadata, name = name) - - var1 = IS.get_time_series(IS.Deterministic, component, name; start_time = initial_time) - var_key1 = IS.get_time_series_by_key(key, component; start_time = initial_time) - @test length(var1) == length(var_key1) - @test IS.get_horizon(var1) == horizon - @test IS.get_horizon(var1) == IS.get_horizon(var_key1) - @test IS.get_initial_timestamp(var1) == initial_time - @test IS.get_initial_timestamp(var1) == IS.get_initial_timestamp(var_key1) - - var2 = IS.get_time_series( - IS.Deterministic, - component, - name; - start_time = initial_time, - count = 2, - ) - var_key2 = - IS.get_time_series_by_key(key, component; start_time = initial_time, count = 2) - @test length(var2) == 2 - @test length(var2) == length(var_key2) - - # Throws errors - @test_throws ArgumentError IS.get_time_series_by_key( - key, - component; - start_time = initial_time, - count = 3, - ) -end - @testset "Test copy_to_new_file! on HDF5" begin sys = IS.SystemData(; time_series_in_memory = false) name = "Component1" @@ -1992,9 +2063,9 @@ end @test data_input == first(values((fdata))) IS.add_time_series!(sys, component, time_series) - orig_file = IS.get_file_path(sys.time_series_storage) - IS.copy_to_new_file!(sys.time_series_storage) - @test orig_file != IS.get_file_path(sys.time_series_storage) + orig_file = IS.get_file_path(sys.time_series_manager.data_store) + IS.copy_to_new_file!(sys.time_series_manager.data_store) + @test orig_file != IS.get_file_path(sys.time_series_manager.data_store) time_series2 = IS.get_time_series(IS.Deterministic, component, name) @test time_series2 isa IS.Deterministic @@ -2018,7 +2089,8 @@ end for compression_enabled in (true, false) compression = IS.CompressionSettings(; enabled = compression_enabled) sys = IS.SystemData(; time_series_in_memory = false, compression = compression) - @test sys.time_series_storage.compression.enabled == compression_enabled + @test sys.time_series_manager.data_store.compression.enabled == + compression_enabled name = "Component1" name = "val" component = IS.TestComponent(name, 5) @@ -2031,10 +2103,14 @@ end data = SortedDict(initial_timestamp => data_input) for i in 1:2 time_series = - IS.Deterministic(; name = "name_$i", resolution = resolution, data = data) + IS.Deterministic(; + name = "name_$i", + resolution = resolution, + data = data, + ) IS.add_time_series!(sys, component, time_series) end - old_file = IS.get_file_path(sys.time_series_storage) + old_file = IS.get_file_path(sys.time_series_manager.data_store) new_file, io = mktemp() close(io) IS.copy_h5_file(old_file, new_file) @@ -2052,10 +2128,6 @@ end fo[IS.HDF5_TS_ROOT_PATH][uuid], fn[IS.HDF5_TS_ROOT_PATH][uuid], ) - compare_attributes( - fo[IS.HDF5_TS_ROOT_PATH][uuid][IS.COMPONENT_REFERENCES_KEY], - fn[IS.HDF5_TS_ROOT_PATH][uuid][IS.COMPONENT_REFERENCES_KEY], - ) old_data = fo[IS.HDF5_TS_ROOT_PATH][uuid]["data"][:, :] new_data = fn[IS.HDF5_TS_ROOT_PATH][uuid]["data"][:, :] @test old_data == new_data @@ -2123,7 +2195,7 @@ end IS.add_time_series!(sys, component, ts1a) ts2a = IS.SingleTimeSeries(ts1a, name2; scaling_factor_multiplier = sfm2) IS.add_time_series!(sys, component, ts2a) - @test IS.get_num_time_series(sys.time_series_storage) == 1 + @test IS.get_num_time_series(sys) == 1 ts1b = IS.get_time_series(IS.SingleTimeSeries, component, name1) ts2b = IS.get_time_series(IS.SingleTimeSeries, component, name2) @test ts1b.data == ts2b.data @@ -2164,7 +2236,10 @@ function test_forecasts_with_shared_component_fields(forecast_type) sfm2 = use_scaling_factor ? IS.get_val2 : nothing if forecast_type <: IS.Deterministic data = - SortedDict(initial_time => rand(horizon), other_time => rand(horizon)) + SortedDict( + initial_time => rand(horizon), + other_time => rand(horizon), + ) forecast1a = IS.Deterministic(; data = data, name = name1, @@ -2173,7 +2248,10 @@ function test_forecasts_with_shared_component_fields(forecast_type) ) elseif forecast_type <: IS.Probabilistic data = - Dict(initial_time => rand(horizon, 99), other_time => ones(horizon, 99)) + Dict( + initial_time => rand(horizon, 99), + other_time => ones(horizon, 99), + ) forecast1a = IS.Probabilistic( name1, data, @@ -2183,21 +2261,34 @@ function test_forecasts_with_shared_component_fields(forecast_type) ) elseif forecast_type <: IS.Scenarios data = - Dict(initial_time => rand(horizon, 99), other_time => ones(horizon, 99)) + Dict( + initial_time => rand(horizon, 99), + other_time => ones(horizon, 99), + ) forecast1a = - IS.Scenarios(name1, data, resolution; scaling_factor_multiplier = sfm1) + IS.Scenarios( + name1, + data, + resolution; + scaling_factor_multiplier = sfm1, + ) else error("Unsupported forecast type: $forecast_type") end IS.add_time_series!(sys, component, forecast1a) - forecast2a = forecast_type(forecast1a, name2; scaling_factor_multiplier = sfm2) + forecast2a = + forecast_type(forecast1a, name2; scaling_factor_multiplier = sfm2) IS.add_time_series!(sys, component, forecast2a) - @test IS.get_num_time_series(sys.time_series_storage) == 1 + @test IS.get_num_time_series(sys) == 1 forecast1b = IS.get_time_series(forecast_type, component, name1) forecast2b = IS.get_time_series(forecast_type, component, name2) @test forecast1b.data == forecast2b.data expected1 = - use_scaling_factor ? data[initial_time] * component.val : data[initial_time] + if use_scaling_factor + data[initial_time] * component.val + else + data[initial_time] + end expected2 = if use_scaling_factor data[initial_time] * component.val2 else @@ -2214,14 +2305,14 @@ function test_forecasts_with_shared_component_fields(forecast_type) initial_time; ) == expected2 IS.remove_time_series!(sys, forecast_type, component, "val") - @test IS.get_num_time_series(sys.time_series_storage) == 1 + @test IS.get_num_time_series(sys) == 1 @test IS.get_time_series_values( component, forecast2b, initial_time; ) == expected2 IS.remove_time_series!(sys, forecast_type, component, "val2") - @test IS.get_num_time_series(sys.time_series_storage) == 0 + @test IS.get_num_time_series(sys) == 0 end end end @@ -2244,7 +2335,7 @@ end ENV[IS.TIME_SERIES_DIRECTORY_ENV_VAR] = path try sys = IS.SystemData() - @test splitpath(sys.time_series_storage.file_path)[1] == path + @test splitpath(sys.time_series_manager.data_store.file_path)[1] == path finally pop!(ENV, IS.TIME_SERIES_DIRECTORY_ENV_VAR) end @@ -2260,7 +2351,9 @@ end @testset "Test custom time series directories" begin @test IS._get_time_series_parent_dir(nothing) == tempdir() @test IS._get_time_series_parent_dir(pwd()) == pwd() - @test_throws ErrorException IS._get_time_series_parent_dir("/some/invalid/directory/") + @test_throws ErrorException IS._get_time_series_parent_dir( + "/some/invalid/directory/", + ) ENV["SIENNA_TIME_SERIES_DIRECTORY"] = pwd() try diff --git a/test/test_time_series_cache.jl b/test/test_time_series_cache.jl index 8ede601ca..cbe63aec7 100644 --- a/test/test_time_series_cache.jl +++ b/test/test_time_series_cache.jl @@ -201,10 +201,10 @@ end interval, ) - forecast = IS.get_time_series(IS.AbstractDeterministic, component, name) + forecast = IS.get_time_series(IS.Deterministic, component, name) initial_times = collect(IS.get_initial_times(forecast)) cache = - IS.ForecastCache(IS.AbstractDeterministic, component, name; cache_size_bytes = 1024) + IS.ForecastCache(IS.Deterministic, component, name; cache_size_bytes = 1024) for (i, ta) in enumerate(cache) @test TimeSeries.timestamp(ta) == diff --git a/test/test_time_series_storage.jl b/test/test_time_series_storage.jl index 75d1c42fb..5937dd781 100644 --- a/test/test_time_series_storage.jl +++ b/test/test_time_series_storage.jl @@ -20,65 +20,22 @@ end function test_add_remove(storage::IS.TimeSeriesStorage) name = "component1" name = "val" - component = IS.TestComponent(name, 5) ts = IS.SingleTimeSeries(; data = create_time_array(), name = "test") - IS.serialize_time_series!(storage, IS.get_uuid(component), name, ts) + IS.serialize_time_series!(storage, ts) ts2 = _deserialize_full(storage, ts) @test TimeSeries.timestamp(IS.get_data(ts2)) == TimeSeries.timestamp(IS.get_data(ts)) @test TimeSeries.values(IS.get_data(ts2)) == TimeSeries.values(IS.get_data(ts)) - component2 = IS.TestComponent("component2", 6) - IS.serialize_time_series!(storage, IS.get_uuid(component2), name, ts) - @test IS.get_num_time_series(storage) == 1 - - IS.remove_time_series!(storage, IS.get_uuid(ts), IS.get_uuid(component2), name) - - ## There should still be one reference to the data. - ts2 = _deserialize_full(storage, ts) - @test IS.get_data(ts2) isa TimeSeries.TimeArray - - IS.remove_time_series!(storage, IS.get_uuid(ts), IS.get_uuid(component), name) - @test_throws ArgumentError _deserialize_full(storage, ts) - return IS.get_num_time_series(storage) == 0 -end - -function test_add_references(storage::IS.TimeSeriesStorage) - name = "val" - component1 = IS.TestComponent("component1", 5) - component2 = IS.TestComponent("component2", 6) - ts = IS.SingleTimeSeries(; data = create_time_array(), name = "test") - ts_uuid = IS.get_uuid(ts) - IS.serialize_time_series!(storage, IS.get_uuid(component1), name, ts) - IS.add_time_series_reference!(storage, IS.get_uuid(component2), name, ts_uuid) - - # Adding duplicate references is not allowed. - @test_throws AssertionError IS.add_time_series_reference!( - storage, - IS.get_uuid(component2), - name, - ts_uuid, - ) - - @test IS.get_num_time_series(storage) == 1 - - IS.remove_time_series!(storage, ts_uuid, IS.get_uuid(component1), name) - - # There should still be one reference to the data. - @test _deserialize_full(storage, ts) isa IS.TimeSeriesData - - IS.remove_time_series!(storage, ts_uuid, IS.get_uuid(component2), name) + IS.remove_time_series!(storage, IS.get_uuid(ts)) @test_throws ArgumentError _deserialize_full(storage, ts) return IS.get_num_time_series(storage) == 0 end function test_get_subset(storage::IS.TimeSeriesStorage) - name = "component1" - name = "val" - component = IS.TestComponent(name, 1) ts = IS.SingleTimeSeries(; data = create_time_array(), name = "test") - IS.serialize_time_series!(storage, IS.get_uuid(component), name, ts) + IS.serialize_time_series!(storage, ts) ts2 = _deserialize_full(storage, ts) @test TimeSeries.timestamp(IS.get_data(ts2)) == TimeSeries.timestamp(IS.get_data(ts)) @@ -98,7 +55,7 @@ function test_get_subset(storage::IS.TimeSeriesStorage) data = SortedDict(initial_time1 => ones(horizon), initial_time2 => ones(horizon)) ts = IS.Deterministic(; data = data, name = name, resolution = resolution) - IS.serialize_time_series!(storage, IS.get_uuid(component), name, ts) + IS.serialize_time_series!(storage, ts) ts_metadata = make_metadata(ts) rows = UnitRange(1, horizon) columns = UnitRange(1, 2) @@ -126,11 +83,8 @@ function test_get_subset(storage::IS.TimeSeriesStorage) end function test_clear(storage::IS.TimeSeriesStorage) - name = "component1" - name = "val" - component = IS.TestComponent(name, 5) ts = IS.SingleTimeSeries(; data = create_time_array(), name = "test") - IS.serialize_time_series!(storage, IS.get_uuid(component), name, ts) + IS.serialize_time_series!(storage, ts) ts2 = _deserialize_full(storage, ts) @test TimeSeries.timestamp(IS.get_data(ts2)) == TimeSeries.timestamp(IS.get_data(ts)) @@ -152,15 +106,6 @@ end test_clear(IS.make_time_series_storage(; in_memory = false, directory = ".")) end -@testset "Test copy time series references" begin - for in_memory in (true, false) - test_add_remove(IS.make_time_series_storage(; in_memory = in_memory)) - test_add_references(IS.make_time_series_storage(; in_memory = in_memory)) - test_get_subset(IS.make_time_series_storage(; in_memory = in_memory)) - test_clear(IS.make_time_series_storage(; in_memory = in_memory)) - end -end - @testset "Test data format version" begin storage = IS.make_time_series_storage(; in_memory = false) @test IS.read_data_format_version(storage) == IS.TIME_SERIES_DATA_FORMAT_VERSION @@ -183,12 +128,6 @@ end compression = compression, ), ) - test_add_references( - IS.make_time_series_storage(; - in_memory = in_memory, - compression = compression, - ), - ) test_get_subset( IS.make_time_series_storage(; in_memory = in_memory, @@ -211,9 +150,8 @@ end @test isempty(storage) name = "component1" name = "val" - component = IS.TestComponent(name, 5) ts = IS.SingleTimeSeries(; data = create_time_array(), name = "test") - IS.serialize_time_series!(storage, IS.get_uuid(component), name, ts) + IS.serialize_time_series!(storage, ts) @test !isempty(storage) end end diff --git a/test/test_utils.jl b/test/test_utils.jl index 5c0da2fc8..dcb7c0a49 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -39,10 +39,11 @@ end struct FakeTimeSeries <: InfrastructureSystems.TimeSeriesData end Base.length(::FakeTimeSeries) = 42 +IS.get_name(::FakeTimeSeries) = "fake" @testset "Test TimeSeriesData printing" begin @test occursin( - "FakeTimeSeries time_series (42)", + "FakeTimeSeries.fake", sprint(show, MIME("text/plain"), FakeTimeSeries()), ) end From 68f0e049d202080edb1d504718da9645e156c751 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Fri, 19 Apr 2024 08:09:43 -0600 Subject: [PATCH 02/19] feat(time-series): Code cleanup --- src/system_data.jl | 4 +-- src/time_series_interface.jl | 41 +--------------------------- src/time_series_manager.jl | 22 ++------------- src/time_series_metadata_store.jl | 4 +-- test/test_supplemental_attributes.jl | 2 ++ test/test_time_series.jl | 30 ++++++++++++++++++++ 6 files changed, 40 insertions(+), 63 deletions(-) diff --git a/src/system_data.jl b/src/system_data.jl index 91a5e9bd4..ad4ebadb9 100644 --- a/src/system_data.jl +++ b/src/system_data.jl @@ -465,10 +465,10 @@ function transform_single_time_series!( if length(resolutions) > 1 # TODO: This needs to support an alternate method where horizon is expressed as a # Period (horizon * resolution) - error( + throw(ConflictingInputsError( "transform_single_time_series! is not yet supported when there is more than " * "one resolution: $resolutions", - ) + )) end remove_time_series!(data, DeterministicSingleTimeSeries) diff --git a/src/time_series_interface.jl b/src/time_series_interface.jl index b3d5126ad..a8bc35fe7 100644 --- a/src/time_series_interface.jl +++ b/src/time_series_interface.jl @@ -66,6 +66,7 @@ function get_time_series_uuid( owner::TimeSeriesOwners, name::AbstractString, ) where {T <: TimeSeriesData} + # TODO: do we need this? metadata_type = time_series_data_to_metadata(T) metadata = get_time_series_metadata(metadata_type, owner, name) return get_time_series_uuid(metadata) @@ -411,46 +412,6 @@ function list_time_series_metadata( ) end -function get_num_time_series(owner::TimeSeriesOwners) - mgr = get_time_series_manager(owner) - if isnothing(mgr) - return (0, 0) - end - - static_ts_count = 0 - forecast_count = 0 - for metadata in list_metadata(mgr.metadata_store, owner) - if metadata isa StaticTimeSeriesMetadata - static_ts_count += 1 - elseif metadata isa ForecastMetadata - forecast_count += 1 - else - error("panic") - end - end - - return (static_ts_count, forecast_count) -end - -function get_num_time_series_by_type(owner::TimeSeriesOwners) - counts = Dict{String, Int}() - mgr = get_time_series_manager(owner) - if isnothing(mgr) - return counts - end - - for metadata in list_metadata(mgr.metadata_store, owner) - type = string(nameof(time_series_metadata_to_data(metadata))) - if haskey(counts, type) - counts[type] += 1 - else - counts[type] = 1 - end - end - - return counts -end - function get_time_series( owner::TimeSeriesOwners, time_series::TimeSeriesData, diff --git a/src/time_series_manager.jl b/src/time_series_manager.jl index 5f1447e78..0a8d37f10 100644 --- a/src/time_series_manager.jl +++ b/src/time_series_manager.jl @@ -32,25 +32,6 @@ function TimeSeriesManager(; return TimeSeriesManager(data_store, metadata_store, read_only) end -function add_metadata!( - mgr::TimeSeriesManager, - component::TimeSeriesOwners, - metadata::TimeSeriesMetadata; - skip_if_present = false, - features..., -) - _throw_if_read_only(mgr) - add_metadata!( - mgr.metadata_store, - metadata, - component; - skip_if_present = skip_if_present, - ) - @debug "Added $(summary(metadata)) to $(summary(component)) " _group = - LOG_GROUP_TIME_SERIES - return -end - function add_time_series!( mgr::TimeSeriesManager, owner::TimeSeriesOwners, @@ -58,6 +39,7 @@ function add_time_series!( skip_if_present = false, features..., ) + _throw_if_read_only(mgr) throw_if_does_not_support_time_series(owner) _check_time_series_params(mgr, time_series) metadata_type = time_series_data_to_metadata(typeof(time_series)) @@ -81,6 +63,8 @@ function add_time_series!( metadata; ) end + @debug "Added $(summary(metadata)) to $(summary(owner)) " _group = + LOG_GROUP_TIME_SERIES return end diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index f2a371423..d0e72987e 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -583,7 +583,7 @@ has_time_series( time_series_type::Type{<:TimeSeriesData}, name::String; features..., -) = has_metadata(store, owner, time_series_type, name, features...) +) = has_metadata(store, owner, time_series_type, name; features...) """ Return a sorted Vector of distinct resolutions for all time series of the given type @@ -946,7 +946,7 @@ _get_owner_category( _get_owner_category(::Union{SupplementalAttribute, Type{<:SupplementalAttribute}}) = "SupplementalAttribute" _get_time_series_category(::Type{<:Forecast}) = "Forecast" -_get_time_series_category(::Type{<:StaticTimeSeries}) = "Forecast" +_get_time_series_category(::Type{<:StaticTimeSeries}) = "StaticTimeSeries" function _make_feature_filter(; features...) data = _make_sorted_feature_array(; features...) diff --git a/test/test_supplemental_attributes.jl b/test/test_supplemental_attributes.jl index e10f8149f..85787768a 100644 --- a/test/test_supplemental_attributes.jl +++ b/test/test_supplemental_attributes.jl @@ -117,4 +117,6 @@ end ts_ = IS.get_time_series(IS.SingleTimeSeries, attribute, "test") @test IS.get_initial_timestamp(ts_) == initial_time end + + @test length(collect(IS.iterate_supplemental_attributes_with_time_series(data))) == 3 end diff --git a/test/test_time_series.jl b/test/test_time_series.jl index 768cd69af..883160fa9 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -387,6 +387,20 @@ end scenario = "low", model_year = "2035", ) isa IS.SingleTimeSeries + @test IS.has_time_series(component, IS.SingleTimeSeries) + @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name) + @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name, scenario="low") + @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name, model_year = "2030", scenario="low") + @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name, model_year = "2030", scenario="low") + @test !IS.has_time_series(component, IS.SingleTimeSeries, ts_name, model_year = "2060", scenario="low") + @test length(IS.list_time_series_info(component)) == 4 + @test IS.list_time_series_info(component)[1].type === IS.SingleTimeSeries + @test Tables.rowtable( + IS.sql( + sys.time_series_manager.metadata_store, + "SELECT COUNT(*) AS count FROM $(IS.METADATA_TABLE_NAME)", + ), + )[1].count == 4 end @testset "Test add Deterministic with features" begin @@ -649,6 +663,22 @@ end resolution * (horizon + 1), ) + # Multiple resolutions is not supported yet + resolution2 = Dates.Hour(1) + dates = create_dates("2020-01-01T00:00:00", resolution2, "2020-01-02T00:00:00") + data = collect(1:length(dates)) + ta = TimeSeries.TimeArray(dates, data, [IS.get_name(component)]) + name = "val2" + ts = IS.SingleTimeSeries(name, ta) + IS.add_time_series!(sys, component, ts) + horizon = 24 + @test_throws IS.ConflictingInputsError IS.transform_single_time_series!( + sys, + IS.DeterministicSingleTimeSeries, + horizon, + resolution2 * (horizon + 1), + ) + # Ensure that attempted removal of nonexistent types works fine counts = IS.get_time_series_counts(sys) IS.remove_time_series!(sys, IS.Probabilistic) From 1ba9637ee8a0e3ae8c029c48766b9e7f9553ec61 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Fri, 19 Apr 2024 09:55:17 -0600 Subject: [PATCH 03/19] feat(time-series): Code cleanup 2 --- src/abstract_time_series.jl | 2 +- src/system_data.jl | 10 +++-- src/time_series_interface.jl | 54 ++++++++++-------------- src/time_series_metadata_store.jl | 9 +++- src/time_series_structs.jl | 12 ++---- src/utils/logging.jl | 2 +- src/utils/print.jl | 6 ++- test/test_printing.jl | 7 ++++ test/test_time_series.jl | 70 ++++++++++++++++++++++++++----- 9 files changed, 112 insertions(+), 60 deletions(-) diff --git a/src/abstract_time_series.jl b/src/abstract_time_series.jl index 675e47d5e..c87d3574a 100644 --- a/src/abstract_time_series.jl +++ b/src/abstract_time_series.jl @@ -31,7 +31,7 @@ abstract type AbstractTimeSeriesParameters <: InfrastructureSystemsType end struct StaticTimeSeriesParameters <: AbstractTimeSeriesParameters end -Base.@kwdef struct ForecastParameters <: AbstractTimeSeriesParameters +@kwdef struct ForecastParameters <: AbstractTimeSeriesParameters horizon::Int initial_timestamp::Dates.DateTime interval::Dates.Period diff --git a/src/system_data.jl b/src/system_data.jl index ad4ebadb9..f6f30dc57 100644 --- a/src/system_data.jl +++ b/src/system_data.jl @@ -465,10 +465,12 @@ function transform_single_time_series!( if length(resolutions) > 1 # TODO: This needs to support an alternate method where horizon is expressed as a # Period (horizon * resolution) - throw(ConflictingInputsError( - "transform_single_time_series! is not yet supported when there is more than " * - "one resolution: $resolutions", - )) + throw( + ConflictingInputsError( + "transform_single_time_series! is not yet supported when there is more than " * + "one resolution: $resolutions", + ), + ) end remove_time_series!(data, DeterministicSingleTimeSeries) diff --git a/src/time_series_interface.jl b/src/time_series_interface.jl index a8bc35fe7..4b4df3482 100644 --- a/src/time_series_interface.jl +++ b/src/time_series_interface.jl @@ -61,17 +61,6 @@ function get_time_series( return deserialize_time_series(T, storage, ts_metadata, rows, columns) end -function get_time_series_uuid( - ::Type{T}, - owner::TimeSeriesOwners, - name::AbstractString, -) where {T <: TimeSeriesData} - # TODO: do we need this? - metadata_type = time_series_data_to_metadata(T) - metadata = get_time_series_metadata(metadata_type, owner, name) - return get_time_series_uuid(metadata) -end - function get_time_series_metadata( ::Type{T}, owner::TimeSeriesOwners, @@ -95,8 +84,17 @@ function get_time_series_array( start_time::Union{Nothing, Dates.DateTime} = nothing, len::Union{Nothing, Int} = nothing, ignore_scaling_factors = false, + features..., ) where {T <: TimeSeriesData} - ts = get_time_series(T, owner, name; start_time = start_time, len = len, count = 1) + ts = get_time_series( + T, + owner, + name; + start_time = start_time, + len = len, + count = 1, + features..., + ) if start_time === nothing start_time = get_initial_timestamp(ts) end @@ -163,9 +161,17 @@ function get_time_series_timestamps( name::AbstractString; start_time::Union{Nothing, Dates.DateTime} = nothing, len::Union{Nothing, Int} = nothing, + features..., ) where {T <: TimeSeriesData} return TimeSeries.timestamp( - get_time_series_array(T, owner, name; start_time = start_time, len = len), + get_time_series_array( + T, + owner, + name; + start_time = start_time, + len = len, + features..., + ), ) end @@ -210,6 +216,7 @@ function get_time_series_values( start_time::Union{Nothing, Dates.DateTime} = nothing, len::Union{Nothing, Int} = nothing, ignore_scaling_factors = false, + features..., ) where {T <: TimeSeriesData} return TimeSeries.values( get_time_series_array( @@ -219,6 +226,7 @@ function get_time_series_values( start_time = start_time, len = len, ignore_scaling_factors = ignore_scaling_factors, + features..., ), ) end @@ -412,26 +420,6 @@ function list_time_series_metadata( ) end -function get_time_series( - owner::TimeSeriesOwners, - time_series::TimeSeriesData, -) - storage = get_time_series_storage(owner) - return get_time_series(storage, get_time_series_uuid(time_series)) -end - -function get_time_series_uuids(owner::TimeSeriesOwners) - mgr = get_time_series_manager(owner) - if isnothing(mgr) - return [] - end - - return [ - (get_time_series_uuid(x), get_name(x)) for - x in list_metadata(mgr.metadata_store, owner) - ] -end - function clear_time_series!(owner::TimeSeriesOwners) mgr = get_time_series_manager(owner) if !isnothing(mgr) diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index d0e72987e..c4bb3681e 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -54,7 +54,11 @@ function _create_metadata_table!(store::TimeSeriesMetadataStore) "owner_type TEXT NOT NULL", "owner_category TEXT NOT NULL", "features TEXT NOT NULL", + # The metadata is included as a convenience for serialization/de-serialization, + # specifically for types: time_series_type and scaling_factor_multplier. + # There is a lot duplication of data. "metadata JSON NOT NULL", + ] schema_text = join(schema, ",") _execute(store, "CREATE TABLE $(METADATA_TABLE_NAME)($(schema_text))") @@ -700,7 +704,9 @@ function remove_metadata!( owner; time_series_type = time_series_type, name = name, - require_full_feature_match = false, # TODO: needs more consideration + # TODO/PERF: This can be made faster by attempting search by a full match + # and then fallback to partial. We likely don't care about this for removing. + require_full_feature_match = false, features..., ) num_deleted = _remove_metadata!(store, where_clause) @@ -739,7 +745,6 @@ end Run a query and return the results in a DataFrame. """ function sql(store::TimeSeriesMetadataStore, query::String) - """Run a SQL query on the time series metadata table.""" return DataFrames.DataFrame(_execute(store, query)) end diff --git a/src/time_series_structs.jl b/src/time_series_structs.jl index cf05d2ced..619d5dfb3 100644 --- a/src/time_series_structs.jl +++ b/src/time_series_structs.jl @@ -1,6 +1,6 @@ const TimeSeriesOwners = Union{InfrastructureSystemsComponent, SupplementalAttribute} -Base.@kwdef struct StaticTimeSeriesInfo <: InfrastructureSystemsType +@kwdef struct StaticTimeSeriesInfo <: InfrastructureSystemsType type::DataType name::String initial_timestamp::Dates.DateTime @@ -20,7 +20,7 @@ function make_time_series_info(metadata::StaticTimeSeriesMetadata) ) end -Base.@kwdef struct ForecastInfo <: InfrastructureSystemsType +@kwdef struct ForecastInfo <: InfrastructureSystemsType type::DataType name::String initial_timestamp::Dates.DateTime @@ -48,7 +48,7 @@ end Provides counts of time series including attachments to components and supplemental attributes. """ -Base.@kwdef struct TimeSeriesCounts +@kwdef struct TimeSeriesCounts components_with_time_series::Int supplemental_attributes_with_time_series::Int static_time_series_count::Int @@ -56,15 +56,11 @@ Base.@kwdef struct TimeSeriesCounts end # TODO: This is now only used in PSY. Consider moving. -struct TimeSeriesKey <: InfrastructureSystemsType +@kwdef struct TimeSeriesKey <: InfrastructureSystemsType time_series_type::Type{<:TimeSeriesData} name::String end -function TimeSeriesKey(; time_series_type::Type{<:TimeSeriesData}, name::String) - return TimeSeriesKey(time_series_type, name) -end - function TimeSeriesKey(data::TimeSeriesData) return TimeSeriesKey(typeof(data), get_name(data)) end diff --git a/src/utils/logging.jl b/src/utils/logging.jl index 3955ba8b3..267404c22 100644 --- a/src/utils/logging.jl +++ b/src/utils/logging.jl @@ -108,7 +108,7 @@ function _is_level_valid(tracker::LogEventTracker, level::Logging.LogLevel) return level in keys(tracker.events) end -Base.@kwdef struct LoggingConfiguration +@kwdef struct LoggingConfiguration console::Bool = true console_stream::IO = stderr console_level::Base.LogLevel = Logging.Error diff --git a/src/utils/print.jl b/src/utils/print.jl index 1800f33d8..97fe5c8de 100644 --- a/src/utils/print.jl +++ b/src/utils/print.jl @@ -273,6 +273,10 @@ function show_components( end function show_time_series(owner::TimeSeriesOwners) + show_time_series(stdout, owner) +end + +function show_time_series(io::IO, owner::TimeSeriesOwners) data_by_type = Dict{Any, Vector{OrderedDict{String, Any}}}() for info in list_time_series_info(owner) if !haskey(data_by_type, info.type) @@ -289,7 +293,7 @@ function show_time_series(owner::TimeSeriesOwners) push!(data_by_type[info.type], data) end for rows in values(data_by_type) - PrettyTables.pretty_table(DataFrame(rows)) + PrettyTables.pretty_table(io, DataFrame(rows)) end end diff --git a/test/test_printing.jl b/test/test_printing.jl index bd456f291..cca2f063b 100644 --- a/test/test_printing.jl +++ b/test/test_printing.jl @@ -22,4 +22,11 @@ end text = String(take!(io)) @test occursin("TestComponent", text) @test occursin("val", text) + + component = first(IS.get_components(IS.TestComponent, sys)) + @test IS.has_time_series(component) + io = IOBuffer() + IS.show_time_series(io, component) + text = String(take!(io)) + @test occursin("SingleTimeSeries", text) end diff --git a/test/test_time_series.jl b/test/test_time_series.jl index 883160fa9..1b690e9d1 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -363,11 +363,41 @@ end rand(365), ) ts_name = "test_c" - data = IS.SingleTimeSeries(; data = data, name = ts_name) - IS.add_time_series!(sys, component, data; scenario = "low", model_year = "2030") - IS.add_time_series!(sys, component, data; scenario = "high", model_year = "2030") - IS.add_time_series!(sys, component, data; scenario = "low", model_year = "2035") - IS.add_time_series!(sys, component, data; scenario = "high", model_year = "2035") + ts = IS.SingleTimeSeries(; data = data, name = ts_name) + IS.add_time_series!(sys, component, ts; scenario = "low", model_year = "2030") + # get_time_series with partial query works if there is only 1. + @test IS.get_time_series(IS.SingleTimeSeries, component, ts_name).data == data + @test IS.get_time_series( + IS.SingleTimeSeries, + component, + ts_name; + scenario = "low", + ).data == data + @test IS.get_time_series( + IS.SingleTimeSeries, + component, + ts_name; + scenario = "low", + model_year = "2030", + ).data == data + @test IS.get_time_series_values( + IS.SingleTimeSeries, + component, + ts_name; + scenario = "low", + model_year = "2030", + ) == TimeSeries.values(data) + @test IS.get_time_series_timestamps( + IS.SingleTimeSeries, + component, + ts_name; + scenario = "low", + model_year = "2030", + ) == TimeSeries.timestamp(data) + + IS.add_time_series!(sys, component, ts; scenario = "high", model_year = "2030") + IS.add_time_series!(sys, component, ts; scenario = "low", model_year = "2035") + IS.add_time_series!(sys, component, ts; scenario = "high", model_year = "2035") @test_throws ArgumentError IS.get_time_series( IS.SingleTimeSeries, @@ -389,10 +419,28 @@ end ) isa IS.SingleTimeSeries @test IS.has_time_series(component, IS.SingleTimeSeries) @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name) - @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name, scenario="low") - @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name, model_year = "2030", scenario="low") - @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name, model_year = "2030", scenario="low") - @test !IS.has_time_series(component, IS.SingleTimeSeries, ts_name, model_year = "2060", scenario="low") + @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name, scenario = "low") + @test IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name, + model_year = "2030", + scenario = "low", + ) + @test IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name, + model_year = "2030", + scenario = "low", + ) + @test !IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + model_year = "2060", + scenario = "low", + ) @test length(IS.list_time_series_info(component)) == 4 @test IS.list_time_series_info(component)[1].type === IS.SingleTimeSeries @test Tables.rowtable( @@ -400,7 +448,7 @@ end sys.time_series_manager.metadata_store, "SELECT COUNT(*) AS count FROM $(IS.METADATA_TABLE_NAME)", ), - )[1].count == 4 + )[1].count == 4 end @testset "Test add Deterministic with features" begin @@ -488,6 +536,8 @@ end scenario = "low", model_year = "2035", )[1].features["model_year"] == "2035" + @test length(IS.list_time_series_info(component)) == 4 + @test IS.list_time_series_info(component)[1].type === IS.Deterministic IS.remove_time_series!(sys, IS.Deterministic, component, ts_name; scenario = "low") @test length( From c33e8ba435fc14db3666e0dcdfcac30d4e993b2a Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Fri, 19 Apr 2024 16:58:39 -0600 Subject: [PATCH 04/19] Remove invalid code --- src/time_series_container.jl | 2 -- src/time_series_metadata_store.jl | 1 - 2 files changed, 3 deletions(-) diff --git a/src/time_series_container.jl b/src/time_series_container.jl index 2c631326f..f83d44c23 100644 --- a/src/time_series_container.jl +++ b/src/time_series_container.jl @@ -9,8 +9,6 @@ function TimeSeriesContainer() return TimeSeriesContainer(nothing) end -getproperty(::TimeSeriesContainer, x) = nothing - get_time_series_manager(x::TimeSeriesContainer) = x.manager function set_time_series_manager!( diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index c4bb3681e..acd9899f8 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -58,7 +58,6 @@ function _create_metadata_table!(store::TimeSeriesMetadataStore) # specifically for types: time_series_type and scaling_factor_multplier. # There is a lot duplication of data. "metadata JSON NOT NULL", - ] schema_text = join(schema, ",") _execute(store, "CREATE TABLE $(METADATA_TABLE_NAME)($(schema_text))") From c4c0bf8fbedc20ce91e279c1c8d263848c9d125d Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Mon, 22 Apr 2024 09:11:58 -0600 Subject: [PATCH 05/19] Implement a SQLite backup function --- src/InfrastructureSystems.jl | 1 + src/time_series_container.jl | 4 +- src/time_series_manager.jl | 24 ++++------- src/time_series_metadata_store.jl | 72 +++++++++++++------------------ 4 files changed, 41 insertions(+), 60 deletions(-) diff --git a/src/InfrastructureSystems.jl b/src/InfrastructureSystems.jl index f45288c0a..5ac087991 100644 --- a/src/InfrastructureSystems.jl +++ b/src/InfrastructureSystems.jl @@ -112,6 +112,7 @@ include("utils/generate_structs.jl") include("utils/lazy_dict_from_iterator.jl") include("utils/logging.jl") include("utils/stdout_redirector.jl") +include("utils/sqlite.jl") include("function_data.jl") include("utils/utils.jl") include("internal.jl") diff --git a/src/time_series_container.jl b/src/time_series_container.jl index f83d44c23..74517366b 100644 --- a/src/time_series_container.jl +++ b/src/time_series_container.jl @@ -5,8 +5,8 @@ mutable struct TimeSeriesContainer manager::Union{Nothing, TimeSeriesManager} end -function TimeSeriesContainer() - return TimeSeriesContainer(nothing) +function TimeSeriesContainer(; manager = nothing) + return TimeSeriesContainer(manager) end get_time_series_manager(x::TimeSeriesContainer) = x.manager diff --git a/src/time_series_manager.jl b/src/time_series_manager.jl index 0a8d37f10..8645128de 100644 --- a/src/time_series_manager.jl +++ b/src/time_series_manager.jl @@ -17,9 +17,7 @@ function TimeSeriesManager(; end if isnothing(metadata_store) - filename, io = mktemp(isnothing(directory) ? tempdir() : directory) - close(io) - metadata_store = TimeSeriesMetadataStore(filename) + metadata_store = TimeSeriesMetadataStore() end if isnothing(data_store) @@ -97,19 +95,13 @@ function Base.deepcopy_internal(mgr::TimeSeriesManager, dict::IdDict) copy_to_new_file!(data_store, dirname(mgr.data_store.file_path)) end - close_temporarily!(mgr.metadata_store) do - # TODO: Change this implementation when SQLite.jl supports backup. - # https://github.com/JuliaDatabases/SQLite.jl/issues/210 - new_db_file, io = mktemp() - close(io) - cp(mgr.metadata_store.db.file, new_db_file; force = true) - metadata_store = from_file(TimeSeriesMetadataStore, new_db_file) - new_mgr = TimeSeriesManager(data_store, metadata_store, mgr.read_only) - dict[mgr] = new_mgr - dict[mgr.data_store] = new_mgr.data_store - dict[mgr.metadata_store] = new_mgr.metadata_store - return new_mgr - end + new_db_file = backup_to_temp(mgr.metadata_store) + metadata_store = TimeSeriesMetadataStore(new_db_file) + new_mgr = TimeSeriesManager(data_store, metadata_store, mgr.read_only) + dict[mgr] = new_mgr + dict[mgr.data_store] = new_mgr.data_store + dict[mgr.metadata_store] = new_mgr.metadata_store + return new_mgr end get_metadata( diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index acd9899f8..0adf16ca3 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -5,23 +5,32 @@ mutable struct TimeSeriesMetadataStore db::SQLite.DB end -function TimeSeriesMetadataStore(filename::AbstractString) - # An ideal solution would be to create an in-memory database and then perform a SQLite - # backup to a file whenever the user serializes the system. However, SQLite.jl does - # not support that feature yet: https://github.com/JuliaDatabases/SQLite.jl/issues/210 - store = TimeSeriesMetadataStore(SQLite.DB(filename)) +""" +Construct a new TimeSeriesMetadataStore with an in-memory database. +""" +function TimeSeriesMetadataStore() + store = TimeSeriesMetadataStore(SQLite.DB()) _create_metadata_table!(store) _create_indexes!(store) @debug "Initializedd new time series metadata table" _group = LOG_GROUP_TIME_SERIES return store end -function from_file(::Type{TimeSeriesMetadataStore}, filename::AbstractString) - store = TimeSeriesMetadataStore(SQLite.DB(filename)) +""" +Load a TimeSeriesMetadataStore from a saved database into an in-memory database. +""" +function TimeSeriesMetadataStore(filename::AbstractString) + src = SQLite.DB(filename) + db = SQLite.DB() + backup(db, src) + store = TimeSeriesMetadataStore(db) @debug "Loaded time series metadata from file" _group = LOG_GROUP_TIME_SERIES filename return store end +""" +Load a TimeSeriesMetadataStore from an HDF5 file into an in-memory database. +""" function from_h5_file(::Type{TimeSeriesMetadataStore}, src::AbstractString, directory) data = HDF5.h5open(src, "r") do file file[HDF5_TS_METADATA_ROOT_PATH][:] @@ -30,7 +39,7 @@ function from_h5_file(::Type{TimeSeriesMetadataStore}, src::AbstractString, dire filename, io = mktemp(isnothing(directory) ? tempdir() : directory) write(io, data) close(io) - return from_file(TimeSeriesMetadataStore, filename) + return TimeSeriesMetadataStore(filename) end function _create_metadata_table!(store::TimeSeriesMetadataStore) @@ -135,6 +144,18 @@ function add_metadata!( return end +""" +Backup the database to a file on the temporary filesystem and return that filename. +""" +function backup_to_temp(store::TimeSeriesMetadataStore) + filename, io = mktemp() + close(io) + dst = SQLite.DB(filename) + backup(dst, store.db) + close(dst) + return filename +end + """ Clear all time series metadata from the store. """ @@ -142,26 +163,6 @@ function clear_metadata!(store::TimeSeriesMetadataStore) _execute(store, "DELETE FROM $METADATA_TABLE_NAME") end -function backup(store::TimeSeriesMetadataStore, filename::String) - # This is an unfortunate implementation. SQLite supports backup but SQLite.jl does not. - # https://github.com/JuliaDatabases/SQLite.jl/issues/210 - # When they address the limitation, search the IS repo for this github issue number - # to fix all locations. - was_open = isopen(store.db) - if was_open - close(store.db) - end - - cp(store.db.file, filename) - @debug "Backed up time series metadata" _group = LOG_GROUP_TIME_SERIES filename - - if was_open - store.db = SQLite.DB(store.db.file) - end - - return -end - function check_params_compatibility( store::TimeSeriesMetadataStore, metadata::ForecastMetadata, @@ -253,15 +254,6 @@ function check_consistency(store::TimeSeriesMetadataStore, ::Type{<:StaticTimeSe return Dates.DateTime(row.initial_timestamp), row.length end -function close_temporarily!(func::Function, store::TimeSeriesMetadataStore) - try - close(store.db) - func() - finally - store.db = SQLite.DB(store.db.file) - end -end - function get_forecast_initial_times(store::TimeSeriesMetadataStore) params = get_forecast_parameters(store) isnothing(params) && return [] @@ -748,11 +740,7 @@ function sql(store::TimeSeriesMetadataStore, query::String) end function to_h5_file(store::TimeSeriesMetadataStore, dst::String) - metadata_path, io = mktemp() - close(io) - rm(metadata_path) - backup(store, metadata_path) - + metadata_path = backup_to_temp(store) data = open(metadata_path, "r") do io read(io) end From 7c807666f120de23d52ba763d1d62a8e27a319e2 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Mon, 22 Apr 2024 17:13:32 -0600 Subject: [PATCH 06/19] Add missing file --- src/utils/sqlite.jl | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/utils/sqlite.jl diff --git a/src/utils/sqlite.jl b/src/utils/sqlite.jl new file mode 100644 index 000000000..9e93c0fe8 --- /dev/null +++ b/src/utils/sqlite.jl @@ -0,0 +1,39 @@ +""" +Backup a SQLite database. +""" +# This has been proposed as a solution to https://github.com/JuliaDatabases/SQLite.jl/issues/210 +# and will be removed when the functionality is part of SQLite.jl. +function backup( + dst::SQLite.DB, + src::SQLite.DB; + dst_name::AbstractString = "main", + src_name::AbstractString = "main", + pages::Int = -1, + sleep::Float64 = 0.25, +) + if src === dst + error("src and dst cannot be the same connection") + end + + C = SQLite.C + num_pages = pages == 0 ? -1 : pages + sleep_ms = sleep * 1000 + ptr = C.sqlite3_backup_init(dst.handle, dst_name, src.handle, src_name) + r = C.SQLITE_OK + try + while r == C.SQLITE_OK || r == C.SQLITE_BUSY || r == C.SQLITE_LOCKED + r = C.sqlite3_backup_step(ptr, num_pages) + @debug "backup iteration: remaining = $(C.sqlite3_backup_remaining(ptr))" + if r == C.SQLITE_BUSY || r == C.SQLITE_LOCKED + C.sqlite3_sleep(sleep_ms) + end + end + finally + C.sqlite3_backup_finish(ptr) + if r != C.SQLITE_DONE + e = SQLite.sqliteexception(src.handle) + C.sqlite3_reset(src.handle) + throw(e) + end + end +end From 80013e9fa51c507dc7a6617cd8d86100a4e7384e Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 23 Apr 2024 13:39:06 -0600 Subject: [PATCH 07/19] Standardize Base.summary for package types --- src/utils/print.jl | 18 +++++------------- test/test_utils.jl | 2 +- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/utils/print.jl b/src/utils/print.jl index 688e66167..7d327ce49 100644 --- a/src/utils/print.jl +++ b/src/utils/print.jl @@ -46,13 +46,11 @@ function Base.show(io::IO, ::MIME"text/html", container::InfrastructureSystemsCo end end -function Base.summary(time_series::TimeSeriesData) - return "$(typeof(time_series)).$(get_name(time_series))" -end - -function Base.summary(time_series::TimeSeriesMetadata) - return "$(typeof(time_series)).$(get_name(time_series))" -end +make_label(type::Type{<:InfrastructureSystemsType}, name) = "$(nameof(type)): $name" +Base.summary(x::InfrastructureSystemsComponent) = make_label(typeof(x), get_name(x)) +Base.summary(x::SupplementalAttribute) = make_label(typeof(x), get_uuid(x)) +Base.summary(x::TimeSeriesData) = make_label(typeof(x), get_name(x)) +Base.summary(x::TimeSeriesMetadata) = make_label(typeof(x), get_name(x)) function Base.show(io::IO, data::SystemData) show(io, data.components) @@ -88,12 +86,6 @@ function show_time_series_data(io::IO, data::SystemData; kwargs...) return end -function Base.summary(ist::InfrastructureSystemsComponent) - # All InfrastructureSystemsComponent subtypes are supposed to implement get_name. - # Some don't. They need to override this function. - return "$(typeof(ist)).$(get_name(ist))" -end - function Base.show(io::IO, ::MIME"text/plain", system_units::SystemUnitsSettings) print(io, summary(system_units), ":") for name in fieldnames(typeof(system_units)) diff --git a/test/test_utils.jl b/test/test_utils.jl index dcb7c0a49..5a7746c6a 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -43,7 +43,7 @@ IS.get_name(::FakeTimeSeries) = "fake" @testset "Test TimeSeriesData printing" begin @test occursin( - "FakeTimeSeries.fake", + "FakeTimeSeries: fake", sprint(show, MIME("text/plain"), FakeTimeSeries()), ) end From 29b82537d404c0c8dcaf051958e8d9799a527f19 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 24 Apr 2024 18:15:28 -0600 Subject: [PATCH 08/19] Change the supertype of TimeSeriesData --- src/abstract_time_series.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/abstract_time_series.jl b/src/abstract_time_series.jl index c87d3574a..2473bfac3 100644 --- a/src/abstract_time_series.jl +++ b/src/abstract_time_series.jl @@ -19,7 +19,7 @@ Abstract type for time series stored in the system. Components store references to these through TimeSeriesMetadata values so that data can reside on storage media instead of memory. """ -abstract type TimeSeriesData <: InfrastructureSystemsComponent end +abstract type TimeSeriesData <: InfrastructureSystemsType end # Subtypes must implement # - Base.length From 1ed7baa4e0c2e40eea8c9cddf9495b1da4105403 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 24 Apr 2024 18:15:51 -0600 Subject: [PATCH 09/19] Remove unused file --- src/time_series_parameters.jl | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/time_series_parameters.jl diff --git a/src/time_series_parameters.jl b/src/time_series_parameters.jl deleted file mode 100644 index 8b1378917..000000000 --- a/src/time_series_parameters.jl +++ /dev/null @@ -1 +0,0 @@ - From c7f73e69eb0b41209bae62020c6b28ed7c477ef4 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 24 Apr 2024 18:16:37 -0600 Subject: [PATCH 10/19] Fix time series range checks --- src/component.jl | 44 --------------- src/time_series_interface.jl | 106 ++++++++++++++++++++++++++++------- 2 files changed, 87 insertions(+), 63 deletions(-) diff --git a/src/component.jl b/src/component.jl index 01367b44a..43cd23575 100644 --- a/src/component.jl +++ b/src/component.jl @@ -8,50 +8,6 @@ function prepare_for_removal!(component::InfrastructureSystemsComponent) return end -""" -Returns an iterator of TimeSeriesData instances attached to the component. - -Note that passing a filter function can be much slower than the other filtering parameters -because it reads time series data from media. - -Call `collect` on the result to get an array. - -# Arguments - - - `owner::InfrastructureSystemsComponent`: component or attribute from which to get time_series - - `filter_func = nothing`: Only return time_series for which this returns true. - - `type = nothing`: Only return time_series with this type. - - `name = nothing`: Only return time_series matching this value. -""" -function get_time_series_multiple( - owner::TimeSeriesOwners, - filter_func = nothing; - type = nothing, - name = nothing, -) - throw_if_does_not_support_time_series(owner) - mgr = get_time_series_manager(owner) - # This is true when the component is not part of a system. - isnothing(mgr) && return () - storage = get_time_series_storage(owner) - - Channel() do channel - for metadata in list_metadata(mgr, owner; time_series_type = type, name = name) - ts = deserialize_time_series( - isnothing(type) ? time_series_metadata_to_data(metadata) : type, - storage, - metadata, - UnitRange(1, length(metadata)), - UnitRange(1, get_count(metadata)), - ) - if !isnothing(filter_func) && !filter_func(ts) - continue - end - put!(channel, ts) - end - end -end - """ Transform all instances of SingleTimeSeries to DeterministicSingleTimeSeries. Do nothing if the component does not contain any instances. diff --git a/src/time_series_interface.jl b/src/time_series_interface.jl index 4b4df3482..973333825 100644 --- a/src/time_series_interface.jl +++ b/src/time_series_interface.jl @@ -61,6 +61,50 @@ function get_time_series( return deserialize_time_series(T, storage, ts_metadata, rows, columns) end +""" +Returns an iterator of TimeSeriesData instances attached to the component or attribute. + +Note that passing a filter function can be much slower than the other filtering parameters +because it reads time series data from media. + +Call `collect` on the result to get an array. + +# Arguments + + - `owner::TimeSeriesOwners`: component or attribute from which to get time_series + - `filter_func = nothing`: Only return time_series for which this returns true. + - `type = nothing`: Only return time_series with this type. + - `name = nothing`: Only return time_series matching this value. +""" +function get_time_series_multiple( + owner::TimeSeriesOwners, + filter_func = nothing; + type = nothing, + name = nothing, +) + throw_if_does_not_support_time_series(owner) + mgr = get_time_series_manager(owner) + # This is true when the component or attribute is not part of a system. + isnothing(mgr) && return () + storage = get_time_series_storage(owner) + + Channel() do channel + for metadata in list_metadata(mgr, owner; time_series_type = type, name = name) + ts = deserialize_time_series( + isnothing(type) ? time_series_metadata_to_data(metadata) : type, + storage, + metadata, + UnitRange(1, length(metadata)), + UnitRange(1, get_count(metadata)), + ) + if !isnothing(filter_func) && !filter_func(ts) + continue + end + put!(channel, ts) + end + end +end + function get_time_series_metadata( ::Type{T}, owner::TimeSeriesOwners, @@ -507,36 +551,62 @@ function _get_rows(start_time, len, ts_metadata::ForecastMetadata) return UnitRange(1, len) end -function _check_start_time(start_time, ts_metadata::TimeSeriesMetadata) +function _check_start_time(start_time, metadata::StaticTimeSeriesMetadata) + return _check_start_time_common(start_time, metadata) +end + +function _check_start_time(start_time, metadata::ForecastMetadata) + actual_start_time = _check_start_time_common(start_time, metadata) + window_count = get_count(metadata) + interval = get_interval(metadata) + time_diff = actual_start_time - get_initial_timestamp(metadata) + if window_count > 1 && + Dates.Millisecond(time_diff) % Dates.Millisecond(interval) != Dates.Second(0) + throw( + ArgumentError( + "start_time=$start_time is not on a multiple of interval=$interval", + ), + ) + end + + return actual_start_time +end + +function _check_start_time_common(start_time, metadata::TimeSeriesMetadata) if start_time === nothing - return get_initial_timestamp(ts_metadata) + return get_initial_timestamp(metadata) end - time_diff = start_time - get_initial_timestamp(ts_metadata) - if time_diff < Dates.Second(0) + if start_time < get_initial_timestamp(metadata) throw( ArgumentError( - "start_time=$start_time is earlier than $(get_initial_timestamp(ts_metadata))", + "start_time = $start_time is earlier than $(get_initial_timestamp(metadata))", ), ) end - if typeof(ts_metadata) <: ForecastMetadata - window_count = get_count(ts_metadata) - interval = get_interval(ts_metadata) - if window_count > 1 && - Dates.Millisecond(time_diff) % Dates.Millisecond(interval) != Dates.Second(0) - throw( - ArgumentError( - "start_time=$start_time is not on a multiple of interval=$interval", - ), - ) - end + last_time = _get_last_user_start_timestamp(metadata) + if start_time > last_time + throw( + ArgumentError( + "start_time = $start_time is greater than the last timestamp $last_time", + ), + ) end return start_time end +function _get_last_user_start_timestamp(metadata::StaticTimeSeriesMetadata) + return get_initial_timestamp(metadata) + + (get_length(metadata) - 1) * get_resolution(metadata) +end + +function _get_last_user_start_timestamp(forecast::ForecastMetadata) + return get_initial_timestamp(forecast) + + (get_count(forecast) - 1) * get_interval(forecast) +end + function get_forecast_window_count(initial_timestamp, interval, resolution, len, horizon) if interval == Dates.Second(0) count = 1 @@ -548,9 +618,7 @@ function get_forecast_window_count(initial_timestamp, interval, resolution, len, diff = Dates.Millisecond(last_initial_time - initial_timestamp) % Dates.Millisecond(interval) - if diff != Dates.Millisecond(0) - last_initial_time -= diff - end + last_initial_time -= diff count = Dates.Millisecond(last_initial_time - initial_timestamp) / Dates.Millisecond(interval) + 1 From 93fa48d986f6d3cee09093ee74d055d5b9498c13 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 24 Apr 2024 18:16:54 -0600 Subject: [PATCH 11/19] Restrict allowed types in time series features --- src/descriptors/structs.json | 16 ++++++++-------- src/generated/DeterministicMetadata.jl | 8 ++++---- src/generated/ProbabilisticMetadata.jl | 8 ++++---- src/generated/ScenariosMetadata.jl | 8 ++++---- src/generated/SingleTimeSeriesMetadata.jl | 8 ++++---- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/descriptors/structs.json b/src/descriptors/structs.json index 0816406e7..f88d6867c 100644 --- a/src/descriptors/structs.json +++ b/src/descriptors/structs.json @@ -51,8 +51,8 @@ }, { "name": "features", - "data_type": "Dict{String, <:Any}", - "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "data_type": "Dict{String, Union{Bool, Int, String}}", + "comment": "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years.", "default": "Dict{String, Any}()" }, { @@ -114,8 +114,8 @@ }, { "name": "features", - "data_type": "Dict{String, <:Any}", - "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "data_type": "Dict{String, Union{Bool, Int, String}}", + "comment": "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years.", "default": "Dict{String, Any}()" }, { @@ -177,8 +177,8 @@ }, { "name": "features", - "data_type": "Dict{String, <:Any}", - "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "data_type": "Dict{String, Union{Bool, Int, String}}", + "comment": "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years.", "default": "Dict{String, Any}()" }, { @@ -225,8 +225,8 @@ }, { "name": "features", - "data_type": "Dict{String, <:Any}", - "comment": "User-defined tags that describe the relationship between a component and a time series array.", + "data_type": "Dict{String, Union{Bool, Int, String}}", + "comment": "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years.", "default": "Dict{String, Any}()" }, { diff --git a/src/generated/DeterministicMetadata.jl b/src/generated/DeterministicMetadata.jl index 432f34244..6886a5226 100644 --- a/src/generated/DeterministicMetadata.jl +++ b/src/generated/DeterministicMetadata.jl @@ -15,7 +15,7 @@ This file is auto-generated. Do not edit. horizon::Int time_series_type::Type{<:AbstractDeterministic} scaling_factor_multiplier::Union{Nothing, Function} - features::Dict{String, <:Any} + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end @@ -31,7 +31,7 @@ A deterministic forecast for a particular data field in a Component. - `horizon::Int`: length of this time series - `time_series_type::Type{<:AbstractDeterministic}`: Type of the time series data associated with this metadata. - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. -- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. +- `features::Dict{String, Union{Bool, Int, String}}`: User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years. - `internal::InfrastructureSystemsInternal` """ mutable struct DeterministicMetadata <: ForecastMetadata @@ -52,8 +52,8 @@ mutable struct DeterministicMetadata <: ForecastMetadata time_series_type::Type{<:AbstractDeterministic} "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} - "User-defined tags that describe the relationship between a component and a time series array." - features::Dict{String, <:Any} + "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years." + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end diff --git a/src/generated/ProbabilisticMetadata.jl b/src/generated/ProbabilisticMetadata.jl index 167055e32..bc0125afc 100644 --- a/src/generated/ProbabilisticMetadata.jl +++ b/src/generated/ProbabilisticMetadata.jl @@ -15,7 +15,7 @@ This file is auto-generated. Do not edit. time_series_uuid::UUIDs.UUID horizon::Int scaling_factor_multiplier::Union{Nothing, Function} - features::Dict{String, <:Any} + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end @@ -31,7 +31,7 @@ A Probabilistic forecast for a particular data field in a Component. - `time_series_uuid::UUIDs.UUID`: reference to time series data - `horizon::Int`: length of this time series - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. -- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. +- `features::Dict{String, Union{Bool, Int, String}}`: User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years. - `internal::InfrastructureSystemsInternal` """ mutable struct ProbabilisticMetadata <: ForecastMetadata @@ -52,8 +52,8 @@ mutable struct ProbabilisticMetadata <: ForecastMetadata horizon::Int "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} - "User-defined tags that describe the relationship between a component and a time series array." - features::Dict{String, <:Any} + "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years." + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end diff --git a/src/generated/ScenariosMetadata.jl b/src/generated/ScenariosMetadata.jl index d1efe20c5..fc6b0e087 100644 --- a/src/generated/ScenariosMetadata.jl +++ b/src/generated/ScenariosMetadata.jl @@ -15,7 +15,7 @@ This file is auto-generated. Do not edit. time_series_uuid::UUIDs.UUID horizon::Int scaling_factor_multiplier::Union{Nothing, Function} - features::Dict{String, <:Any} + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end @@ -31,7 +31,7 @@ A Discrete Scenario Based time series for a particular data field in a Component - `time_series_uuid::UUIDs.UUID`: reference to time series data - `horizon::Int`: length of this time series - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. -- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. +- `features::Dict{String, Union{Bool, Int, String}}`: User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years. - `internal::InfrastructureSystemsInternal` """ mutable struct ScenariosMetadata <: ForecastMetadata @@ -52,8 +52,8 @@ mutable struct ScenariosMetadata <: ForecastMetadata horizon::Int "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} - "User-defined tags that describe the relationship between a component and a time series array." - features::Dict{String, <:Any} + "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years." + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end diff --git a/src/generated/SingleTimeSeriesMetadata.jl b/src/generated/SingleTimeSeriesMetadata.jl index 30e76a12c..989a4bb2c 100644 --- a/src/generated/SingleTimeSeriesMetadata.jl +++ b/src/generated/SingleTimeSeriesMetadata.jl @@ -12,7 +12,7 @@ This file is auto-generated. Do not edit. time_series_uuid::UUIDs.UUID length::Int scaling_factor_multiplier::Union{Nothing, Function} - features::Dict{String, <:Any} + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end @@ -25,7 +25,7 @@ A TimeSeries Data object in contigous form. - `time_series_uuid::UUIDs.UUID`: reference to time series data - `length::Int`: length of this time series - `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series data are scaling factors. Called on the associated component to convert the values. -- `features::Dict{String, <:Any}`: User-defined tags that describe the relationship between a component and a time series array. +- `features::Dict{String, Union{Bool, Int, String}}`: User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years. - `internal::InfrastructureSystemsInternal` """ mutable struct SingleTimeSeriesMetadata <: StaticTimeSeriesMetadata @@ -40,8 +40,8 @@ mutable struct SingleTimeSeriesMetadata <: StaticTimeSeriesMetadata length::Int "Applicable when the time series data are scaling factors. Called on the associated component to convert the values." scaling_factor_multiplier::Union{Nothing, Function} - "User-defined tags that describe the relationship between a component and a time series array." - features::Dict{String, <:Any} + "User-defined tags that differentiate multiple time series arrays that represent the same component attribute, such as different arrays for different scenarios or years." + features::Dict{String, Union{Bool, Int, String}} internal::InfrastructureSystemsInternal end From 1ff92bd08ecb49ee8e8e97595b87931039e8bcfb Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 24 Apr 2024 18:18:25 -0600 Subject: [PATCH 12/19] Fix bug when building filter query with non-strings --- src/time_series_manager.jl | 14 ++- src/time_series_metadata_store.jl | 18 ++-- test/test_time_series.jl | 137 +++++++++++++++++++++++++----- 3 files changed, 137 insertions(+), 32 deletions(-) diff --git a/src/time_series_manager.jl b/src/time_series_manager.jl index 8645128de..985ad654e 100644 --- a/src/time_series_manager.jl +++ b/src/time_series_manager.jl @@ -46,7 +46,13 @@ function add_time_series!( metadata_exists = has_metadata(mgr.metadata_store, owner, metadata) if metadata_exists && !skip_if_present - throw(ArgumentError("$(summary(metadata)) is already stored")) + msg = if isempty(features) + "$(summary(metadata)) is already stored" + else + fmsg = join(["$k = $v" for (k, v) in features], ", ") + "$(summary(metadata)) with features $fmsg is already stored" + end + throw(ArgumentError(msg)) end if !data_exists @@ -243,8 +249,8 @@ function compare_values( ) match = true for name in fieldnames(TimeSeriesManager) - val_x = getfield(x, name) - val_y = getfield(y, name) + val_x = getproperty(x, name) + val_y = getproperty(y, name) if name == :data_store && typeof(val_x) != typeof(val_y) @warn "Cannot compare $(typeof(val_x)) and $(typeof(val_y))" # TODO 1.0: workaround for not being able to convert Hdf5TimeSeriesStorage to @@ -253,7 +259,7 @@ function compare_values( end if !compare_values(val_x, val_y; compare_uuids = compare_uuids, exclude = exclude) - @error "TimeSeriesManager field = $name does not match" getfield(x, name) getfield( + @error "TimeSeriesManager field = $name does not match" getproperty(x, name) getproperty( y, name, ) diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index 0adf16ca3..75cde8e11 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -109,17 +109,13 @@ function _create_indexes!(store::TimeSeriesMetadataStore) end """ -Add metadata to the store. +Add metadata to the store. The caller must check if there are duplicates. """ function add_metadata!( store::TimeSeriesMetadataStore, owner::TimeSeriesOwners, metadata::TimeSeriesMetadata; ) - if has_metadata(store, owner, metadata) - throw(ArgumentError("time_series $(summary(metadata)) is already stored")) - end - check_params_compatibility(store, metadata) owner_category = _get_owner_category(owner) ts_type = time_series_metadata_to_data(metadata) @@ -511,7 +507,7 @@ function has_metadata( metadata::TimeSeriesMetadata, ) features = Dict(Symbol(k) => v for (k, v) in get_features(metadata)) - return has_metadata( + return _try_has_time_series_metadata_by_full_params( store, owner, time_series_metadata_to_data(metadata), @@ -942,10 +938,16 @@ _get_time_series_category(::Type{<:StaticTimeSeries}) = "StaticTimeSeries" function _make_feature_filter(; features...) data = _make_sorted_feature_array(; features...) - return join((["metadata->>'\$.features.$k' = '$v'" for (k, v) in data]), "AND ") + return join( + (["metadata->>'\$.features.$k' = $(_make_val_str(v))" for (k, v) in data]), + " AND ", + ) end -function _make_features_string(features::Dict{String, <:Any}) +_make_val_str(val::Union{Bool, Int}) = string(val) +_make_val_str(val::String) = "'$val'" + +function _make_features_string(features::Dict{String, Union{Bool, Int, String}}) key_names = sort!(collect(keys(features))) data = [Dict(k => features[k]) for k in key_names] return JSON3.write(data) diff --git a/test/test_time_series.jl b/test/test_time_series.jl index 1b690e9d1..fcd7f1b67 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -47,6 +47,12 @@ start_time = other_time, count = 2, ) + @test_throws ArgumentError IS.get_time_series( + IS.Deterministic, + component, + name; + start_time = other_time + resolution, + ) count = IS.get_count(var2) @test count == 2 @@ -451,6 +457,97 @@ end )[1].count == 4 end +@testset "Test add with features with mixed types" begin + sys = IS.SystemData() + name = "Component1" + component = IS.TestComponent(name, 5) + IS.add_component!(sys, component) + + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + + data = TimeSeries.TimeArray( + range(initial_time; length = 365, step = resolution), + rand(365), + ) + ts_name = "test" + ts = IS.SingleTimeSeries(; data = data, name = ts_name) + IS.add_time_series!(sys, component, ts; scenario = "low", model_year = "2030") + @test IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + scenario = "low", + model_year = "2030", + ) + @test !IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + scenario = "low", + model_year = 2030, + ) + IS.add_time_series!(sys, component, ts; scenario = "low", model_year = 2030) + @test IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + scenario = "low", + model_year = 2030, + ) + IS.add_time_series!(sys, component, ts; scenario = "low", model_year = 2035) + @test IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + scenario = "low", + model_year = 2035, + ) + @test !IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + scenario = "low", + model_year = "2035", + ) + IS.add_time_series!(sys, component, ts; scenario = "low", model_year = "2035") + @test IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + scenario = "low", + model_year = "2035", + ) + IS.add_time_series!(sys, component, ts; scenario = "low", some_condition = true) + @test IS.has_time_series(component, IS.SingleTimeSeries, ts_name; some_condition = true) + @test !IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + some_condition = "true", + ) + IS.add_time_series!(sys, component, ts; scenario = "low", some_condition = "false") + @test !IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + some_condition = false, + ) + IS.add_time_series!(sys, component, ts; scenario = "low", some_condition = false) + @test IS.has_time_series( + component, + IS.SingleTimeSeries, + ts_name; + some_condition = false, + ) + @test_throws MethodError IS.add_time_series!( + sys, + component, + ts; + scenario = Dict("key" => "val"), + ) +end + @testset "Test add Deterministic with features" begin sys = IS.SystemData() name = "Component1" @@ -948,26 +1045,26 @@ function _test_add_single_time_series_type(test_value, type_name) #_test_add_single_time_series_helper(component, initial_time) end -#@testset "Test add SingleTimeSeries with LinearFunctionData Cost" begin -# _test_add_single_time_series_type( -# repeat([IS.LinearFunctionData(3.14)], 365), -# "LinearFunctionData", -# ) -#end -# -#@testset "Test add SingleTimeSeries with QuadraticFunctionData Cost" begin -# _test_add_single_time_series_type( -# repeat([IS.QuadraticFunctionData(999.0, 1.0, 0.0)], 365), -# "QuadraticFunctionData", -# ) -#end -# -#@testset "Test add SingleTimeSeries with PiecewiseLinearPointData Cost" begin -# _test_add_single_time_series_type( -# repeat([IS.PiecewiseLinearPointData(repeat([(999.0, 1.0)], 5))], 365), -# "PiecewiseLinearPointData", -# ) -#end +@testset "Test add SingleTimeSeries with LinearFunctionData Cost" begin + _test_add_single_time_series_type( + repeat([IS.LinearFunctionData(3.14)], 365), + "LinearFunctionData", + ) +end + +@testset "Test add SingleTimeSeries with QuadraticFunctionData Cost" begin + _test_add_single_time_series_type( + repeat([IS.QuadraticFunctionData(999.0, 1.0, 0.0)], 365), + "QuadraticFunctionData", + ) +end + +@testset "Test add SingleTimeSeries with PiecewiseLinearPointData Cost" begin + _test_add_single_time_series_type( + repeat([IS.PiecewiseLinearPointData(repeat([(999.0, 1.0)], 5))], 365), + "PiecewiseLinearPointData", + ) +end @testset "Test read_time_series_file_metadata" begin file = joinpath(FORECASTS_DIR, "ComponentsAsColumnsNoTime.json") From b74317dd1eea679449d36ae63af6f3f118bdd24a Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Wed, 24 Apr 2024 18:19:33 -0600 Subject: [PATCH 13/19] Code cleanup --- src/components.jl | 8 ++++---- src/hdf5_time_series_storage.jl | 2 +- src/system_data.jl | 4 ++-- src/time_series_storage.jl | 1 - src/time_series_structs.jl | 6 +++--- src/utils/print.jl | 4 ++-- src/utils/utils.jl | 2 +- test/test_utils.jl | 2 +- 8 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/components.jl b/src/components.jl index 5d389af89..4959cae9e 100644 --- a/src/components.jl +++ b/src/components.jl @@ -363,11 +363,11 @@ function compare_values( name in exclude && continue # This gets validated in SystemData. name == :time_series_manager && continue - val_x = getfield(x, name) - val_y = getfield(y, name) + val_x = getproperty(x, name) + val_y = getproperty(y, name) if !compare_values(val_x, val_y; compare_uuids = compare_uuids, exclude = exclude) - val_x = getfield(x, name) - val_y = getfield(y, name) + val_x = getproperty(x, name) + val_y = getproperty(y, name) @error "Components field = $name does not match" val_x val_y match = false end diff --git a/src/hdf5_time_series_storage.jl b/src/hdf5_time_series_storage.jl index efdfe6d3d..70b7a0a0c 100644 --- a/src/hdf5_time_series_storage.jl +++ b/src/hdf5_time_series_storage.jl @@ -859,7 +859,7 @@ function compare_values( for ((uuid_x, data_x), (uuid_y, data_y)) in zip(item_x, item_y) if uuid_x != uuid_y - @error "UUIDs doesn't match" uuid_x uuid_y + @error "UUIDs don't match" uuid_x uuid_y return false end if data_x != data_y diff --git a/src/system_data.jl b/src/system_data.jl index 00d5cf028..7e07358ec 100644 --- a/src/system_data.jl +++ b/src/system_data.jl @@ -321,8 +321,8 @@ function compare_values( # the components. continue end - val_x = getfield(x, name) - val_y = getfield(y, name) + val_x = getproperty(x, name) + val_y = getproperty(y, name) if !compare_values(val_x, val_y; compare_uuids = compare_uuids, exclude = exclude) @error "SystemData field = $name does not match" getproperty(x, name) getproperty( y, diff --git a/src/time_series_storage.jl b/src/time_series_storage.jl index 46823486f..b693e4662 100644 --- a/src/time_series_storage.jl +++ b/src/time_series_storage.jl @@ -9,7 +9,6 @@ All subtypes must implement: - deserialize_time_series - get_compression_settings - get_num_time_series - - is_read_only - remove_time_series! - serialize_time_series! - replace_component_uuid! diff --git a/src/time_series_structs.jl b/src/time_series_structs.jl index 619d5dfb3..7532f0849 100644 --- a/src/time_series_structs.jl +++ b/src/time_series_structs.jl @@ -1,7 +1,7 @@ const TimeSeriesOwners = Union{InfrastructureSystemsComponent, SupplementalAttribute} @kwdef struct StaticTimeSeriesInfo <: InfrastructureSystemsType - type::DataType + type::Type{<:TimeSeriesData} name::String initial_timestamp::Dates.DateTime resolution::Dates.Period @@ -21,7 +21,7 @@ function make_time_series_info(metadata::StaticTimeSeriesMetadata) end @kwdef struct ForecastInfo <: InfrastructureSystemsType - type::DataType + type::Type{<:TimeSeriesData} name::String initial_timestamp::Dates.DateTime resolution::Dates.Period @@ -70,7 +70,7 @@ function deserialize_struct(::Type{TimeSeriesKey}, data::Dict) for field_name in fieldnames(TimeSeriesKey) val = data[string(field_name)] if field_name == :time_series_type - val = getfield(InfrastructureSystems, Symbol(strip_module_name(val))) + val = getproperty(InfrastructureSystems, Symbol(strip_module_name(val))) end vals[field_name] = val end diff --git a/src/utils/print.jl b/src/utils/print.jl index 7d327ce49..70ac16c4c 100644 --- a/src/utils/print.jl +++ b/src/utils/print.jl @@ -97,11 +97,11 @@ end function Base.show(io::IO, ::MIME"text/plain", ist::InfrastructureSystemsComponent) print(io, summary(ist), ":") for name in fieldnames(typeof(ist)) - obj = getfield(ist, name) + obj = getproperty(ist, name) if obj isa InfrastructureSystemsInternal || obj isa TimeSeriesContainer continue elseif obj isa InfrastructureSystemsType - val = summary(getfield(ist, name)) + val = summary(getproperty(ist, name)) elseif obj isa Vector{<:InfrastructureSystemsComponent} val = summary(getproperty(ist, name)) else diff --git a/src/utils/utils.jl b/src/utils/utils.jl index b67ebd133..e04d19dec 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -564,7 +564,7 @@ transform_array_for_hdf( transform_array_for_hdf(data::Vector{T}) where {T <: FunctionData} = throw(ArgumentError("Not currently implemented for $T")) -to_namedtuple(val) = (; (x => getfield(val, x) for x in fieldnames(typeof(val)))...) +to_namedtuple(val) = (; (x => getproperty(val, x) for x in fieldnames(typeof(val)))...) function compute_file_hash(path::String, files::Vector{String}) data = Dict("files" => []) diff --git a/test/test_utils.jl b/test/test_utils.jl index 5a7746c6a..0fc33bcb4 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -44,6 +44,6 @@ IS.get_name(::FakeTimeSeries) = "fake" @testset "Test TimeSeriesData printing" begin @test occursin( "FakeTimeSeries: fake", - sprint(show, MIME("text/plain"), FakeTimeSeries()), + sprint(show, MIME("text/plain"), summary(FakeTimeSeries())), ) end From 66f94fb10d0896e45ed5e8bb87466093923aec7b Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Thu, 25 Apr 2024 07:32:46 -0600 Subject: [PATCH 14/19] Fix indexes in SQL table --- src/time_series_metadata_store.jl | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index 75cde8e11..7ce4c16fc 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -12,7 +12,7 @@ function TimeSeriesMetadataStore() store = TimeSeriesMetadataStore(SQLite.DB()) _create_metadata_table!(store) _create_indexes!(store) - @debug "Initializedd new time series metadata table" _group = LOG_GROUP_TIME_SERIES + @debug "Initialized new time series metadata table" _group = LOG_GROUP_TIME_SERIES return store end @@ -64,7 +64,7 @@ function _create_metadata_table!(store::TimeSeriesMetadataStore) "owner_category TEXT NOT NULL", "features TEXT NOT NULL", # The metadata is included as a convenience for serialization/de-serialization, - # specifically for types: time_series_type and scaling_factor_multplier. + # specifically for types: time_series_type and scaling_factor_mulitplier. # There is a lot duplication of data. "metadata JSON NOT NULL", ] @@ -83,20 +83,12 @@ function _create_indexes!(store::TimeSeriesMetadataStore) # 2. Optimize for checks at system.add_time_series. Use all fields and features. # 3. Optimize for returning all metadata for a time series UUID. SQLite.createindex!(store.db, METADATA_TABLE_NAME, "by_id", "id"; unique = true) - SQLite.createindex!(store.db, METADATA_TABLE_NAME, "by_c", "owner_uuid"; unique = false) - SQLite.createindex!( - store.db, - METADATA_TABLE_NAME, - "by_c_n_tst", - ["owner_uuid", "name", "time_series_type"]; - unique = false, - ) SQLite.createindex!( store.db, METADATA_TABLE_NAME, "by_c_n_tst_features", ["owner_uuid", "name", "time_series_type", "features"]; - unique = false, + unique = true, ) SQLite.createindex!( store.db, From 8e826af309103d9bffe5617c188341335e6921ce Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Thu, 25 Apr 2024 07:33:20 -0600 Subject: [PATCH 15/19] Add test of time_series_read_only --- src/time_series_manager.jl | 4 ++++ test/test_serialization.jl | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/time_series_manager.jl b/src/time_series_manager.jl index 985ad654e..f56839edd 100644 --- a/src/time_series_manager.jl +++ b/src/time_series_manager.jl @@ -256,6 +256,10 @@ function compare_values( # TODO 1.0: workaround for not being able to convert Hdf5TimeSeriesStorage to # InMemoryTimeSeriesStorage continue + elseif name == :read_only + # Skip this because users can change it during deserialization and we test it + # separately. + continue end if !compare_values(val_x, val_y; compare_uuids = compare_uuids, exclude = exclude) diff --git a/test/test_serialization.jl b/test/test_serialization.jl index c6273f11c..7b739c170 100644 --- a/test/test_serialization.jl +++ b/test/test_serialization.jl @@ -64,16 +64,31 @@ end directory end +function _make_time_series() + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + data = TimeSeries.TimeArray( + range(initial_time; length = 2, step = resolution), + ones(2), + ) + data = IS.SingleTimeSeries(; data = data, name = "ts") +end + @testset "Test JSON serialization of with read-only time series" begin sys = create_system_data_shared_time_series(; time_series_in_memory = false) - sys2, result = validate_serialization(sys) + sys2, result = validate_serialization(sys; time_series_read_only = true) @test result + + component = first(IS.get_components(IS.TestComponent, sys2)) + @test_throws ArgumentError IS.add_time_series!(sys, component, _make_time_series()) end @testset "Test JSON serialization of with mutable time series" begin sys = create_system_data_shared_time_series(; time_series_in_memory = false) sys2, result = validate_serialization(sys; time_series_read_only = false) @test result + component = first(IS.get_components(IS.TestComponent, sys2)) + IS.add_time_series!(sys2, component, _make_time_series()) end @testset "Test JSON serialization with no time series" begin From b611c3b17ecbca031f225c9bfc7b09137793b71b Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Thu, 25 Apr 2024 13:27:02 -0600 Subject: [PATCH 16/19] Use JSONB format for metadata --- src/time_series_metadata_store.jl | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index 7ce4c16fc..d1cd721a2 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -9,6 +9,8 @@ end Construct a new TimeSeriesMetadataStore with an in-memory database. """ function TimeSeriesMetadataStore() + # This metadata is not expected to exceed system memory, so create an in-memory + # database so that it is faster. This could be changed. store = TimeSeriesMetadataStore(SQLite.DB()) _create_metadata_table!(store) _create_indexes!(store) @@ -64,8 +66,8 @@ function _create_metadata_table!(store::TimeSeriesMetadataStore) "owner_category TEXT NOT NULL", "features TEXT NOT NULL", # The metadata is included as a convenience for serialization/de-serialization, - # specifically for types: time_series_type and scaling_factor_mulitplier. - # There is a lot duplication of data. + # specifically for types and their modules: time_series_type and scaling_factor_mulitplier. + # There is duplication of data, but it saves a lot of code. "metadata JSON NOT NULL", ] schema_text = join(schema, ",") @@ -82,7 +84,6 @@ function _create_indexes!(store::TimeSeriesMetadataStore) # 1c. time series for one component/attribute with all features # 2. Optimize for checks at system.add_time_series. Use all fields and features. # 3. Optimize for returning all metadata for a time series UUID. - SQLite.createindex!(store.db, METADATA_TABLE_NAME, "by_id", "id"; unique = true) SQLite.createindex!( store.db, METADATA_TABLE_NAME, @@ -121,7 +122,7 @@ function add_metadata!( ts_category, features, ) - params = chop(repeat("?,", length(vals))) + params = repeat("?,", length(vals) - 1) * "jsonb(?)" SQLite.DBInterface.execute( store.db, "INSERT INTO $METADATA_TABLE_NAME VALUES($params)", @@ -623,7 +624,11 @@ function list_metadata( name = name, features..., ) - query = "SELECT metadata FROM $METADATA_TABLE_NAME WHERE $where_clause" + query = """ + SELECT json(metadata) AS metadata + FROM $METADATA_TABLE_NAME + WHERE $where_clause + """ table = Tables.rowtable(_execute(store, query)) return [_deserialize_metadata(x.metadata) for x in table] end @@ -844,7 +849,7 @@ function _try_get_time_series_metadata_by_full_params( owner, time_series_type, name, - "metadata"; + "json(metadata) AS metadata"; features..., ) len = length(rows) From 37b2e67c71212f39b61fe4fcef4f5dea5a7a79d6 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Fri, 26 Apr 2024 11:27:33 -0600 Subject: [PATCH 17/19] Address PR comments --- src/time_series_interface.jl | 19 +++++++++++++++ src/time_series_metadata_store.jl | 40 +++++++++++++++---------------- src/time_series_storage.jl | 1 - src/time_series_structs.jl | 6 +++-- test/test_time_series.jl | 26 ++++++++++++++++---- 5 files changed, 65 insertions(+), 27 deletions(-) diff --git a/src/time_series_interface.jl b/src/time_series_interface.jl index 973333825..65f68aa55 100644 --- a/src/time_series_interface.jl +++ b/src/time_series_interface.jl @@ -61,6 +61,25 @@ function get_time_series( return deserialize_time_series(T, storage, ts_metadata, rows, columns) end +function get_time_series( + owner::TimeSeriesOwners, + info::AbstractTimeSeriesInfo, + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Nothing, Int} = nothing, + count::Union{Nothing, Int} = nothing, +) + features = Dict{Symbol, Any}(Symbol(k) => v for (k, v) in info.features) + return get_time_series( + info.type, + owner, + info.name; + start_time = start_time, + len = len, + count = count, + features..., + ) +end + """ Returns an iterator of TimeSeriesData instances attached to the component or attribute. diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index d1cd721a2..f78592cb3 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -219,7 +219,7 @@ check_consistency(::TimeSeriesMetadataStore, ::Type{<:Forecast}) = nothing Throw InvalidValue if the SingleTimeSeries arrays have different initial times or lengths. Return the initial timestamp and length as a tuple. """ -function check_consistency(store::TimeSeriesMetadataStore, ::Type{<:StaticTimeSeries}) +function check_consistency(store::TimeSeriesMetadataStore, ::Type{SingleTimeSeries}) query = """ SELECT DISTINCT initial_timestamp @@ -243,6 +243,9 @@ function check_consistency(store::TimeSeriesMetadataStore, ::Type{<:StaticTimeSe return Dates.DateTime(row.initial_timestamp), row.length end +# check_consistency is not implemented on StaticTimeSeries because new types may have +# different requirments than SingleTimeSeries. Let future developers make that decision. + function get_forecast_initial_times(store::TimeSeriesMetadataStore) params = get_forecast_parameters(store) isnothing(params) && return [] @@ -307,7 +310,7 @@ function get_forecast_initial_timestamp(store::TimeSeriesMetadataStore) """ table = Tables.rowtable(_execute(store, query)) return if isempty(table) - Dates.DateTime(Dates.Minute(0)) + nothing else Dates.DateTime(table[1].initial_timestamp) end @@ -323,7 +326,7 @@ function get_forecast_interval(store::TimeSeriesMetadataStore) """ table = Tables.rowtable(_execute(store, query)) return if isempty(table) - Dates.Period(Dates.Millisecond(0)) + Dates.Millisecond(0) else Dates.Millisecond(table[1].interval_ms) end @@ -359,8 +362,8 @@ function get_metadata( len = length(metadata_items) if len == 0 if time_series_type === Deterministic - # This is a hack to account for the fact that we allow non-standard behavior - # with DeterministicSingleTimeSeries. + # This is a hack to account for the fact that we allow users to use + # Deterministic interchangeably with DeterministicSingleTimeSeries. try return get_metadata( store, @@ -532,7 +535,7 @@ function has_metadata( name = name, features..., ) - query = "SELECT COUNT(*) AS count FROM $METADATA_TABLE_NAME WHERE $where_clause" + query = "SELECT COUNT(*) AS count FROM $METADATA_TABLE_NAME $where_clause" return _execute_count(store, query) > 0 end @@ -540,7 +543,7 @@ end Return True if there is time series matching the UUID. """ function has_time_series(store::TimeSeriesMetadataStore, time_series_uuid::Base.UUID) - where_clause = "time_series_uuid = '$time_series_uuid'" + where_clause = "WHERE time_series_uuid = '$time_series_uuid'" return _has_time_series(store, where_clause) end @@ -548,7 +551,7 @@ function has_time_series( store::TimeSeriesMetadataStore, owner::TimeSeriesOwners, ) - where_clause = _make_owner_where_clause(owner) + where_clause = "WHERE owner_uuid = '$(get_uuid(owner))'" return _has_time_series(store, where_clause) end @@ -606,7 +609,7 @@ function list_matching_time_series_uuids( name = name, features..., ) - query = "SELECT DISTINCT time_series_uuid FROM $METADATA_TABLE_NAME WHERE $where_clause" + query = "SELECT DISTINCT time_series_uuid FROM $METADATA_TABLE_NAME $where_clause" table = Tables.columntable(_execute(store, query)) return Base.UUID.(table.time_series_uuid) end @@ -627,7 +630,7 @@ function list_metadata( query = """ SELECT json(metadata) AS metadata FROM $METADATA_TABLE_NAME - WHERE $where_clause + $where_clause """ table = Tables.rowtable(_execute(store, query)) return [_deserialize_metadata(x.metadata) for x in table] @@ -696,8 +699,8 @@ function remove_metadata!( num_deleted = _remove_metadata!(store, where_clause) if num_deleted == 0 if time_series_type === Deterministic - # This is a hack to account for the fact that we allow non-standard behavior - # with DeterministicSingleTimeSeries. + # This is a hack to account for the fact that we allow users to use + # Deterministic interchangeably with DeterministicSingleTimeSeries. remove_metadata!( store, owner; @@ -821,7 +824,7 @@ function _execute_count(store::TimeSeriesMetadataStore, query::AbstractString) end function _has_time_series(store::TimeSeriesMetadataStore, where_clause::String) - query = "SELECT COUNT(*) AS count FROM $METADATA_TABLE_NAME WHERE $where_clause" + query = "SELECT COUNT(*) AS count FROM $METADATA_TABLE_NAME $where_clause" return _execute_count(store, query) > 0 end @@ -829,7 +832,7 @@ function _remove_metadata!( store::TimeSeriesMetadataStore, where_clause::AbstractString, ) - _execute(store, "DELETE FROM $METADATA_TABLE_NAME WHERE $where_clause") + _execute(store, "DELETE FROM $METADATA_TABLE_NAME $where_clause") table = Tables.rowtable(_execute(store, "SELECT CHANGES() AS changes")) @assert_op length(table) == 1 @debug "Deleted $(table[1].changes) rows from the time series metadata table" _group = @@ -895,7 +898,7 @@ function _try_time_series_metadata_by_full_params( require_full_feature_match = true, features..., ) - query = "SELECT $column FROM $METADATA_TABLE_NAME WHERE $where_clause" + query = "SELECT $column FROM $METADATA_TABLE_NAME $where_clause" return Tables.rowtable(_execute(store, query)) end @@ -956,9 +959,6 @@ function _make_features_string(; features...) return JSON3.write(data) end -_make_owner_where_clause(owner::TimeSeriesOwners) = - "owner_uuid = '$(get_uuid(owner))'" - function _make_sorted_feature_array(; features...) key_names = sort!(collect(string.(keys(features)))) return [(key, features[Symbol(key)]) for key in key_names] @@ -973,7 +973,7 @@ function _make_where_clause( ) vals = String[] if !isnothing(owner) - push!(vals, _make_owner_where_clause(owner)) + push!(vals, "owner_uuid = '$(get_uuid(owner))'") end if !isnothing(name) push!(vals, "name = '$name'") @@ -990,7 +990,7 @@ function _make_where_clause( push!(vals, val) end - return "(" * join(vals, " AND ") * ")" + return isempty(vals) ? "" : "WHERE (" * join(vals, " AND ") * ")" end function _make_where_clause(owner::TimeSeriesOwners, metadata::TimeSeriesMetadata) diff --git a/src/time_series_storage.jl b/src/time_series_storage.jl index b693e4662..6d69e1094 100644 --- a/src/time_series_storage.jl +++ b/src/time_series_storage.jl @@ -11,7 +11,6 @@ All subtypes must implement: - get_num_time_series - remove_time_series! - serialize_time_series! - - replace_component_uuid! - Base.isempty """ abstract type TimeSeriesStorage end diff --git a/src/time_series_structs.jl b/src/time_series_structs.jl index 7532f0849..f4c9618d2 100644 --- a/src/time_series_structs.jl +++ b/src/time_series_structs.jl @@ -1,6 +1,8 @@ const TimeSeriesOwners = Union{InfrastructureSystemsComponent, SupplementalAttribute} -@kwdef struct StaticTimeSeriesInfo <: InfrastructureSystemsType +abstract type AbstractTimeSeriesInfo <: InfrastructureSystemsType end + +@kwdef struct StaticTimeSeriesInfo <: AbstractTimeSeriesInfo type::Type{<:TimeSeriesData} name::String initial_timestamp::Dates.DateTime @@ -20,7 +22,7 @@ function make_time_series_info(metadata::StaticTimeSeriesMetadata) ) end -@kwdef struct ForecastInfo <: InfrastructureSystemsType +@kwdef struct ForecastInfo <: AbstractTimeSeriesInfo type::Type{<:TimeSeriesData} name::String initial_timestamp::Dates.DateTime diff --git a/test/test_time_series.jl b/test/test_time_series.jl index fcd7f1b67..c5df6fadb 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -455,6 +455,9 @@ end "SELECT COUNT(*) AS count FROM $(IS.METADATA_TABLE_NAME)", ), )[1].count == 4 + for info in IS.list_time_series_info(component) + @test IS.get_data(IS.get_time_series(component, info)) == data + end end @testset "Test add with features with mixed types" begin @@ -546,6 +549,21 @@ end ts; scenario = Dict("key" => "val"), ) + # Duplicate features in different order. + @test_throws ArgumentError IS.add_time_series!( + sys, + component, + ts; + scenario = "low", + model_year = "2035", + ) + @test_throws ArgumentError IS.add_time_series!( + sys, + component, + ts; + model_year = "2035", + scenario = "low", + ) end @testset "Test add Deterministic with features" begin @@ -1039,10 +1057,10 @@ function _test_add_single_time_series_type(test_value, type_name) ) data = IS.SingleTimeSeries(; data = data_series, name = "test_c") IS.add_time_series!(sys, component, data) - #ts = IS.get_time_series(IS.SingleTimeSeries, component, "test_c";) - #@test IS.get_data_type(ts) == type_name - #@test reshape(TimeSeries.values(IS.get_data(ts)), 365) == TimeSeries.values(data_series) - #_test_add_single_time_series_helper(component, initial_time) + ts = IS.get_time_series(IS.SingleTimeSeries, component, "test_c";) + @test IS.get_data_type(ts) == type_name + @test reshape(TimeSeries.values(IS.get_data(ts)), 365) == TimeSeries.values(data_series) + _test_add_single_time_series_helper(component, initial_time) end @testset "Test add SingleTimeSeries with LinearFunctionData Cost" begin From a9bd86b6be22e1a57857a7eb45f3b93f0f18f6bf Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Fri, 26 Apr 2024 12:18:00 -0600 Subject: [PATCH 18/19] Fix bug with removing time series metadata --- src/time_series_metadata_store.jl | 3 ++- test/test_time_series.jl | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index f78592cb3..98c51375e 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -994,10 +994,11 @@ function _make_where_clause( end function _make_where_clause(owner::TimeSeriesOwners, metadata::TimeSeriesMetadata) + features = Dict(Symbol(k) => v for (k, v) in get_features(metadata)) return _make_where_clause( owner; time_series_type = time_series_metadata_to_data(metadata), name = get_name(metadata), - get_features(metadata)..., + features..., ) end diff --git a/test/test_time_series.jl b/test/test_time_series.jl index c5df6fadb..e32cde80d 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -458,6 +458,8 @@ end for info in IS.list_time_series_info(component) @test IS.get_data(IS.get_time_series(component, info)) == data end + IS.remove_time_series!(sys, IS.SingleTimeSeries) + @test isempty(IS.list_time_series_info(component)) end @testset "Test add with features with mixed types" begin From 5c89df2d832ac210f12d58eccce564572889497e Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Fri, 26 Apr 2024 12:38:14 -0600 Subject: [PATCH 19/19] Return nothing for forecast parameters if there are no forecasts --- src/time_series_metadata_store.jl | 6 +++--- src/time_series_storage.jl | 1 - src/utils/utils.jl | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/time_series_metadata_store.jl b/src/time_series_metadata_store.jl index 98c51375e..f99a7f864 100644 --- a/src/time_series_metadata_store.jl +++ b/src/time_series_metadata_store.jl @@ -285,7 +285,7 @@ function get_forecast_window_count(store::TimeSeriesMetadataStore) LIMIT 1 """ table = Tables.rowtable(_execute(store, query)) - return isempty(table) ? 0 : table[1].window_count + return isempty(table) ? nothing : table[1].window_count end function get_forecast_horizon(store::TimeSeriesMetadataStore) @@ -297,7 +297,7 @@ function get_forecast_horizon(store::TimeSeriesMetadataStore) LIMIT 1 """ table = Tables.rowtable(_execute(store, query)) - return isempty(table) ? 0 : table[1].horizon + return isempty(table) ? nothing : table[1].horizon end function get_forecast_initial_timestamp(store::TimeSeriesMetadataStore) @@ -326,7 +326,7 @@ function get_forecast_interval(store::TimeSeriesMetadataStore) """ table = Tables.rowtable(_execute(store, query)) return if isempty(table) - Dates.Millisecond(0) + nothing else Dates.Millisecond(table[1].interval_ms) end diff --git a/src/time_series_storage.jl b/src/time_series_storage.jl index 6d69e1094..15bce57cf 100644 --- a/src/time_series_storage.jl +++ b/src/time_series_storage.jl @@ -4,7 +4,6 @@ Abstract type for time series storage implementations. All subtypes must implement: - - check_read_only - clear_time_series! - deserialize_time_series - get_compression_settings diff --git a/src/utils/utils.jl b/src/utils/utils.jl index e04d19dec..3899eac61 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -4,6 +4,7 @@ import JSON3 const HASH_FILENAME = "check.sha256" +# TODO DT: possibly incorrect g_cached_subtypes = Dict{DataType, Vector{DataType}}() """