Updates on Schema (#13)

* add definitions for fuels and prime movers * fix comment * add PrettyTables * include exports * add prime movers and fuels * added ext and new variable types * change costs to FunctionData * import enums * remove include for definitions * corrected property names and types * export fuels and techs * remove extra stuff from IS * add empty function for parsing * add test schema file to git ignore * changed structs to Technologies * add internal field to structs * remove time series names * add serialization * updating to add SQLite to the import for .db reading into structs * creating the base functions that will be used to populate the structs * Added db to dataframe method to generate structs * adding in temporary mapping functionality for prime movers. * add function to populate structs from db * wrote parser function * formatter * corrected prime mover mapper * clean up old comments * clean up comments * add DB functions to exports * add DataFrames * removed deserialization from add_technology! * adds structs to a portfolio * removed unnecessary dict in dataframe_to_structs * removing unnecessary code * change SupplyTechnology costs to ValueCurve * add support_time_series --------- Co-authored-by: Jerry Potts <[email protected]> Co-authored-by: Pradyumna Rao <[email protected]>
NREL-Sienna · Jul 3, 2024 · 8e7f5ed · 8e7f5ed
1 parent 0779f7a
commit 8e7f5ed
Show file tree

Hide file tree

Showing 12 changed files with 321 additions and 80 deletions.
diff --git a/.gitignore b/.gitignore
@@ -123,3 +123,4 @@ $RECYCLE.BIN/
 
 ## Acknowledgements
 # Many thanks to `https://gitignore.io/`, written and maintained by Joe Blau, which contributed much material to this gitignore file.
+SiennaInvestSchema.json
diff --git a/Project.toml b/Project.toml
@@ -4,12 +4,14 @@ authors = ["Jose Daniel Lara"]
 version = "0.1.0"
 
 [deps]
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 InfrastructureSystems = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 JSONSchema = "7d188eb4-7ad8-530c-ae41-71a32a6d4692"
 PowerSystems = "bcd98974-b02a-5e2f-9ee0-a103f5c450dd"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
 
 [compat]
 DocStringExtensions = "~0.9"

diff --git a/src/PowerSystemsInvestmentsPortfolios.jl b/src/PowerSystemsInvestmentsPortfolios.jl
@@ -3,17 +3,14 @@ module PowerSystemsInvestmentsPortfolios
 import InfrastructureSystems
 # TODO: Some of these re-exports may cause name collisions with PowerSystems
 import InfrastructureSystems:
-    add_time_series,
     to_json,
     from_json,
     serialize,
     deserialize,
-    get_time_series,
     has_time_series,
     get_time_series_array,
     get_time_series_timestamps,
     get_time_series_values,
-    get_time_series_names,
     InfrastructureSystemsInternal,
     CompressionSettings,
     CompressionTypes,
@@ -25,27 +22,48 @@ import InfrastructureSystems:
 import PowerSystems
 import JSONSchema
 import JSON3
+import PrettyTables
+import SQLite
+import DataFrames
 
 export Portfolio
+export Technology
 export SupplyTechnology
 export TransportTechnology
-export DemandTechnology
 export StorageTechnology
+export DemandRequirement
+export DemandsideTechnology
 
 export add_technology!
 export add_technologies!
 export read_json_data
 export generate_invest_structs
 export generate_structs
+export db_to_dataframes
+export map_prime_mover
+export dataframe_to_structs
+export db_to_portfolio_parser
 
 const PSY = PowerSystems
 const IS = InfrastructureSystems
 const MU = IS.Mustache
 
+##### Imports #####
+
+import PowerSystems: ThermalFuels, PrimeMovers, StorageTech
+
+##### Exports #####
+
+export ThermalFuels
+export PrimeMovers
+export StorageTech
+
 include("models/technologies.jl")
 include("models/generated/includes.jl")
 include("portfolio.jl")
+include("serialization.jl")
 include("generate_structs.jl")
+include("utils/print.jl")
 
 using DocStringExtensions
 

diff --git a/src/generate_structs.jl b/src/generate_structs.jl
@@ -98,7 +98,8 @@ function generate_invest_structs(directory, data::JSONSchema.Schema; print_resul
             end
 
             param["kwarg_value"] = ""
-            if !isnothing(get(param, "default", nothing))
+            if !isnothing(get(values, "default", nothing))
+                param["default"] = values["default"]
                 param["kwarg_value"] = "=" * param["default"]
             elseif !isnothing(get(param, "internal_default", nothing))
                 param["kwarg_value"] = "=" * string(param["internal_default"])
@@ -179,3 +180,131 @@ function generate_structs(
     generate_invest_structs(output_directory, data; print_results=print_results)
     return
 end
+
+"""
+The following function imports from the database and generates the structs for a portfolio
+@input database_filepath::AbstractString: The path to the database file
+@input schema_JSON_filepath::AbstractString: The path to the schema JSON file
+"""
+function db_to_portfolio_parser(database_filepath::AbstractString)
+
+    #Goal will be be able to read in database and populate structs simultaneously
+
+    dfs = db_to_dataframes(database_filepath)
+    p = dataframe_to_structs(dfs)
+
+    return p
+end
+
+"""
+The following function imports from the database and creates a dictionary
+of DataFrames for each table in the database
+
+# Example usage:
+
+db_path = "/Users/prao/GitHub_Repos/SiennaInvest/PowerSystemsInvestmentsPortfoliosTestData/RTS_GMLC.db"
+dataframes_all = db_to_dataframes(db_path)
+
+# Access a specific DataFrame
+
+supply_technologies_df = dataframes_all["supply_technologies"]
+"""
+function db_to_dataframes(db_path::String)
+    # Connect to the SQLite database
+    db = SQLite.DB(db_path)
+
+    # Get a list of tables in the database
+    tables = SQLite.tables(db)
+
+    # Create a dictionary to store DataFrames for each table
+    dfs = Dict{String, DataFrame}()
+
+    #Will adjust queries to only pull a subset of data
+    for table in tables
+        table_name = table.name
+        # Read each table into a DataFrame
+        query = "SELECT * FROM $table_name"
+        df = DataFrame(DBInterface.execute(db, query))
+        dfs[table_name] = df
+    end
+
+    # Close the database connection
+    SQLite.close(db)
+
+    return dfs
+end
+
+# TODO: Figure out more permanent solution for mapping prime movers
+"""
+Function to map prime mover types to PrimeMovers
+"""
+function map_prime_mover(prime_mover::String)
+    mapping_dict = Dict(
+        "CT" => PrimeMovers.CT,
+        "STEAM" => PrimeMovers.ST,
+        "CC" => PrimeMovers.CC,
+        "SYNC_COND" => PrimeMovers.OT,
+        "NUCLEAR" => PrimeMovers.ST,
+        "HYDRO" => PrimeMovers.HA,
+        "ROR" => PrimeMovers.IC,
+        "PV" => PrimeMovers.PVe,
+        "CSP" => PrimeMovers.CP,
+        "RTPV" => PrimeMovers.PVe,
+        "WIND" => PrimeMovers.WT,
+        "STORAGE" => PrimeMovers.BA,
+    )
+
+    return mapping_dict[prime_mover]
+end
+
+function dataframe_to_structs(df_dict::Dict)
+
+    #Initialize Portfolio
+    p = Portfolio(0.07)
+
+    #Temporary measure for small database, will go into
+    #more complex query based methods once database is expanded
+
+    #Populate SupplyTechnology structs from database
+    for row in eachrow(df_dict["supply_technologies"])
+        t = SupplyTechnology{ThermalStandard}(;
+            #Data pulled from DB
+            name=string(row["technology_id"]),
+            gen_ID=string(row["technology_id"]),
+            capital_cost=LinearFunctionData(row["capital_cost"]),
+            variable_cost=LinearFunctionData(row["vom_cost"]),
+            balancing_topology=string(row["balancing_topology"]),
+            operations_cost=LinearFunctionData(row["fom_cost"]),
+            prime_mover_type=map_prime_mover(row["prime_mover"]),
+
+            #Placeholder values
+            base_power=100.0,
+            minimum_required_capacity=0.0,
+            available=true,
+            initial_capacity=200.0,
+            fuel=ThermalFuels.COAL,
+            power_systems_type="ThermalStandard",
+            maximum_capacity=10000.0,
+            capacity_factor=0.98,
+        )
+        add_technology!(p, t)
+    end
+
+    #Populate DemandRequirement structs from database
+    for row in eachrow(df_dict["demand_requirements"])
+        d = DemandRequirement{ElectricLoad}(
+            #Data pulled from DB
+            name=string(row["entity_attribute_id"]),
+            region=string(row["area"]),
+            peak_load=row["peak_load"],
+
+            #Placeholder values
+            load_growth=0.05,
+            available=true,
+            power_systems_type="ElectricLoad",
+        )
+        add_technology!(p, d)
+    end
+
+    return p
+end
diff --git a/src/models/generated/DemandRequirement.jl b/src/models/generated/DemandRequirement.jl
@@ -5,10 +5,12 @@ This file is auto-generated. Do not edit.
 #! format: off
 
 """
-    mutable struct DemandRequirement{T <: PSY.StaticInjection} <: InfrastructureSystemsComponent
+    mutable struct DemandRequirement{T <: PSY.StaticInjection} <: Technology
         load_growth::Float64
         name::String
         power_systems_type::String
+        internal::InfrastructureSystemsInternal
+        ext::Dict
         region::String
         available::Bool
         peak_load::Float64
@@ -17,20 +19,26 @@ This file is auto-generated. Do not edit.
 
 
 # Arguments
-- `load_growth::Float64`: Annual load growth (%)
+- `load_growth::Float64`: (default: `1.0`) Annual load growth (%)
 - `name::String`: The technology name
 - `power_systems_type::String`: maps to a valid PowerSystems.jl for PCM modeling
+- `internal::InfrastructureSystemsInternal`: (default: `InfrastructureSystemsInternal()`) Internal field
+- `ext::Dict`: (default: `Dict()`) Option for providing additional data
 - `region::String`: Corresponding region for peak demand
-- `available::Bool`: identifies whether the technology is available
-- `peak_load::Float64`: Demand value (MW) for single timepoint (for now)
+- `available::Bool`: (default: `true`) identifies whether the technology is available
+- `peak_load::Float64`: (default: `100.0`) Demand value (MW) for single timepoint (for now)
 """
-mutable struct DemandRequirement{T <: PSY.StaticInjection} <: InfrastructureSystemsComponent
+mutable struct DemandRequirement{T <: PSY.StaticInjection} <: Technology
     "Annual load growth (%)"
     load_growth::Float64
     "The technology name"
     name::String
     "maps to a valid PowerSystems.jl for PCM modeling"
     power_systems_type::String
+    "Internal field"
+    internal::InfrastructureSystemsInternal
+    "Option for providing additional data"
+    ext::Dict
     "Corresponding region for peak demand"
     region::String
     "identifies whether the technology is available"
@@ -40,8 +48,8 @@ mutable struct DemandRequirement{T <: PSY.StaticInjection} <: InfrastructureSyst
 end
 
 
-function DemandRequirement{T}(; load_growth, name, power_systems_type, region, available, peak_load, ) where T <: PSY.StaticInjection
-    DemandRequirement{T}(load_growth, name, power_systems_type, region, available, peak_load, )
+function DemandRequirement{T}(; load_growth=1.0, name, power_systems_type, internal=InfrastructureSystemsInternal(), ext=Dict(), region, available=true, peak_load=100.0, ) where T <: PSY.StaticInjection
+    DemandRequirement{T}(load_growth, name, power_systems_type, internal, ext, region, available, peak_load, )
 end
 
 """Get [`DemandRequirement`](@ref) `load_growth`."""
@@ -50,6 +58,10 @@ get_load_growth(value::DemandRequirement) = value.load_growth
 get_name(value::DemandRequirement) = value.name
 """Get [`DemandRequirement`](@ref) `power_systems_type`."""
 get_power_systems_type(value::DemandRequirement) = value.power_systems_type
+"""Get [`DemandRequirement`](@ref) `internal`."""
+get_internal(value::DemandRequirement) = value.internal
+"""Get [`DemandRequirement`](@ref) `ext`."""
+get_ext(value::DemandRequirement) = value.ext
 """Get [`DemandRequirement`](@ref) `region`."""
 get_region(value::DemandRequirement) = value.region
 """Get [`DemandRequirement`](@ref) `available`."""
@@ -63,6 +75,10 @@ set_load_growth!(value::DemandRequirement, val) = value.load_growth = val
 set_name!(value::DemandRequirement, val) = value.name = val
 """Set [`DemandRequirement`](@ref) `power_systems_type`."""
 set_power_systems_type!(value::DemandRequirement, val) = value.power_systems_type = val
+"""Set [`DemandRequirement`](@ref) `internal`."""
+set_internal!(value::DemandRequirement, val) = value.internal = val
+"""Set [`DemandRequirement`](@ref) `ext`."""
+set_ext!(value::DemandRequirement, val) = value.ext = val
 """Set [`DemandRequirement`](@ref) `region`."""
 set_region!(value::DemandRequirement, val) = value.region = val
 """Set [`DemandRequirement`](@ref) `available`."""

diff --git a/src/models/generated/DemandsideTechnology.jl b/src/models/generated/DemandsideTechnology.jl
@@ -5,9 +5,11 @@ This file is auto-generated. Do not edit.
 #! format: off
 
 """
-    mutable struct DemandsideTechnology{T <: PSY.StaticInjection} <: InfrastructureSystemsComponent
+    mutable struct DemandSideTechnology{T <: PSY.StaticInjection} <: Technology
         name::String
         power_systems_type::String
+        internal::InfrastructureSystemsInternal
+        ext::Dict
         available::Bool
     end
 
@@ -16,32 +18,46 @@ This file is auto-generated. Do not edit.
 # Arguments
 - `name::String`: The technology name
 - `power_systems_type::String`: maps to a valid PowerSystems.jl for PCM modeling
+- `internal::InfrastructureSystemsInternal`: (default: `InfrastructureSystemsInternal()`) Internal field
+- `ext::Dict`: (default: `Dict()`) Option for providing additional data
 - `available::Bool`: identifies whether the technology is available
 """
-mutable struct DemandsideTechnology{T <: PSY.StaticInjection} <: InfrastructureSystemsComponent
+mutable struct DemandSideTechnology{T <: PSY.StaticInjection} <: Technology
     "The technology name"
     name::String
     "maps to a valid PowerSystems.jl for PCM modeling"
     power_systems_type::String
+    "Internal field"
+    internal::InfrastructureSystemsInternal
+    "Option for providing additional data"
+    ext::Dict
     "identifies whether the technology is available"
     available::Bool
 end
 
 
-function DemandsideTechnology{T}(; name, power_systems_type, available, ) where T <: PSY.StaticInjection
-    DemandsideTechnology{T}(name, power_systems_type, available, )
+function DemandSideTechnology{T}(; name, power_systems_type, internal=InfrastructureSystemsInternal(), ext=Dict(), available, ) where T <: PSY.StaticInjection
+    DemandSideTechnology{T}(name, power_systems_type, internal, ext, available, )
 end
 
-"""Get [`DemandsideTechnology`](@ref) `name`."""
-get_name(value::DemandsideTechnology) = value.name
-"""Get [`DemandsideTechnology`](@ref) `power_systems_type`."""
-get_power_systems_type(value::DemandsideTechnology) = value.power_systems_type
-"""Get [`DemandsideTechnology`](@ref) `available`."""
-get_available(value::DemandsideTechnology) = value.available
-
-"""Set [`DemandsideTechnology`](@ref) `name`."""
-set_name!(value::DemandsideTechnology, val) = value.name = val
-"""Set [`DemandsideTechnology`](@ref) `power_systems_type`."""
-set_power_systems_type!(value::DemandsideTechnology, val) = value.power_systems_type = val
-"""Set [`DemandsideTechnology`](@ref) `available`."""
-set_available!(value::DemandsideTechnology, val) = value.available = val
+"""Get [`DemandSideTechnology`](@ref) `name`."""
+get_name(value::DemandSideTechnology) = value.name
+"""Get [`DemandSideTechnology`](@ref) `power_systems_type`."""
+get_power_systems_type(value::DemandSideTechnology) = value.power_systems_type
+"""Get [`DemandSideTechnology`](@ref) `internal`."""
+get_internal(value::DemandSideTechnology) = value.internal
+"""Get [`DemandSideTechnology`](@ref) `ext`."""
+get_ext(value::DemandSideTechnology) = value.ext
+"""Get [`DemandSideTechnology`](@ref) `available`."""
+get_available(value::DemandSideTechnology) = value.available
+
+"""Set [`DemandSideTechnology`](@ref) `name`."""
+set_name!(value::DemandSideTechnology, val) = value.name = val
+"""Set [`DemandSideTechnology`](@ref) `power_systems_type`."""
+set_power_systems_type!(value::DemandSideTechnology, val) = value.power_systems_type = val
+"""Set [`DemandSideTechnology`](@ref) `internal`."""
+set_internal!(value::DemandSideTechnology, val) = value.internal = val
+"""Set [`DemandSideTechnology`](@ref) `ext`."""
+set_ext!(value::DemandSideTechnology, val) = value.ext = val
+"""Set [`DemandSideTechnology`](@ref) `available`."""
+set_available!(value::DemandSideTechnology, val) = value.available = val
Original file line number	Diff line number	Diff line change
Expand Up		@@ -123,3 +123,4 @@ $RECYCLE.BIN/

		## Acknowledgements
		# Many thanks to `https://gitignore.io/`, written and maintained by Joe Blau, which contributed much material to this gitignore file.
		SiennaInvestSchema.json