Full time series reconstruction

CHANGELOG.md

@@ -14,7 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Add objective scaler for addressing problem ill-conditioning (#667)
 - Add workflow that ensures that CHANGELOG.md and the version number are updated (#711)
-- Print GenX version at startup and export it to disk (#712) 
+- Print GenX version at startup and export it to disk (#712)
+- Added the option to output results with time series reconstructed for the entire year (#700)
 
 ## [0.4.0] - 2024-03-18
 

Project.toml

@@ -1,7 +1,7 @@
 name = "GenX"
 uuid = "5d317b1e-30ec-4ed6-a8ce-8d2d88d7cfac"
 authors = ["Bonaldo, Luca", "Chakrabarti, Sambuddha", "Cheng, Fangwei", "Ding, Yifu", "Jenkins, Jesse D.", "Luo, Qian", "Macdonald, Ruaridh", "Mallapragada, Dharik", "Manocha, Aneesha", "Mantegna, Gabe ", "Morris, Jack", "Patankar, Neha", "Pecci, Filippo", "Schwartz, Aaron", "Schwartz, Jacob", "Schivley, Greg", "Sepulveda, Nestor", "Xu, Qingyu", "Zhou, Justin"]
-version = "0.4.0-dev.6"
+version = "0.4.0-dev.7"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"

docs/src/Model_Reference/TDR.md

@@ -8,4 +8,8 @@ Order = [:type, :function]
 
 ```@docs
 GenX.run_timedomainreduction!
+```
+
+```@docs
+GenX.full_time_series_reconstruction
 ```

docs/src/User_Guide/model_configuration.md

@@ -92,5 +92,9 @@ The following tables summarize the model settings parameters and their default/p
 | WriteOutputs | Flag for writing the model outputs with hourly resolution or just the annual sum.|
 || "full" = write the model outputs with hourly resolution.|
 || "annual" = write only the annual sum of the model outputs.|
+| OutputFullTimeSeries | Flag for writing the full time series of the model outputs.|
+||1 = write the full time series of the model outputs.|
+||0 = write only the reduced time series of the model outputs.|
+| OutputFullTimeSeriesFolder | Name of the folder where the full time series of the model outputs will be stored inside the results directory (default: Full_TimeSeries).|
 
 The next step in configuring a GenX model is to specify the solver settings parameters using a `[solver_name]_settings.yml` file inside the `settings` folder. The solver settings parameters are solver specific and are described in the following section.

example_systems/1_three_zones/settings/genx_settings.yml

@@ -10,3 +10,4 @@ ParameterScale: 1 # Turn on parameter scaling wherein demand, capacity and power
 WriteShadowPrices: 1 # Write shadow prices of LP or relaxed MILP; 0 = not active; 1 = active
 UCommit: 2 # Unit committment of thermal power plants; 0 = not active; 1 = active using integer clestering; 2 = active using linearized clustering
 TimeDomainReduction: 1 # Time domain reduce (i.e. cluster) inputs based on Demand_data.csv, Generators_variability.csv, and Fuels_data.csv; 0 = not active (use input data as provided); 0 = active (cluster input data, or use data that has already been clustered)
+OutputFullTimeSeries: 1

src/GenX.jl

@@ -72,6 +72,7 @@ include_all_in_folder("write_outputs")
 
 include("time_domain_reduction/time_domain_reduction.jl")
 include("time_domain_reduction/precluster.jl")
+include("time_domain_reduction/full_time_series_reconstruction.jl")
 
 include_all_in_folder("multi_stage")
 include_all_in_folder("additional_tools")

src/case_runners/case_runner.jl

@@ -187,6 +187,7 @@ function run_genx_case_multistage!(case::AbstractString, mysetup::Dict, optimize
 
     # Step 4) Write final outputs from each stage
     for p in 1:mysetup["MultiStageSettingsDict"]["NumStages"]
+        mysetup["MultiStageSettingsDict"]["CurStage"] = p
         outpath_cur = joinpath(outpath, "results_p$p")
         write_outputs(model_dict[p], outpath_cur, mysetup, inputs_dict[p])
     end

src/configure_settings/configure_settings.jl

@@ -16,6 +16,8 @@ function default_settings()
         "UCommit" => 0,
         "TimeDomainReduction" => 0,
         "TimeDomainReductionFolder" => "TDR_results",
+        "OutputFullTimeSeries" => 0,
+        "OutputFullTimeSeriesFolder" => "Full_TimeSeries",
         "ModelingToGenerateAlternatives" => 0,
         "ModelingtoGenerateAlternativeSlack" => 0.1,
         "MGAAnnualGeneration" => 0,

src/time_domain_reduction/full_time_series_reconstruction.jl

@@ -0,0 +1,71 @@
+"""
+    full_time_series_reconstruction(path::AbstractString, setup::Dict, DF::DataFrame)
+
+Internal function for performing the reconstruction. This function returns a DataFrame with the full series reconstruction. 
+
+# Arguments
+- `path` (AbstractString): Path input to the results folder
+- `setup` (Dict): Case setup
+- `DF` (DataFrame): DataFrame to be reconstructed
+
+# Returns
+- `reconDF` (DataFrame): DataFrame with the full series reconstruction
+"""
+function full_time_series_reconstruction(
+        path::AbstractString, setup::Dict, DF::DataFrame)
+    if setup["MultiStage"] == 1
+        dirs = splitpath(path)
+        case = joinpath(dirs[.!occursin.("result", dirs)])  # Get the case folder without the "results" folder(s)
+        cur_stage = setup["MultiStageSettingsDict"]["CurStage"]
+        TDRpath = joinpath(case, "inputs", string("inputs_p", cur_stage),
+            setup["TimeDomainReductionFolder"])
+    else
+        case = path[1:findlast('/', path)]
+        TDRpath = joinpath(case, setup["TimeDomainReductionFolder"])
+    end
+    # Read Period map file Period_map.csv
+    Period_map = CSV.read(joinpath(TDRpath, "Period_map.csv"), DataFrame)
+
+    # Read time domain reduction settings file time_domain_reduction_settings.yml
+    myTDRsetup = YAML.load(open(joinpath(
+        case, "settings/time_domain_reduction_settings.yml")))
+
+    # Define Timesteps per Representative Period and Weight Total
+    TimestepsPerRepPeriod = myTDRsetup["TimestepsPerRepPeriod"]
+    WeightTotal = myTDRsetup["WeightTotal"]
+    # Calculate the number of total periods the original time series was split into (will usually be 52)
+    numPeriods = floor(Int, WeightTotal / TimestepsPerRepPeriod)
+
+    # Get a matrix of the input DataFrame
+    DFMatrix = Matrix(DF)
+    # Initialize an array to add the reconstructed data to
+    recon = ["t$t" for t in 1:(TimestepsPerRepPeriod * numPeriods)]
+
+    # Find the index of the row with the first time step
+    t1 = findfirst(x -> x == "t1", DF[!, 1])
+
+    # Reconstruction of all hours of the year from TDR
+    for j in range(2, ncol(DF))
+        col = DF[t1:end, j]
+        recon_col = []
+        for i in range(1, numPeriods)
+            index = Period_map[i, "Rep_Period_Index"]
+            recon_temp = col[(TimestepsPerRepPeriod * index - (TimestepsPerRepPeriod - 1)):(TimestepsPerRepPeriod * index)]
+            recon_col = [recon_col; recon_temp]
+        end
+        recon = [recon recon_col]
+    end
+    reconDF = DataFrame(recon, :auto)
+
+    # Insert rows that were above "t1" in the original DataFrame (e.g. "Zone" and "AnnualSum") if present
+    for i in range(1, t1 - 1)
+        insert!(reconDF, i, DFMatrix[i, 1:end], promote = true)
+    end
+
+    # Repeat the last rows of the year to fill in the gap (should be 24 hours for non-leap year)
+    end_diff = WeightTotal - nrow(reconDF) + 1
+    new_rows = reconDF[(nrow(reconDF) - end_diff):nrow(reconDF), 1:end]
+    new_rows[!, 1] = ["t$t" for t in (WeightTotal - end_diff):WeightTotal]
+    reconDF = [reconDF; new_rows]
+    return reconDF
+end

src/write_outputs/transmission/write_transmission_flows.jl

@@ -23,6 +23,11 @@ function write_transmission_flows(path::AbstractString,
         auxNew_Names = [Symbol("Line"); [Symbol("t$t") for t in 1:T]]
         rename!(dfFlow, auxNew_Names)
         CSV.write(filepath, dftranspose(dfFlow, false), writeheader = false)
+
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(path, setup, dfFlow, "flow")
+            @info("Writing Full Time Series for Transmission Flows")
+        end
     end
     return nothing
 end

src/write_outputs/transmission/write_transmission_losses.jl

@@ -30,6 +30,11 @@ function write_transmission_losses(path::AbstractString,
         CSV.write(joinpath(path, "tlosses.csv"),
             dftranspose(dfTLosses, false),
             writeheader = false)
+
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(path, setup, dfTLosses, "tlosses")
+            @info("Writing Full Time Series for Time Losses")
+        end
     end
     return nothing
 end

src/write_outputs/ucommit/write_commit.jl

@@ -11,4 +11,9 @@ function write_commit(path::AbstractString, inputs::Dict, setup::Dict, EP::Model
     auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("t$t") for t in 1:T]]
     rename!(dfCommit, auxNew_Names)
     CSV.write(joinpath(path, "commit.csv"), dftranspose(dfCommit, false), header = false)
+
+    if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+        write_full_time_series_reconstruction(path, setup, dfCommit, "commit")
+        @info("Writing Full Time Series for Commitment")
+    end
 end

src/write_outputs/ucommit/write_shutdown.jl

@@ -13,7 +13,11 @@ function write_shutdown(path::AbstractString, inputs::Dict, setup::Dict, EP::Mod
     if setup["WriteOutputs"] == "annual"
         write_annual(filepath, dfShutdown)
     else # setup["WriteOutputs"] == "full"
-        write_fulltimeseries(filepath, shut, dfShutdown)
+        df_Shutdown = write_fulltimeseries(filepath, shut, dfShutdown)
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(path, setup, df_Shutdown, "shutdown")
+            @info("Writing Full Time Series for Shutdown")
+        end
     end
     return nothing
 end

src/write_outputs/ucommit/write_start.jl

@@ -12,7 +12,11 @@ function write_start(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
     if setup["WriteOutputs"] == "annual"
         write_annual(filepath, dfStart)
     else # setup["WriteOutputs"] == "full"	
-        write_fulltimeseries(filepath, start, dfStart)
+        df_Start = write_fulltimeseries(filepath, start, dfStart)
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(path, setup, df_Start, "start")
+            @info("Writing Full Time Series for Startup")
+        end
     end
     return nothing
 end

src/write_outputs/write_charge.jl

@@ -41,7 +41,12 @@ function write_charge(path::AbstractString, inputs::Dict, setup::Dict, EP::Model
     if setup["WriteOutputs"] == "annual"
         write_annual(filepath, dfCharge)
     else # setup["WriteOutputs"] == "full"
-        write_fulltimeseries(filepath, charge, dfCharge)
+        df_Charge = write_fulltimeseries(filepath, charge, dfCharge)
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(
+                path, setup, df_Charge, "charge")
+            @info("Writing Full Time Series for Charge")
+        end
     end
     return nothing
 end

src/write_outputs/write_co2.jl

@@ -31,7 +31,13 @@ function write_co2_emissions_plant(path::AbstractString,
     if setup["WriteOutputs"] == "annual"
         write_annual(filepath, dfEmissions_plant)
     else # setup["WriteOutputs"] == "full"
-        write_fulltimeseries(filepath, emissions_plant, dfEmissions_plant)
+        df_Emissions_plant = write_fulltimeseries(
+            filepath, emissions_plant, dfEmissions_plant)
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(
+                path, setup, df_Emissions_plant, "emissions_plant")
+            @info("Writing Full Time Series for Emissions Plant")
+        end
     end
     return nothing
 end

src/write_outputs/write_curtailment.jl

@@ -57,7 +57,11 @@ function write_curtailment(path::AbstractString, inputs::Dict, setup::Dict, EP::
     if setup["WriteOutputs"] == "annual"
         write_annual(filename, dfCurtailment)
     else # setup["WriteOutputs"] == "full"
-        write_fulltimeseries(filename, curtailment, dfCurtailment)
+        df_Curtailment = write_fulltimeseries(filename, curtailment, dfCurtailment)
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(path, setup, df_Curtailment, "curtail")
+            @info("Writing Full Time Series for Curtailment")
+        end
     end
     return nothing
 end

src/write_outputs/write_emissions.jl

@@ -124,6 +124,11 @@ function write_emissions(path::AbstractString, inputs::Dict, setup::Dict, EP::Mo
             CSV.write(joinpath(path, "emissions.csv"),
                 dftranspose(dfEmissions, false),
                 writeheader = false)
+
+            if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+                write_full_time_series_reconstruction(path, setup, dfEmissions, "emissions")
+                @info("Writing Full Time Series for Emissions")
+            end
         end
     end
     return nothing

src/write_outputs/write_fuel_consumption.jl

@@ -85,6 +85,12 @@ function write_fuel_consumption_ts(path::AbstractString,
         DataFrame(tempts, [Symbol("t$t") for t in 1:T]))
     CSV.write(joinpath(path, "FuelConsumption_plant_MMBTU.csv"),
         dftranspose(dfPlantFuel_TS, false), header = false)
+
+    if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+        write_full_time_series_reconstruction(
+            path, setup, dfPlantFuel_TS, "FuelConsumption_plant_MMBTU")
+        @info("Writing Full Time Series for Fuel Consumption")
+    end
 end
 
 function write_fuel_consumption_tot(path::AbstractString,

src/write_outputs/write_nse.jl

@@ -37,6 +37,11 @@ function write_nse(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
         dfNse = vcat(dfNse, total)
 
         CSV.write(joinpath(path, "nse.csv"), dftranspose(dfNse, false), writeheader = false)
+
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(path, setup, dfNse, "nse")
+            @info("Writing Full Time Series for NSE")
+        end
     end
     return nothing
 end

src/write_outputs/write_outputs.jl

@@ -23,6 +23,10 @@ function write_outputs(EP::Model, path::AbstractString, setup::Dict, inputs::Dic
         mkpath(path)
     end
 
+    if setup["OutputFullTimeSeries"] == 1
+        mkpath(joinpath(path, setup["OutputFullTimeSeriesFolder"]))
+    end
+
     # https://jump.dev/MathOptInterface.jl/v0.9.10/apireference/#MathOptInterface.TerminationStatusCode
     status = termination_status(EP)
 
@@ -503,8 +507,9 @@ function write_fulltimeseries(fullpath::AbstractString,
     total = DataFrame(["Total" 0 sum(dfOut[!, :AnnualSum]) fill(0.0, (1, T))], auxNew_Names)
     total[!, 4:(T + 3)] .= sum(dataOut, dims = 1)
     dfOut = vcat(dfOut, total)
+
     CSV.write(fullpath, dftranspose(dfOut, false), writeheader = false)
-    return nothing
+    return dfOut
 end
 
 """
@@ -546,4 +551,35 @@ function write_system_env_summary(path::AbstractString)
     )
 
     YAML.write_file(joinpath(path, "system_summary.yml"), env_summary)
-end
+end
+
+@doc raw"""write_full_time_series_reconstruction(path::AbstractString,
+                            setup::Dict,
+                            DF::DataFrame,
+                            name::String)
+Create a DataFrame with all 8,760 hours of the year from the reduced output.
+
+This function calls `full_time_series_reconstruction()``, which uses Period_map.csv to create a new DataFrame with 8,760 time steps, as well as other pre-existing rows such as "Zone".
+For each 52 weeks of the year, the corresponding representative week is taken from the input DataFrame and copied into the new DataFrame. Representative periods that 
+represent more than one week will appear multiple times in the output. 
+
+Note: Currently, TDR only gives the representative periods in Period_map for 52 weeks, when a (non-leap) year is 52 weeks + 24 hours. This function takes the last 24 hours of 
+the time series and copies them to get up to all 8,760 hours in a year.
+
+This function is called when output files with time series data (e.g. power.csv, emissions.csv) are created, if the setup key "OutputFullTimeSeries" is set to "1".
+
+# Arguments
+- `path` (AbstractString): Path input to the results folder
+- `setup` (Dict): Case setup
+- `DF` (DataFrame): DataFrame to be reconstructed
+- `name` (String): Name desired for the .csv file
+
+"""
+function write_full_time_series_reconstruction(
+        path::AbstractString, setup::Dict, DF::DataFrame, name::String)
+    FullTimeSeriesFolder = setup["OutputFullTimeSeriesFolder"]
+    output_path = joinpath(path, FullTimeSeriesFolder)
+    dfOut_full = full_time_series_reconstruction(path, setup, dftranspose(DF, false))
+    CSV.write(joinpath(output_path, "$name.csv"), dfOut_full, header = false)
+    return nothing
+end

src/write_outputs/write_power.jl

@@ -24,7 +24,11 @@ function write_power(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
     if setup["WriteOutputs"] == "annual"
         write_annual(filepath, dfPower)
     else # setup["WriteOutputs"] == "full"
-        write_fulltimeseries(filepath, power, dfPower)
+        df_Power = write_fulltimeseries(filepath, power, dfPower)
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(path, setup, df_Power, "power")
+            @info("Writing Full Time Series for Power")
+        end
     end
 
     return dfPower #Shouldn't this be return nothing

src/write_outputs/write_power_balance.jl

@@ -96,6 +96,12 @@ function write_power_balance(path::AbstractString, inputs::Dict, setup::Dict, EP
         CSV.write(joinpath(path, "power_balance.csv"),
             dftranspose(dfPowerBalance, false),
             writeheader = false)
+
+        if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+            write_full_time_series_reconstruction(
+                path, setup, dfPowerBalance, "power_balance")
+            @info("Writing Full Time Series for Power Balance")
+        end
     end
     return nothing
 end

src/write_outputs/write_price.jl

@@ -21,6 +21,11 @@ function write_price(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
     CSV.write(joinpath(path, "prices.csv"),
         dftranspose(dfPrice, false),
         writeheader = false)
+
+    if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+        write_full_time_series_reconstruction(path, setup, dfPrice, "prices")
+        @info("Writing Full Time Series for Price")
+    end
     return nothing
 end
 

src/write_outputs/write_reliability.jl

@@ -20,4 +20,9 @@ function write_reliability(path::AbstractString, inputs::Dict, setup::Dict, EP::
     CSV.write(joinpath(path, "reliability.csv"),
         dftranspose(dfReliability, false),
         header = false)
+
+    if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+        write_full_time_series_reconstruction(path, setup, dfReliability, "reliability")
+        @info("Writing Full Time Series for Reliability")
+    end
 end

src/write_outputs/write_storage.jl

@@ -39,4 +39,9 @@ function write_storage(path::AbstractString, inputs::Dict, setup::Dict, EP::Mode
     auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("t$t") for t in 1:T]]
     rename!(dfStorage, auxNew_Names)
     CSV.write(joinpath(path, "storage.csv"), dftranspose(dfStorage, false), header = false)
+
+    if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1
+        write_full_time_series_reconstruction(path, setup, dfStorage, "storage")
+        @info("Writing Full Time Series for Storage")
+    end
 end