Flexible Output Writing (#617)

This commit adds the option of deciding which files to write and which ones to skip. Also, among the time-series files, it gives users the option to choose between writing the hourly outputs versus annual values. 

It does so by adding a new settings parameter called "WriteOutputs" that can be set to annual or full and also introducing a new YAML file called output_settings.yml 

---------

Co-authored-by: Chakrabarti, Sambuddha (Sam) <[email protected]>

Example_Systems/RealSystemExample/ISONE_Singlezone/Settings/genx_settings.yml

@@ -1,8 +1,8 @@
 OverwriteResults: 0 # Overwrite existing results in output folder or create a new one; 0 = create new folder; 1 = overwrite existing results
 PrintModel: 0 # Write the model formulation as an output; 0 = active; 1 = not active
-NetworkExpansion: 0 # Transmission network expansionl; 0 = not active; 1 = active systemwide
+NetworkExpansion: 1 # Transmission network expansionl; 0 = not active; 1 = active systemwide
 Trans_Loss_Segments: 1 # Number of segments used in piecewise linear approximation of transmission losses; 1 = linear, >2 = piecewise quadratic
-Reserves: 0 # Regulation (primary) and operating (secondary) reserves; 0 = not active, 1 = active systemwide
+Reserves: 1 # Regulation (primary) and operating (secondary) reserves; 0 = not active, 1 = active systemwide
 EnergyShareRequirement: 1 # Minimum qualifying renewables penetration; 0 = not active; 1 = active systemwide
 CapacityReserveMargin: 1 # Number of capacity reserve margin constraints; 0 = not active; 1 = active systemwide
 CO2Cap: 1 # CO2 emissions cap; 0 = not active (no CO2 emission limit); 1 = mass-based emission limit constraint; 2 = demand + rate-based emission limit constraint; 3 = generation + rate-based emission limit constraint
@@ -18,3 +18,4 @@ ModelingToGenerateAlternatives: 0 # Modeling to generate alternatives; 0 = not a
 ModelingtoGenerateAlternativeSlack: 0.1 # Slack value as a fraction of least-cost objective in budget constraint used for evaluating alternative model solutions; positive float value
 ModelingToGenerateAlternativeIterations: 3 # Number of MGA iterations with maximization and minimization objective
 MethodofMorris: 0 #Flag for turning on the Method of Morris analysis
+WriteOutputs: "annual"

Example_Systems/RealSystemExample/ISONE_Singlezone/Settings/output_settings.yml

@@ -0,0 +1,27 @@
+WriteCosts: false
+WriteCapacity: false
+WriteCapacityValue: false
+WriteCapacityFactor: false
+WriteCharge: false
+WriteChargingCost: false
+WriteCO2: false
+WriteCO2Cap: false
+WriteCommit: false
+WriteCurtailment: false
+WriteEmissions: false
+WriteEnergyRevenue: false
+WriteESRPrices: false
+WriteESRRevenue: false
+WriteFuelConsumption: false
+WriteHourlyMatchingPrices: false
+WriteHydrogenPrices: false
+WriteMaintenance: false
+WriteMaxCapReq: false
+WriteMinCapReq: false
+WriteNetRevenue: false
+WriteNSE: false
+WriteNWExpansion: false
+WriteTransmissionFlows: false
+WriteTransmissionLosses: false
+WriteVirtualDischarge: false
+WriteVREStor: false

docs/src/data_documentation.md

@@ -70,13 +70,18 @@ Note that all settings parameters are case sensitive.
 |PrintModel | Flag for printing the model equations as .lp file.|
 ||1 = including the model equation as an output|
 ||0 = the model equation won't be included as an output|
+| WriteOutputs | Flag for writing the model outputs with hourly resolution or just the annual sum.|
+|| "full" = write the model outputs with hourly resolution.|
+|| "annual" = write only the annual sum of the model outputs.|
 
 Additionally, Solver related settings parameters are specified in the appropriate .yml file (e.g. `gurobi_settings.yml` or `cplex_settings.yml`),
 which should be located in the current working directory.
 Note that GenX supplies default settings for most solver settings in the various solver-specific functions found in the `src/configure_solver/` directory.
 To overwrite default settings, you can specify the below Solver specific settings.
 Settings are specific to each solver.
 
+(Optional) The user can also select the output files that they want to export using the `output_settings.yml` file. This file containes a list of `yes/no` options for each output file, and should be located in the `Settings` folder. By default, if `output_settings.yml` is not included, GenX will export all output files. 
+
 ###### Table 1b: Summary of the Solver settings parameters
 ---
 |**Settings Parameter** | **Description**|

src/case_runners/case_runner.jl

@@ -14,8 +14,9 @@ end
 case - folder for the case
 """
 function run_genx_case!(case::AbstractString, optimizer::Any=HiGHS.Optimizer)
-    genx_settings = get_settings_path(case, "genx_settings.yml") #Settings YAML file path
-    mysetup = configure_settings(genx_settings) # mysetup dictionary stores settings and GenX-specific parameters
+    genx_settings = get_settings_path(case, "genx_settings.yml") # Settings YAML file path
+    writeoutput_settings = get_settings_path(case, "output_settings.yml") # Write-output settings YAML file path
+    mysetup = configure_settings(genx_settings, writeoutput_settings) # mysetup dictionary stores settings and GenX-specific parameters
 
     if mysetup["MultiStage"] == 0
         run_genx_case_simple!(case, mysetup, optimizer)

src/configure_settings/configure_settings.jl

@@ -24,26 +24,34 @@ function default_settings()
         "IncludeLossesInESR" => 0,
         "HydrogenHourlyMatching" => 0,
         "EnableJuMPStringNames" => false,
+        "HydrogenHourlyMatching" => 0,
+        "WriteOutputs" => "full",
         "ComputeConflicts" => 0,
         "ResourcePath" => "Resources",
     )
 end
 
-function configure_settings(settings_path::String)
+function configure_settings(settings_path::String, output_settings_path::String)
     println("Configuring Settings")
     model_settings = YAML.load(open(settings_path))
 
     settings = default_settings()
-
     merge!(settings, model_settings)
 
+    output_settings = configure_writeoutput(output_settings_path, settings)
+    settings["WriteOutputsSettingsDict"] = output_settings
+
     validate_settings!(settings)
     return settings
 end
 
 function validate_settings!(settings::Dict{Any,Any})
     # Check for any settings combinations that are not allowed.
     # If we find any then make a response and issue a note to the user.
+
+    # make WriteOutputs setting lowercase and check for valid value
+    settings["WriteOutputs"] = lowercase(settings["WriteOutputs"])
+    @assert settings["WriteOutputs"] ∈ ["annual", "full"]
 
     if "OperationWrapping" in keys(settings)
         @warn """The behavior of the TimeDomainReduction and OperationWrapping
@@ -57,3 +65,79 @@ function validate_settings!(settings::Dict{Any,Any})
     end
 
 end
+
+function default_writeoutput()
+    Dict{String,Bool}(
+        "WriteCosts" => true,
+        "WriteCapacity" => true,
+        "WriteCapacityValue" => true,
+        "WriteCapacityFactor" => true,
+        "WriteCharge" => true,
+        "WriteChargingCost" => true,
+        "WriteCO2" => true,
+        "WriteCO2Cap" => true,
+        "WriteCommit" => true,
+        "WriteCurtailment" => true,
+        "WriteEmissions" => true,
+        "WriteEnergyRevenue" => true,
+        "WriteESRPrices" => true,
+        "WriteESRRevenue" => true,
+        "WriteFuelConsumption" => true,
+        "WriteHourlyMatchingPrices" => true,
+        "WriteHydrogenPrices" => true,
+        "WriteMaintenance" => true,
+        "WriteMaxCapReq" => true,
+        "WriteMinCapReq" => true,
+        "WriteNetRevenue" => true,
+        "WriteNSE" => true,
+        "WriteNWExpansion" => true,
+        "WriteOpWrapLDSdStor" => true,
+        "WriteOpWrapLDSStorInit" => true,
+        "WritePower" => true,
+        "WritePowerBalance" => true,
+        "WritePrice" => true,
+        "WriteReg" => true,
+        "WriteReliability" => true,
+        "WriteReserveMargin" => true,
+        "WriteReserveMarginRevenue" => true,
+        "WriteReserveMarginSlack" => true,
+        "WriteReserveMarginWithWeights" => true,
+        "WriteRsv" => true,
+        "WriteShutdown" => true,
+        "WriteStart" => true,
+        "WriteStatus" => true,
+        "WriteStorage" => true,
+        "WriteStorageDual" => true,
+        "WriteSubsidyRevenue" => true,
+        "WriteTimeWeights" => true,
+        "WriteTransmissionFlows" => true,
+        "WriteTransmissionLosses" => true,
+        "WriteVirtualDischarge" => true,
+        "WriteVREStor" => true
+    )
+end
+
+function configure_writeoutput(output_settings_path::String, settings::Dict)
+
+    writeoutput = default_writeoutput()
+
+    # don't write files with hourly data if settings["WriteOutputs"] == "annual"
+    if settings["WriteOutputs"] == "annual"
+        writeoutput["WritePrice"] = false
+        writeoutput["WriteReliability"] = false
+        writeoutput["WriteStorage"] = false
+        writeoutput["WriteStorageDual"] = false
+        writeoutput["WriteTimeWeights"] = false
+        writeoutput["WriteCommit"] = false
+        writeoutput["WriteCapacityValue"] = false
+        writeoutput["WriteReserveMargin"] = false
+        writeoutput["WriteReserveMarginWithWeights"] = false
+    end
+
+    # read in YAML file if provided
+    if isfile(output_settings_path)
+        model_writeoutput = YAML.load(open(output_settings_path))
+        merge!(writeoutput, model_writeoutput)
+    end
+    return writeoutput
+end

src/multi_stage/dual_dynamic_programming.jl

@@ -336,7 +336,7 @@ function write_multi_stage_outputs(stats_d::Dict, outpath::String, settings_d::D
     	write_multi_stage_network_expansion(outpath, multi_stage_settings_d)
     end
     write_multi_stage_costs(outpath, multi_stage_settings_d, inputs_dict)
-    write_multi_stage_stats(outpath, stats_d)
+    multi_stage_settings_d["Myopic"] == 0 && write_multi_stage_stats(outpath, stats_d)
     write_multi_stage_settings(outpath, settings_d)
 
 end

src/write_outputs/capacity_reserve_margin/write_reserve_margin.jl

@@ -5,5 +5,5 @@ function write_reserve_margin(path::AbstractString, setup::Dict, EP::Model)
 	end
 	dfResMar = DataFrame(temp_ResMar, :auto)
 	CSV.write(joinpath(path, "ReserveMargin.csv"), dfResMar)
-	return dfResMar
+	return nothing
 end

src/write_outputs/capacity_reserve_margin/write_reserve_margin_slack.jl

@@ -4,14 +4,22 @@ function write_reserve_margin_slack(path::AbstractString, inputs::Dict, setup::D
     dfResMar_slack = DataFrame(CRM_Constraint = [Symbol("CapRes_$res") for res = 1:NCRM], 
                                 AnnualSum = value.(EP[:eCapResSlack_Year]),
                                 Penalty = value.(EP[:eCCapResSlack]))
-    temp_ResMar_slack = value.(EP[:vCapResSlack])
+
     if setup["ParameterScale"] == 1
         dfResMar_slack.AnnualSum .*= ModelScalingFactor # Convert GW to MW
         dfResMar_slack.Penalty .*= ModelScalingFactor^2 # Convert Million $ to $
-        temp_ResMar_slack .*= ModelScalingFactor # Convert GW to MW
     end
-    dfResMar_slack = hcat(dfResMar_slack, DataFrame(temp_ResMar_slack, [Symbol("t$t") for t in 1:T]))
-    CSV.write(joinpath(path, "ReserveMargin_prices_and_penalties.csv"), dftranspose(dfResMar_slack, false), header=false)
-    return dfResMar_slack
+
+    if setup["WriteOutputs"] == "annual"
+        CSV.write(joinpath(path, "ReserveMargin_prices_and_penalties.csv"), dfResMar_slack)
+    else     # setup["WriteOutputs"] == "full"
+        temp_ResMar_slack = value.(EP[:vCapResSlack])
+        if setup["ParameterScale"] == 1
+            temp_ResMar_slack .*= ModelScalingFactor # Convert GW to MW
+        end
+        dfResMar_slack = hcat(dfResMar_slack, DataFrame(temp_ResMar_slack, [Symbol("t$t") for t in 1:T]))
+        CSV.write(joinpath(path, "ReserveMargin_prices_and_penalties.csv"), dftranspose(dfResMar_slack, false), writeheader=false)
+    end
+    return nothing
 end
 

src/write_outputs/capacity_reserve_margin/write_virtual_discharge.jl

@@ -5,30 +5,25 @@ Function for writing the "virtual" discharge of each storage technology. Virtual
 	allow storage resources to contribute to the capacity reserve margin without actually discharging.
 """
 function write_virtual_discharge(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
-	gen = inputs["RESOURCES"]
-	zones = zone_id.(gen)
 
 	G = inputs["G"]     # Number of resources (generators, storage, DR, and DERs)
 	T = inputs["T"]     # Number of time steps (hours)
 	STOR_ALL = inputs["STOR_ALL"]
 
-	dfVirtualDischarge = DataFrame(Resource = inputs["RESOURCE_NAMES"], Zone = zones, AnnualSum = Array{Union{Missing,Float64}}(undef, G))
-	virtual_discharge = zeros(G,T)
-
 	scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1
-	if !isempty(STOR_ALL)
-	    virtual_discharge[STOR_ALL, :] = (value.(EP[:vCAPRES_discharge][STOR_ALL, :]).data - value.(EP[:vCAPRES_charge][STOR_ALL, :]).data) * scale_factor
-	end
+
+	resources = inputs["RESOURCE_NAMES"][STOR_ALL]
+	zones = inputs["R_ZONES"][STOR_ALL]
+	virtual_discharge = (value.(EP[:vCAPRES_discharge][STOR_ALL, :].data) - value.(EP[:vCAPRES_charge][STOR_ALL, :].data)) * scale_factor
 
+	dfVirtualDischarge = DataFrame(Resource = resources, Zone = zones)
 	dfVirtualDischarge.AnnualSum .= virtual_discharge * inputs["omega"]
-	dfVirtualDischarge = hcat(dfVirtualDischarge, DataFrame(virtual_discharge, :auto))
-	auxNew_Names=[Symbol("Resource");Symbol("Zone");Symbol("AnnualSum");[Symbol("t$t") for t in 1:T]]
-	rename!(dfVirtualDischarge,auxNew_Names)
-	total = DataFrame(["Total" 0 sum(dfVirtualDischarge[!,:AnnualSum]) fill(0.0, (1,T))], :auto)
 
-	total[:, 4:T+3] .= sum(virtual_discharge, dims = 1)
-	rename!(total,auxNew_Names)
-	dfVirtualDischarge = vcat(dfVirtualDischarge, total)
-	CSV.write(joinpath(path, "virtual_discharge.csv"), dftranspose(dfVirtualDischarge, false), header=false)
-	return dfVirtualDischarge
-end
+	filepath = joinpath(path, "virtual_discharge.csv")
+	if setup["WriteOutputs"] == "annual"
+		write_annual(filepath, dfVirtualDischarge)
+	else 	# setup["WriteOutputs"] == "full"
+		write_fulltimeseries(filepath, virtual_discharge, dfVirtualDischarge)
+	end
+	return nothing
+end 

src/write_outputs/co2_cap/write_co2_cap.jl

@@ -21,5 +21,5 @@ function write_co2_cap(path::AbstractString, inputs::Dict, setup::Dict, EP::Mode
 
     CSV.write(joinpath(path, "CO2_prices_and_penalties.csv"), dfCO2Price)
 
-    return dfCO2Price
+    return nothing
 end

src/write_outputs/hydrogen/write_hourly_matching_prices.jl

@@ -13,5 +13,5 @@ function write_hourly_matching_prices(path::AbstractString, inputs::Dict, setup:
 
 	CSV.write(joinpath(path, "hourly_matching_prices.csv"), dftranspose(dfHourlyMatchPrices, false), header=false)
 
-	return dfHourlyMatchPrices
+	return nothing
 end

src/write_outputs/hydrogen/write_hydrogen_prices.jl

@@ -3,5 +3,5 @@ function write_hydrogen_prices(path::AbstractString, inputs::Dict, setup::Dict,
 	dfHydrogenPrice = DataFrame(Hydrogen_Price_Per_Tonne = convert(Array{Float64}, dual.(EP[:cHydrogenMin])*scale_factor))
 
 	CSV.write(joinpath(path, "hydrogen_prices.csv"), dfHydrogenPrice)
-	return dfHydrogenPrice
+	return nothing
 end

src/write_outputs/reserves/write_reg.jl

@@ -1,23 +1,20 @@
 function write_reg(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
-	gen = inputs["RESOURCES"]
-	zones = zone_id.(gen)
-
-	G = inputs["G"]     # Number of resources (generators, storage, DR, and DERs)
-	T = inputs["T"]     # Number of time steps (hours)
 	REG = inputs["REG"]
 	scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1
+
+	resources = inputs["RESOURCE_NAMES"][REG]
+	zones = inputs["R_ZONES"][REG]
 	# Regulation contributions for each resource in each time step
-	dfReg = DataFrame(Resource = inputs["RESOURCE_NAMES"], Zone = zones)
-	reg = zeros(G,T)
-	reg[REG, :] = value.(EP[:vREG][REG, :])
-	dfReg.AnnualSum = (reg*scale_factor) * inputs["omega"]
-	dfReg = hcat(dfReg, DataFrame(reg*scale_factor, :auto))
-	auxNew_Names=[Symbol("Resource");Symbol("Zone");Symbol("AnnualSum");[Symbol("t$t") for t in 1:T]]
-	rename!(dfReg,auxNew_Names)
+	reg = value.(EP[:vREG][REG, :].data) * scale_factor
+
+	dfReg = DataFrame(Resource = resources, Zone = zones)
+	dfReg.AnnualSum = reg * inputs["omega"]
 
-	total = DataFrame(["Total" 0 sum(dfReg.AnnualSum) fill(0.0, (1,T))], :auto)
-	total[!, 4:T+3] .= sum(reg, dims = 1)
-	rename!(total,auxNew_Names)
-	dfReg = vcat(dfReg, total)
-	CSV.write(joinpath(path, "reg.csv"), dftranspose(dfReg, false), header=false)
+	filepath = joinpath(path, "reg.csv")
+	if setup["WriteOutputs"] == "annual"
+		write_annual(filepath, dfReg)
+	else 	# setup["WriteOutputs"] == "full"
+		write_fulltimeseries(filepath, reg, dfReg)
+	end
+	return nothing
 end

src/write_outputs/reserves/write_rsv.jl

@@ -1,28 +1,31 @@
 function write_rsv(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
-	gen = inputs["RESOURCES"]
-	zones = zone_id.(gen)
-
-	G = inputs["G"]     # Number of resources (generators, storage, DR, and DERs)
-	T = inputs["T"]     # Number of time steps (hours)
 	RSV = inputs["RSV"]
 	scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1
-	dfRsv = DataFrame(Resource = inputs["RESOURCE_NAMES"], Zone = zones)
-	rsv = zeros(G,T)
-	unmet_vec = zeros(T)
-	rsv[RSV, :] = value.(EP[:vRSV][RSV, :]) * scale_factor
-	unmet_vec = value.(EP[:vUNMET_RSV]) * scale_factor
-	total_unmet = sum(unmet_vec)
+
+	resources = inputs["RESOURCE_NAMES"][RSV]
+	zones = inputs["R_ZONES"][RSV]
+	rsv = value.(EP[:vRSV][RSV, :].data) * scale_factor
+
+	dfRsv = DataFrame(Resource = resources, Zone = zones)
+
 	dfRsv.AnnualSum = rsv * inputs["omega"]
-	dfRsv = hcat(dfRsv, DataFrame(rsv, :auto))
-	auxNew_Names=[Symbol("Resource");Symbol("Zone");Symbol("AnnualSum");[Symbol("t$t") for t in 1:T]]
-	rename!(dfRsv,auxNew_Names)
 
-	total = DataFrame(["Total" 0 sum(dfRsv.AnnualSum) zeros(1, T)], :auto)
-	unmet = DataFrame(["unmet" 0 total_unmet zeros(1, T)], :auto)
-	total[!, 4:T+3] .= sum(rsv, dims = 1)
-	unmet[!, 4:T+3] .= transpose(unmet_vec)
-	rename!(total,auxNew_Names)
-	rename!(unmet,auxNew_Names)
-	dfRsv = vcat(dfRsv, unmet, total)
-	CSV.write(joinpath(path, "reg_dn.csv"), dftranspose(dfRsv, false), header=false)
+	if setup["WriteOutputs"] == "annual"
+		write_annual(joinpath(path, "reg_dn.csv"), dfRsv)
+	else 	# setup["WriteOutputs"] == "full"
+		unmet_vec = value.(EP[:vUNMET_RSV]) * scale_factor
+		total_unmet = sum(unmet_vec)
+		dfRsv = hcat(dfRsv, DataFrame(rsv, :auto))
+		auxNew_Names=[Symbol("Resource");Symbol("Zone");Symbol("AnnualSum");[Symbol("t$t") for t in 1:T]]
+		rename!(dfRsv,auxNew_Names)
+
+		total = DataFrame(["Total" 0 sum(dfRsv.AnnualSum) zeros(1, T)], :auto)
+		unmet = DataFrame(["unmet" 0 total_unmet zeros(1, T)], :auto)
+		total[!, 4:T+3] .= sum(rsv, dims = 1)
+		unmet[!, 4:T+3] .= transpose(unmet_vec)
+		rename!(total,auxNew_Names)
+		rename!(unmet,auxNew_Names)
+		dfRsv = vcat(dfRsv, unmet, total)
+		CSV.write(joinpath(path, "reg_dn.csv"), dftranspose(dfRsv, false), writeheader=false)
+	end
 end