Merge remote-tracking branch 'origin/develop' into multi_fuels

.github/workflows/ci.yml

@@ -0,0 +1,42 @@
+name: CI
+
+on: 
+  pull_request:
+  push:
+    branches: [main, develop]
+    tags: ['*']
+
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name}}
+    runs-on: ${{ matrix.os }}
+    strategy: 
+      fail-fast: false
+      matrix:
+        version:
+          - "1.6" # LTS (64-bit Linux)
+          - "1.7"
+          - "1.8" 
+          - '1.9' # latest stable Julia release (Linux)
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+        include: # additional tests [Julia-nightly] (Linux) 
+          - os: ubuntu-latest
+            version: 'nightly'
+            arch: x64
+            allow_failure: true
+    steps:
+      - uses: actions/checkout@v3
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v1
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v3
+        with:
+          file: lcov.info

.gitignore

@@ -19,6 +19,8 @@ gurobi.log
 cplex.log
 Highs.log
 
+test/Logs
+
 /docs/build/
 
 # Microsoft office temporary files
@@ -42,3 +44,7 @@ Example_Systems/SmallNewEngland/ThreeZones_full/*
 
 Example_Systems/SmallNewEngland/ThreeZones_MultiStage/Inputs/*
 test_get_retirement_period.jl
+
+# Test files
+test/TDR/TDR_Results_test
+test/MethodofMorris/morris.csv

CHANGELOG.md

@@ -22,6 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add PR template (#516)
 - Validation ensures that resource flags (THERM, HYDRO, LDS etc) are self-consistent (#513).
 - Maintenance formulation for thermal-commit plants (#556).
+- Add new tests for GenX: three-zone, multi-stage, electrolyzer, VRE+storage, 
+  piecewise_fuel+CO2, and TDR (#563 and #578).
+
 
 ### Fixed
 - Set MUST_RUN=1 for RealSystemExample/small_hydro plants (#517).
@@ -43,6 +46,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `Load` and `LoadWeight` keys in the `time_domain_reduction_settings.yml` file are renamed to `Demand`, `DemandWeight`.
 - The `New_Build` column in `Generators_data.csv` has been separated into two: `New_Build` and `Can_Retire` (#392).
   Values in each column are {0,1}.
+- Separate proprietary JuMP solvers from the GenX package.
+  This allows users of Gurobi or CPLEX to use them without modifying
+  the source code of the GenX package directly. This is a key step in publishing
+  GenX as a proper Julia package. This does require change to the Run.jl files,
+  to specify the solver. (#531)
+- In the examples, change Reg_Max and Rsv_Max of any MUST_RUN generators to 0.
+  This mitigates but does not fully fix (#576).
 
 ### Deprecated
 - The above `load` keys, which generally refer to electrical demand, are being deprecated.

Example_Systems/Electrolyzer_Example/Settings/genx_settings.yml

@@ -62,11 +62,8 @@ StorageLosses: 1
 # 1 = yes
 HydrogenHourlyMatching: 1
 
-## Solver settings
-# ----------------
-
-# Available solvers: Gurobi, CPLEX, Clps
-# Solver: HiGHS
+## Solution settings
+# ------------------
 
 # Internally scale demand, capacity and power variables in GW rather than MW.
 # 0 = not active

Example_Systems/MethodofMorrisExample/OneZone/Settings/genx_settings.yml

@@ -9,7 +9,6 @@ CO2Cap: 0 # CO2 emissions cap; 0 = not active (no CO2 emission limit); 1 = mass-
 StorageLosses: 1 # Energy Share Requirement and CO2 constraints account for energy lost; 0 = not active (DO NOT account for energy lost); 1 = active systemwide (DO account for energy lost)
 MinCapReq: 0  # Activate minimum technology carveout constraints; 0 = not active; 1 = active
 MaxCapReq: 0  # Activate maximum technology carveout constraints; 0 = not active; 1 = active
-Solver: HiGHS # Available solvers: Gurobi, CPLEX, Clp, Cbc
 ParameterScale: 1 # Turn on parameter scaling wherein demand, capacity and power variables are defined in GW rather than MW. 0 = not active; 1 = active systemwide
 WriteShadowPrices: 1 # Write shadow prices of LP or relaxed MILP; 0 = not active; 1 = active
 UCommit: 0 # Unit committment of thermal power plants; 0 = not active; 1 = active using integer clestering; 2 = active using linearized clustering

Example_Systems/PiecewiseFuel_CO2_Example/Settings/genx_settings.yml

@@ -60,14 +60,11 @@ CO2Cap: 1
 # Hydrogen electrolyzer hourly supply matching required?
 # 0 = constraint ot active
 # 1 = constraint active for all electrolyzer resources
-#HydrogenHourlyMatching: 0
+# HydrogenHourlyMatching: 0
 
-## Solver settings
+## Solution settings
 # ----------------
 
-# Available solvers: Gurobi, CPLEX, Clps
-#Solver: HiGHS
-
 # Internally scale demand, capacity and power variables in GW rather than MW.
 # 0 = not active
 # 1 = active systemwide
@@ -76,11 +73,11 @@ ParameterScale: 1
 # Time domain reduce (i.e. cluster) inputs based on demand, resource availability profiles, and fuel prices;
 # 0 = not active (use input data as provided)
 # 1 = active (cluster input data, or use data that has already been clustered)
-#TimeDomainReduction: 0
+# TimeDomainReduction: 0
 
 # Directory where results from time domain reduction will be saved.
 # If results already exist, these will be used without running time domain reduction script again.
-#TimeDomainReductionFolder: "TDR_Results"
+# TimeDomainReductionFolder: "TDR_Results"
 
 ## Output settings
 # ----------------

Example_Systems/ProprietarySolver/CO2_cap.csv

@@ -0,0 +1,2 @@
+,Network_zones,CO_2_Cap_Zone_1,CO_2_Max_tons_MWh_1,CO_2_Max_Mtons_1
+NE,z1,1,0.05,0.018

Example_Systems/ProprietarySolver/Capacity_reserve_margin.csv

@@ -0,0 +1,2 @@
+,Network_zones,CapRes_1
+NE,z1,0.156

Example_Systems/ProprietarySolver/Demand_data.csv

@@ -0,0 +1,25 @@
+Voll,Demand_Segment,Cost_of_Demand_Curtailment_per_MW,Max_Demand_Curtailment,Rep_Periods,Timesteps_per_Rep_Period,Sub_Weights,Time_Index,Demand_MW_z1
+50000,1,1,1,1,24,8760,1,11162
+,,,,,,,2,10556
+,,,,,,,3,10105
+,,,,,,,4,9878
+,,,,,,,5,9843
+,,,,,,,6,10017
+,,,,,,,7,10390
+,,,,,,,8,10727
+,,,,,,,9,11298
+,,,,,,,10,11859
+,,,,,,,11,12196
+,,,,,,,12,12321
+,,,,,,,13,12381
+,,,,,,,14,12270
+,,,,,,,15,12149
+,,,,,,,16,12219
+,,,,,,,17,13410
+,,,,,,,18,14539
+,,,,,,,19,14454
+,,,,,,,20,14012
+,,,,,,,21,13494
+,,,,,,,22,12772
+,,,,,,,23,11877
+,,,,,,,24,10874

Example_Systems/ProprietarySolver/Fuels_data.csv

@@ -0,0 +1,26 @@
+Time_Index,NG
+0,0.05306
+1,5.28
+2,5.28
+3,5.28
+4,5.28
+5,5.28
+6,5.28
+7,5.28
+8,5.28
+9,5.28
+10,5.28
+11,5.28
+12,5.28
+13,5.28
+14,5.28
+15,5.28
+16,5.28
+17,5.28
+18,5.28
+19,5.28
+20,5.28
+21,5.28
+22,5.28
+23,5.28
+24,5.28

Example_Systems/ProprietarySolver/Generators_data.csv

@@ -0,0 +1,5 @@
+Resource,Zone,THERM,MUST_RUN,STOR,FLEX,HYDRO,VRE,LDS,Num_VRE_Bins,New_Build,Can_Retire,Existing_Cap_MW,Existing_Cap_MWh,Existing_Charge_Cap_MW,Max_Cap_MW,Max_Cap_MWh,Max_Charge_Cap_MW,Min_Cap_MW,Min_Cap_MWh,Min_Charge_Cap_MW,Inv_Cost_per_MWyr,Inv_Cost_per_MWhyr,Inv_Cost_Charge_per_MWyr,Fixed_OM_Cost_per_MWyr,Fixed_OM_Cost_per_MWhyr,Fixed_OM_Cost_Charge_per_MWyr,Var_OM_Cost_per_MWh,Var_OM_Cost_per_MWh_In,Heat_Rate_MMBTU_per_MWh,Fuel,Cap_Size,Start_Cost_per_MW,Start_Fuel_MMBTU_per_MW,Up_Time,Down_Time,Ramp_Up_Percentage,Ramp_Dn_Percentage,Min_Power,Self_Disch,Eff_Up,Eff_Down,Min_Duration,Max_Duration,MaxCapTag_1,MaxCapTag_2,MaxCapTag_3,Resource_Type,CapRes_1,region,cluster
+natural_gas_combined_cycle,1,1,0,0,0,0,0,0,0,1,0,0,0,0,-1,-1,-1,0,0,0,65400,0,0,10287,0,0,3.55,0,7.43,NG,250,91,2,6,6,0.64,0.64,0.468,0,1,1,0,0,0,0,0,natural_gas_fired_combined_cycle,0.93,NE,1
+solar_pv,1,0,0,0,0,0,1,0,1,1,0,0,0,0,-1,-1,-1,0,0,0,85300,0,0,18760,0,0,0,0,9.13,None,0,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,solar_photovoltaic,0.8,NE,1
+onshore_wind,1,0,0,0,0,0,1,0,1,1,0,0,0,0,-1,-1,-1,0,0,0,97200,0,0,43205,0,0,0.1,0,9.12,None,0,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,onshore_wind_turbine,0.8,NE,1
+battery,1,0,0,1,0,0,0,0,0,1,0,0,0,0,-1,-1,-1,0,0,0,19584,22494,0,4895,5622,0,0.15,0.15,0,None,0,0,0,0,0,1,1,0,0,0.92,0.92,1,10,0,0,1,battery_mid,0.95,NE,0

Example_Systems/ProprietarySolver/Generators_variability.csv

@@ -0,0 +1,25 @@
+Time_Index,solar_pv,onshore_wind
+1,0,0.889717042
+2,0,0.877715468
+3,0,0.903424203
+4,0,0.895153165
+5,0,0.757258117
+6,0,0.630928695
+7,0,0.557177782
+8,0,0.6072492
+9,0.1779,0.423417866
+10,0.429,0.007470775
+11,0.5748,0.002535942
+12,0.6484,0.002153709
+13,0.6208,0.00445132
+14,0.596,0.007711587
+15,0.5013,0.100848213
+16,0.3311,0.201802149
+17,0.0642,0.141933054
+18,0,0.567022562
+19,0,0.946024895
+20,0,0.923394203
+21,0,0.953386247
+22,0,0.929205418
+23,0,0.849528909
+24,0,0.665570974

Example_Systems/ProprietarySolver/Maximum_capacity_requirement.csv

@@ -0,0 +1,4 @@
+MaxCapReqConstraint,ConstraintDescription,Max_MW
+1,PV,50000
+2,Wind,100000
+3,Batteries,60000

Example_Systems/ProprietarySolver/README.md

@@ -0,0 +1,5 @@
+# Proprietary solver example
+
+This demonstrates how to use a proprietary solver such as Gurobi or CPLEX, instead of the default solver HiGHS.
+
+See `Run.jl` and the `Solver` setting in `Settings/genx_settings.yml`.

Example_Systems/ProprietarySolver/Run.jl

@@ -0,0 +1,5 @@
+using GenX, Gurobi
+
+optimizer = Gurobi.Optimizer
+
+run_genx_case!(dirname(@__FILE__),optimizer)

Example_Systems/ProprietarySolver/Settings/cplex_settings.yml

@@ -0,0 +1,10 @@
+# CPLEX Solver Parameters
+Feasib_Tol: 1.0e-05 # Constraint (primal) feasibility tolerances.
+Optimal_Tol: 1e-5 # Dual feasibility tolerances.
+Pre_Solve: 1 # Controls presolve level.
+TimeLimit: 110000 # Limits total time solver.
+MIPGap: 1e-3 # Relative (p.u. of optimal) mixed integer optimality tolerance for MIP problems (ignored otherwise).
+Method: 2 # Algorithm used to solve continuous models (including MIP root relaxation).
+BarConvTol: 1.0e-08 # Barrier convergence tolerance (determines when barrier terminates).
+NumericFocus: 0 # Numerical precision emphasis.
+SolutionType: 2 # Solution type for LP or QP.

Example_Systems/ProprietarySolver/Settings/genx_settings.yml

@@ -0,0 +1,98 @@
+## Model settings
+# --------------
+
+# Number of segments used in piecewise linear approximation of transmission losses;
+#  1 = linear
+# >2 = piecewise quadratic
+Trans_Loss_Segments: 1
+
+# Unit committment of thermal power plants;
+# 0 = not active
+# 1 = active using integer clustering
+# 2 = active using linearized clustering
+UCommit: 2
+
+## Policy settings
+# ---------------
+
+# Regulation (primary) and operating (secondary) reserves
+# 0 = not active
+# 1 = active systemwide
+# Reserves: 0
+
+# Minimum qualifying renewables penetration
+# 0 = not active
+# 1 = active systemwide
+# EnergyShareRequirement: 0
+
+# Number of capacity reserve margin constraints
+# 0 = not active
+# 1 = active systemwide
+CapacityReserveMargin: 1
+
+# CO2 emissions cap
+# 0 = not active (no CO2 emission limit)
+# 1 = mass-based emission limit constraint
+# 2 = demand + rate-based emission limit constraint
+# 3 = generation + rate-based emission limit constraint
+CO2Cap: 2
+
+# Let lost energy be accounted for in Energy Share Requirement and CO2 constraints
+# 0 = not active (DO NOT account for energy lost);
+# 1 = active systemwide (DO account for energy lost)
+StorageLosses: 1
+
+# Activate minimum technology carveout constraints
+# 0 = not active
+# 1 = active
+# MinCapReq: 0
+
+# Activate maximum technology limits
+# 0 = not active
+# 1 = active
+MaxCapReq: 1
+#
+# Transmission network expansion
+# 0 = not active
+# 1 = active systemwide
+# NetworkExpansion: 0
+
+# Hydrogen electrolyzer hourly supply matching required?
+# 0 = no
+# 1 = yes
+# HydrogenHourlyMatching: 0
+
+## Solution settings
+# ----------------
+
+# Internally scale demand, capacity and power variables in GW rather than MW.
+# 0 = not active
+# 1 = active systemwide
+ParameterScale: 1
+
+# Time domain reduce (i.e. cluster) inputs based on demand, resource availability profiles, and fuel prices;
+# 0 = not active (use input data as provided)
+# 1 = active (cluster input data, or use data that has already been clustered)
+# TimeDomainReduction: 0
+
+# Directory where results from time domain reduction will be saved.
+# If results already exist, these will be used without running time domain reduction script again.
+# TimeDomainReductionFolder: "TDR_Results"
+
+## Output settings
+# ----------------
+
+# Write shadow prices of LP or relaxed MILP
+# 0 = not active
+# 1 = active
+WriteShadowPrices: 1
+
+# Overwrite existing results in output folder or create a new one;
+# 0 = create new folder
+# 1 = overwrite existing results
+# OverwriteResults: 0
+
+# Write the model formulation as an output;
+# 0 = active
+# 1 = not active
+# PrintModel: 0

Example_Systems/ProprietarySolver/Settings/gurobi_settings.yml

@@ -0,0 +1,15 @@
+# Gurobi Solver Parameters
+# Common solver settings
+Feasib_Tol: 1.0e-05           # Constraint (primal) feasibility tolerances.
+Optimal_Tol: 1e-5             # Dual feasibility tolerances.
+TimeLimit: 110000             # Limits total time solver.
+Pre_Solve: 1                  # Controls presolve level.
+Method: 2                  # Algorithm used to solve continuous models (including MIP root relaxation).
+
+#Gurobi-specific solver settings
+MIPGap: 1e-3                  # Relative (p.u. of optimal) mixed integer optimality tolerance for MIP problems (ignored otherwise).
+BarConvTol: 1.0e-08           # Barrier convergence tolerance (determines when barrier terminates).
+NumericFocus: 0               # Numerical precision emphasis.
+Crossover: 0                # Barrier crossver strategy.
+PreDual: 0                    # Decides whether presolve should pass the primal or dual linear programming problem to the LP optimization algorithm.
+AggFill: 10                   # Allowed fill during presolve aggregation.