Include month resolution to hurdle rates

db/csvs_test_examples/temporal/10_1horizon_1period_2months/horizon_params.csv

@@ -0,0 +1,2 @@
+subproblem_id,balancing_type_horizon,horizon,boundary
+1,day,202001,circular

db/csvs_test_examples/temporal/10_1horizon_1period_2months/horizon_timepoints.csv

@@ -0,0 +1,2 @@
+subproblem_id,stage_id,balancing_type_horizon,horizon,tmp_start,tmp_end
+1,1,day,202001,20200101,20200102

db/csvs_test_examples/temporal/10_1horizon_1period_2months/period_params.csv

@@ -0,0 +1,2 @@
+period,discount_factor,period_start_year,period_end_year
+2020,1,2020,2021

db/csvs_test_examples/temporal/10_1horizon_1period_2months/structure.csv

@@ -0,0 +1,3 @@
+subproblem_id,stage_id,timepoint,period,number_of_hours_in_timepoint,timepoint_weight,previous_stage_timepoint_map,spinup_or_lookahead,linked_timepoint,month,hour_of_day,timestamp,ignore_horizon_day
+1,1,20200101,2020,1,4380,,0,,1,1,2001-01-20 0:00,202001
+1,1,20200102,2020,1,4380,,0,,2,1,2001-01-20 1:00,202001

db/csvs_test_examples/transmission/transmission_hurdle_rates/1_transmission_hurdle_rates_1.csv

@@ -1,5 +1,5 @@
-transmission_line,period,hurdle_rate_positive_direction_per_mwh,hurdle_rate_negative_direction_per_mwh
-Tx_1,2020,5,5
-Tx_1,2025,5,5
-Tx_new,2020,5,5
-Tx_new,2025,5,5
+transmission_line,period,month,hurdle_rate_positive_direction_per_mwh,hurdle_rate_negative_direction_per_mwh
+Tx1,2020,1,5,5
+Tx1,2020,2,8,8
+Tx_new,2020,1,5,5
+Tx_new,2020,2,8,8

db/csvs_test_examples/transmission/transmission_hurdle_rates/2_tx_hurdle_rates_no_months.csv

@@ -0,0 +1,3 @@
+transmission_line,period,month,hurdle_rate_positive_direction_per_mwh,hurdle_rate_negative_direction_per_mwh
+Tx1,2020,0,5,5
+Tx_new,2020,0,5,5

db/csvs_test_examples/transmission/transmission_portfolios/1_transmission_portfolios_nonew.csv → db/csvs_test_examples/transmission/transmission_portfolios/4_transmission_portfolios_new_and_nonew.csv

@@ -1,2 +1,3 @@
-transmission_line,capacity_type
-Tx1,tx_spec
+transmission_line,capacity_type
+Tx1,tx_spec
+Tx_new,tx_new_lin

db/db_schema.sql

@@ -1840,9 +1840,10 @@ CREATE TABLE inputs_transmission_hurdle_rates (
 transmission_hurdle_rate_scenario_id INTEGER,
 transmission_line VARCHAR(64),
 period INTEGER,
+month INTEGER,
 hurdle_rate_positive_direction_per_mwh FLOAT,
 hurdle_rate_negative_direction_per_mwh FLOAT,
-PRIMARY KEY (transmission_hurdle_rate_scenario_id, transmission_line, period),
+PRIMARY KEY (transmission_hurdle_rate_scenario_id, transmission_line, period, month),
 FOREIGN KEY (transmission_hurdle_rate_scenario_id) REFERENCES
 subscenarios_transmission_hurdle_rates (transmission_hurdle_rate_scenario_id)
 );
@@ -3409,6 +3410,7 @@ transmission_line VARCHAR(64),
 load_zone_from VARCHAR(64),
 load_zone_to VARCHAR(64),
 period INTEGER,
+month INTEGER,
 subproblem_id INTEGER,
 stage_id INTEGER,
 timepoint INTEGER,

gridpath/transmission/operations/hurdle_costs.py

@@ -25,10 +25,11 @@
 from builtins import str
 import csv
 import os.path
+
+import pandas as pd
 from pyomo.environ import Param, Var, Constraint, NonNegativeReals, Expression, value
 
 from db.common_functions import spin_on_database_lock
-from gridpath.auxiliary.auxiliary import cursor_to_df
 from gridpath.auxiliary.db_interface import setup_results_import
 from gridpath.auxiliary.validations import (
     write_validation_to_database,
@@ -37,7 +38,8 @@
     validate_values,
     validate_missing_inputs,
 )
-
+from gridpath.auxiliary.auxiliary import cursor_to_df
+from gridpath.temporal.operations.timepoints import get_inputs_from_database as get_temporal_inputs_from_database
 
 def add_model_components(m, d, scenario_directory, subproblem, stage):
     """
@@ -52,7 +54,7 @@ def add_model_components(m, d, scenario_directory, subproblem, stage):
     | | *Defaults to*: :code:`0`                                              |
     |                                                                         |
     | The transmission line's hurdle rate in $ per MWh for its positive       |
-    | flows in each operational period.                                       |
+    | flows in each month of each operational period.                                       |
     +-------------------------------------------------------------------------+
     | | :code:`hurdle_rate_neg_dir_per_mwh`                                   |
     | | *Defined over*: :code:`TX_OPR_PRDS`                                   |
@@ -115,14 +117,16 @@ def add_model_components(m, d, scenario_directory, subproblem, stage):
 
     m.hurdle_rate_pos_dir_per_mwh = Param(
         m.TX_LINES,
-        m.PERIODS,  # TODO: chanage to TX_OPR_PRDS?
+        m.PERIODS,
+        m.MONTHS, # TODO: chanage to TX_OPR_PRDS?
         within=NonNegativeReals,
         default=0,
     )
 
     m.hurdle_rate_neg_dir_per_mwh = Param(
         m.TX_LINES,
-        m.PERIODS,  # TODO: chanage to TX_OPR_PRDS?
+        m.PERIODS,
+        m.MONTHS,# TODO: chanage to TX_OPR_PRDS?
         within=NonNegativeReals,
         default=0,
     )
@@ -157,13 +161,13 @@ def hurdle_cost_pos_dir_rule(mod, tx, tmp):
     Hurdle_Cost_Pos_Dir must be non-negative, so will be 0
     when Transmit_Power is negative (flow in the negative direction).
     """
-    if mod.hurdle_rate_pos_dir_per_mwh[tx, mod.period[tmp]] == 0:
+    if mod.hurdle_rate_pos_dir_per_mwh[tx, mod.period[tmp], mod.month[tmp]] == 0:
         return Constraint.Skip
     else:
         return (
             mod.Hurdle_Cost_Pos_Dir[tx, tmp]
             >= mod.Transmit_Power_MW[tx, tmp]
-            * mod.hurdle_rate_pos_dir_per_mwh[tx, mod.period[tmp]]
+            * mod.hurdle_rate_pos_dir_per_mwh[tx, mod.period[tmp], mod.month[tmp]]
         )
 
 
@@ -175,19 +179,53 @@ def hurdle_cost_neg_dir_rule(mod, tx, tmp):
     Hurdle_Cost_Neg_Dir must be non-negative, so will be 0
     when Transmit_Power is positive (flow in the positive direction).
     """
-    if mod.hurdle_rate_neg_dir_per_mwh[tx, mod.period[tmp]] == 0:
+    if mod.hurdle_rate_neg_dir_per_mwh[tx, mod.period[tmp], mod.month[tmp]] == 0:
         return Constraint.Skip
     else:
         return (
             mod.Hurdle_Cost_Neg_Dir[tx, tmp]
             >= -mod.Transmit_Power_MW[tx, tmp]
-            * mod.hurdle_rate_neg_dir_per_mwh[tx, mod.period[tmp]]
+            * mod.hurdle_rate_neg_dir_per_mwh[tx, mod.period[tmp], mod.month[tmp]]
         )
 
 
 # Input-Output
 ###############################################################################
 
+def format_monthly_hurdle_data(hrs_df_unformatted, periods_months_df):
+    """ Fill monthly hurdle rates with default values where applicable
+
+    :param hrs_df_unformatted: DataFrame with hurdle rates
+    :param periods_months_df: DataFrame with period and month columns
+    :return:
+    pandas.DataFrame with monthly hurdle rates for all relevant months
+    """
+    period_months_dict = {
+        period: periods_months_df[periods_months_df["period"]==period]["month"].unique().tolist()
+        for period in periods_months_df["period"].unique()
+    }
+    hrs_df_formatted = pd.DataFrame(columns=hrs_df_unformatted.columns)
+    for tx_line, df_per_tx_line in hrs_df_unformatted.groupby("transmission_line"):
+        for period, df_per_tx_line_per_period in df_per_tx_line.groupby("period"):
+            months = df_per_tx_line_per_period.month.unique().tolist()
+            expected_months = period_months_dict[period]
+            if set(months) == set(expected_months):
+                hrs_df_formatted = pd.concat([hrs_df_formatted, df_per_tx_line_per_period], axis=0, ignore_index=True)
+            elif months == [0]:
+                df_expanded = df_per_tx_line_per_period.loc[df_per_tx_line_per_period.index.repeat(len(expected_months))]
+                df_expanded["month"] = expected_months
+                hrs_df_formatted = pd.concat([hrs_df_formatted, df_expanded], axis=0, ignore_index=True)
+            else:
+                raise ValueError(
+                    """Hurdle rates for transmission line {} and period {} aren't specified for all relevant months
+                    (got {}, expected {} or 0). 
+                    Set month to 0 if inputs are the same for each month 
+                    or make sure all modelled months are included.""".format(
+                        tx_line, period, months, expected_months
+                    ),
+                )
+    return hrs_df_formatted
+
 
 def load_model_data(m, d, data_portal, scenario_directory, subproblem, stage):
     """
@@ -200,22 +238,40 @@ def load_model_data(m, d, data_portal, scenario_directory, subproblem, stage):
     :param stage:
     :return:
     """
-    data_portal.load(
-        filename=os.path.join(
-            scenario_directory,
-            str(subproblem),
-            str(stage),
-            "inputs",
-            "transmission_hurdle_rates.tab",
-        ),
-        select=(
-            "transmission_line",
-            "period",
-            "hurdle_rate_positive_direction_per_mwh",
-            "hurdle_rate_negative_direction_per_mwh",
-        ),
-        param=(m.hurdle_rate_pos_dir_per_mwh, m.hurdle_rate_neg_dir_per_mwh),
+
+    hurdle_rate_pos_dir_per_mwh_dict = {}
+    hurdle_rate_neg_dir_per_mwh_dict = {}
+
+    hrs_file = os.path.join(
+        scenario_directory, str(subproblem), str(stage), "inputs", "transmission_hurdle_rates.tab",
     )
+    hrs_df_unformatted = pd.read_csv(hrs_file, sep="\t")
+
+    # The timepoints.tab file indicates what periods have months
+    timepoints_file = os.path.join(
+        scenario_directory, str(subproblem), str(stage), "inputs", "timepoints.tab"
+    )
+    timepoints_df = pd.read_csv(timepoints_file, sep="\t")
+
+    hrs_df = format_monthly_hurdle_data(hrs_df_unformatted, timepoints_df)
+
+    for tx_line, hrs_df_tx_slice in hrs_df.groupby("transmission_line"):
+        for period, hrs_df_period_slice in hrs_df_tx_slice.groupby("period"):
+            for month, hrs_df_month_slice in hrs_df_period_slice.groupby("month"):
+                hurdle_rate_pos_dir_per_mwh_dict.update(
+                    {
+                        (tx_line, period, month): hrs_df_month_slice.loc[
+                                                  :, "hurdle_rate_positive_direction_per_mwh"].values[0]
+                    }
+                )
+                hurdle_rate_neg_dir_per_mwh_dict.update(
+                    {
+                        (tx_line, period, month): hrs_df_month_slice.loc[
+                                                  :, "hurdle_rate_negative_direction_per_mwh"].values[0]
+                    }
+                )
+    data_portal.data()["hurdle_rate_pos_dir_per_mwh"] = hurdle_rate_pos_dir_per_mwh_dict
+    data_portal.data()["hurdle_rate_neg_dir_per_mwh"] = hurdle_rate_neg_dir_per_mwh_dict
 
 
 def export_results(scenario_directory, subproblem, stage, m, d):
@@ -244,6 +300,7 @@ def export_results(scenario_directory, subproblem, stage, m, d):
             [
                 "tx_line",
                 "period",
+                "month",
                 "timepoint",
                 "timepoint_weight",
                 "number_of_hours_in_timepoint",
@@ -258,6 +315,7 @@ def export_results(scenario_directory, subproblem, stage, m, d):
                 [
                     tx,
                     m.period[tmp],
+                    m.month[tmp],
                     tmp,
                     m.tmp_weight[tmp],
                     m.hrs_in_tmp[tmp],
@@ -285,29 +343,15 @@ def get_inputs_from_database(scenario_id, subscenarios, subproblem, stage, conn)
     stage = 1 if stage == "" else stage
     c = conn.cursor()
     hurdle_rates = c.execute(
-        """SELECT transmission_line, period, 
+        """SELECT transmission_line, period, month,
         hurdle_rate_positive_direction_per_mwh,
         hurdle_rate_negative_direction_per_mwh
-        FROM inputs_transmission_portfolios
-        CROSS JOIN
-            (SELECT period
-            FROM inputs_temporal_periods
-            WHERE temporal_scenario_id = {}) AS relevant_periods 
-        LEFT OUTER JOIN
-            (SELECT transmission_line, period, 
-            hurdle_rate_positive_direction_per_mwh,
-            hurdle_rate_negative_direction_per_mwh
-            FROM inputs_transmission_hurdle_rates
-            WHERE transmission_hurdle_rate_scenario_id = {}) AS relevant_hrs
-        USING (transmission_line, period)
-        WHERE transmission_portfolio_scenario_id = {};
+        FROM inputs_transmission_hurdle_rates 		
+        WHERE transmission_hurdle_rate_scenario_id = {};
         """.format(
-            subscenarios.TEMPORAL_SCENARIO_ID,
             subscenarios.TRANSMISSION_HURDLE_RATE_SCENARIO_ID,
-            subscenarios.TRANSMISSION_PORTFOLIO_SCENARIO_ID,
         )
     )
-
     return hurdle_rates
 
 
@@ -324,11 +368,9 @@ def write_model_inputs(
     :param conn: database connection
     :return:
     """
-
     hurdle_rates = get_inputs_from_database(
         scenario_id, subscenarios, subproblem, stage, conn
     )
-
     with open(
         os.path.join(
             scenario_directory,
@@ -347,6 +389,7 @@ def write_model_inputs(
             [
                 "transmission_line",
                 "period",
+                "month",
                 "hurdle_rate_positive_direction_per_mwh",
                 "hurdle_rate_negative_direction_per_mwh",
             ]
@@ -394,19 +437,21 @@ def import_results_into_database(
         for row in reader:
             tx_line = row[0]
             period = row[1]
-            timepoint = row[2]
-            timepoint_weight = row[3]
-            number_of_hours_in_timepoint = row[4]
-            lz_from = row[5]
-            lz_to = row[6]
-            hurdle_cost_positve_direction = row[7]
-            hurdle_cost_negative_direction = row[8]
+            month = row[2]
+            timepoint = row[3]
+            timepoint_weight = row[4]
+            number_of_hours_in_timepoint = row[5]
+            lz_from = row[6]
+            lz_to = row[7]
+            hurdle_cost_positve_direction = row[8]
+            hurdle_cost_negative_direction = row[9]
 
             results.append(
                 (
                     scenario_id,
                     tx_line,
                     period,
+                    month,
                     subproblem,
                     stage,
                     timepoint,
@@ -420,12 +465,12 @@ def import_results_into_database(
             )
     insert_temp_sql = """
         INSERT INTO temp_results_transmission_hurdle_costs{}
-        (scenario_id, transmission_line, period, subproblem_id, stage_id,
+        (scenario_id, transmission_line, period, month, subproblem_id, stage_id,
         timepoint, timepoint_weight,
         number_of_hours_in_timepoint,
         load_zone_from, load_zone_to, 
         hurdle_cost_positive_direction, hurdle_cost_negative_direction)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
         """.format(
         scenario_id
     )
@@ -434,12 +479,12 @@ def import_results_into_database(
     # Insert sorted results into permanent results table
     insert_sql = """
         INSERT INTO results_transmission_hurdle_costs
-        (scenario_id, transmission_line, period, subproblem_id, stage_id, 
+        (scenario_id, transmission_line, period, month, subproblem_id, stage_id, 
         timepoint, timepoint_weight, number_of_hours_in_timepoint,
         load_zone_from, load_zone_to, hurdle_cost_positive_direction,
         hurdle_cost_negative_direction)
         SELECT
-        scenario_id, transmission_line, period, subproblem_id, stage_id,
+        scenario_id, transmission_line, period, month, subproblem_id, stage_id,
         timepoint, timepoint_weight, number_of_hours_in_timepoint,
         load_zone_from, load_zone_to, hurdle_cost_positive_direction,
         hurdle_cost_negative_direction
@@ -535,8 +580,18 @@ def validate_inputs(scenario_id, subscenarios, subproblem, stage, conn):
     hurdle_rates = get_inputs_from_database(
         scenario_id, subscenarios, subproblem, stage, conn
     )
-
-    df = cursor_to_df(hurdle_rates)
+    hrs_df_unformatted = cursor_to_df(hurdle_rates)
+    periods_months = conn.execute(
+        """SELECT DISTINCT period, month
+        FROM inputs_temporal
+        WHERE temporal_scenario_id = {}
+        AND subproblem_id = {}
+        AND stage_id = {};""".format(
+            subscenarios.TEMPORAL_SCENARIO_ID, subproblem, stage
+        )
+    )
+    periods_months_df = cursor_to_df(periods_months)
+    df = format_monthly_hurdle_data(hrs_df_unformatted, periods_months_df)
 
     # Get expected dtypes
     expected_dtypes = get_expected_dtypes(

tests/test_data/inputs/transmission_hurdle_rates.tab

@@ -1,5 +1,7 @@
-transmission_line	period	hurdle_rate_positive_direction_per_mwh	hurdle_rate_negative_direction_per_mwh
-Tx1	2020	1.0	1.0
-Tx1	2030	1.0	1.0
-Tx_New	2030	0.0	0.0
-Tx_New	2030	0.0	0.0
+transmission_line	period	month	hurdle_rate_positive_direction_per_mwh	hurdle_rate_negative_direction_per_mwh
+Tx1	2020	5	1.5	5.1
+Tx1	2020	9	1.9	9.1
+Tx1	2030	4	31.4	34.4
+Tx1	2030	7	31.7	37.1
+Tx_New	2020	0	20.20	20.0
+Tx_New	2030	0	30.30	30.0