From b17a12d9002ec8d6932e267327e6cf2c519fb4e7 Mon Sep 17 00:00:00 2001 From: Behnam Date: Sun, 3 Feb 2019 22:20:42 +0100 Subject: [PATCH 01/44] main function f_addNewYear added for adding new time steps --- toolbox/add_year/f_addNewYear.py | 1220 ++++++++++++++++++++++++++++++ 1 file changed, 1220 insertions(+) create mode 100644 toolbox/add_year/f_addNewYear.py diff --git a/toolbox/add_year/f_addNewYear.py b/toolbox/add_year/f_addNewYear.py new file mode 100644 index 000000000..a0e4dd173 --- /dev/null +++ b/toolbox/add_year/f_addNewYear.py @@ -0,0 +1,1220 @@ +# -*- coding: utf-8 -*- +""" +Description: + This functionality adds new time steps to an existing MESSAGE scenario + (hereafter "reference scenario"). This is done by creating a new empty + scenario (hereafter "new scenario") and: + - Copying all sets from reference scenario and adding new time steps to + relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") + - Copying all parameters from reference scenario, adding new time steps to + relevant parameters, calculating missing values for the added time steps. + +Sections of this code: + I. Required python packages are imported and ixmp platform loaded. + II. Generic utilities for dataframe manipulation + III. The main class called "addNewYear" + IV. Submodule "addSets" for adding and modifying the sets + V. Submodule "addPar" for copying and modifying each parameter + VI. The submodule "addPar" calls two utility functions ("df_interpolate" + and "df_interpolate_2D") for calculating missing values. + VII. Code for running the script as "main" + + +Usage: + This script can be used either: + A) By running directly from the command line, example: + --------------------------------------------------------------------------- + python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" + --years_new "[2015,2025,2035,2045]" + --------------------------------------------------------------------------- + (Other input arguments are optional. For more info see Section V below.) + + B) By calling the class "addNewYear" from other python scripts +""" +# %% I) Importing required packages and loading ixmp platform + +import numpy as np +import pandas as pd +from timeit import default_timer as timer +import argparse +import ixmp as ix +import message_ix + +mp = ix.Platform(dbtype='HSQLDB') # Loading ixmp Platform + +# %% II) Required utility functions for dataframe manupulation +# II.A) Utility function for interpolation/extrapolation of two numbers, +# lists or series (x: time steps, y: data points) + + +def intpol(y1, y2, x1, x2, x): + if x2 == x1 and y2 != y1: + print('>>> Warning <<<: No difference between x1 and x2,' + + 'returned empty!!!') + return [] + elif x2 == x1 and y2 == y1: + return y1 + else: + y = y1 + ((y2 - y1) / (x2 - x1)) * (x - x1) + return y + +# II.B) Utility function for slicing a MultiIndex dataframe and +# setting a value to a specific level +# df: dataframe, idx: list of indexes, level: string, locator: list, +# value: integer/string + + +def f_slice(df, idx, level, locator, value): + if locator: + df = df.reset_index().loc[df.reset_index()[level].isin(locator)].copy() + else: + df = df.reset_index().copy() + if value: + df[level] = value + return df.set_index(idx) + +# II.C) A function for creating a mask for removing extra values from a +# dataframe + + +def f_mask(df, index, count, value): + df.loc[index, df.columns > (df.loc[[index]].notnull().cumsum( + axis=1) == count).idxmax(axis=1).values[0]] = value + +# II.D) Function for unifroming the "unit" in different years to prevent +# mistakes in indexing and grouping + + +def unit_uniform(df): + column = [x for x in df.columns if x in ['commodity', 'emission']] + if column: + com_list = set(df[column[0]]) + for com in com_list: + df.loc[df[column[0]] == com, 'unit'] = df.loc[ + df[column[0]] == com, 'unit'].mode()[0] + else: + df['unit'] = df['unit'].mode()[0] + return df + +# %% III) The main class + + +class addNewYear(object): + ''' This class does the following: + A. calls function "addSets" to add and modify required sets. + B. calls function "addPar" to add new years and modifications + to each parameter if needed. + + Parameters: + ----------- + sc_ref : string + reference scenario (MESSAGE model/scenario instance) + sc_new : string + new scenario for adding new years and required modifications + (MESSAGE model/scenario instance) + yrs_new : list of integers + new years to be added + firstyear_new : integer, default None + a new first model year for new scenario (optional) + macro : boolean, default False + a flag to add new years to macro parameters (True) or not (False) + baseyear_macro : integer, default None + a new base year for macro + parameter: list of strings, default all + parameters for adding new years + rewrite: boolean, default True + a flag to permit rewriting a parameter in new scenario when adding + new years (True) or not (False) + check_unit: boolean, default False + a flag to uniform the units under each commodity (if there is + inconsistency between units in two model years) + extrapol_neg: float, default None + a number to multiply by the data of the previous timestep, + when extrapolation is negative + bound_extend: boolean, default True + a flag to permit the duplication of the data from the previous + timestep, when there is only one data point for interpolation + (e.g., permitting the extension of a bound to 2025, when there + is only one value in 2020) + ''' + + def __init__(self, sc_ref, sc_new, years_new, firstyear_new=None, + lastyear_new=None, macro=False, baseyear_macro=None, + parameter=all, region=all, rewrite=True, unit_check=True, + extrapol_neg=None, bound_extend=True): + + # III.A) Adding sets and required modifications + years_new = sorted([x for x in years_new if str(x) + not in set(sc_ref.set('year'))]) + self.addSets( + sc_ref, + sc_new, + years_new, + firstyear_new, + lastyear_new, + baseyear_macro) + # -------------------------------------------------------------------------- + # III.B) Adding parameters and calculating the missing values for the + # additonal years + if parameter == all: + par_list = sorted(sc_ref.par_list()) + elif isinstance(parameter, list): + par_list = parameter + elif isinstance(parameter, str): + par_list = [parameter] + else: + print( + 'Parameters should be defined in a list of strings or as' + + 'a single string!') + + if 'technical_lifetime' in par_list: + par_list.insert( + 0, par_list.pop( + par_list.index('technical_lifetime'))) + + if region == all: + reg_list = sc_ref.set('node').tolist() + elif isinstance(region, list): + reg_list = region + elif isinstance(region, str): + reg_list = [region] + else: + print('Regions should be defined in a list of strings or as' + + 'a single string!') + + # List of parameters to be ignored (even not copied to the new + # scenario) + par_ignore = ['duration_period'] + par_list = [x for x in par_list if x not in par_ignore] + + if not macro: + par_macro = [ + 'demand_MESSAGE', + 'price_MESSAGE', + 'cost_MESSAGE', + 'gdp_calibrate', + 'historical_gdp', + 'MERtoPPP', + 'kgdp', + 'kpvs', + 'depr', + 'drate', + 'esub', + 'lotol', + 'p_ref', + 'lakl', + 'prfconst', + 'grow', + 'aeei', + 'aeei_factor', + 'gdp_rate'] + par_list = [x for x in par_list if x not in par_macro] + + for parname in par_list: + # For historical parameters extrapolation permitted (e.g., from + # 2010 to 2015) + if 'historical' in parname: + extrapol = True + yrs_new = [ + x for x in years_new if x < int( + sc_new.set( + 'cat_year', { + 'type_year': 'firstmodelyear'})['year'])] + else: + extrapol = False + yrs_new = years_new + + if 'bound' in parname: + bound_ext = bound_extend + else: + bound_ext = True + + year_list = [x for x in sc_ref.idx_sets(parname) if 'year' in x] + + if len(year_list) == 2 or parname in ['land_output']: + # The loop over "node" is only for reducing the size of tables + for node in reg_list: + self.addPar( + sc_ref, + sc_new, + yrs_new, + parname, + [node], + extrapolate=extrapol, + rewrite=rewrite, + unit_check=unit_check, + extrapol_neg=extrapol_neg, + bound_extend=bound_ext) + else: + self.addPar( + sc_ref, + sc_new, + yrs_new, + parname, + reg_list, + extrapolate=extrapol, + rewrite=rewrite, + unit_check=unit_check, + extrapol_neg=extrapol_neg, + bound_extend=bound_ext) + + sc_new.set_as_default() + print('> All required parameters were successfully ' + + 'added to the new scenario.') + + # %% Submodules needed for running the main function + # IV) Adding new years to sets + def addSets( + self, + sc_ref, + sc_new, + years_new, + firstyear_new=None, + lastyear_new=None, + baseyear_macro=None): + ''' + Description: + Adding required sets and relevant modifications: + This function adds additional years to an existing scenario, + by starting to make a new scenario from scratch. + After modification of the year-related sets, this function copeis + all other sets from the "reference" scenario + to the "new" scenario. + + Input arguments: + Please see the description for the input arguments under the main + function "addNewYear" + + Usage: + This module is called by function "addNewYear" + ''' + # IV.A) Treatment of the additional years in the year-related sets + + # A.1. Set - year + yrs_old = list(map(int, sc_ref.set('year'))) + horizon_new = sorted(yrs_old + years_new) + sc_new.add_set('year', [str(yr) for yr in horizon_new]) + + # A.2. Set _ type_year + yr_typ = sc_ref.set('type_year').tolist() + sc_new.add_set('type_year', sorted( + yr_typ + [str(yr) for yr in years_new])) + + # A.3. Set _ cat_year + yr_cat = sc_ref.set('cat_year') + + # A.4. Changing the first year if needed + if firstyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'firstmodelyear', 'year'] = firstyear_new + if lastyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'lastmodelyear', 'year'] = lastyear_new + + # A.5. Changing the base year and initialization year of macro if a new + # year specified + if not yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'] = baseyear_macro + if not yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year' + ].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'] = baseyear_macro + + yr_pair = [] + for yr in years_new: + yr_pair.append([yr, yr]) + yr_pair.append(['cumulative', yr]) + + yr_cat = yr_cat.append( + pd.DataFrame( + yr_pair, + columns=[ + 'type_year', + 'year']), + ignore_index=True).sort_values('year').reset_index( + drop=True) + + # A.6. Changing the cumulative years based on the new first model year + firstyear_new = int( + yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', 'year']) + yr_cat = yr_cat.drop(yr_cat.loc[ + (yr_cat['type_year'] == 'cumulative') & ( + yr_cat['year'] < firstyear_new)].index) + sc_new.add_set('cat_year', yr_cat) + + # IV.B) Copying all other sets + set_list = [s for s in sc_ref.set_list() if 'year' not in s] + # Sets with one index set + index_list = [ + x for x in set_list if not isinstance( + sc_ref.set(x), pd.DataFrame)] + for set_name in index_list: + if set_name not in sc_new.set_list(): + sc_new.init_set(set_name, idx_sets=None, idx_names=None) + sc_new.add_set(set_name, sc_ref.set(set_name).tolist()) + + # The rest of the sets + for set_name in [x for x in set_list if x not in index_list]: + if set_name not in sc_new.set_list() and not [ + x for x in sc_ref.idx_sets( + set_name) if x not in sc_ref.set_list()]: + sc_new.init_set(set_name, + idx_sets=sc_ref.idx_sets(set_name).tolist(), + idx_names=sc_ref.idx_names(set_name).tolist()) + sc_new.add_set(set_name, sc_ref.set(set_name)) + + sc_new.commit('sets added!') + print('> All the sets updated and added to the new scenario.') + + # %% V) Adding new years to parameters + + def addPar( + self, + sc_ref, + sc_new, + yrs_new, + parname, + region_list, + extrapolate=False, + rewrite=True, + unit_check=True, + extrapol_neg=None, + bound_extend=True): + ''' Adding required parameters and relevant modifications: + Description: + This function adds additional years to a parameter. + The value of the parameter for additional years is calculated + mainly by interpolating and extrapolating of data from existing + years. + + Input arguments: + Please see the description for the input arguments under the + main function "addNewYear" + + Usage: + This module is called by function "addNewYear" + ''' + + # V.A) Initialization and checks + + par_list_new = sc_new.par_list().tolist() + idx_names = sc_ref.idx_names(parname) + horizon = sorted([int(x) for x in list(set(sc_ref.set('year')))]) + node_col = [ + x for x in idx_names if x in [ + 'node', + 'node_loc', + 'node_rel']] + + if parname not in par_list_new: + sc_new.check_out() + sc_new.init_par( + parname, + idx_sets=sc_ref.idx_sets(parname).tolist(), + idx_names=sc_ref.idx_names(parname).tolist()) + sc_new.commit('New parameter initiated!') + + if node_col: + par_old = sc_ref.par(parname, {node_col[0]: region_list}) + par_new = sc_new.par(parname, {node_col[0]: region_list}) + sort_order = [ + node_col[0], + 'technology', + 'commodity', + 'year_vtg', + 'year_act'] + nodes = par_old[node_col[0]].unique().tolist() + else: + par_old = sc_ref.par(parname) + par_new = sc_new.par(parname) + sort_order = ['technology', 'commodity', 'year_vtg', 'year_act'] + nodes = ['N/A'] + + if not par_new.empty and not rewrite: + print( + '> Parameter "' + parname + '" already has data ' + 'in new scenario and left unchanged for node/s: {}.'.format( + region_list)) + return + if par_old.empty: + print( + '> Parameter "' + parname + '" is empty in reference scenario ' + 'for node/s: {}!'.format(region_list)) + return + + # Sorting the data to make it ready for dataframe manupulation + sort_order = [x for x in sort_order if x in idx_names] + if sort_order: + par_old = par_old.sort_values(sort_order).reset_index(drop=True) + + sc_new.check_out() + if not par_new.empty and rewrite: + print( + '> Parameter "' + parname + '" is being removed from new ' + 'scenario to be updated for node/s in {}...'.format(nodes)) + sc_new.remove_par(parname, par_new) + + col_list = sc_ref.idx_names(parname).tolist() + year_list = [ + c for c in col_list if c in [ + 'year', + 'year_vtg', + 'year_act', + 'year_rel']] + + # A uniform "unit" for values in different years to prevent mistakes in + # indexing and grouping + if 'unit' in col_list and unit_check: + par_old = unit_uniform(par_old) + # -------------------------------------------------------------------------------------------------------- + # V.B) Treatment of the new years in the specified parameter based on + # the time-related dimension of that parameter + # V.B.1) Parameters with no time component + if len(year_list) == 0: + sc_new.add_par(parname, par_old) + sc_new.commit(parname) + print( + '> Parameter "' + parname + '" just copied to new scenario ' + 'since has no time-related entries.') + + # V.B.2) Parameters with one dimension related to time + elif len(year_list) == 1: + year_col = year_list[0] + df = par_old.copy() + df_y = self.df_interpolate( + df, + yrs_new, + horizon, + year_col, + 'value', + extrapolate=extrapolate, + extrapol_neg=extrapol_neg, + bound_extend=bound_extend) + sc_new.add_par(parname, df_y) + sc_new.commit(' ') + print( + '> Parameter "' + parname + '" copied and new years ' + 'added for node/s: "{}".'.format(nodes)) + + # V.B.3) Parameters with two dimensions related to time (such as + # 'input','output', etc.) + elif len(year_list) == 2: + year_col = 'year_act' + node_col = 'node_loc' + year_ref = [x for x in year_list if x != year_col][0] + + year_diff = [x for x in horizon[1:- + 1] if horizon[ + horizon.index(x) + 1] - + horizon[horizon.index(x)] > horizon[ + horizon.index(x)] - horizon[ + horizon.index(x) - 1]] + print( + '> Parameter "{}" is being added for node/s "{}"...'.format( + parname, + nodes)) + + # Flagging technologies that have lifetime for adding new timesteps + firstyear_new = int( + sc_new.set( + 'cat_year', { + 'type_year': 'firstmodelyear'})['year']) + min_step = min(np.diff( + sorted([int(x) for x in set(sc_new.set('year')) if int( + x) > firstyear_new]))) + par_tec = sc_new.par( + 'technical_lifetime', { + 'node_loc': nodes}) + # Technologies with lifetime bigger than minimum time interval + par_tec = par_tec.loc[par_tec['value'] > min_step] + df = par_old.copy() + + if parname == 'relation_activity': + tec_list = [] + else: + tec_list = [t for t in list(set(df[ + 'technology'])) if t in list(set(par_tec[ + 'technology']))] + + df_y = self.df_interpolate_2d( + df, + yrs_new, + horizon, + year_ref, + year_col, + tec_list, + par_tec, + value_col='value', + extrapolate=extrapolate, + extrapol_neg=extrapol_neg, + year_diff=year_diff) + sc_new.add_par(parname, df_y) + sc_new.commit(parname) + print( + '> Parameter "' + parname + '" copied and new years added ' + 'for node/s: "{}".'.format(nodes)) + + # %% VI) Required functions + # VI.A) A function to add new years to a datafarme by interpolation and + # (extrapolation if needed) + + def df_interpolate( + self, + df, + yrs_new, + horizon, + year_col, + value_col='value', + extrapolate=False, + extrapol_neg=None, + bound_extend=True): + ''' + Description: + This function receives a parameter data as a dataframe, and adds + new data for the additonal years by interpolation and + extrapolation. + + Input arguments: + df_par (dataframe): the dataframe of the parameter to which new + years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_col (string): the header of the column to which the new years + should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This function is called by function "addPar" + ''' + horizon_new = sorted(horizon + yrs_new) + idx = [x for x in df.columns if x not in [year_col, value_col]] + if not df.empty: + df2 = df.pivot_table(index=idx, columns=year_col, values=value_col) + + # To sort the new years smaller than the first year for + # extrapolation (e.g. 2025 values are calculated first; then + # values of 2015 based on 2020 and 2025) + year_before = sorted( + [x for x in yrs_new if x < min(df2.columns)], reverse=True) + if year_before and extrapolate: + for y in year_before: + yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y))) + + for yr in yrs_new: + if yr > max(horizon): + extrapol = True + else: + extrapol = extrapolate + + # a) If this new year is after the current range of modeled + # years, do extrapolation + if extrapol: + if yr == horizon_new[horizon_new.index( + max(df2.columns)) + 1]: + year_pre = max([x for x in df2.columns if x < yr]) + + if len([x for x in df2.columns if x < yr]) >= 2: + year_pp = max( + [x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol(df2[year_pre], df2[year_pp], + year_pre, year_pp, yr) + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) + + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & (df2[ + year_pre] >= 0), + year_pre] * extrapol_neg + else: + df2[yr] = df2[year_pre] + + # b) If this new year is before the current range of modeled + # years, do extrapolation + elif yr < min(df2.columns) and extrapol: + year_next = min([x for x in df2.columns if x > yr]) + + if len([x for x in df2.columns if x > yr]) >= 2: + year_nn = horizon[horizon.index(yr) + 2] + df2[yr] = intpol( + df2[year_next], df2[year_nn], + year_next, year_nn, yr) + df2[yr][np.isinf(df2[year_next])] = df2[year_next] + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_next] >= 0)].empty: + df2.loc[(df2[yr] < 0) & + (df2[year_next] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_next] >= 0), + year_next] * extrapol_neg + + elif bound_extend: + df2[yr] = df2[year_next] + + # c) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(df2.columns): + year_pre = max([x for x in df2.columns if x < yr]) + year_next = min([x for x in df2.columns if x > yr]) + df2[yr] = intpol( + df2[year_pre], df2[year_next], year_pre, year_next, yr) + + # Extrapolate for new years if the value exists for the + # previous year but not for the next years + # TODO: here is the place that should be changed if the + # new year should go the time step before the existing one + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2[yr] = df2[yr].fillna( + intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr)) + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre] >= 0), + year_pre] * extrapol_neg + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + + df2 = pd.melt( + df2.reset_index(), + id_vars=idx, + value_vars=[ + x for x in df2.columns if x not in idx], + var_name=year_col, + value_name=value_col).dropna( + subset=[value_col]).reset_index( + drop=True) + df2 = df2.sort_values(idx).reset_index(drop=True) + else: + print( + '+++ WARNING: The submitted dataframe is empty, so returned' + + 'empty results!!! +++') + df2 = df + return df2 + + # %% VI.B) A function to interpolate the data for new time steps in + # parameters with two dimensions related to time + + def df_interpolate_2d( + self, + df, + yrs_new, + horizon, + year_ref, + year_col, + tec_list, + par_tec, + value_col='value', + extrapolate=False, + extrapol_neg=None, + year_diff=None): + ''' + Description: + This function receives a dataframe that has 2 time-related columns + (e.g., "input" or "relation_activity"), and adds new data for the + additonal years in both time-related columns by interpolation + and extrapolation. + + Input arguments: + df (dataframe): the parameter to which new years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_ref (string): the header of the first column to which the new + years should be added, for example, "year_vtg" + year_col (string): the header of the second column to which the + new years should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This utility is called by function "addPar" + ''' + def f_index(df1, df2): return df1.loc[df1.index.isin( + df2.index)] # For checking the index of two dataframes + + idx = [x for x in df.columns if x not in [year_col, value_col]] + if df.empty: + return df + print( + '+++ WARNING: The submitted dataframe is empty, so' + + 'returned empty results!!! +++') + + df_tec = df.loc[df['technology'].isin(tec_list)] + df2 = df.pivot_table(index=idx, columns=year_col, values='value') + df2_tec = df_tec.pivot_table( + index=idx, columns=year_col, values='value') + + # ------------------------------------------------------------------------------ + # First, changing the time interval for the transition period + # (e.g., year 2010 in old R11 model transits from 5 year to 10 year) + horizon_new = sorted(horizon + + [x for x in yrs_new if x not in horizon]) + yr_diff_new = [x for x in horizon_new[1:- + 1] if horizon_new[ + horizon_new.index(x) + + 1] - + horizon_new[horizon_new.index(x)] > horizon_new[ + horizon_new.index(x)] - horizon_new[ + horizon_new.index(x) - 1]] + + if year_diff and tec_list: + if isinstance(year_diff, list): + year_diff = year_diff[0] + + # Removing data from old transition year + if not yr_diff_new or year_diff not in yr_diff_new: + year_next = [x for x in df2.columns if x > year_diff][0] + + df_pre = f_slice( + df2_tec, idx, year_ref, [year_diff], year_diff) + df_next = f_slice( + df2_tec, idx, year_ref, [year_next], year_diff) + df_count = pd.DataFrame({ + 'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[df_next.index.isin( + df_pre.index)].count(axis=1)}, + index=df_pre.index) + df_y = df_count.loc[df_count['c_pre'] == df_count[ + 'c_next'] + 1] + + for i in df_y.index: + df2.loc[i, df2.columns > (df2.loc[[i]].notnull().cumsum( + axis=1) == df_count[ + 'c_next'][i]).idxmax(axis=1).values[0]] = np.nan + + # Generating duration_period_sum matrix for masking + df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) + for i in df_dur.index: + for j in [x for x in df_dur.columns if x > i]: + df_dur.loc[i, j] = j - i + + # Adding data for new transition year + if yr_diff_new and tec_list and year_diff not in yr_diff_new: + yrs = [x for x in horizon if x <= yr_diff_new[0]] + year_next = min([x for x in df2.columns if x > yr_diff_new[0]]) + df_yrs = f_slice(df2_tec, idx, year_ref, yrs, []) + if yr_diff_new[0] in df2.columns: + df_yrs = df_yrs.loc[~pd.isna(df_yrs[yr_diff_new[0]]), :] + df_yrs = df_yrs.append( + f_slice( + df2_tec, + idx, + year_ref, + [year_next], + []), + ignore_index=False).reset_index( + ).sort_values(idx).set_index(idx) + + for yr in sorted( + [x for x in list(set(df_yrs.reset_index( + )[year_ref])) if x < year_next]): + yr_next = min([x for x in horizon_new if x > yr]) + d = f_slice(df_yrs, idx, year_ref, [yr], []) + d_n = f_slice(df_yrs, idx, year_ref, [yr_next], yr) + + if d_n[year_next].loc[~pd.isna(d_n[year_next])].empty: + if [x for x in horizon_new if x > yr_next]: + yr_nn = min([x for x in horizon_new if x > yr_next]) + else: + yr_nn = yr_next + d_n = f_slice(df_yrs, idx, year_ref, [yr_nn], yr) + d_n = d_n.loc[d_n.index.isin(d.index), :] + d = d.loc[d.index.isin(d_n.index), :] + d[d.isnull() & d_n.notnull()] = d_n + df2.loc[df2.index.isin(d.index), :] = d + + df_dur.loc[df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] = df_dur.loc[ + df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] - ( + yr_diff_new[0] - horizon_new[horizon_new.index( + yr_diff_new[0]) - 1]) + # -------------------------------------------------------------------------- + # Second, adding year_act of new years when year_vtg is in the existing + # years + for yr in yrs_new: + if yr > max(horizon): + extrapol = True + else: + extrapol = extrapolate + + # a) If this new year is greater than the current range of modeled + # years, do extrapolation + if yr > horizon_new[horizon_new.index( + max(df2.columns))] and extrapol: + year_pre = max([x for x in df2.columns if x < yr]) + year_pp = max([x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr) + df2[yr][np.isinf(df2[year_pre].shift(+1)) + ] = df2[year_pre].shift(+1) + df2[yr] = df2[yr].fillna(df2[year_pre]) + + if yr - horizon_new[horizon_new.index(yr) - 1] >= horizon_new[ + horizon_new.index(yr) - 1] - horizon_new[ + horizon_new.index(yr) - 2]: + + df2[yr].loc[pd.isna(df2[year_pre].shift(+1)) + & ~pd.isna(df2[year_pp].shift(+1))] = np.nan + if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0), + year_pre] * extrapol_neg + + # b) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(df2.columns): + year_pre = max([x for x in df2.columns if x < yr]) + year_next = min([x for x in df2.columns if x > yr]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_next], + year_pre, + year_next, + yr) + df2_t = df2.loc[df2_tec.index, :].copy() + + # This part calculates the missing value if only the previous + # timestep has a value (and not the next) + if tec_list: + df2_t[yr].loc[(pd.isna(df2_t[yr])) & (~pd.isna(df2_t[ + year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_next].shift(-1), + year_pre, year_next, yr) + + # Treating technologies with phase-out in model years + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2_t[yr].loc[(pd.isna(df2_t[yr])) & ( + pd.isna(df2_t[year_pre].shift(-1))) & ( + ~pd.isna(df2_t[year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_pp], + year_pre, year_pp, yr) + + if extrapol_neg and not df2_t[yr].loc[( + df2_t[yr] < 0) & (df2_t[year_pre] >= 0)].empty: + df2_t.loc[(df2_t[yr] < 0) & (df2_t[year_pre] >= 0), + yr] = df2_t.loc[(df2_t[yr] < 0) & ( + df2_t[year_pre] >= 0), + year_pre] * extrapol_neg + df2.loc[df2_tec.index, :] = df2_t + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + df2 = df2.reindex(sorted(df2.columns), axis=1) + # -------------------------------------------------------------------------- + # Third, adding year_vtg of new years and their respective year_act for + # both existing and new years + for yr in yrs_new: + # a) If this new year is after the current range of modeled years, + # do extrapolation + if yr > max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_pp = horizon_new[horizon_new.index(yr) - 2] + df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) + df_pp = f_slice(df2, idx, year_ref, [year_pp], yr) + df_yr = intpol( + df_pre, + f_index( + df_pp, + df_pre), + year_pre, + year_pp, + yr) + df_yr[np.isinf(df_pre)] = df_pre + + # For those technolofies with one value for each year + df_yr.loc[pd.isna(df_yr[yr])] = intpol( + df_pre, df_pp.shift(+1, axis=1), + year_pre, year_pp, yr).shift(+1, axis=1) + df_yr[pd.isna(df_yr)] = df_pre + + if extrapol_neg: + df_yr[(df_yr < 0) & (df_pre >= 0)] = df_pre * extrapol_neg + df_yr.loc[:, df_yr.columns < yr] = np.nan + + # c) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_next = min([x for x in horizon if x > yr]) + + df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) + df_next = f_slice(df2, idx, year_ref, [year_next], yr) + df_yr = pd.concat(( + df_pre, + df_next.loc[df_next.index.isin(df_pre.index)]), + axis=0).groupby(level=idx).mean() + df_yr[yr] = df_yr[yr].fillna( + df_yr[[year_pre, year_next]].mean(axis=1)) + df_yr[np.isinf(df_pre)] = df_pre + + # Creating a mask to remove extra values + df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[ + df_next.index.isin(df_pre.index)].count(axis=1)}, + index=df_yr.index) + + for i in df_yr.index: + # Mainly for cases of two new consecutive years (like 2022 + # and 2024) + if ~np.isnan( + df_count['c_next'][i]) and df_count[ + 'c_pre'][i] >= df_count['c_next'][i] + 2: + df_yr[year_pre] = np.nan + + # For technologies phasing out before the end of horizon + # (like nuc_lc) + elif np.isnan(df_count['c_next'][i]): + df_yr.loc[i, :] = df_pre.loc[i, :].shift(+1) + if tec_list: + f_mask(df_yr, i, df_count['c_pre'][i] + 1, np.nan) + else: + f_mask(df_yr, i, df_count['c_pre'][i], np.nan) + + # For the rest + else: + df_yr[year_pre] = np.nan + f_mask(df_yr, i, df_count['c_pre'][i], np.nan) + + else: + continue + + df2 = df2.append(df_yr) + df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() + # --------------------------------------------------------------------------- + # Forth: final masking based on technical lifetime + if tec_list: + + df3 = df2.copy() + for y in sorted([x for x in list( + set(df2.index.get_level_values(year_ref)) + ) if x in df_dur.index]): + df3.loc[df3.index.get_level_values(year_ref).isin([y]), + df3.columns.isin(df_dur.columns)] = df_dur.loc[ + y, df_dur.columns.isin(df3.columns)].values + + df3 = df3.reset_index().set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + par_tec = par_tec.set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + + for i in [x for x in par_tec.index if x in df3.index]: + df3.loc[i, 'lifetime'] = par_tec.loc[i, 'value'].copy() + + df3 = df3.reset_index().set_index(idx).dropna( + subset=['lifetime']).sort_index() + for i in df3.index: + df2.loc[i, df3.loc[i, :] >= int( + df3.loc[i, 'lifetime'])] = np.nan + + # Removing extra values from non-lifetime technologies + for i in [x for x in df2.index if x not in df3.index]: + df2.loc[i, df2.columns > df2.loc[[i]].index.get_level_values( + year_ref)[0]] = np.nan + + df_par = pd.melt( + df2.reset_index(), + id_vars=idx, + value_vars=[ + x for x in df2.columns if x not in idx], + var_name=year_col, + value_name='value').dropna( + subset=['value']) + df_par = df_par.sort_values(idx).reset_index(drop=True) + return df_par + + +# %% VII) Input arguments related to running the script from command line +# Text formatting for long help texts +class SmartFormatter(argparse.HelpFormatter): + + def _split_lines(self, text, width): + if text.startswith('B|'): + return text[2:].splitlines() + return argparse.HelpFormatter._split_lines(self, text, width) + + +def read_args(): + + annon = """ + Adding additional years to a MESSAGE model/scenario instance + + Example: + python f_addYear.py --model_ref "Austria_tutorial" --scen_ref "test_core" + --scen_new "test_5y" --years_new "[2015,2025,2035,2045]" + + """ + parser = argparse.ArgumentParser( + description=annon, + formatter_class=SmartFormatter) + + parser.add_argument('--model_ref', + help="B| --model_ref: string\n" + "reference model name") + parser.add_argument('--scen_ref', + help="B| --scen_ref: string\n" + "reference scenario name") + parser.add_argument('--version_ref', + help="B| --version_ref: integer (standard=None)\n" + "version number of reference scenario") + parser.add_argument('--model_new', + help='B| --model_new: string (standard=None)\n' + 'new model name') + parser.add_argument('--scen_new', + help='B| --scen_new: string (optional)\n' + 'new scenario name') + parser.add_argument('--create_new', + help='B| --create_new: string (standard=True)\n' + 'True, for creating a new scenario\n' + 'False, for using an existing scenario') + parser.add_argument('--years_new', + help='--years_new : list\n new years to be added') + parser.add_argument('--firstyear_new', + help='B| --firstyear_new: integer (standard=None)\n' + 'new first model year') + parser.add_argument('--lastyear_new', + help='B| --lastyear_new: integer (standard=None)\n' + 'new last model year') + parser.add_argument('--macro', + help='B| --macro: string (standard=False)\n' + 'True, for adding new years to macro parameters\n' + 'False, for ignoring new years for macro parameters') + parser.add_argument('--baseyear_macro', + help='B| --baseyear_macro : integer (standard=None)\n' + 'new base year for macro\n') + parser.add_argument('--parameter', + help='B| --parameter: list (standard=all)\n' + 'list of parameters for adding new years to them\n' + 'all, for all the parameters') + parser.add_argument('--region', + help='B| --region: list (standard=all)\n' + 'list of regions for adding new years to them\n' + 'all, for all the regions') + parser.add_argument('--rewrite', + help='B| --rewrite: string (standard=True)\n' + 'rewriting parameters in the new scenario') + parser.add_argument('--unit_check', + help='B|--unit_check: string (standard=False)\n' + 'checking units before adding new years') + parser.add_argument('--extrapol_neg', + help='B| --extrapol_neg: string (standard=None)\n' + 'treating negative values from extrapolation of\n' + 'two positive values:\n' + 'None: does nothing\n' + 'integer: multiplier to adjacent value' + 'replacing negative one') + parser.add_argument('--bound_extend', + help='B| --bound_extend : string (standard=True)\n' + 'copying data from previous timestep if only one data' + 'available for interpolation:\n') + + args = parser.parse_args() + return args + + +# %% If run as main script +if __name__ == '__main__': + start = timer() + print('>> Running the script f_addYears.py...') + args = read_args() + model_ref = args.model_ref + scen_ref = args.scen_ref + version_ref = int(args.version_ref) if args.version_ref else None + + model_new = args.model_new if args.model_new else args.model_ref + scen_new = args.scen_new if args.scen_new else str(args.scen_ref + '_5y') + create_new = args.create_new if args.create_new else True + print(args.years_new) + years_new = list(map(int, args.years_new.strip('[]').split(','))) + firstyear_new = int(args.firstyear_new) if args.firstyear_new else None + lastyear_new = int(args.lastyear_new) if args.lastyear_new else None + macro = args.macro if args.macro else False + baseyear_macro = int(args.baseyear_macro) if args.baseyear_macro else None + + parameter = list(map(str, args.parameter.strip( + '[]').split(','))) if args.parameter else all + region = list(map(str, args.region.strip('[]').split(',')) + ) if args.region else all + rewrite = args.rewrite if args.rewrite else True + unit_check = args.unit_check if args.unit_check else True + extrapol_neg = args.extrapol_neg if args.extrapol_neg else 0.5 + bound_extend = args.bound_extend if args.bound_extend else True + + # Loading the reference scenario and creating a new scenario to add the + # additional years + if version_ref: + sc_ref = message_ix.Scenario( + mp, + model=model_ref, + scen=scen_ref, + version=version_ref) + else: + sc_ref = message_ix.Scenario(mp, model=model_ref, scen=scen_ref) + + if create_new: + sc_new = message_ix.Scenario( + mp, + model=model_new, + scen=scen_new, + version='new', + scheme='MESSAGE', + annotation='5 year modelling') + else: + sc_new = message_ix.Scenario(mp, model=model_new, scen=scen_new) + if sc_new.has_solution(): + sc_new.remove_solution() + sc_new.check_out() + + # Calling the main function + addNewYear( + sc_ref, + sc_new, + years_new, + firstyear_new, + lastyear_new, + macro, + baseyear_macro, + parameter, + region, + rewrite, + unit_check, + extrapol_neg, + bound_extend) + end = timer() + mp.close_db() + print('> Elapsed time for adding new years:', round((end - start) / 60), + 'min and', round((end - start) % 60, 1), 'sec.') + print('> New scenario with additional years is: "{}"|"{}"|{}'.format( + sc_new.model, sc_new.scenario, str(sc_new.version))) + +# Sample input arguments from commandline +# python f_addNewYear.py --model_ref "CD_Links_SSP2" --scen_ref "baseline" +# --years_new "[2015,2025,2035,2045,2055,2120]" From 24cbd19eed8c132a3b8afd4b71e0c67777acd4e4 Mon Sep 17 00:00:00 2001 From: Behnam Date: Sun, 3 Feb 2019 22:20:59 +0100 Subject: [PATCH 02/44] test for function f_addNewYear added --- tests/test_feature_add_year.py | 93 ++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tests/test_feature_add_year.py diff --git a/tests/test_feature_add_year.py b/tests/test_feature_add_year.py new file mode 100644 index 000000000..216b64f62 --- /dev/null +++ b/tests/test_feature_add_year.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# Importing required packages +import os +import math +import ixmp as ix +import message_ix +from conftest import here + +db_dir = os.path.join(here, 'testdb') +test_db = os.path.join(db_dir, 'ixmptest') +test_mp = ix.Platform(dbprops=test_db, dbtype='HSQLDB') + +# the path to where the main script is (based on message_ix folder hirerarchy) +file_path = os.path.join(here, '..', 'toolbox', 'add_year') +os.chdir(file_path) +from f_addNewYear import addNewYear + + +def test_addNewYear(test_mp): + msg_multiyear_args = ('canning problem (MESSAGE scheme)', 'multi-year') + sc_ref = message_ix.Scenario(test_mp, *msg_multiyear_args) + assert sc_ref.par('technical_lifetime')[ + 'technology'].isin(['canning_plant']).any() + sc_ref.solve() + + # Building a new scenario and adding new years + sc_new = message_ix.Scenario(test_mp, + model='foo', + scenario='bar', + version='new', + annonation=' ') + + # Using the function addNewYear + years_new = [2015, 2025] + + addNewYear( + sc_ref, + sc_new, + years_new, + ) + + # 1. Testing the set "year" for the new years + horizon_old = sorted([int(x) for x in sc_ref.set('year')]) + horizon_test = sorted(horizon_old + years_new) + + horizon_new = sorted([int(x) for x in sc_new.set('year')]) + + # Asserting if the lists are equal + assert (horizon_test == horizon_new) + + # 2. Testing parameter "technical_lifetime" + df_tec = sc_ref.par( + 'technical_lifetime', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_vtg': [2020, 2030]}) + value_ref = df_tec['value'].mean() + + df_tec_new = sc_new.par( + 'technical_lifetime', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_vtg': 2025}) + + value_new = float(df_tec_new['value']) + + # Asserting if the missing data is generated accurately by interpolation + # of adjacent data points + assert math.isclose(value_ref, value_new, rel_tol=1e-04) + + # 3. Testing parameter "output" + df_tec = sc_ref.par( + 'output', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_act': 2030, + 'year_vtg': [2020, 2030]}) + + value_ref = df_tec['value'].mean() + + df_tec_new = sc_new.par( + 'output', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_act': 2025, + 'year_vtg': 2025}) + + # Asserting if the missing data is generated or not + assert df_tec_new['value'].tolist() + + # Asserting if the missing data is generated accurately by interpolation + value_new = float(df_tec_new['value']) + assert math.isclose(value_ref, value_new, rel_tol=1e-04) From bd9fc75dfab96c683fc1ca5a1ebcda0636935c6b Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 08:53:29 +0100 Subject: [PATCH 03/44] Create README.md --- toolbox/add_year/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 toolbox/add_year/README.md diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md new file mode 100644 index 000000000..fffd07300 --- /dev/null +++ b/toolbox/add_year/README.md @@ -0,0 +1,23 @@ +# Description +This functionality adds new time steps to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: +- Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") +- Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. + +# Main steps +1. An existing scenario is loaded and the desired new years is specified. The new years can be in between existing years or beyond the existing model horizon. +2. A new (empty) scenario is created for adding the new time steps. +3. The new years added to the relevant sets. This include: +- adding new years to the set "year" +- adding new years to the set "cat_year" +- changing "first_modelyear" and "last_modelyear" if needed. + +Usage: + This script can be used either: + A) By running directly from the command line, example: + --------------------------------------------------------------------------- + python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" + --years_new "[2015,2025,2035,2045]" + --------------------------------------------------------------------------- + (Other input arguments are optional. For more info see Section V below.) + + B) By calling the class "addNewYear" from other python scripts From 221480d7e4feb64f4300929e89acf079c2002dcd Mon Sep 17 00:00:00 2001 From: Behnam Date: Mon, 4 Feb 2019 08:53:59 +0100 Subject: [PATCH 04/44] release notes updated --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2277e149a..4bc779e73 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -115,3 +115,4 @@ jdbc.pwd = ixmp - [#65](https://github.com/iiasa/message_ix/pull/65): Bugfix for downloading tutorials. Now downloads current installed version by default. - [#60](https://github.com/iiasa/message_ix/pull/60): Add basic ability to write and read model input to/from Excel - [#59](https://github.com/iiasa/message_ix/pull/59): Added MacOSX CI support +- [#**](https://github.com/iiasa/message_ix/pull/**): A feature for adding new time steps to a scenario From 61afa10945a1be6ba2aa5f0489c98f2213784bb9 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:02:43 +0100 Subject: [PATCH 05/44] Update README.md --- toolbox/add_year/README.md | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index fffd07300..4efd0127d 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -3,13 +3,29 @@ This functionality adds new time steps to an existing scenario (hereafter "refer - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. +# Main features +- It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. +- The new years can be consecutive, between existing years, and after the model horizon. +- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter for modifications. + # Main steps -1. An existing scenario is loaded and the desired new years is specified. The new years can be in between existing years or beyond the existing model horizon. +1. An existing scenario is loaded and the desired new years is specified. 2. A new (empty) scenario is created for adding the new time steps. -3. The new years added to the relevant sets. This include: -- adding new years to the set "year" -- adding new years to the set "cat_year" -- changing "first_modelyear" and "last_modelyear" if needed. +3. The new years are added to the relevant sets: +- adding new years to the set "year" and "type_year" +- changing "firstmodelyear", "lastmodelyear", "baseyear_macro", and "initializeyear_macro" if needed. +- modifying the set "cat_year" for the new years. +4. The new years are added to the index sets of relevant parameters, and the missing data for the new years are calculated based on interpolation of adjacent data points. The following steps are applied: +- each non-empty parameter is loaded from the reference scenario +- the year-related indexes of the parameter are identified (either 0, 1, and 2 index under MESSAGE scheme) +- the new years are added to the parameter, and the missing data is calculated based on the number of year-related indexes. For example, the new years are added to index "year_vtg" in parameter "inv_cost", while these new years are added both to "year_vtg" and "year_act" in parameter "output". +- the missing data is calculated by interpolation. +- for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. +5. The changes are commited and saved to the new scenario. + +# Notice: +I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. + Usage: This script can be used either: From fdc1a06431edbfa5ebd51dc24adfc7593cd4e04c Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:40:52 +0100 Subject: [PATCH 06/44] Update README.md --- toolbox/add_year/README.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 4efd0127d..8912b69ba 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -6,7 +6,7 @@ This functionality adds new time steps to an existing scenario (hereafter "refer # Main features - It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. - The new years can be consecutive, between existing years, and after the model horizon. -- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter for modifications. +- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. # Main steps 1. An existing scenario is loaded and the desired new years is specified. @@ -26,14 +26,9 @@ This functionality adds new time steps to an existing scenario (hereafter "refer # Notice: I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. - -Usage: - This script can be used either: - A) By running directly from the command line, example: - --------------------------------------------------------------------------- - python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" - --years_new "[2015,2025,2035,2045]" - --------------------------------------------------------------------------- - (Other input arguments are optional. For more info see Section V below.) - - B) By calling the class "addNewYear" from other python scripts +# Usage: +This script can be used either: +A) By running directly from the command line, example: +python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" +(For the full list of input arguments see the explanation in the code. +B) By calling the class "addNewYear" from another python script. From 22a0ab244c895cc6296ac02a5bdaea48b4ec0241 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:44:06 +0100 Subject: [PATCH 07/44] Update README.md --- toolbox/add_year/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 8912b69ba..f873bdd65 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -1,14 +1,14 @@ -# Description +## Description This functionality adds new time steps to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. -# Main features +## Main features - It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. - The new years can be consecutive, between existing years, and after the model horizon. - The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. -# Main steps +## Main steps 1. An existing scenario is loaded and the desired new years is specified. 2. A new (empty) scenario is created for adding the new time steps. 3. The new years are added to the relevant sets: @@ -23,12 +23,14 @@ This functionality adds new time steps to an existing scenario (hereafter "refer - for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. 5. The changes are commited and saved to the new scenario. -# Notice: +## Notice: I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. -# Usage: +## Usage: This script can be used either: A) By running directly from the command line, example: -python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" + ``` + python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" + ``` (For the full list of input arguments see the explanation in the code. B) By calling the class "addNewYear" from another python script. From f19a7bd085839676ee66731bf807c47239a4dbad Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:45:35 +0100 Subject: [PATCH 08/44] Update README.md --- toolbox/add_year/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index f873bdd65..93a3f9b24 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -32,5 +32,5 @@ A) By running directly from the command line, example: ``` python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" ``` -(For the full list of input arguments see the explanation in the code. +For the full list of input arguments see the explanation in the code. B) By calling the class "addNewYear" from another python script. From dfc963077ecc4f11d062ea0c25b4825853c556f3 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:46:54 +0100 Subject: [PATCH 09/44] Update README.md --- toolbox/add_year/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 93a3f9b24..169c81740 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -28,9 +28,10 @@ I. This functionality in the current format does not ensure that the new scenari ## Usage: This script can be used either: -A) By running directly from the command line, example: +- By running directly from the command line, example: ``` python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" ``` + For the full list of input arguments see the explanation in the code. -B) By calling the class "addNewYear" from another python script. +- By calling the class "addNewYear" from another python script. From 017be480ad30250ace06976386ee8384aaea1680 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:58:47 +0100 Subject: [PATCH 10/44] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4bc779e73..270ca0c71 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -115,4 +115,4 @@ jdbc.pwd = ixmp - [#65](https://github.com/iiasa/message_ix/pull/65): Bugfix for downloading tutorials. Now downloads current installed version by default. - [#60](https://github.com/iiasa/message_ix/pull/60): Add basic ability to write and read model input to/from Excel - [#59](https://github.com/iiasa/message_ix/pull/59): Added MacOSX CI support -- [#**](https://github.com/iiasa/message_ix/pull/**): A feature for adding new time steps to a scenario +- [#161](https://github.com/iiasa/message_ix/pull/161): A feature for adding new time steps to a scenario From a9bf7753880be811ca17accea783ad47bcb19faf Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 15:57:14 +0100 Subject: [PATCH 11/44] Update f_addNewYear.py --- toolbox/add_year/f_addNewYear.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolbox/add_year/f_addNewYear.py b/toolbox/add_year/f_addNewYear.py index a0e4dd173..5f840a60e 100644 --- a/toolbox/add_year/f_addNewYear.py +++ b/toolbox/add_year/f_addNewYear.py @@ -880,8 +880,8 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( horizon_new.index(yr) - 1] - horizon_new[ horizon_new.index(yr) - 2]: - df2[yr].loc[pd.isna(df2[year_pre].shift(+1)) - & ~pd.isna(df2[year_pp].shift(+1))] = np.nan + df2[yr].loc[(pd.isna(df2[year_pre].shift(+1))) & ( + ~pd.isna(df2[year_pp].shift(+1)))] = np.nan if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( df2[year_pre].shift(+1) >= 0)].empty: df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), From c6d4a22d5d9bd5907d62eaab5c7cb2651156102d Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Tue, 9 Apr 2019 18:04:12 +0200 Subject: [PATCH 12/44] Update README.md --- toolbox/add_year/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 169c81740..e8df02917 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -1,5 +1,5 @@ ## Description -This functionality adds new time steps to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: +This functionality adds new modeling years to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. From a66e566e83308608b27347fc309dc7014354ab56 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 2 May 2019 10:44:01 +0200 Subject: [PATCH 13/44] Use message_ix.tools instead of toolbox namespace --- message_ix/tools/__init__.py | 0 {toolbox => message_ix/tools}/add_year/README.md | 0 .../tools/add_year/__init__.py | 0 tests/{test_feature_add_year.py => tools/test_add_year.py} | 7 ++----- 4 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 message_ix/tools/__init__.py rename {toolbox => message_ix/tools}/add_year/README.md (100%) rename toolbox/add_year/f_addNewYear.py => message_ix/tools/add_year/__init__.py (100%) rename tests/{test_feature_add_year.py => tools/test_add_year.py} (93%) diff --git a/message_ix/tools/__init__.py b/message_ix/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/toolbox/add_year/README.md b/message_ix/tools/add_year/README.md similarity index 100% rename from toolbox/add_year/README.md rename to message_ix/tools/add_year/README.md diff --git a/toolbox/add_year/f_addNewYear.py b/message_ix/tools/add_year/__init__.py similarity index 100% rename from toolbox/add_year/f_addNewYear.py rename to message_ix/tools/add_year/__init__.py diff --git a/tests/test_feature_add_year.py b/tests/tools/test_add_year.py similarity index 93% rename from tests/test_feature_add_year.py rename to tests/tools/test_add_year.py index 216b64f62..0a422098f 100644 --- a/tests/test_feature_add_year.py +++ b/tests/tools/test_add_year.py @@ -6,15 +6,12 @@ import message_ix from conftest import here +from message_ix.tools.add_year import addNewYear + db_dir = os.path.join(here, 'testdb') test_db = os.path.join(db_dir, 'ixmptest') test_mp = ix.Platform(dbprops=test_db, dbtype='HSQLDB') -# the path to where the main script is (based on message_ix folder hirerarchy) -file_path = os.path.join(here, '..', 'toolbox', 'add_year') -os.chdir(file_path) -from f_addNewYear import addNewYear - def test_addNewYear(test_mp): msg_multiyear_args = ('canning problem (MESSAGE scheme)', 'multi-year') From aae58ae707b4bf607bef6ab6961ab1e85c351e09 Mon Sep 17 00:00:00 2001 From: Behnam Date: Sun, 3 Feb 2019 22:20:42 +0100 Subject: [PATCH 14/44] main function f_addNewYear added for adding new time steps --- toolbox/add_year/f_addNewYear.py | 1220 ++++++++++++++++++++++++++++++ 1 file changed, 1220 insertions(+) create mode 100644 toolbox/add_year/f_addNewYear.py diff --git a/toolbox/add_year/f_addNewYear.py b/toolbox/add_year/f_addNewYear.py new file mode 100644 index 000000000..a0e4dd173 --- /dev/null +++ b/toolbox/add_year/f_addNewYear.py @@ -0,0 +1,1220 @@ +# -*- coding: utf-8 -*- +""" +Description: + This functionality adds new time steps to an existing MESSAGE scenario + (hereafter "reference scenario"). This is done by creating a new empty + scenario (hereafter "new scenario") and: + - Copying all sets from reference scenario and adding new time steps to + relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") + - Copying all parameters from reference scenario, adding new time steps to + relevant parameters, calculating missing values for the added time steps. + +Sections of this code: + I. Required python packages are imported and ixmp platform loaded. + II. Generic utilities for dataframe manipulation + III. The main class called "addNewYear" + IV. Submodule "addSets" for adding and modifying the sets + V. Submodule "addPar" for copying and modifying each parameter + VI. The submodule "addPar" calls two utility functions ("df_interpolate" + and "df_interpolate_2D") for calculating missing values. + VII. Code for running the script as "main" + + +Usage: + This script can be used either: + A) By running directly from the command line, example: + --------------------------------------------------------------------------- + python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" + --years_new "[2015,2025,2035,2045]" + --------------------------------------------------------------------------- + (Other input arguments are optional. For more info see Section V below.) + + B) By calling the class "addNewYear" from other python scripts +""" +# %% I) Importing required packages and loading ixmp platform + +import numpy as np +import pandas as pd +from timeit import default_timer as timer +import argparse +import ixmp as ix +import message_ix + +mp = ix.Platform(dbtype='HSQLDB') # Loading ixmp Platform + +# %% II) Required utility functions for dataframe manupulation +# II.A) Utility function for interpolation/extrapolation of two numbers, +# lists or series (x: time steps, y: data points) + + +def intpol(y1, y2, x1, x2, x): + if x2 == x1 and y2 != y1: + print('>>> Warning <<<: No difference between x1 and x2,' + + 'returned empty!!!') + return [] + elif x2 == x1 and y2 == y1: + return y1 + else: + y = y1 + ((y2 - y1) / (x2 - x1)) * (x - x1) + return y + +# II.B) Utility function for slicing a MultiIndex dataframe and +# setting a value to a specific level +# df: dataframe, idx: list of indexes, level: string, locator: list, +# value: integer/string + + +def f_slice(df, idx, level, locator, value): + if locator: + df = df.reset_index().loc[df.reset_index()[level].isin(locator)].copy() + else: + df = df.reset_index().copy() + if value: + df[level] = value + return df.set_index(idx) + +# II.C) A function for creating a mask for removing extra values from a +# dataframe + + +def f_mask(df, index, count, value): + df.loc[index, df.columns > (df.loc[[index]].notnull().cumsum( + axis=1) == count).idxmax(axis=1).values[0]] = value + +# II.D) Function for unifroming the "unit" in different years to prevent +# mistakes in indexing and grouping + + +def unit_uniform(df): + column = [x for x in df.columns if x in ['commodity', 'emission']] + if column: + com_list = set(df[column[0]]) + for com in com_list: + df.loc[df[column[0]] == com, 'unit'] = df.loc[ + df[column[0]] == com, 'unit'].mode()[0] + else: + df['unit'] = df['unit'].mode()[0] + return df + +# %% III) The main class + + +class addNewYear(object): + ''' This class does the following: + A. calls function "addSets" to add and modify required sets. + B. calls function "addPar" to add new years and modifications + to each parameter if needed. + + Parameters: + ----------- + sc_ref : string + reference scenario (MESSAGE model/scenario instance) + sc_new : string + new scenario for adding new years and required modifications + (MESSAGE model/scenario instance) + yrs_new : list of integers + new years to be added + firstyear_new : integer, default None + a new first model year for new scenario (optional) + macro : boolean, default False + a flag to add new years to macro parameters (True) or not (False) + baseyear_macro : integer, default None + a new base year for macro + parameter: list of strings, default all + parameters for adding new years + rewrite: boolean, default True + a flag to permit rewriting a parameter in new scenario when adding + new years (True) or not (False) + check_unit: boolean, default False + a flag to uniform the units under each commodity (if there is + inconsistency between units in two model years) + extrapol_neg: float, default None + a number to multiply by the data of the previous timestep, + when extrapolation is negative + bound_extend: boolean, default True + a flag to permit the duplication of the data from the previous + timestep, when there is only one data point for interpolation + (e.g., permitting the extension of a bound to 2025, when there + is only one value in 2020) + ''' + + def __init__(self, sc_ref, sc_new, years_new, firstyear_new=None, + lastyear_new=None, macro=False, baseyear_macro=None, + parameter=all, region=all, rewrite=True, unit_check=True, + extrapol_neg=None, bound_extend=True): + + # III.A) Adding sets and required modifications + years_new = sorted([x for x in years_new if str(x) + not in set(sc_ref.set('year'))]) + self.addSets( + sc_ref, + sc_new, + years_new, + firstyear_new, + lastyear_new, + baseyear_macro) + # -------------------------------------------------------------------------- + # III.B) Adding parameters and calculating the missing values for the + # additonal years + if parameter == all: + par_list = sorted(sc_ref.par_list()) + elif isinstance(parameter, list): + par_list = parameter + elif isinstance(parameter, str): + par_list = [parameter] + else: + print( + 'Parameters should be defined in a list of strings or as' + + 'a single string!') + + if 'technical_lifetime' in par_list: + par_list.insert( + 0, par_list.pop( + par_list.index('technical_lifetime'))) + + if region == all: + reg_list = sc_ref.set('node').tolist() + elif isinstance(region, list): + reg_list = region + elif isinstance(region, str): + reg_list = [region] + else: + print('Regions should be defined in a list of strings or as' + + 'a single string!') + + # List of parameters to be ignored (even not copied to the new + # scenario) + par_ignore = ['duration_period'] + par_list = [x for x in par_list if x not in par_ignore] + + if not macro: + par_macro = [ + 'demand_MESSAGE', + 'price_MESSAGE', + 'cost_MESSAGE', + 'gdp_calibrate', + 'historical_gdp', + 'MERtoPPP', + 'kgdp', + 'kpvs', + 'depr', + 'drate', + 'esub', + 'lotol', + 'p_ref', + 'lakl', + 'prfconst', + 'grow', + 'aeei', + 'aeei_factor', + 'gdp_rate'] + par_list = [x for x in par_list if x not in par_macro] + + for parname in par_list: + # For historical parameters extrapolation permitted (e.g., from + # 2010 to 2015) + if 'historical' in parname: + extrapol = True + yrs_new = [ + x for x in years_new if x < int( + sc_new.set( + 'cat_year', { + 'type_year': 'firstmodelyear'})['year'])] + else: + extrapol = False + yrs_new = years_new + + if 'bound' in parname: + bound_ext = bound_extend + else: + bound_ext = True + + year_list = [x for x in sc_ref.idx_sets(parname) if 'year' in x] + + if len(year_list) == 2 or parname in ['land_output']: + # The loop over "node" is only for reducing the size of tables + for node in reg_list: + self.addPar( + sc_ref, + sc_new, + yrs_new, + parname, + [node], + extrapolate=extrapol, + rewrite=rewrite, + unit_check=unit_check, + extrapol_neg=extrapol_neg, + bound_extend=bound_ext) + else: + self.addPar( + sc_ref, + sc_new, + yrs_new, + parname, + reg_list, + extrapolate=extrapol, + rewrite=rewrite, + unit_check=unit_check, + extrapol_neg=extrapol_neg, + bound_extend=bound_ext) + + sc_new.set_as_default() + print('> All required parameters were successfully ' + + 'added to the new scenario.') + + # %% Submodules needed for running the main function + # IV) Adding new years to sets + def addSets( + self, + sc_ref, + sc_new, + years_new, + firstyear_new=None, + lastyear_new=None, + baseyear_macro=None): + ''' + Description: + Adding required sets and relevant modifications: + This function adds additional years to an existing scenario, + by starting to make a new scenario from scratch. + After modification of the year-related sets, this function copeis + all other sets from the "reference" scenario + to the "new" scenario. + + Input arguments: + Please see the description for the input arguments under the main + function "addNewYear" + + Usage: + This module is called by function "addNewYear" + ''' + # IV.A) Treatment of the additional years in the year-related sets + + # A.1. Set - year + yrs_old = list(map(int, sc_ref.set('year'))) + horizon_new = sorted(yrs_old + years_new) + sc_new.add_set('year', [str(yr) for yr in horizon_new]) + + # A.2. Set _ type_year + yr_typ = sc_ref.set('type_year').tolist() + sc_new.add_set('type_year', sorted( + yr_typ + [str(yr) for yr in years_new])) + + # A.3. Set _ cat_year + yr_cat = sc_ref.set('cat_year') + + # A.4. Changing the first year if needed + if firstyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'firstmodelyear', 'year'] = firstyear_new + if lastyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'lastmodelyear', 'year'] = lastyear_new + + # A.5. Changing the base year and initialization year of macro if a new + # year specified + if not yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'] = baseyear_macro + if not yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year' + ].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'] = baseyear_macro + + yr_pair = [] + for yr in years_new: + yr_pair.append([yr, yr]) + yr_pair.append(['cumulative', yr]) + + yr_cat = yr_cat.append( + pd.DataFrame( + yr_pair, + columns=[ + 'type_year', + 'year']), + ignore_index=True).sort_values('year').reset_index( + drop=True) + + # A.6. Changing the cumulative years based on the new first model year + firstyear_new = int( + yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', 'year']) + yr_cat = yr_cat.drop(yr_cat.loc[ + (yr_cat['type_year'] == 'cumulative') & ( + yr_cat['year'] < firstyear_new)].index) + sc_new.add_set('cat_year', yr_cat) + + # IV.B) Copying all other sets + set_list = [s for s in sc_ref.set_list() if 'year' not in s] + # Sets with one index set + index_list = [ + x for x in set_list if not isinstance( + sc_ref.set(x), pd.DataFrame)] + for set_name in index_list: + if set_name not in sc_new.set_list(): + sc_new.init_set(set_name, idx_sets=None, idx_names=None) + sc_new.add_set(set_name, sc_ref.set(set_name).tolist()) + + # The rest of the sets + for set_name in [x for x in set_list if x not in index_list]: + if set_name not in sc_new.set_list() and not [ + x for x in sc_ref.idx_sets( + set_name) if x not in sc_ref.set_list()]: + sc_new.init_set(set_name, + idx_sets=sc_ref.idx_sets(set_name).tolist(), + idx_names=sc_ref.idx_names(set_name).tolist()) + sc_new.add_set(set_name, sc_ref.set(set_name)) + + sc_new.commit('sets added!') + print('> All the sets updated and added to the new scenario.') + + # %% V) Adding new years to parameters + + def addPar( + self, + sc_ref, + sc_new, + yrs_new, + parname, + region_list, + extrapolate=False, + rewrite=True, + unit_check=True, + extrapol_neg=None, + bound_extend=True): + ''' Adding required parameters and relevant modifications: + Description: + This function adds additional years to a parameter. + The value of the parameter for additional years is calculated + mainly by interpolating and extrapolating of data from existing + years. + + Input arguments: + Please see the description for the input arguments under the + main function "addNewYear" + + Usage: + This module is called by function "addNewYear" + ''' + + # V.A) Initialization and checks + + par_list_new = sc_new.par_list().tolist() + idx_names = sc_ref.idx_names(parname) + horizon = sorted([int(x) for x in list(set(sc_ref.set('year')))]) + node_col = [ + x for x in idx_names if x in [ + 'node', + 'node_loc', + 'node_rel']] + + if parname not in par_list_new: + sc_new.check_out() + sc_new.init_par( + parname, + idx_sets=sc_ref.idx_sets(parname).tolist(), + idx_names=sc_ref.idx_names(parname).tolist()) + sc_new.commit('New parameter initiated!') + + if node_col: + par_old = sc_ref.par(parname, {node_col[0]: region_list}) + par_new = sc_new.par(parname, {node_col[0]: region_list}) + sort_order = [ + node_col[0], + 'technology', + 'commodity', + 'year_vtg', + 'year_act'] + nodes = par_old[node_col[0]].unique().tolist() + else: + par_old = sc_ref.par(parname) + par_new = sc_new.par(parname) + sort_order = ['technology', 'commodity', 'year_vtg', 'year_act'] + nodes = ['N/A'] + + if not par_new.empty and not rewrite: + print( + '> Parameter "' + parname + '" already has data ' + 'in new scenario and left unchanged for node/s: {}.'.format( + region_list)) + return + if par_old.empty: + print( + '> Parameter "' + parname + '" is empty in reference scenario ' + 'for node/s: {}!'.format(region_list)) + return + + # Sorting the data to make it ready for dataframe manupulation + sort_order = [x for x in sort_order if x in idx_names] + if sort_order: + par_old = par_old.sort_values(sort_order).reset_index(drop=True) + + sc_new.check_out() + if not par_new.empty and rewrite: + print( + '> Parameter "' + parname + '" is being removed from new ' + 'scenario to be updated for node/s in {}...'.format(nodes)) + sc_new.remove_par(parname, par_new) + + col_list = sc_ref.idx_names(parname).tolist() + year_list = [ + c for c in col_list if c in [ + 'year', + 'year_vtg', + 'year_act', + 'year_rel']] + + # A uniform "unit" for values in different years to prevent mistakes in + # indexing and grouping + if 'unit' in col_list and unit_check: + par_old = unit_uniform(par_old) + # -------------------------------------------------------------------------------------------------------- + # V.B) Treatment of the new years in the specified parameter based on + # the time-related dimension of that parameter + # V.B.1) Parameters with no time component + if len(year_list) == 0: + sc_new.add_par(parname, par_old) + sc_new.commit(parname) + print( + '> Parameter "' + parname + '" just copied to new scenario ' + 'since has no time-related entries.') + + # V.B.2) Parameters with one dimension related to time + elif len(year_list) == 1: + year_col = year_list[0] + df = par_old.copy() + df_y = self.df_interpolate( + df, + yrs_new, + horizon, + year_col, + 'value', + extrapolate=extrapolate, + extrapol_neg=extrapol_neg, + bound_extend=bound_extend) + sc_new.add_par(parname, df_y) + sc_new.commit(' ') + print( + '> Parameter "' + parname + '" copied and new years ' + 'added for node/s: "{}".'.format(nodes)) + + # V.B.3) Parameters with two dimensions related to time (such as + # 'input','output', etc.) + elif len(year_list) == 2: + year_col = 'year_act' + node_col = 'node_loc' + year_ref = [x for x in year_list if x != year_col][0] + + year_diff = [x for x in horizon[1:- + 1] if horizon[ + horizon.index(x) + 1] - + horizon[horizon.index(x)] > horizon[ + horizon.index(x)] - horizon[ + horizon.index(x) - 1]] + print( + '> Parameter "{}" is being added for node/s "{}"...'.format( + parname, + nodes)) + + # Flagging technologies that have lifetime for adding new timesteps + firstyear_new = int( + sc_new.set( + 'cat_year', { + 'type_year': 'firstmodelyear'})['year']) + min_step = min(np.diff( + sorted([int(x) for x in set(sc_new.set('year')) if int( + x) > firstyear_new]))) + par_tec = sc_new.par( + 'technical_lifetime', { + 'node_loc': nodes}) + # Technologies with lifetime bigger than minimum time interval + par_tec = par_tec.loc[par_tec['value'] > min_step] + df = par_old.copy() + + if parname == 'relation_activity': + tec_list = [] + else: + tec_list = [t for t in list(set(df[ + 'technology'])) if t in list(set(par_tec[ + 'technology']))] + + df_y = self.df_interpolate_2d( + df, + yrs_new, + horizon, + year_ref, + year_col, + tec_list, + par_tec, + value_col='value', + extrapolate=extrapolate, + extrapol_neg=extrapol_neg, + year_diff=year_diff) + sc_new.add_par(parname, df_y) + sc_new.commit(parname) + print( + '> Parameter "' + parname + '" copied and new years added ' + 'for node/s: "{}".'.format(nodes)) + + # %% VI) Required functions + # VI.A) A function to add new years to a datafarme by interpolation and + # (extrapolation if needed) + + def df_interpolate( + self, + df, + yrs_new, + horizon, + year_col, + value_col='value', + extrapolate=False, + extrapol_neg=None, + bound_extend=True): + ''' + Description: + This function receives a parameter data as a dataframe, and adds + new data for the additonal years by interpolation and + extrapolation. + + Input arguments: + df_par (dataframe): the dataframe of the parameter to which new + years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_col (string): the header of the column to which the new years + should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This function is called by function "addPar" + ''' + horizon_new = sorted(horizon + yrs_new) + idx = [x for x in df.columns if x not in [year_col, value_col]] + if not df.empty: + df2 = df.pivot_table(index=idx, columns=year_col, values=value_col) + + # To sort the new years smaller than the first year for + # extrapolation (e.g. 2025 values are calculated first; then + # values of 2015 based on 2020 and 2025) + year_before = sorted( + [x for x in yrs_new if x < min(df2.columns)], reverse=True) + if year_before and extrapolate: + for y in year_before: + yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y))) + + for yr in yrs_new: + if yr > max(horizon): + extrapol = True + else: + extrapol = extrapolate + + # a) If this new year is after the current range of modeled + # years, do extrapolation + if extrapol: + if yr == horizon_new[horizon_new.index( + max(df2.columns)) + 1]: + year_pre = max([x for x in df2.columns if x < yr]) + + if len([x for x in df2.columns if x < yr]) >= 2: + year_pp = max( + [x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol(df2[year_pre], df2[year_pp], + year_pre, year_pp, yr) + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) + + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & (df2[ + year_pre] >= 0), + year_pre] * extrapol_neg + else: + df2[yr] = df2[year_pre] + + # b) If this new year is before the current range of modeled + # years, do extrapolation + elif yr < min(df2.columns) and extrapol: + year_next = min([x for x in df2.columns if x > yr]) + + if len([x for x in df2.columns if x > yr]) >= 2: + year_nn = horizon[horizon.index(yr) + 2] + df2[yr] = intpol( + df2[year_next], df2[year_nn], + year_next, year_nn, yr) + df2[yr][np.isinf(df2[year_next])] = df2[year_next] + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_next] >= 0)].empty: + df2.loc[(df2[yr] < 0) & + (df2[year_next] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_next] >= 0), + year_next] * extrapol_neg + + elif bound_extend: + df2[yr] = df2[year_next] + + # c) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(df2.columns): + year_pre = max([x for x in df2.columns if x < yr]) + year_next = min([x for x in df2.columns if x > yr]) + df2[yr] = intpol( + df2[year_pre], df2[year_next], year_pre, year_next, yr) + + # Extrapolate for new years if the value exists for the + # previous year but not for the next years + # TODO: here is the place that should be changed if the + # new year should go the time step before the existing one + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2[yr] = df2[yr].fillna( + intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr)) + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre] >= 0), + year_pre] * extrapol_neg + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + + df2 = pd.melt( + df2.reset_index(), + id_vars=idx, + value_vars=[ + x for x in df2.columns if x not in idx], + var_name=year_col, + value_name=value_col).dropna( + subset=[value_col]).reset_index( + drop=True) + df2 = df2.sort_values(idx).reset_index(drop=True) + else: + print( + '+++ WARNING: The submitted dataframe is empty, so returned' + + 'empty results!!! +++') + df2 = df + return df2 + + # %% VI.B) A function to interpolate the data for new time steps in + # parameters with two dimensions related to time + + def df_interpolate_2d( + self, + df, + yrs_new, + horizon, + year_ref, + year_col, + tec_list, + par_tec, + value_col='value', + extrapolate=False, + extrapol_neg=None, + year_diff=None): + ''' + Description: + This function receives a dataframe that has 2 time-related columns + (e.g., "input" or "relation_activity"), and adds new data for the + additonal years in both time-related columns by interpolation + and extrapolation. + + Input arguments: + df (dataframe): the parameter to which new years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_ref (string): the header of the first column to which the new + years should be added, for example, "year_vtg" + year_col (string): the header of the second column to which the + new years should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This utility is called by function "addPar" + ''' + def f_index(df1, df2): return df1.loc[df1.index.isin( + df2.index)] # For checking the index of two dataframes + + idx = [x for x in df.columns if x not in [year_col, value_col]] + if df.empty: + return df + print( + '+++ WARNING: The submitted dataframe is empty, so' + + 'returned empty results!!! +++') + + df_tec = df.loc[df['technology'].isin(tec_list)] + df2 = df.pivot_table(index=idx, columns=year_col, values='value') + df2_tec = df_tec.pivot_table( + index=idx, columns=year_col, values='value') + + # ------------------------------------------------------------------------------ + # First, changing the time interval for the transition period + # (e.g., year 2010 in old R11 model transits from 5 year to 10 year) + horizon_new = sorted(horizon + + [x for x in yrs_new if x not in horizon]) + yr_diff_new = [x for x in horizon_new[1:- + 1] if horizon_new[ + horizon_new.index(x) + + 1] - + horizon_new[horizon_new.index(x)] > horizon_new[ + horizon_new.index(x)] - horizon_new[ + horizon_new.index(x) - 1]] + + if year_diff and tec_list: + if isinstance(year_diff, list): + year_diff = year_diff[0] + + # Removing data from old transition year + if not yr_diff_new or year_diff not in yr_diff_new: + year_next = [x for x in df2.columns if x > year_diff][0] + + df_pre = f_slice( + df2_tec, idx, year_ref, [year_diff], year_diff) + df_next = f_slice( + df2_tec, idx, year_ref, [year_next], year_diff) + df_count = pd.DataFrame({ + 'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[df_next.index.isin( + df_pre.index)].count(axis=1)}, + index=df_pre.index) + df_y = df_count.loc[df_count['c_pre'] == df_count[ + 'c_next'] + 1] + + for i in df_y.index: + df2.loc[i, df2.columns > (df2.loc[[i]].notnull().cumsum( + axis=1) == df_count[ + 'c_next'][i]).idxmax(axis=1).values[0]] = np.nan + + # Generating duration_period_sum matrix for masking + df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) + for i in df_dur.index: + for j in [x for x in df_dur.columns if x > i]: + df_dur.loc[i, j] = j - i + + # Adding data for new transition year + if yr_diff_new and tec_list and year_diff not in yr_diff_new: + yrs = [x for x in horizon if x <= yr_diff_new[0]] + year_next = min([x for x in df2.columns if x > yr_diff_new[0]]) + df_yrs = f_slice(df2_tec, idx, year_ref, yrs, []) + if yr_diff_new[0] in df2.columns: + df_yrs = df_yrs.loc[~pd.isna(df_yrs[yr_diff_new[0]]), :] + df_yrs = df_yrs.append( + f_slice( + df2_tec, + idx, + year_ref, + [year_next], + []), + ignore_index=False).reset_index( + ).sort_values(idx).set_index(idx) + + for yr in sorted( + [x for x in list(set(df_yrs.reset_index( + )[year_ref])) if x < year_next]): + yr_next = min([x for x in horizon_new if x > yr]) + d = f_slice(df_yrs, idx, year_ref, [yr], []) + d_n = f_slice(df_yrs, idx, year_ref, [yr_next], yr) + + if d_n[year_next].loc[~pd.isna(d_n[year_next])].empty: + if [x for x in horizon_new if x > yr_next]: + yr_nn = min([x for x in horizon_new if x > yr_next]) + else: + yr_nn = yr_next + d_n = f_slice(df_yrs, idx, year_ref, [yr_nn], yr) + d_n = d_n.loc[d_n.index.isin(d.index), :] + d = d.loc[d.index.isin(d_n.index), :] + d[d.isnull() & d_n.notnull()] = d_n + df2.loc[df2.index.isin(d.index), :] = d + + df_dur.loc[df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] = df_dur.loc[ + df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] - ( + yr_diff_new[0] - horizon_new[horizon_new.index( + yr_diff_new[0]) - 1]) + # -------------------------------------------------------------------------- + # Second, adding year_act of new years when year_vtg is in the existing + # years + for yr in yrs_new: + if yr > max(horizon): + extrapol = True + else: + extrapol = extrapolate + + # a) If this new year is greater than the current range of modeled + # years, do extrapolation + if yr > horizon_new[horizon_new.index( + max(df2.columns))] and extrapol: + year_pre = max([x for x in df2.columns if x < yr]) + year_pp = max([x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr) + df2[yr][np.isinf(df2[year_pre].shift(+1)) + ] = df2[year_pre].shift(+1) + df2[yr] = df2[yr].fillna(df2[year_pre]) + + if yr - horizon_new[horizon_new.index(yr) - 1] >= horizon_new[ + horizon_new.index(yr) - 1] - horizon_new[ + horizon_new.index(yr) - 2]: + + df2[yr].loc[pd.isna(df2[year_pre].shift(+1)) + & ~pd.isna(df2[year_pp].shift(+1))] = np.nan + if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0), + year_pre] * extrapol_neg + + # b) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(df2.columns): + year_pre = max([x for x in df2.columns if x < yr]) + year_next = min([x for x in df2.columns if x > yr]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_next], + year_pre, + year_next, + yr) + df2_t = df2.loc[df2_tec.index, :].copy() + + # This part calculates the missing value if only the previous + # timestep has a value (and not the next) + if tec_list: + df2_t[yr].loc[(pd.isna(df2_t[yr])) & (~pd.isna(df2_t[ + year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_next].shift(-1), + year_pre, year_next, yr) + + # Treating technologies with phase-out in model years + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2_t[yr].loc[(pd.isna(df2_t[yr])) & ( + pd.isna(df2_t[year_pre].shift(-1))) & ( + ~pd.isna(df2_t[year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_pp], + year_pre, year_pp, yr) + + if extrapol_neg and not df2_t[yr].loc[( + df2_t[yr] < 0) & (df2_t[year_pre] >= 0)].empty: + df2_t.loc[(df2_t[yr] < 0) & (df2_t[year_pre] >= 0), + yr] = df2_t.loc[(df2_t[yr] < 0) & ( + df2_t[year_pre] >= 0), + year_pre] * extrapol_neg + df2.loc[df2_tec.index, :] = df2_t + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + df2 = df2.reindex(sorted(df2.columns), axis=1) + # -------------------------------------------------------------------------- + # Third, adding year_vtg of new years and their respective year_act for + # both existing and new years + for yr in yrs_new: + # a) If this new year is after the current range of modeled years, + # do extrapolation + if yr > max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_pp = horizon_new[horizon_new.index(yr) - 2] + df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) + df_pp = f_slice(df2, idx, year_ref, [year_pp], yr) + df_yr = intpol( + df_pre, + f_index( + df_pp, + df_pre), + year_pre, + year_pp, + yr) + df_yr[np.isinf(df_pre)] = df_pre + + # For those technolofies with one value for each year + df_yr.loc[pd.isna(df_yr[yr])] = intpol( + df_pre, df_pp.shift(+1, axis=1), + year_pre, year_pp, yr).shift(+1, axis=1) + df_yr[pd.isna(df_yr)] = df_pre + + if extrapol_neg: + df_yr[(df_yr < 0) & (df_pre >= 0)] = df_pre * extrapol_neg + df_yr.loc[:, df_yr.columns < yr] = np.nan + + # c) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_next = min([x for x in horizon if x > yr]) + + df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) + df_next = f_slice(df2, idx, year_ref, [year_next], yr) + df_yr = pd.concat(( + df_pre, + df_next.loc[df_next.index.isin(df_pre.index)]), + axis=0).groupby(level=idx).mean() + df_yr[yr] = df_yr[yr].fillna( + df_yr[[year_pre, year_next]].mean(axis=1)) + df_yr[np.isinf(df_pre)] = df_pre + + # Creating a mask to remove extra values + df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[ + df_next.index.isin(df_pre.index)].count(axis=1)}, + index=df_yr.index) + + for i in df_yr.index: + # Mainly for cases of two new consecutive years (like 2022 + # and 2024) + if ~np.isnan( + df_count['c_next'][i]) and df_count[ + 'c_pre'][i] >= df_count['c_next'][i] + 2: + df_yr[year_pre] = np.nan + + # For technologies phasing out before the end of horizon + # (like nuc_lc) + elif np.isnan(df_count['c_next'][i]): + df_yr.loc[i, :] = df_pre.loc[i, :].shift(+1) + if tec_list: + f_mask(df_yr, i, df_count['c_pre'][i] + 1, np.nan) + else: + f_mask(df_yr, i, df_count['c_pre'][i], np.nan) + + # For the rest + else: + df_yr[year_pre] = np.nan + f_mask(df_yr, i, df_count['c_pre'][i], np.nan) + + else: + continue + + df2 = df2.append(df_yr) + df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() + # --------------------------------------------------------------------------- + # Forth: final masking based on technical lifetime + if tec_list: + + df3 = df2.copy() + for y in sorted([x for x in list( + set(df2.index.get_level_values(year_ref)) + ) if x in df_dur.index]): + df3.loc[df3.index.get_level_values(year_ref).isin([y]), + df3.columns.isin(df_dur.columns)] = df_dur.loc[ + y, df_dur.columns.isin(df3.columns)].values + + df3 = df3.reset_index().set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + par_tec = par_tec.set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + + for i in [x for x in par_tec.index if x in df3.index]: + df3.loc[i, 'lifetime'] = par_tec.loc[i, 'value'].copy() + + df3 = df3.reset_index().set_index(idx).dropna( + subset=['lifetime']).sort_index() + for i in df3.index: + df2.loc[i, df3.loc[i, :] >= int( + df3.loc[i, 'lifetime'])] = np.nan + + # Removing extra values from non-lifetime technologies + for i in [x for x in df2.index if x not in df3.index]: + df2.loc[i, df2.columns > df2.loc[[i]].index.get_level_values( + year_ref)[0]] = np.nan + + df_par = pd.melt( + df2.reset_index(), + id_vars=idx, + value_vars=[ + x for x in df2.columns if x not in idx], + var_name=year_col, + value_name='value').dropna( + subset=['value']) + df_par = df_par.sort_values(idx).reset_index(drop=True) + return df_par + + +# %% VII) Input arguments related to running the script from command line +# Text formatting for long help texts +class SmartFormatter(argparse.HelpFormatter): + + def _split_lines(self, text, width): + if text.startswith('B|'): + return text[2:].splitlines() + return argparse.HelpFormatter._split_lines(self, text, width) + + +def read_args(): + + annon = """ + Adding additional years to a MESSAGE model/scenario instance + + Example: + python f_addYear.py --model_ref "Austria_tutorial" --scen_ref "test_core" + --scen_new "test_5y" --years_new "[2015,2025,2035,2045]" + + """ + parser = argparse.ArgumentParser( + description=annon, + formatter_class=SmartFormatter) + + parser.add_argument('--model_ref', + help="B| --model_ref: string\n" + "reference model name") + parser.add_argument('--scen_ref', + help="B| --scen_ref: string\n" + "reference scenario name") + parser.add_argument('--version_ref', + help="B| --version_ref: integer (standard=None)\n" + "version number of reference scenario") + parser.add_argument('--model_new', + help='B| --model_new: string (standard=None)\n' + 'new model name') + parser.add_argument('--scen_new', + help='B| --scen_new: string (optional)\n' + 'new scenario name') + parser.add_argument('--create_new', + help='B| --create_new: string (standard=True)\n' + 'True, for creating a new scenario\n' + 'False, for using an existing scenario') + parser.add_argument('--years_new', + help='--years_new : list\n new years to be added') + parser.add_argument('--firstyear_new', + help='B| --firstyear_new: integer (standard=None)\n' + 'new first model year') + parser.add_argument('--lastyear_new', + help='B| --lastyear_new: integer (standard=None)\n' + 'new last model year') + parser.add_argument('--macro', + help='B| --macro: string (standard=False)\n' + 'True, for adding new years to macro parameters\n' + 'False, for ignoring new years for macro parameters') + parser.add_argument('--baseyear_macro', + help='B| --baseyear_macro : integer (standard=None)\n' + 'new base year for macro\n') + parser.add_argument('--parameter', + help='B| --parameter: list (standard=all)\n' + 'list of parameters for adding new years to them\n' + 'all, for all the parameters') + parser.add_argument('--region', + help='B| --region: list (standard=all)\n' + 'list of regions for adding new years to them\n' + 'all, for all the regions') + parser.add_argument('--rewrite', + help='B| --rewrite: string (standard=True)\n' + 'rewriting parameters in the new scenario') + parser.add_argument('--unit_check', + help='B|--unit_check: string (standard=False)\n' + 'checking units before adding new years') + parser.add_argument('--extrapol_neg', + help='B| --extrapol_neg: string (standard=None)\n' + 'treating negative values from extrapolation of\n' + 'two positive values:\n' + 'None: does nothing\n' + 'integer: multiplier to adjacent value' + 'replacing negative one') + parser.add_argument('--bound_extend', + help='B| --bound_extend : string (standard=True)\n' + 'copying data from previous timestep if only one data' + 'available for interpolation:\n') + + args = parser.parse_args() + return args + + +# %% If run as main script +if __name__ == '__main__': + start = timer() + print('>> Running the script f_addYears.py...') + args = read_args() + model_ref = args.model_ref + scen_ref = args.scen_ref + version_ref = int(args.version_ref) if args.version_ref else None + + model_new = args.model_new if args.model_new else args.model_ref + scen_new = args.scen_new if args.scen_new else str(args.scen_ref + '_5y') + create_new = args.create_new if args.create_new else True + print(args.years_new) + years_new = list(map(int, args.years_new.strip('[]').split(','))) + firstyear_new = int(args.firstyear_new) if args.firstyear_new else None + lastyear_new = int(args.lastyear_new) if args.lastyear_new else None + macro = args.macro if args.macro else False + baseyear_macro = int(args.baseyear_macro) if args.baseyear_macro else None + + parameter = list(map(str, args.parameter.strip( + '[]').split(','))) if args.parameter else all + region = list(map(str, args.region.strip('[]').split(',')) + ) if args.region else all + rewrite = args.rewrite if args.rewrite else True + unit_check = args.unit_check if args.unit_check else True + extrapol_neg = args.extrapol_neg if args.extrapol_neg else 0.5 + bound_extend = args.bound_extend if args.bound_extend else True + + # Loading the reference scenario and creating a new scenario to add the + # additional years + if version_ref: + sc_ref = message_ix.Scenario( + mp, + model=model_ref, + scen=scen_ref, + version=version_ref) + else: + sc_ref = message_ix.Scenario(mp, model=model_ref, scen=scen_ref) + + if create_new: + sc_new = message_ix.Scenario( + mp, + model=model_new, + scen=scen_new, + version='new', + scheme='MESSAGE', + annotation='5 year modelling') + else: + sc_new = message_ix.Scenario(mp, model=model_new, scen=scen_new) + if sc_new.has_solution(): + sc_new.remove_solution() + sc_new.check_out() + + # Calling the main function + addNewYear( + sc_ref, + sc_new, + years_new, + firstyear_new, + lastyear_new, + macro, + baseyear_macro, + parameter, + region, + rewrite, + unit_check, + extrapol_neg, + bound_extend) + end = timer() + mp.close_db() + print('> Elapsed time for adding new years:', round((end - start) / 60), + 'min and', round((end - start) % 60, 1), 'sec.') + print('> New scenario with additional years is: "{}"|"{}"|{}'.format( + sc_new.model, sc_new.scenario, str(sc_new.version))) + +# Sample input arguments from commandline +# python f_addNewYear.py --model_ref "CD_Links_SSP2" --scen_ref "baseline" +# --years_new "[2015,2025,2035,2045,2055,2120]" From d8ad78fafec9bba3e2d01f1e909f6a795d6443d2 Mon Sep 17 00:00:00 2001 From: Behnam Date: Sun, 3 Feb 2019 22:20:59 +0100 Subject: [PATCH 15/44] test for function f_addNewYear added --- tests/test_feature_add_year.py | 93 ++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tests/test_feature_add_year.py diff --git a/tests/test_feature_add_year.py b/tests/test_feature_add_year.py new file mode 100644 index 000000000..216b64f62 --- /dev/null +++ b/tests/test_feature_add_year.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# Importing required packages +import os +import math +import ixmp as ix +import message_ix +from conftest import here + +db_dir = os.path.join(here, 'testdb') +test_db = os.path.join(db_dir, 'ixmptest') +test_mp = ix.Platform(dbprops=test_db, dbtype='HSQLDB') + +# the path to where the main script is (based on message_ix folder hirerarchy) +file_path = os.path.join(here, '..', 'toolbox', 'add_year') +os.chdir(file_path) +from f_addNewYear import addNewYear + + +def test_addNewYear(test_mp): + msg_multiyear_args = ('canning problem (MESSAGE scheme)', 'multi-year') + sc_ref = message_ix.Scenario(test_mp, *msg_multiyear_args) + assert sc_ref.par('technical_lifetime')[ + 'technology'].isin(['canning_plant']).any() + sc_ref.solve() + + # Building a new scenario and adding new years + sc_new = message_ix.Scenario(test_mp, + model='foo', + scenario='bar', + version='new', + annonation=' ') + + # Using the function addNewYear + years_new = [2015, 2025] + + addNewYear( + sc_ref, + sc_new, + years_new, + ) + + # 1. Testing the set "year" for the new years + horizon_old = sorted([int(x) for x in sc_ref.set('year')]) + horizon_test = sorted(horizon_old + years_new) + + horizon_new = sorted([int(x) for x in sc_new.set('year')]) + + # Asserting if the lists are equal + assert (horizon_test == horizon_new) + + # 2. Testing parameter "technical_lifetime" + df_tec = sc_ref.par( + 'technical_lifetime', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_vtg': [2020, 2030]}) + value_ref = df_tec['value'].mean() + + df_tec_new = sc_new.par( + 'technical_lifetime', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_vtg': 2025}) + + value_new = float(df_tec_new['value']) + + # Asserting if the missing data is generated accurately by interpolation + # of adjacent data points + assert math.isclose(value_ref, value_new, rel_tol=1e-04) + + # 3. Testing parameter "output" + df_tec = sc_ref.par( + 'output', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_act': 2030, + 'year_vtg': [2020, 2030]}) + + value_ref = df_tec['value'].mean() + + df_tec_new = sc_new.par( + 'output', { + 'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_act': 2025, + 'year_vtg': 2025}) + + # Asserting if the missing data is generated or not + assert df_tec_new['value'].tolist() + + # Asserting if the missing data is generated accurately by interpolation + value_new = float(df_tec_new['value']) + assert math.isclose(value_ref, value_new, rel_tol=1e-04) From 82cd3c33b694ca99308aadf0a3f437f18831aca1 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 08:53:29 +0100 Subject: [PATCH 16/44] Create README.md --- toolbox/add_year/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 toolbox/add_year/README.md diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md new file mode 100644 index 000000000..fffd07300 --- /dev/null +++ b/toolbox/add_year/README.md @@ -0,0 +1,23 @@ +# Description +This functionality adds new time steps to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: +- Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") +- Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. + +# Main steps +1. An existing scenario is loaded and the desired new years is specified. The new years can be in between existing years or beyond the existing model horizon. +2. A new (empty) scenario is created for adding the new time steps. +3. The new years added to the relevant sets. This include: +- adding new years to the set "year" +- adding new years to the set "cat_year" +- changing "first_modelyear" and "last_modelyear" if needed. + +Usage: + This script can be used either: + A) By running directly from the command line, example: + --------------------------------------------------------------------------- + python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" + --years_new "[2015,2025,2035,2045]" + --------------------------------------------------------------------------- + (Other input arguments are optional. For more info see Section V below.) + + B) By calling the class "addNewYear" from other python scripts From c2ff8d9433e1b2e266515dc029e1cddf148420fa Mon Sep 17 00:00:00 2001 From: Behnam Date: Mon, 4 Feb 2019 08:53:59 +0100 Subject: [PATCH 17/44] release notes updated --- RELEASE_NOTES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d9ca46f18..98dd7c4a9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,7 @@ # Next Release +- [#**](https://github.com/iiasa/message_ix/pull/**): A feature for adding new time steps to a scenario - [#182](https://github.com/iiasa/message_ix/pull/182): Fix cross-platform cloning. - [#173](https://github.com/iiasa/message_ix/pull/173): The `solve` command now takes additional arguments when solving with CPLEX. The `cplex.opt` file is now generated on the fly during the solve command and removed after successfully solving. - [#154](https://github.com/iiasa/message_ix/pull/154): Enable documentation build on ReadTheDocs. @@ -119,4 +120,4 @@ jdbc.pwd = ixmp - [#65](https://github.com/iiasa/message_ix/pull/65): Bugfix for downloading tutorials. Now downloads current installed version by default. - [#60](https://github.com/iiasa/message_ix/pull/60): Add basic ability to write and read model input to/from Excel - [#59](https://github.com/iiasa/message_ix/pull/59): Added MacOSX CI support -- [#187](https://github.com/iiasa/message_ix/pull/187): Test for cumulative bound on emissions \ No newline at end of file +- [#187](https://github.com/iiasa/message_ix/pull/187): Test for cumulative bound on emissions From 80ab09823dd648ce4bb9ddba16a55e207878e55b Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:02:43 +0100 Subject: [PATCH 18/44] Update README.md --- toolbox/add_year/README.md | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index fffd07300..4efd0127d 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -3,13 +3,29 @@ This functionality adds new time steps to an existing scenario (hereafter "refer - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. +# Main features +- It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. +- The new years can be consecutive, between existing years, and after the model horizon. +- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter for modifications. + # Main steps -1. An existing scenario is loaded and the desired new years is specified. The new years can be in between existing years or beyond the existing model horizon. +1. An existing scenario is loaded and the desired new years is specified. 2. A new (empty) scenario is created for adding the new time steps. -3. The new years added to the relevant sets. This include: -- adding new years to the set "year" -- adding new years to the set "cat_year" -- changing "first_modelyear" and "last_modelyear" if needed. +3. The new years are added to the relevant sets: +- adding new years to the set "year" and "type_year" +- changing "firstmodelyear", "lastmodelyear", "baseyear_macro", and "initializeyear_macro" if needed. +- modifying the set "cat_year" for the new years. +4. The new years are added to the index sets of relevant parameters, and the missing data for the new years are calculated based on interpolation of adjacent data points. The following steps are applied: +- each non-empty parameter is loaded from the reference scenario +- the year-related indexes of the parameter are identified (either 0, 1, and 2 index under MESSAGE scheme) +- the new years are added to the parameter, and the missing data is calculated based on the number of year-related indexes. For example, the new years are added to index "year_vtg" in parameter "inv_cost", while these new years are added both to "year_vtg" and "year_act" in parameter "output". +- the missing data is calculated by interpolation. +- for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. +5. The changes are commited and saved to the new scenario. + +# Notice: +I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. + Usage: This script can be used either: From a49e88c3be9c2fa92fd53828617e4808d7a222b8 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:40:52 +0100 Subject: [PATCH 19/44] Update README.md --- toolbox/add_year/README.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 4efd0127d..8912b69ba 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -6,7 +6,7 @@ This functionality adds new time steps to an existing scenario (hereafter "refer # Main features - It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. - The new years can be consecutive, between existing years, and after the model horizon. -- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter for modifications. +- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. # Main steps 1. An existing scenario is loaded and the desired new years is specified. @@ -26,14 +26,9 @@ This functionality adds new time steps to an existing scenario (hereafter "refer # Notice: I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. - -Usage: - This script can be used either: - A) By running directly from the command line, example: - --------------------------------------------------------------------------- - python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" - --years_new "[2015,2025,2035,2045]" - --------------------------------------------------------------------------- - (Other input arguments are optional. For more info see Section V below.) - - B) By calling the class "addNewYear" from other python scripts +# Usage: +This script can be used either: +A) By running directly from the command line, example: +python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" +(For the full list of input arguments see the explanation in the code. +B) By calling the class "addNewYear" from another python script. From 072cc5eb14ed40c993f00ec69db551552c645591 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:44:06 +0100 Subject: [PATCH 20/44] Update README.md --- toolbox/add_year/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 8912b69ba..f873bdd65 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -1,14 +1,14 @@ -# Description +## Description This functionality adds new time steps to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. -# Main features +## Main features - It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. - The new years can be consecutive, between existing years, and after the model horizon. - The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. -# Main steps +## Main steps 1. An existing scenario is loaded and the desired new years is specified. 2. A new (empty) scenario is created for adding the new time steps. 3. The new years are added to the relevant sets: @@ -23,12 +23,14 @@ This functionality adds new time steps to an existing scenario (hereafter "refer - for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. 5. The changes are commited and saved to the new scenario. -# Notice: +## Notice: I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. -# Usage: +## Usage: This script can be used either: A) By running directly from the command line, example: -python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" + ``` + python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" + ``` (For the full list of input arguments see the explanation in the code. B) By calling the class "addNewYear" from another python script. From 9bf566e17d867af13f73f1323f4d8ce90cdfdc03 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:45:35 +0100 Subject: [PATCH 21/44] Update README.md --- toolbox/add_year/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index f873bdd65..93a3f9b24 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -32,5 +32,5 @@ A) By running directly from the command line, example: ``` python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" ``` -(For the full list of input arguments see the explanation in the code. +For the full list of input arguments see the explanation in the code. B) By calling the class "addNewYear" from another python script. From 35813ab358498a91b8e23063b72e5baab408742e Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:46:54 +0100 Subject: [PATCH 22/44] Update README.md --- toolbox/add_year/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 93a3f9b24..169c81740 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -28,9 +28,10 @@ I. This functionality in the current format does not ensure that the new scenari ## Usage: This script can be used either: -A) By running directly from the command line, example: +- By running directly from the command line, example: ``` python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" ``` + For the full list of input arguments see the explanation in the code. -B) By calling the class "addNewYear" from another python script. +- By calling the class "addNewYear" from another python script. From fafb9607c2c194d68572f1f4785fda63bceb1399 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 11:58:47 +0100 Subject: [PATCH 23/44] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 98dd7c4a9..ccea0cce1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,7 +1,7 @@ # Next Release -- [#**](https://github.com/iiasa/message_ix/pull/**): A feature for adding new time steps to a scenario +- [#161](https://github.com/iiasa/message_ix/pull/161): A feature for adding new time steps to a scenario - [#182](https://github.com/iiasa/message_ix/pull/182): Fix cross-platform cloning. - [#173](https://github.com/iiasa/message_ix/pull/173): The `solve` command now takes additional arguments when solving with CPLEX. The `cplex.opt` file is now generated on the fly during the solve command and removed after successfully solving. - [#154](https://github.com/iiasa/message_ix/pull/154): Enable documentation build on ReadTheDocs. From 5ac25a9c26d24305357183782cf2975b9745f266 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Mon, 4 Feb 2019 15:57:14 +0100 Subject: [PATCH 24/44] Update f_addNewYear.py --- toolbox/add_year/f_addNewYear.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolbox/add_year/f_addNewYear.py b/toolbox/add_year/f_addNewYear.py index a0e4dd173..5f840a60e 100644 --- a/toolbox/add_year/f_addNewYear.py +++ b/toolbox/add_year/f_addNewYear.py @@ -880,8 +880,8 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( horizon_new.index(yr) - 1] - horizon_new[ horizon_new.index(yr) - 2]: - df2[yr].loc[pd.isna(df2[year_pre].shift(+1)) - & ~pd.isna(df2[year_pp].shift(+1))] = np.nan + df2[yr].loc[(pd.isna(df2[year_pre].shift(+1))) & ( + ~pd.isna(df2[year_pp].shift(+1)))] = np.nan if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( df2[year_pre].shift(+1) >= 0)].empty: df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), From f7726352849be76cc63d2315539b850d10880070 Mon Sep 17 00:00:00 2001 From: behnam2015 <30926636+behnam2015@users.noreply.github.com> Date: Tue, 9 Apr 2019 18:04:12 +0200 Subject: [PATCH 25/44] Update README.md --- toolbox/add_year/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolbox/add_year/README.md b/toolbox/add_year/README.md index 169c81740..e8df02917 100644 --- a/toolbox/add_year/README.md +++ b/toolbox/add_year/README.md @@ -1,5 +1,5 @@ ## Description -This functionality adds new time steps to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: +This functionality adds new modeling years to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. From 5d3701a58ce218a3e6a8003cb2a7044260a0c77e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 2 May 2019 10:44:01 +0200 Subject: [PATCH 26/44] Use message_ix.tools instead of toolbox namespace --- message_ix/tools/__init__.py | 0 {toolbox => message_ix/tools}/add_year/README.md | 0 .../tools/add_year/__init__.py | 0 tests/{test_feature_add_year.py => tools/test_add_year.py} | 7 ++----- 4 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 message_ix/tools/__init__.py rename {toolbox => message_ix/tools}/add_year/README.md (100%) rename toolbox/add_year/f_addNewYear.py => message_ix/tools/add_year/__init__.py (100%) rename tests/{test_feature_add_year.py => tools/test_add_year.py} (93%) diff --git a/message_ix/tools/__init__.py b/message_ix/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/toolbox/add_year/README.md b/message_ix/tools/add_year/README.md similarity index 100% rename from toolbox/add_year/README.md rename to message_ix/tools/add_year/README.md diff --git a/toolbox/add_year/f_addNewYear.py b/message_ix/tools/add_year/__init__.py similarity index 100% rename from toolbox/add_year/f_addNewYear.py rename to message_ix/tools/add_year/__init__.py diff --git a/tests/test_feature_add_year.py b/tests/tools/test_add_year.py similarity index 93% rename from tests/test_feature_add_year.py rename to tests/tools/test_add_year.py index 216b64f62..0a422098f 100644 --- a/tests/test_feature_add_year.py +++ b/tests/tools/test_add_year.py @@ -6,15 +6,12 @@ import message_ix from conftest import here +from message_ix.tools.add_year import addNewYear + db_dir = os.path.join(here, 'testdb') test_db = os.path.join(db_dir, 'ixmptest') test_mp = ix.Platform(dbprops=test_db, dbtype='HSQLDB') -# the path to where the main script is (based on message_ix folder hirerarchy) -file_path = os.path.join(here, '..', 'toolbox', 'add_year') -os.chdir(file_path) -from f_addNewYear import addNewYear - def test_addNewYear(test_mp): msg_multiyear_args = ('canning problem (MESSAGE scheme)', 'multi-year') From cf8f2e7b55aec3b947867779815c8796a686138a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 2 May 2019 12:01:42 +0200 Subject: [PATCH 27/44] Rewrite add_year CLI using Click. --- message_ix/tools/add_year/__init__.py | 176 +------------------------- message_ix/tools/add_year/__main__.py | 171 +++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 175 deletions(-) create mode 100644 message_ix/tools/add_year/__main__.py diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 5f840a60e..17eb2e85a 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -35,12 +35,6 @@ import numpy as np import pandas as pd -from timeit import default_timer as timer -import argparse -import ixmp as ix -import message_ix - -mp = ix.Platform(dbtype='HSQLDB') # Loading ixmp Platform # %% II) Required utility functions for dataframe manupulation # II.A) Utility function for interpolation/extrapolation of two numbers, @@ -671,7 +665,7 @@ def df_interpolate( # Extrapolate for new years if the value exists for the # previous year but not for the next years - # TODO: here is the place that should be changed if the + # TODO: here is the place that should be changed if the # new year should go the time step before the existing one if [x for x in df2.columns if x < year_pre]: year_pp = max([x for x in df2.columns if x < year_pre]) @@ -1050,171 +1044,3 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( subset=['value']) df_par = df_par.sort_values(idx).reset_index(drop=True) return df_par - - -# %% VII) Input arguments related to running the script from command line -# Text formatting for long help texts -class SmartFormatter(argparse.HelpFormatter): - - def _split_lines(self, text, width): - if text.startswith('B|'): - return text[2:].splitlines() - return argparse.HelpFormatter._split_lines(self, text, width) - - -def read_args(): - - annon = """ - Adding additional years to a MESSAGE model/scenario instance - - Example: - python f_addYear.py --model_ref "Austria_tutorial" --scen_ref "test_core" - --scen_new "test_5y" --years_new "[2015,2025,2035,2045]" - - """ - parser = argparse.ArgumentParser( - description=annon, - formatter_class=SmartFormatter) - - parser.add_argument('--model_ref', - help="B| --model_ref: string\n" - "reference model name") - parser.add_argument('--scen_ref', - help="B| --scen_ref: string\n" - "reference scenario name") - parser.add_argument('--version_ref', - help="B| --version_ref: integer (standard=None)\n" - "version number of reference scenario") - parser.add_argument('--model_new', - help='B| --model_new: string (standard=None)\n' - 'new model name') - parser.add_argument('--scen_new', - help='B| --scen_new: string (optional)\n' - 'new scenario name') - parser.add_argument('--create_new', - help='B| --create_new: string (standard=True)\n' - 'True, for creating a new scenario\n' - 'False, for using an existing scenario') - parser.add_argument('--years_new', - help='--years_new : list\n new years to be added') - parser.add_argument('--firstyear_new', - help='B| --firstyear_new: integer (standard=None)\n' - 'new first model year') - parser.add_argument('--lastyear_new', - help='B| --lastyear_new: integer (standard=None)\n' - 'new last model year') - parser.add_argument('--macro', - help='B| --macro: string (standard=False)\n' - 'True, for adding new years to macro parameters\n' - 'False, for ignoring new years for macro parameters') - parser.add_argument('--baseyear_macro', - help='B| --baseyear_macro : integer (standard=None)\n' - 'new base year for macro\n') - parser.add_argument('--parameter', - help='B| --parameter: list (standard=all)\n' - 'list of parameters for adding new years to them\n' - 'all, for all the parameters') - parser.add_argument('--region', - help='B| --region: list (standard=all)\n' - 'list of regions for adding new years to them\n' - 'all, for all the regions') - parser.add_argument('--rewrite', - help='B| --rewrite: string (standard=True)\n' - 'rewriting parameters in the new scenario') - parser.add_argument('--unit_check', - help='B|--unit_check: string (standard=False)\n' - 'checking units before adding new years') - parser.add_argument('--extrapol_neg', - help='B| --extrapol_neg: string (standard=None)\n' - 'treating negative values from extrapolation of\n' - 'two positive values:\n' - 'None: does nothing\n' - 'integer: multiplier to adjacent value' - 'replacing negative one') - parser.add_argument('--bound_extend', - help='B| --bound_extend : string (standard=True)\n' - 'copying data from previous timestep if only one data' - 'available for interpolation:\n') - - args = parser.parse_args() - return args - - -# %% If run as main script -if __name__ == '__main__': - start = timer() - print('>> Running the script f_addYears.py...') - args = read_args() - model_ref = args.model_ref - scen_ref = args.scen_ref - version_ref = int(args.version_ref) if args.version_ref else None - - model_new = args.model_new if args.model_new else args.model_ref - scen_new = args.scen_new if args.scen_new else str(args.scen_ref + '_5y') - create_new = args.create_new if args.create_new else True - print(args.years_new) - years_new = list(map(int, args.years_new.strip('[]').split(','))) - firstyear_new = int(args.firstyear_new) if args.firstyear_new else None - lastyear_new = int(args.lastyear_new) if args.lastyear_new else None - macro = args.macro if args.macro else False - baseyear_macro = int(args.baseyear_macro) if args.baseyear_macro else None - - parameter = list(map(str, args.parameter.strip( - '[]').split(','))) if args.parameter else all - region = list(map(str, args.region.strip('[]').split(',')) - ) if args.region else all - rewrite = args.rewrite if args.rewrite else True - unit_check = args.unit_check if args.unit_check else True - extrapol_neg = args.extrapol_neg if args.extrapol_neg else 0.5 - bound_extend = args.bound_extend if args.bound_extend else True - - # Loading the reference scenario and creating a new scenario to add the - # additional years - if version_ref: - sc_ref = message_ix.Scenario( - mp, - model=model_ref, - scen=scen_ref, - version=version_ref) - else: - sc_ref = message_ix.Scenario(mp, model=model_ref, scen=scen_ref) - - if create_new: - sc_new = message_ix.Scenario( - mp, - model=model_new, - scen=scen_new, - version='new', - scheme='MESSAGE', - annotation='5 year modelling') - else: - sc_new = message_ix.Scenario(mp, model=model_new, scen=scen_new) - if sc_new.has_solution(): - sc_new.remove_solution() - sc_new.check_out() - - # Calling the main function - addNewYear( - sc_ref, - sc_new, - years_new, - firstyear_new, - lastyear_new, - macro, - baseyear_macro, - parameter, - region, - rewrite, - unit_check, - extrapol_neg, - bound_extend) - end = timer() - mp.close_db() - print('> Elapsed time for adding new years:', round((end - start) / 60), - 'min and', round((end - start) % 60, 1), 'sec.') - print('> New scenario with additional years is: "{}"|"{}"|{}'.format( - sc_new.model, sc_new.scenario, str(sc_new.version))) - -# Sample input arguments from commandline -# python f_addNewYear.py --model_ref "CD_Links_SSP2" --scen_ref "baseline" -# --years_new "[2015,2025,2035,2045,2055,2120]" diff --git a/message_ix/tools/add_year/__main__.py b/message_ix/tools/add_year/__main__.py new file mode 100644 index 000000000..166c3d2f8 --- /dev/null +++ b/message_ix/tools/add_year/__main__.py @@ -0,0 +1,171 @@ +"""Add additional years to a MESSAGE model/scenario instance + +\b +Examples: +$ python f_addYear.py --model_ref Austria_tutorial --scen_ref test_core \ + --scen_new test_5y --years_new 2015,2025,2035,2045 +$ python f_addNewYear.py --model_ref CD_Links_SSP2 --scen_ref baseline \ + --years_new "[2015,2025,2035,2045,2055,2120]" + +If --create_new=False is given, the target Scenario must already exist. + +If --extrapol_neg is an integer, negative extrapolated values are are replaced +with the product of the integer and an (which?) adjacent value. By default, +negative values may be produced. + +If --bound_extend is True (the default), data from previous timestep is copied +if only one data point is available for extrapolation. + +""" +from functools import partial +from timeit import default_timer as timer + +import click +import ixmp +import message_ix + +from . import addNewYear + + +def split_value(ctx, param, value, type=str): + """Callback for validation of comma-separated parameter values. + + *value* is parsed as "[v1,v2,v3,...]" where: + + - the square brackets are optional, and + - each value is of type *type*. + """ + try: + if value is None: + value = '' + return list(map(type, value.strip('[]').split(','))) + except ValueError: + raise click.BadParameter(param.human_readable_name, value) + + +@click.command(help=__doc__) +@click.option('--model_ref', help='reference model name', required=True) +@click.option('--scen_ref', help='reference scenario name', required=True) +@click.option('--version_ref', help='version number of reference scenario', + default=None, type=int) +@click.option('--model_new', help='new model name', default=None) +@click.option('--scen_new', help='new scenario name', default=None) +@click.option('--create_new', help='create new scenario', type=bool, + default=True, show_default=True) +@click.option('--years_new', help='new years to be added', + callback=partial(split_value, type=int)) +@click.option('--firstyear_new', help='new first model year', type=int, + default=None) +@click.option('--lastyear_new', help='new last model year', type=int, + default=None) +@click.option('--macro', help='also add years to MACRO parameters', type=bool, + default=False, show_default=True) +@click.option('--baseyear_macro', help='new base year for MACRO', type=int, + default=None) +@click.option('--parameter', help='names of parameters to add years', + callback=split_value, default='all', show_default=True) +@click.option('--region', help='names of regions to add years', + callback=split_value, default='all', show_default=True) +@click.option('--rewrite', help='rewrite parameters in the new scenario', + type=bool, default=True, show_default=True) +@click.option('--unit_check', help='check units before adding new years', + type=bool, default=False, show_default=True) +@click.option('--extrapol_neg', help='handle negative extrapolated values', + type=float, default=0.5, show_default=True) +@click.option('--bound_extend', help='copy data from previous timestep', + type=bool, default=True, show_default=True) +@click.option('--dry-run', help='Only parse arguments & exit.', is_flag=True) +def main(model_ref, scen_ref, version_ref, model_new, scen_new, create_new, + years_new, firstyear_new, lastyear_new, macro, baseyear_macro, + parameter, region, rewrite, unit_check, extrapol_neg, bound_extend, + dry_run): + + start = timer() + print('>> Running the script f_addYears.py...') + + # Handle default arguments + ref_kw = dict(model=model_ref, scen=scen_ref) + if version_ref: + ref_kw['version'] = version_ref + + if model_new is None: + model_new = model_ref + + if scen_new is None: + # FIXME is this a good default? + scen_new = scen_ref + '_5y' + + new_kw = dict(model=model_new, scen=scen_new) + if create_new: + new_kw.update(dict( + version='new', + scheme='MESSAGE', + annotation='5 year modelling', + )) + + # Output for debugging + print(years_new) + + if dry_run: + # Print arguments debugging and return + print( + 'sc_ref:', ref_kw, + 'sc_new:', new_kw, + 'years_new:', years_new, + 'firstyear_new:', firstyear_new, + 'lastyear_new:', lastyear_new, + 'macro:', macro, + 'baseyear_macro:', baseyear_macro, + 'parameter:', parameter, + 'region:', region, + 'rewrite:', rewrite, + 'unit_check:', unit_check, + 'extrapol_neg:', extrapol_neg, + 'bound_extend:', bound_extend, + sep='\n') + return + + # Load the ixmp Platform + mp = ixmp.Platform(dbtype='HSQLDB') + + # Loading the reference scenario and creating a new scenario to add the + # additional years + sc_ref = message_ix.Scenario(mp, **ref_kw) + + if create_new: + sc_new = message_ix.Scenario(mp, **new_kw) + else: + sc_new = message_ix.Scenario(mp, **new_kw) + if sc_new.has_solution(): + sc_new.remove_solution() + sc_new.check_out() + + # Calling the main function + addNewYear( + sc_ref=sc_ref, + sc_new=sc_new, + years_new=years_new, + firstyear_new=firstyear_new, + lastyear_new=lastyear_new, + macro=macro, + baseyear_macro=baseyear_macro, + parameter=parameter, + region=region, + rewrite=rewrite, + unit_check=unit_check, + extrapol_neg=extrapol_neg, + bound_extend=bound_extend) + + end = timer() + + mp.close_db() + + print('> Elapsed time for adding new years:', round((end - start) / 60), + 'min and', round((end - start) % 60, 1), 'sec.') + + print('> New scenario with additional years is: "{}"|"{}"|{}'.format( + sc_new.model, sc_new.scenario, str(sc_new.version))) + + +# Execute the script +main() From e9c16493e97b8e08baf0922aa76ad21888c3ef50 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 2 May 2019 13:52:06 +0200 Subject: [PATCH 28/44] Link the README for the new tool into the docs build --- doc/source/guide.rst | 9 +++ doc/source/tools/add_year.rst | 1 + .../tools/add_year/{README.md => README.rst} | 62 +++++++++++++------ 3 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 doc/source/tools/add_year.rst rename message_ix/tools/add_year/{README.md => README.rst} (53%) diff --git a/doc/source/guide.rst b/doc/source/guide.rst index 302c3ff83..64ecac1c7 100644 --- a/doc/source/guide.rst +++ b/doc/source/guide.rst @@ -8,3 +8,12 @@ common model design choices. efficiency debugging + +Model-building tools +-------------------- + +.. toctree:: + :maxdepth: 1 + :glob: + + tools/* diff --git a/doc/source/tools/add_year.rst b/doc/source/tools/add_year.rst new file mode 100644 index 000000000..fb38e444a --- /dev/null +++ b/doc/source/tools/add_year.rst @@ -0,0 +1 @@ +.. include:: ../../../message_ix/tools/add_year/README.rst diff --git a/message_ix/tools/add_year/README.md b/message_ix/tools/add_year/README.rst similarity index 53% rename from message_ix/tools/add_year/README.md rename to message_ix/tools/add_year/README.rst index e8df02917..0b67c2109 100644 --- a/message_ix/tools/add_year/README.md +++ b/message_ix/tools/add_year/README.rst @@ -1,37 +1,63 @@ -## Description +Add model years to an existing Scenario +======================================= + +Description +----------- This functionality adds new modeling years to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: + - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. -## Main features +Main features +------------- - It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. - The new years can be consecutive, between existing years, and after the model horizon. - The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. -## Main steps +Main steps +---------- 1. An existing scenario is loaded and the desired new years is specified. 2. A new (empty) scenario is created for adding the new time steps. 3. The new years are added to the relevant sets: -- adding new years to the set "year" and "type_year" -- changing "firstmodelyear", "lastmodelyear", "baseyear_macro", and "initializeyear_macro" if needed. -- modifying the set "cat_year" for the new years. + + - adding new years to the set "year" and "type_year" + - changing "firstmodelyear", "lastmodelyear", "baseyear_macro", and "initializeyear_macro" if needed. + - modifying the set "cat_year" for the new years. + 4. The new years are added to the index sets of relevant parameters, and the missing data for the new years are calculated based on interpolation of adjacent data points. The following steps are applied: -- each non-empty parameter is loaded from the reference scenario -- the year-related indexes of the parameter are identified (either 0, 1, and 2 index under MESSAGE scheme) -- the new years are added to the parameter, and the missing data is calculated based on the number of year-related indexes. For example, the new years are added to index "year_vtg" in parameter "inv_cost", while these new years are added both to "year_vtg" and "year_act" in parameter "output". -- the missing data is calculated by interpolation. -- for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. + + - each non-empty parameter is loaded from the reference scenario + - the year-related indexes of the parameter are identified (either 0, 1, and 2 index under MESSAGE scheme) + - the new years are added to the parameter, and the missing data is calculated based on the number of year-related indexes. For example, the new years are added to index "year_vtg" in parameter "inv_cost", while these new years are added both to "year_vtg" and "year_act" in parameter "output". + - the missing data is calculated by interpolation. + - for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. + 5. The changes are commited and saved to the new scenario. -## Notice: +Notice +------ I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. -## Usage: +Usage +----- This script can be used either: -- By running directly from the command line, example: - ``` - python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" - ``` + +- By running directly from the command line, example:: -For the full list of input arguments see the explanation in the code. + $ python -m message_ix.tools.add_year \ + --model_ref MESSAGE_Model \ + --scen_ref baseline \ + --years_new 2015,2025,2035,2045 + + For the full list of input arguments, run:: + + $ python -m message_ix.tools.add_year --help + - By calling the class "addNewYear" from another python script. + +API +--- + +.. automodule:: message_ix.tools.add_year + :members: + From 3cec109e3203f20c11610cb7ac323ac173898653 Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 2 May 2019 18:34:08 +0200 Subject: [PATCH 29/44] cleanup of main function --- message_ix/tools/add_year/__init__.py | 1693 ++++++++++++------------- 1 file changed, 812 insertions(+), 881 deletions(-) diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 17eb2e85a..eb6062495 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -13,9 +13,9 @@ I. Required python packages are imported and ixmp platform loaded. II. Generic utilities for dataframe manipulation III. The main class called "addNewYear" - IV. Submodule "addSets" for adding and modifying the sets - V. Submodule "addPar" for copying and modifying each parameter - VI. The submodule "addPar" calls two utility functions ("df_interpolate" + IV. Submodule "add_year_set" for adding and modifying the sets + V. Submodule "add_year_par" for copying and modifying each parameter + VI. The submodule "add_year_par" calls two utility functions ("df_interpolate" and "df_interpolate_2D") for calculating missing values. VII. Code for running the script as "main" @@ -58,7 +58,7 @@ def intpol(y1, y2, x1, x2, x): # value: integer/string -def f_slice(df, idx, level, locator, value): +def slice_df(df, idx, level, locator, value): if locator: df = df.reset_index().loc[df.reset_index()[level].isin(locator)].copy() else: @@ -71,7 +71,7 @@ def f_slice(df, idx, level, locator, value): # dataframe -def f_mask(df, index, count, value): +def mask_df(df, index, count, value): df.loc[index, df.columns > (df.loc[[index]].notnull().cumsum( axis=1) == count).idxmax(axis=1).values[0]] = value @@ -93,10 +93,13 @@ def unit_uniform(df): # %% III) The main class -class addNewYear(object): +def addNewYear(sc_ref, sc_new, years_new, firstyear_new=None, + lastyear_new=None, macro=False, baseyear_macro=None, + parameter='all', region='all', rewrite=True, unit_check=True, + extrapol_neg=None, bound_extend=True): ''' This class does the following: - A. calls function "addSets" to add and modify required sets. - B. calls function "addPar" to add new years and modifications + A. calls function "add_year_set" to add and modify required sets. + B. calls function "add_year_par" to add new years and modifications to each parameter if needed. Parameters: @@ -131,584 +134,337 @@ class addNewYear(object): (e.g., permitting the extension of a bound to 2025, when there is only one value in 2020) ''' - - def __init__(self, sc_ref, sc_new, years_new, firstyear_new=None, - lastyear_new=None, macro=False, baseyear_macro=None, - parameter=all, region=all, rewrite=True, unit_check=True, - extrapol_neg=None, bound_extend=True): - - # III.A) Adding sets and required modifications - years_new = sorted([x for x in years_new if str(x) - not in set(sc_ref.set('year'))]) - self.addSets( - sc_ref, - sc_new, - years_new, - firstyear_new, - lastyear_new, - baseyear_macro) - # -------------------------------------------------------------------------- - # III.B) Adding parameters and calculating the missing values for the - # additonal years - if parameter == all: - par_list = sorted(sc_ref.par_list()) - elif isinstance(parameter, list): - par_list = parameter - elif isinstance(parameter, str): - par_list = [parameter] + # III.A) Adding sets and required modifications + years_new = sorted([x for x in years_new if str(x) + not in set(sc_ref.set('year'))]) + add_year_set(sc_ref, sc_new, years_new, firstyear_new, lastyear_new, + baseyear_macro) + # ------------------------------------------------------------------------- + # III.B) Adding parameters and calculating the missing values for the + # additonal years + if parameter == 'all': + par_list = sorted(sc_ref.par_list()) + elif isinstance(parameter, list): + par_list = parameter + elif isinstance(parameter, str): + par_list = [parameter] + else: + print('Parameters should be defined in a list of strings or as' + ' a single string!') + + if 'technical_lifetime' in par_list: + par_list.insert(0, par_list.pop(par_list.index('technical_lifetime'))) + + if region == 'all': + reg_list = sc_ref.set('node').tolist() + elif isinstance(region, list): + reg_list = region + elif isinstance(region, str): + reg_list = [region] + else: + print('Regions should be defined in a list of strings or as' + ' a single string!') + + # List of parameters to be ignored (even not copied to the new + # scenario) + par_ignore = ['duration_period'] + par_list = [x for x in par_list if x not in par_ignore] + + if not macro: + par_macro = ['demand_MESSAGE', 'price_MESSAGE', 'cost_MESSAGE', + 'gdp_calibrate', 'historical_gdp', 'MERtoPPP', 'kgdp', + 'kpvs', 'depr', 'drate', 'esub', 'lotol', 'p_ref', 'lakl', + 'prfconst', 'grow', 'aeei', 'aeei_factor', 'gdp_rate'] + par_list = [x for x in par_list if x not in par_macro] + + firstyear = sc_new.set('cat_year', {'type_year': 'firstmodelyear'})['year'] + for parname in par_list: + # For historical parameters extrapolation permitted (e.g., from + # 2010 to 2015) + if 'historical' in parname: + extrapol = True + yrs_new = [x for x in years_new if x < int(firstyear)] else: - print( - 'Parameters should be defined in a list of strings or as' + - 'a single string!') - - if 'technical_lifetime' in par_list: - par_list.insert( - 0, par_list.pop( - par_list.index('technical_lifetime'))) - - if region == all: - reg_list = sc_ref.set('node').tolist() - elif isinstance(region, list): - reg_list = region - elif isinstance(region, str): - reg_list = [region] + extrapol = False + yrs_new = years_new + + if 'bound' in parname: + bound_ext = bound_extend else: - print('Regions should be defined in a list of strings or as' + - 'a single string!') - - # List of parameters to be ignored (even not copied to the new - # scenario) - par_ignore = ['duration_period'] - par_list = [x for x in par_list if x not in par_ignore] - - if not macro: - par_macro = [ - 'demand_MESSAGE', - 'price_MESSAGE', - 'cost_MESSAGE', - 'gdp_calibrate', - 'historical_gdp', - 'MERtoPPP', - 'kgdp', - 'kpvs', - 'depr', - 'drate', - 'esub', - 'lotol', - 'p_ref', - 'lakl', - 'prfconst', - 'grow', - 'aeei', - 'aeei_factor', - 'gdp_rate'] - par_list = [x for x in par_list if x not in par_macro] - - for parname in par_list: - # For historical parameters extrapolation permitted (e.g., from - # 2010 to 2015) - if 'historical' in parname: - extrapol = True - yrs_new = [ - x for x in years_new if x < int( - sc_new.set( - 'cat_year', { - 'type_year': 'firstmodelyear'})['year'])] - else: - extrapol = False - yrs_new = years_new + bound_ext = True - if 'bound' in parname: - bound_ext = bound_extend - else: - bound_ext = True - - year_list = [x for x in sc_ref.idx_sets(parname) if 'year' in x] - - if len(year_list) == 2 or parname in ['land_output']: - # The loop over "node" is only for reducing the size of tables - for node in reg_list: - self.addPar( - sc_ref, - sc_new, - yrs_new, - parname, - [node], - extrapolate=extrapol, - rewrite=rewrite, - unit_check=unit_check, - extrapol_neg=extrapol_neg, - bound_extend=bound_ext) - else: - self.addPar( - sc_ref, - sc_new, - yrs_new, - parname, - reg_list, - extrapolate=extrapol, - rewrite=rewrite, - unit_check=unit_check, - extrapol_neg=extrapol_neg, - bound_extend=bound_ext) - - sc_new.set_as_default() - print('> All required parameters were successfully ' + - 'added to the new scenario.') - - # %% Submodules needed for running the main function - # IV) Adding new years to sets - def addSets( - self, - sc_ref, - sc_new, - years_new, - firstyear_new=None, - lastyear_new=None, - baseyear_macro=None): - ''' - Description: - Adding required sets and relevant modifications: - This function adds additional years to an existing scenario, - by starting to make a new scenario from scratch. - After modification of the year-related sets, this function copeis - all other sets from the "reference" scenario - to the "new" scenario. - - Input arguments: - Please see the description for the input arguments under the main - function "addNewYear" - - Usage: - This module is called by function "addNewYear" - ''' - # IV.A) Treatment of the additional years in the year-related sets - - # A.1. Set - year - yrs_old = list(map(int, sc_ref.set('year'))) - horizon_new = sorted(yrs_old + years_new) - sc_new.add_set('year', [str(yr) for yr in horizon_new]) - - # A.2. Set _ type_year - yr_typ = sc_ref.set('type_year').tolist() - sc_new.add_set('type_year', sorted( - yr_typ + [str(yr) for yr in years_new])) - - # A.3. Set _ cat_year - yr_cat = sc_ref.set('cat_year') - - # A.4. Changing the first year if needed - if firstyear_new: - yr_cat.loc[yr_cat['type_year'] == - 'firstmodelyear', 'year'] = firstyear_new - if lastyear_new: - yr_cat.loc[yr_cat['type_year'] == - 'lastmodelyear', 'year'] = lastyear_new - - # A.5. Changing the base year and initialization year of macro if a new - # year specified - if not yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'].empty and baseyear_macro: - yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'] = baseyear_macro - if not yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year' - ].empty and baseyear_macro: - yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year'] = baseyear_macro - - yr_pair = [] - for yr in years_new: - yr_pair.append([yr, yr]) - yr_pair.append(['cumulative', yr]) - - yr_cat = yr_cat.append( - pd.DataFrame( - yr_pair, - columns=[ - 'type_year', - 'year']), - ignore_index=True).sort_values('year').reset_index( - drop=True) - - # A.6. Changing the cumulative years based on the new first model year - firstyear_new = int( - yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', 'year']) - yr_cat = yr_cat.drop(yr_cat.loc[ - (yr_cat['type_year'] == 'cumulative') & ( - yr_cat['year'] < firstyear_new)].index) - sc_new.add_set('cat_year', yr_cat) - - # IV.B) Copying all other sets - set_list = [s for s in sc_ref.set_list() if 'year' not in s] - # Sets with one index set - index_list = [ - x for x in set_list if not isinstance( - sc_ref.set(x), pd.DataFrame)] - for set_name in index_list: - if set_name not in sc_new.set_list(): - sc_new.init_set(set_name, idx_sets=None, idx_names=None) - sc_new.add_set(set_name, sc_ref.set(set_name).tolist()) - - # The rest of the sets - for set_name in [x for x in set_list if x not in index_list]: - if set_name not in sc_new.set_list() and not [ - x for x in sc_ref.idx_sets( - set_name) if x not in sc_ref.set_list()]: - sc_new.init_set(set_name, - idx_sets=sc_ref.idx_sets(set_name).tolist(), - idx_names=sc_ref.idx_names(set_name).tolist()) - sc_new.add_set(set_name, sc_ref.set(set_name)) - - sc_new.commit('sets added!') - print('> All the sets updated and added to the new scenario.') - - # %% V) Adding new years to parameters - - def addPar( - self, - sc_ref, - sc_new, - yrs_new, - parname, - region_list, - extrapolate=False, - rewrite=True, - unit_check=True, - extrapol_neg=None, - bound_extend=True): - ''' Adding required parameters and relevant modifications: - Description: - This function adds additional years to a parameter. - The value of the parameter for additional years is calculated - mainly by interpolating and extrapolating of data from existing - years. - - Input arguments: - Please see the description for the input arguments under the - main function "addNewYear" - - Usage: - This module is called by function "addNewYear" - ''' - - # V.A) Initialization and checks - - par_list_new = sc_new.par_list().tolist() - idx_names = sc_ref.idx_names(parname) - horizon = sorted([int(x) for x in list(set(sc_ref.set('year')))]) - node_col = [ - x for x in idx_names if x in [ - 'node', - 'node_loc', - 'node_rel']] - - if parname not in par_list_new: - sc_new.check_out() - sc_new.init_par( - parname, - idx_sets=sc_ref.idx_sets(parname).tolist(), - idx_names=sc_ref.idx_names(parname).tolist()) - sc_new.commit('New parameter initiated!') - - if node_col: - par_old = sc_ref.par(parname, {node_col[0]: region_list}) - par_new = sc_new.par(parname, {node_col[0]: region_list}) - sort_order = [ - node_col[0], - 'technology', - 'commodity', - 'year_vtg', - 'year_act'] - nodes = par_old[node_col[0]].unique().tolist() + year_list = [x for x in sc_ref.idx_sets(parname) if 'year' in x] + + if len(year_list) == 2 or parname in ['land_output']: + # The loop over "node" is only for reducing the size of tables + for node in reg_list: + add_year_par(sc_ref, sc_new, yrs_new, parname, [node], + extrapol, rewrite, unit_check, extrapol_neg, bound_ext) else: - par_old = sc_ref.par(parname) - par_new = sc_new.par(parname) - sort_order = ['technology', 'commodity', 'year_vtg', 'year_act'] - nodes = ['N/A'] - - if not par_new.empty and not rewrite: - print( - '> Parameter "' + parname + '" already has data ' - 'in new scenario and left unchanged for node/s: {}.'.format( - region_list)) - return - if par_old.empty: - print( - '> Parameter "' + parname + '" is empty in reference scenario ' - 'for node/s: {}!'.format(region_list)) - return - - # Sorting the data to make it ready for dataframe manupulation - sort_order = [x for x in sort_order if x in idx_names] - if sort_order: - par_old = par_old.sort_values(sort_order).reset_index(drop=True) + add_year_par(sc_ref, sc_new, yrs_new, parname, reg_list, + extrapol, rewrite, unit_check, extrapol_neg, bound_ext) + + sc_new.set_as_default() + print('> All required parameters were successfully ' + + 'added to the new scenario.') + +# %% Submodules needed for running the main function +# IV) Adding new years to sets +def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, + baseyear_macro=None): + ''' + Description: + Adding required sets and relevant modifications: + This function adds additional years to an existing scenario, + by starting to make a new scenario from scratch. + After modification of the year-related sets, this function copeis + all other sets from the "reference" scenario + to the "new" scenario. + + Input arguments: + Please see the description for the input arguments under the main + function "addNewYear" + + Usage: + This module is called by function "addNewYear" + ''' +# IV.A) Treatment of the additional years in the year-related sets + + # A.1. Set - year + yrs_old = list(map(int, sc_ref.set('year'))) + horizon_new = sorted(yrs_old + years_new) + sc_new.add_set('year', [str(yr) for yr in horizon_new]) + + # A.2. Set _ type_year + yr_typ = sc_ref.set('type_year').tolist() + sc_new.add_set('type_year', sorted(yr_typ + [str(yr) for yr in years_new])) + + # A.3. Set _ cat_year + yr_cat = sc_ref.set('cat_year') + + # A.4. Changing the first year if needed + if firstyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'firstmodelyear', 'year'] = firstyear_new + if lastyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'lastmodelyear', 'year'] = lastyear_new + + # A.5. Changing the base year and initialization year of macro if a new + # year specified + if not yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'] = baseyear_macro + if not yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'] = baseyear_macro + + yr_pair = [] + for yr in years_new: + yr_pair.append([yr, yr]) + yr_pair.append(['cumulative', yr]) + + yr_cat = yr_cat.append(pd.DataFrame(yr_pair, + columns=['type_year', 'year']), + ignore_index=True + ).sort_values('year' + ).reset_index(drop=True) + + # A.6. Changing the cumulative years based on the new first model year + firstyear_new = int(yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', + 'year']) + yr_cat = yr_cat.drop(yr_cat.loc[(yr_cat['type_year'] == 'cumulative') & ( + yr_cat['year'] < firstyear_new)].index) + sc_new.add_set('cat_year', yr_cat) + + # IV.B) Copying all other sets + set_list = [s for s in sc_ref.set_list() if 'year' not in s] + # Sets with one index set + index_list = [x for x in set_list if not isinstance(sc_ref.set(x), + pd.DataFrame)] + for set_name in index_list: + if set_name not in sc_new.set_list(): + sc_new.init_set(set_name, idx_sets=None, idx_names=None) + sc_new.add_set(set_name, sc_ref.set(set_name).tolist()) + + # The rest of the sets + + for set_name in [x for x in set_list if x not in index_list]: + new_set = [x for x in sc_ref.idx_sets(set_name + ) if x not in sc_ref.set_list()] + if set_name not in sc_new.set_list() and not new_set: + sc_new.init_set(set_name, + idx_sets=sc_ref.idx_sets(set_name).tolist(), + idx_names=sc_ref.idx_names(set_name).tolist()) + sc_new.add_set(set_name, sc_ref.set(set_name)) + + sc_new.commit('sets added!') + print('> All the sets updated and added to the new scenario.') + +# %% V) Adding new years to parameters + +def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, + extrapolate=False, rewrite=True, unit_check=True, + extrapol_neg=None, bound_extend=True): + ''' Adding required parameters and relevant modifications: + Description: + This function adds additional years to a parameter. + The value of the parameter for additional years is calculated + mainly by interpolating and extrapolating of data from existing + years. + + Input arguments: + Please see the description for the input arguments under the + main function "addNewYear" + + Usage: + This module is called by function "addNewYear" + ''' + +# V.A) Initialization and checks + + par_list_new = sc_new.par_list().tolist() + idx_names = sc_ref.idx_names(parname) + horizon = sorted([int(x) for x in list(set(sc_ref.set('year')))]) + node_col = [ + x for x in idx_names if x in [ + 'node', + 'node_loc', + 'node_rel']] + if parname not in par_list_new: sc_new.check_out() - if not par_new.empty and rewrite: - print( - '> Parameter "' + parname + '" is being removed from new ' - 'scenario to be updated for node/s in {}...'.format(nodes)) - sc_new.remove_par(parname, par_new) - - col_list = sc_ref.idx_names(parname).tolist() - year_list = [ - c for c in col_list if c in [ - 'year', - 'year_vtg', - 'year_act', - 'year_rel']] - - # A uniform "unit" for values in different years to prevent mistakes in - # indexing and grouping - if 'unit' in col_list and unit_check: - par_old = unit_uniform(par_old) - # -------------------------------------------------------------------------------------------------------- - # V.B) Treatment of the new years in the specified parameter based on - # the time-related dimension of that parameter - # V.B.1) Parameters with no time component - if len(year_list) == 0: - sc_new.add_par(parname, par_old) - sc_new.commit(parname) - print( - '> Parameter "' + parname + '" just copied to new scenario ' - 'since has no time-related entries.') - - # V.B.2) Parameters with one dimension related to time - elif len(year_list) == 1: - year_col = year_list[0] - df = par_old.copy() - df_y = self.df_interpolate( - df, - yrs_new, - horizon, - year_col, - 'value', - extrapolate=extrapolate, - extrapol_neg=extrapol_neg, - bound_extend=bound_extend) - sc_new.add_par(parname, df_y) - sc_new.commit(' ') - print( - '> Parameter "' + parname + '" copied and new years ' - 'added for node/s: "{}".'.format(nodes)) - - # V.B.3) Parameters with two dimensions related to time (such as - # 'input','output', etc.) - elif len(year_list) == 2: - year_col = 'year_act' - node_col = 'node_loc' - year_ref = [x for x in year_list if x != year_col][0] - - year_diff = [x for x in horizon[1:- - 1] if horizon[ - horizon.index(x) + 1] - - horizon[horizon.index(x)] > horizon[ - horizon.index(x)] - horizon[ - horizon.index(x) - 1]] - print( - '> Parameter "{}" is being added for node/s "{}"...'.format( - parname, - nodes)) - - # Flagging technologies that have lifetime for adding new timesteps - firstyear_new = int( - sc_new.set( - 'cat_year', { - 'type_year': 'firstmodelyear'})['year']) - min_step = min(np.diff( - sorted([int(x) for x in set(sc_new.set('year')) if int( - x) > firstyear_new]))) - par_tec = sc_new.par( - 'technical_lifetime', { - 'node_loc': nodes}) - # Technologies with lifetime bigger than minimum time interval - par_tec = par_tec.loc[par_tec['value'] > min_step] - df = par_old.copy() - - if parname == 'relation_activity': - tec_list = [] - else: - tec_list = [t for t in list(set(df[ - 'technology'])) if t in list(set(par_tec[ - 'technology']))] - - df_y = self.df_interpolate_2d( - df, - yrs_new, - horizon, - year_ref, - year_col, - tec_list, - par_tec, - value_col='value', - extrapolate=extrapolate, - extrapol_neg=extrapol_neg, - year_diff=year_diff) - sc_new.add_par(parname, df_y) - sc_new.commit(parname) - print( - '> Parameter "' + parname + '" copied and new years added ' - 'for node/s: "{}".'.format(nodes)) - - # %% VI) Required functions - # VI.A) A function to add new years to a datafarme by interpolation and - # (extrapolation if needed) - - def df_interpolate( - self, + sc_new.init_par( + parname, + idx_sets=sc_ref.idx_sets(parname).tolist(), + idx_names=sc_ref.idx_names(parname).tolist()) + sc_new.commit('New parameter initiated!') + + if node_col: + par_old = sc_ref.par(parname, {node_col[0]: region_list}) + par_new = sc_new.par(parname, {node_col[0]: region_list}) + sort_order = [ + node_col[0], + 'technology', + 'commodity', + 'year_vtg', + 'year_act'] + nodes = par_old[node_col[0]].unique().tolist() + else: + par_old = sc_ref.par(parname) + par_new = sc_new.par(parname) + sort_order = ['technology', 'commodity', 'year_vtg', 'year_act'] + nodes = ['N/A'] + + if not par_new.empty and not rewrite: + print( + '> Parameter "' + parname + '" already has data ' + 'in new scenario and left unchanged for node/s: {}.'.format( + region_list)) + return + if par_old.empty: + print( + '> Parameter "' + parname + '" is empty in reference scenario ' + 'for node/s: {}!'.format(region_list)) + return + + # Sorting the data to make it ready for dataframe manupulation + sort_order = [x for x in sort_order if x in idx_names] + if sort_order: + par_old = par_old.sort_values(sort_order).reset_index(drop=True) + + sc_new.check_out() + if not par_new.empty and rewrite: + print( + '> Parameter "' + parname + '" is being removed from new ' + 'scenario to be updated for node/s in {}...'.format(nodes)) + sc_new.remove_par(parname, par_new) + + col_list = sc_ref.idx_names(parname).tolist() + year_list = [ + c for c in col_list if c in [ + 'year', + 'year_vtg', + 'year_act', + 'year_rel']] + + # A uniform "unit" for values in different years to prevent mistakes in + # indexing and grouping + if 'unit' in col_list and unit_check: + par_old = unit_uniform(par_old) +# -------------------------------------------------------------------------------------------------------- +# V.B) Treatment of the new years in the specified parameter based on +# the time-related dimension of that parameter +# V.B.1) Parameters with no time component + if len(year_list) == 0: + sc_new.add_par(parname, par_old) + sc_new.commit(parname) + print( + '> Parameter "' + parname + '" just copied to new scenario ' + 'since has no time-related entries.') + +# V.B.2) Parameters with one dimension related to time + elif len(year_list) == 1: + year_col = year_list[0] + df = par_old.copy() + df_y = self.df_interpolate( df, yrs_new, horizon, year_col, - value_col='value', - extrapolate=False, - extrapol_neg=None, - bound_extend=True): - ''' - Description: - This function receives a parameter data as a dataframe, and adds - new data for the additonal years by interpolation and - extrapolation. - - Input arguments: - df_par (dataframe): the dataframe of the parameter to which new - years to be added - yrs_new (list of integers): new years to be added - horizon (list of integers): the horizon of the reference scenario - year_col (string): the header of the column to which the new years - should be added, for example, "year_act" - value_col (string): the header of the column containing values - extrapolate: if True, the extrapolation is allowed when a new year - is outside the parameter years - extrapol_neg: if True, negative values obtained by extrapolation - are allowed. - - Usage: - This function is called by function "addPar" - ''' - horizon_new = sorted(horizon + yrs_new) - idx = [x for x in df.columns if x not in [year_col, value_col]] - if not df.empty: - df2 = df.pivot_table(index=idx, columns=year_col, values=value_col) - - # To sort the new years smaller than the first year for - # extrapolation (e.g. 2025 values are calculated first; then - # values of 2015 based on 2020 and 2025) - year_before = sorted( - [x for x in yrs_new if x < min(df2.columns)], reverse=True) - if year_before and extrapolate: - for y in year_before: - yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y))) - - for yr in yrs_new: - if yr > max(horizon): - extrapol = True - else: - extrapol = extrapolate - - # a) If this new year is after the current range of modeled - # years, do extrapolation - if extrapol: - if yr == horizon_new[horizon_new.index( - max(df2.columns)) + 1]: - year_pre = max([x for x in df2.columns if x < yr]) - - if len([x for x in df2.columns if x < yr]) >= 2: - year_pp = max( - [x for x in df2.columns if x < year_pre]) - - df2[yr] = intpol(df2[year_pre], df2[year_pp], - year_pre, year_pp, yr) - - if bound_extend: - df2[yr] = df2[yr].fillna(df2[year_pre]) - - df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_pre] >= 0)].empty: - df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), - yr] = df2.loc[(df2[yr] < 0) & (df2[ - year_pre] >= 0), - year_pre] * extrapol_neg - else: - df2[yr] = df2[year_pre] - - # b) If this new year is before the current range of modeled - # years, do extrapolation - elif yr < min(df2.columns) and extrapol: - year_next = min([x for x in df2.columns if x > yr]) - - if len([x for x in df2.columns if x > yr]) >= 2: - year_nn = horizon[horizon.index(yr) + 2] - df2[yr] = intpol( - df2[year_next], df2[year_nn], - year_next, year_nn, yr) - df2[yr][np.isinf(df2[year_next])] = df2[year_next] - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_next] >= 0)].empty: - df2.loc[(df2[yr] < 0) & - (df2[year_next] >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_next] >= 0), - year_next] * extrapol_neg - - elif bound_extend: - df2[yr] = df2[year_next] - - # c) Otherise, do intrapolation - elif yr > min(df2.columns) and yr < max(df2.columns): - year_pre = max([x for x in df2.columns if x < yr]) - year_next = min([x for x in df2.columns if x > yr]) - df2[yr] = intpol( - df2[year_pre], df2[year_next], year_pre, year_next, yr) - - # Extrapolate for new years if the value exists for the - # previous year but not for the next years - # TODO: here is the place that should be changed if the - # new year should go the time step before the existing one - if [x for x in df2.columns if x < year_pre]: - year_pp = max([x for x in df2.columns if x < year_pre]) - df2[yr] = df2[yr].fillna( - intpol( - df2[year_pre], - df2[year_pp], - year_pre, - year_pp, - yr)) - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_pre] >= 0)].empty: - df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_pre] >= 0), - year_pre] * extrapol_neg + 'value', + extrapolate=extrapolate, + extrapol_neg=extrapol_neg, + bound_extend=bound_extend) + sc_new.add_par(parname, df_y) + sc_new.commit(' ') + print( + '> Parameter "' + parname + '" copied and new years ' + 'added for node/s: "{}".'.format(nodes)) + +# V.B.3) Parameters with two dimensions related to time (such as +# 'input','output', etc.) + elif len(year_list) == 2: + year_col = 'year_act' + node_col = 'node_loc' + year_ref = [x for x in year_list if x != year_col][0] + + year_diff = [x for x in horizon[1:- + 1] if horizon[ + horizon.index(x) + 1] - + horizon[horizon.index(x)] > horizon[ + horizon.index(x)] - horizon[ + horizon.index(x) - 1]] + print( + '> Parameter "{}" is being added for node/s "{}"...'.format( + parname, + nodes)) - if bound_extend: - df2[yr] = df2[yr].fillna(df2[year_pre]) - df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - - df2 = pd.melt( - df2.reset_index(), - id_vars=idx, - value_vars=[ - x for x in df2.columns if x not in idx], - var_name=year_col, - value_name=value_col).dropna( - subset=[value_col]).reset_index( - drop=True) - df2 = df2.sort_values(idx).reset_index(drop=True) + # Flagging technologies that have lifetime for adding new timesteps + firstyear_new = int( + sc_new.set( + 'cat_year', { + 'type_year': 'firstmodelyear'})['year']) + min_step = min(np.diff( + sorted([int(x) for x in set(sc_new.set('year')) if int( + x) > firstyear_new]))) + par_tec = sc_new.par( + 'technical_lifetime', { + 'node_loc': nodes}) + # Technologies with lifetime bigger than minimum time interval + par_tec = par_tec.loc[par_tec['value'] > min_step] + df = par_old.copy() + + if parname == 'relation_activity': + tec_list = [] else: - print( - '+++ WARNING: The submitted dataframe is empty, so returned' + - 'empty results!!! +++') - df2 = df - return df2 - - # %% VI.B) A function to interpolate the data for new time steps in - # parameters with two dimensions related to time + tec_list = [t for t in list(set(df[ + 'technology'])) if t in list(set(par_tec[ + 'technology']))] - def df_interpolate_2d( - self, + df_y = self.df_interpolate_2d( df, yrs_new, horizon, @@ -717,330 +473,505 @@ def df_interpolate_2d( tec_list, par_tec, value_col='value', - extrapolate=False, - extrapol_neg=None, - year_diff=None): - ''' - Description: - This function receives a dataframe that has 2 time-related columns - (e.g., "input" or "relation_activity"), and adds new data for the - additonal years in both time-related columns by interpolation - and extrapolation. - - Input arguments: - df (dataframe): the parameter to which new years to be added - yrs_new (list of integers): new years to be added - horizon (list of integers): the horizon of the reference scenario - year_ref (string): the header of the first column to which the new - years should be added, for example, "year_vtg" - year_col (string): the header of the second column to which the - new years should be added, for example, "year_act" - value_col (string): the header of the column containing values - extrapolate: if True, the extrapolation is allowed when a new year - is outside the parameter years - extrapol_neg: if True, negative values obtained by extrapolation - are allowed. - - Usage: - This utility is called by function "addPar" - ''' - def f_index(df1, df2): return df1.loc[df1.index.isin( - df2.index)] # For checking the index of two dataframes - - idx = [x for x in df.columns if x not in [year_col, value_col]] - if df.empty: - return df - print( - '+++ WARNING: The submitted dataframe is empty, so' + - 'returned empty results!!! +++') - - df_tec = df.loc[df['technology'].isin(tec_list)] - df2 = df.pivot_table(index=idx, columns=year_col, values='value') - df2_tec = df_tec.pivot_table( - index=idx, columns=year_col, values='value') - - # ------------------------------------------------------------------------------ - # First, changing the time interval for the transition period - # (e.g., year 2010 in old R11 model transits from 5 year to 10 year) - horizon_new = sorted(horizon + - [x for x in yrs_new if x not in horizon]) - yr_diff_new = [x for x in horizon_new[1:- - 1] if horizon_new[ - horizon_new.index(x) + - 1] - - horizon_new[horizon_new.index(x)] > horizon_new[ - horizon_new.index(x)] - horizon_new[ - horizon_new.index(x) - 1]] - - if year_diff and tec_list: - if isinstance(year_diff, list): - year_diff = year_diff[0] - - # Removing data from old transition year - if not yr_diff_new or year_diff not in yr_diff_new: - year_next = [x for x in df2.columns if x > year_diff][0] - - df_pre = f_slice( - df2_tec, idx, year_ref, [year_diff], year_diff) - df_next = f_slice( - df2_tec, idx, year_ref, [year_next], year_diff) - df_count = pd.DataFrame({ - 'c_pre': df_pre.count(axis=1), - 'c_next': df_next.loc[df_next.index.isin( - df_pre.index)].count(axis=1)}, - index=df_pre.index) - df_y = df_count.loc[df_count['c_pre'] == df_count[ - 'c_next'] + 1] - - for i in df_y.index: - df2.loc[i, df2.columns > (df2.loc[[i]].notnull().cumsum( - axis=1) == df_count[ - 'c_next'][i]).idxmax(axis=1).values[0]] = np.nan - - # Generating duration_period_sum matrix for masking - df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) - for i in df_dur.index: - for j in [x for x in df_dur.columns if x > i]: - df_dur.loc[i, j] = j - i - - # Adding data for new transition year - if yr_diff_new and tec_list and year_diff not in yr_diff_new: - yrs = [x for x in horizon if x <= yr_diff_new[0]] - year_next = min([x for x in df2.columns if x > yr_diff_new[0]]) - df_yrs = f_slice(df2_tec, idx, year_ref, yrs, []) - if yr_diff_new[0] in df2.columns: - df_yrs = df_yrs.loc[~pd.isna(df_yrs[yr_diff_new[0]]), :] - df_yrs = df_yrs.append( - f_slice( - df2_tec, - idx, - year_ref, - [year_next], - []), - ignore_index=False).reset_index( - ).sort_values(idx).set_index(idx) - - for yr in sorted( - [x for x in list(set(df_yrs.reset_index( - )[year_ref])) if x < year_next]): - yr_next = min([x for x in horizon_new if x > yr]) - d = f_slice(df_yrs, idx, year_ref, [yr], []) - d_n = f_slice(df_yrs, idx, year_ref, [yr_next], yr) - - if d_n[year_next].loc[~pd.isna(d_n[year_next])].empty: - if [x for x in horizon_new if x > yr_next]: - yr_nn = min([x for x in horizon_new if x > yr_next]) - else: - yr_nn = yr_next - d_n = f_slice(df_yrs, idx, year_ref, [yr_nn], yr) - d_n = d_n.loc[d_n.index.isin(d.index), :] - d = d.loc[d.index.isin(d_n.index), :] - d[d.isnull() & d_n.notnull()] = d_n - df2.loc[df2.index.isin(d.index), :] = d - - df_dur.loc[df_dur.index <= yr_diff_new[0], - df_dur.columns >= year_next] = df_dur.loc[ - df_dur.index <= yr_diff_new[0], - df_dur.columns >= year_next] - ( - yr_diff_new[0] - horizon_new[horizon_new.index( - yr_diff_new[0]) - 1]) - # -------------------------------------------------------------------------- - # Second, adding year_act of new years when year_vtg is in the existing - # years + extrapolate=extrapolate, + extrapol_neg=extrapol_neg, + year_diff=year_diff) + sc_new.add_par(parname, df_y) + sc_new.commit(parname) + print( + '> Parameter "' + parname + '" copied and new years added ' + 'for node/s: "{}".'.format(nodes)) + +# %% VI) Required functions +# VI.A) A function to add new years to a datafarme by interpolation and +# (extrapolation if needed) + +def df_interpolate( + self, + df, + yrs_new, + horizon, + year_col, + value_col='value', + extrapolate=False, + extrapol_neg=None, + bound_extend=True): + ''' + Description: + This function receives a parameter data as a dataframe, and adds + new data for the additonal years by interpolation and + extrapolation. + + Input arguments: + df_par (dataframe): the dataframe of the parameter to which new + years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_col (string): the header of the column to which the new years + should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This function is called by function "add_year_par" + ''' + horizon_new = sorted(horizon + yrs_new) + idx = [x for x in df.columns if x not in [year_col, value_col]] + if not df.empty: + df2 = df.pivot_table(index=idx, columns=year_col, values=value_col) + + # To sort the new years smaller than the first year for + # extrapolation (e.g. 2025 values are calculated first; then + # values of 2015 based on 2020 and 2025) + year_before = sorted( + [x for x in yrs_new if x < min(df2.columns)], reverse=True) + if year_before and extrapolate: + for y in year_before: + yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y))) + for yr in yrs_new: if yr > max(horizon): extrapol = True else: extrapol = extrapolate - # a) If this new year is greater than the current range of modeled + # a) If this new year is after the current range of modeled # years, do extrapolation - if yr > horizon_new[horizon_new.index( - max(df2.columns))] and extrapol: - year_pre = max([x for x in df2.columns if x < yr]) - year_pp = max([x for x in df2.columns if x < year_pre]) + if extrapol: + if yr == horizon_new[horizon_new.index( + max(df2.columns)) + 1]: + year_pre = max([x for x in df2.columns if x < yr]) - df2[yr] = intpol( - df2[year_pre], - df2[year_pp], - year_pre, - year_pp, - yr) - df2[yr][np.isinf(df2[year_pre].shift(+1)) - ] = df2[year_pre].shift(+1) - df2[yr] = df2[yr].fillna(df2[year_pre]) - - if yr - horizon_new[horizon_new.index(yr) - 1] >= horizon_new[ - horizon_new.index(yr) - 1] - horizon_new[ - horizon_new.index(yr) - 2]: - - df2[yr].loc[(pd.isna(df2[year_pre].shift(+1))) & ( - ~pd.isna(df2[year_pp].shift(+1)))] = np.nan - if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( - df2[year_pre].shift(+1) >= 0)].empty: - df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_pre].shift(+1) >= 0), - year_pre] * extrapol_neg - - # b) Otherise, do intrapolation + if len([x for x in df2.columns if x < yr]) >= 2: + year_pp = max( + [x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol(df2[year_pre], df2[year_pp], + year_pre, year_pp, yr) + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) + + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & (df2[ + year_pre] >= 0), + year_pre] * extrapol_neg + else: + df2[yr] = df2[year_pre] + + # b) If this new year is before the current range of modeled + # years, do extrapolation + elif yr < min(df2.columns) and extrapol: + year_next = min([x for x in df2.columns if x > yr]) + + if len([x for x in df2.columns if x > yr]) >= 2: + year_nn = horizon[horizon.index(yr) + 2] + df2[yr] = intpol( + df2[year_next], df2[year_nn], + year_next, year_nn, yr) + df2[yr][np.isinf(df2[year_next])] = df2[year_next] + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_next] >= 0)].empty: + df2.loc[(df2[yr] < 0) & + (df2[year_next] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_next] >= 0), + year_next] * extrapol_neg + + elif bound_extend: + df2[yr] = df2[year_next] + + # c) Otherise, do intrapolation elif yr > min(df2.columns) and yr < max(df2.columns): year_pre = max([x for x in df2.columns if x < yr]) year_next = min([x for x in df2.columns if x > yr]) - df2[yr] = intpol( - df2[year_pre], - df2[year_next], - year_pre, - year_next, - yr) - df2_t = df2.loc[df2_tec.index, :].copy() - - # This part calculates the missing value if only the previous - # timestep has a value (and not the next) - if tec_list: - df2_t[yr].loc[(pd.isna(df2_t[yr])) & (~pd.isna(df2_t[ - year_pre]))] = intpol( - df2_t[year_pre], - df2_t[year_next].shift(-1), - year_pre, year_next, yr) - - # Treating technologies with phase-out in model years - if [x for x in df2.columns if x < year_pre]: - year_pp = max([x for x in df2.columns if x < year_pre]) - df2_t[yr].loc[(pd.isna(df2_t[yr])) & ( - pd.isna(df2_t[year_pre].shift(-1))) & ( - ~pd.isna(df2_t[year_pre]))] = intpol( - df2_t[year_pre], - df2_t[year_pp], - year_pre, year_pp, yr) - - if extrapol_neg and not df2_t[yr].loc[( - df2_t[yr] < 0) & (df2_t[year_pre] >= 0)].empty: - df2_t.loc[(df2_t[yr] < 0) & (df2_t[year_pre] >= 0), - yr] = df2_t.loc[(df2_t[yr] < 0) & ( - df2_t[year_pre] >= 0), - year_pre] * extrapol_neg - df2.loc[df2_tec.index, :] = df2_t + df2[year_pre], df2[year_next], year_pre, year_next, yr) + + # Extrapolate for new years if the value exists for the + # previous year but not for the next years + # TODO: here is the place that should be changed if the + # new year should go the time step before the existing one + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2[yr] = df2[yr].fillna( + intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr)) + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre] >= 0), + year_pre] * extrapol_neg + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - df2 = df2.reindex(sorted(df2.columns), axis=1) - # -------------------------------------------------------------------------- - # Third, adding year_vtg of new years and their respective year_act for - # both existing and new years - for yr in yrs_new: - # a) If this new year is after the current range of modeled years, - # do extrapolation - if yr > max(horizon): - year_pre = horizon_new[horizon_new.index(yr) - 1] - year_pp = horizon_new[horizon_new.index(yr) - 2] - df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) - df_pp = f_slice(df2, idx, year_ref, [year_pp], yr) - df_yr = intpol( - df_pre, - f_index( - df_pp, - df_pre), - year_pre, - year_pp, - yr) - df_yr[np.isinf(df_pre)] = df_pre - - # For those technolofies with one value for each year - df_yr.loc[pd.isna(df_yr[yr])] = intpol( - df_pre, df_pp.shift(+1, axis=1), - year_pre, year_pp, yr).shift(+1, axis=1) - df_yr[pd.isna(df_yr)] = df_pre - - if extrapol_neg: - df_yr[(df_yr < 0) & (df_pre >= 0)] = df_pre * extrapol_neg - df_yr.loc[:, df_yr.columns < yr] = np.nan - - # c) Otherise, do intrapolation - elif yr > min(df2.columns) and yr < max(horizon): - year_pre = horizon_new[horizon_new.index(yr) - 1] - year_next = min([x for x in horizon if x > yr]) - - df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) - df_next = f_slice(df2, idx, year_ref, [year_next], yr) - df_yr = pd.concat(( - df_pre, - df_next.loc[df_next.index.isin(df_pre.index)]), - axis=0).groupby(level=idx).mean() - df_yr[yr] = df_yr[yr].fillna( - df_yr[[year_pre, year_next]].mean(axis=1)) - df_yr[np.isinf(df_pre)] = df_pre - - # Creating a mask to remove extra values - df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), - 'c_next': df_next.loc[ - df_next.index.isin(df_pre.index)].count(axis=1)}, - index=df_yr.index) - - for i in df_yr.index: - # Mainly for cases of two new consecutive years (like 2022 - # and 2024) - if ~np.isnan( - df_count['c_next'][i]) and df_count[ - 'c_pre'][i] >= df_count['c_next'][i] + 2: - df_yr[year_pre] = np.nan - - # For technologies phasing out before the end of horizon - # (like nuc_lc) - elif np.isnan(df_count['c_next'][i]): - df_yr.loc[i, :] = df_pre.loc[i, :].shift(+1) - if tec_list: - f_mask(df_yr, i, df_count['c_pre'][i] + 1, np.nan) - else: - f_mask(df_yr, i, df_count['c_pre'][i], np.nan) - - # For the rest - else: - df_yr[year_pre] = np.nan - f_mask(df_yr, i, df_count['c_pre'][i], np.nan) - else: - continue - - df2 = df2.append(df_yr) - df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() - # --------------------------------------------------------------------------- - # Forth: final masking based on technical lifetime - if tec_list: - - df3 = df2.copy() - for y in sorted([x for x in list( - set(df2.index.get_level_values(year_ref)) - ) if x in df_dur.index]): - df3.loc[df3.index.get_level_values(year_ref).isin([y]), - df3.columns.isin(df_dur.columns)] = df_dur.loc[ - y, df_dur.columns.isin(df3.columns)].values - - df3 = df3.reset_index().set_index( - ['node_loc', 'technology', year_ref]).sort_index(level=1) - par_tec = par_tec.set_index( - ['node_loc', 'technology', year_ref]).sort_index(level=1) - - for i in [x for x in par_tec.index if x in df3.index]: - df3.loc[i, 'lifetime'] = par_tec.loc[i, 'value'].copy() - - df3 = df3.reset_index().set_index(idx).dropna( - subset=['lifetime']).sort_index() - for i in df3.index: - df2.loc[i, df3.loc[i, :] >= int( - df3.loc[i, 'lifetime'])] = np.nan - - # Removing extra values from non-lifetime technologies - for i in [x for x in df2.index if x not in df3.index]: - df2.loc[i, df2.columns > df2.loc[[i]].index.get_level_values( - year_ref)[0]] = np.nan - - df_par = pd.melt( + df2 = pd.melt( df2.reset_index(), id_vars=idx, value_vars=[ x for x in df2.columns if x not in idx], var_name=year_col, - value_name='value').dropna( - subset=['value']) - df_par = df_par.sort_values(idx).reset_index(drop=True) - return df_par + value_name=value_col).dropna( + subset=[value_col]).reset_index( + drop=True) + df2 = df2.sort_values(idx).reset_index(drop=True) + else: + print( + '+++ WARNING: The submitted dataframe is empty, so returned' + + 'empty results!!! +++') + df2 = df + return df2 + +# %% VI.B) A function to interpolate the data for new time steps in +# parameters with two dimensions related to time + +def df_interpolate_2d( + self, + df, + yrs_new, + horizon, + year_ref, + year_col, + tec_list, + par_tec, + value_col='value', + extrapolate=False, + extrapol_neg=None, + year_diff=None): + ''' + Description: + This function receives a dataframe that has 2 time-related columns + (e.g., "input" or "relation_activity"), and adds new data for the + additonal years in both time-related columns by interpolation + and extrapolation. + + Input arguments: + df (dataframe): the parameter to which new years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_ref (string): the header of the first column to which the new + years should be added, for example, "year_vtg" + year_col (string): the header of the second column to which the + new years should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This utility is called by function "add_year_par" + ''' + def f_index(df1, df2): return df1.loc[df1.index.isin( + df2.index)] # For checking the index of two dataframes + + idx = [x for x in df.columns if x not in [year_col, value_col]] + if df.empty: + return df + print( + '+++ WARNING: The submitted dataframe is empty, so' + + 'returned empty results!!! +++') + + df_tec = df.loc[df['technology'].isin(tec_list)] + df2 = df.pivot_table(index=idx, columns=year_col, values='value') + df2_tec = df_tec.pivot_table( + index=idx, columns=year_col, values='value') + + # ------------------------------------------------------------------------------ + # First, changing the time interval for the transition period + # (e.g., year 2010 in old R11 model transits from 5 year to 10 year) + horizon_new = sorted(horizon + + [x for x in yrs_new if x not in horizon]) + yr_diff_new = [x for x in horizon_new[1:- + 1] if horizon_new[ + horizon_new.index(x) + + 1] - + horizon_new[horizon_new.index(x)] > horizon_new[ + horizon_new.index(x)] - horizon_new[ + horizon_new.index(x) - 1]] + + if year_diff and tec_list: + if isinstance(year_diff, list): + year_diff = year_diff[0] + + # Removing data from old transition year + if not yr_diff_new or year_diff not in yr_diff_new: + year_next = [x for x in df2.columns if x > year_diff][0] + + df_pre = slice_df( + df2_tec, idx, year_ref, [year_diff], year_diff) + df_next = slice_df( + df2_tec, idx, year_ref, [year_next], year_diff) + df_count = pd.DataFrame({ + 'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[df_next.index.isin( + df_pre.index)].count(axis=1)}, + index=df_pre.index) + df_y = df_count.loc[df_count['c_pre'] == df_count[ + 'c_next'] + 1] + + for i in df_y.index: + df2.loc[i, df2.columns > (df2.loc[[i]].notnull().cumsum( + axis=1) == df_count[ + 'c_next'][i]).idxmax(axis=1).values[0]] = np.nan + + # Generating duration_period_sum matrix for masking + df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) + for i in df_dur.index: + for j in [x for x in df_dur.columns if x > i]: + df_dur.loc[i, j] = j - i + + # Adding data for new transition year + if yr_diff_new and tec_list and year_diff not in yr_diff_new: + yrs = [x for x in horizon if x <= yr_diff_new[0]] + year_next = min([x for x in df2.columns if x > yr_diff_new[0]]) + df_yrs = slice_df(df2_tec, idx, year_ref, yrs, []) + if yr_diff_new[0] in df2.columns: + df_yrs = df_yrs.loc[~pd.isna(df_yrs[yr_diff_new[0]]), :] + df_yrs = df_yrs.append( + slice_df( + df2_tec, + idx, + year_ref, + [year_next], + []), + ignore_index=False).reset_index( + ).sort_values(idx).set_index(idx) + + for yr in sorted( + [x for x in list(set(df_yrs.reset_index( + )[year_ref])) if x < year_next]): + yr_next = min([x for x in horizon_new if x > yr]) + d = slice_df(df_yrs, idx, year_ref, [yr], []) + d_n = slice_df(df_yrs, idx, year_ref, [yr_next], yr) + + if d_n[year_next].loc[~pd.isna(d_n[year_next])].empty: + if [x for x in horizon_new if x > yr_next]: + yr_nn = min([x for x in horizon_new if x > yr_next]) + else: + yr_nn = yr_next + d_n = slice_df(df_yrs, idx, year_ref, [yr_nn], yr) + d_n = d_n.loc[d_n.index.isin(d.index), :] + d = d.loc[d.index.isin(d_n.index), :] + d[d.isnull() & d_n.notnull()] = d_n + df2.loc[df2.index.isin(d.index), :] = d + + df_dur.loc[df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] = df_dur.loc[ + df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] - ( + yr_diff_new[0] - horizon_new[horizon_new.index( + yr_diff_new[0]) - 1]) + # -------------------------------------------------------------------------- + # Second, adding year_act of new years when year_vtg is in the existing + # years + for yr in yrs_new: + if yr > max(horizon): + extrapol = True + else: + extrapol = extrapolate + + # a) If this new year is greater than the current range of modeled + # years, do extrapolation + if yr > horizon_new[horizon_new.index( + max(df2.columns))] and extrapol: + year_pre = max([x for x in df2.columns if x < yr]) + year_pp = max([x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr) + df2[yr][np.isinf(df2[year_pre].shift(+1)) + ] = df2[year_pre].shift(+1) + df2[yr] = df2[yr].fillna(df2[year_pre]) + + if yr - horizon_new[horizon_new.index(yr) - 1] >= horizon_new[ + horizon_new.index(yr) - 1] - horizon_new[ + horizon_new.index(yr) - 2]: + + df2[yr].loc[(pd.isna(df2[year_pre].shift(+1))) & ( + ~pd.isna(df2[year_pp].shift(+1)))] = np.nan + if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0), + year_pre] * extrapol_neg + + # b) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(df2.columns): + year_pre = max([x for x in df2.columns if x < yr]) + year_next = min([x for x in df2.columns if x > yr]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_next], + year_pre, + year_next, + yr) + df2_t = df2.loc[df2_tec.index, :].copy() + + # This part calculates the missing value if only the previous + # timestep has a value (and not the next) + if tec_list: + df2_t[yr].loc[(pd.isna(df2_t[yr])) & (~pd.isna(df2_t[ + year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_next].shift(-1), + year_pre, year_next, yr) + + # Treating technologies with phase-out in model years + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2_t[yr].loc[(pd.isna(df2_t[yr])) & ( + pd.isna(df2_t[year_pre].shift(-1))) & ( + ~pd.isna(df2_t[year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_pp], + year_pre, year_pp, yr) + + if extrapol_neg and not df2_t[yr].loc[( + df2_t[yr] < 0) & (df2_t[year_pre] >= 0)].empty: + df2_t.loc[(df2_t[yr] < 0) & (df2_t[year_pre] >= 0), + yr] = df2_t.loc[(df2_t[yr] < 0) & ( + df2_t[year_pre] >= 0), + year_pre] * extrapol_neg + df2.loc[df2_tec.index, :] = df2_t + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + df2 = df2.reindex(sorted(df2.columns), axis=1) + # -------------------------------------------------------------------------- + # Third, adding year_vtg of new years and their respective year_act for + # both existing and new years + for yr in yrs_new: + # a) If this new year is after the current range of modeled years, + # do extrapolation + if yr > max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_pp = horizon_new[horizon_new.index(yr) - 2] + df_pre = slice_df(df2, idx, year_ref, [year_pre], yr) + df_pp = slice_df(df2, idx, year_ref, [year_pp], yr) + df_yr = intpol( + df_pre, + f_index( + df_pp, + df_pre), + year_pre, + year_pp, + yr) + df_yr[np.isinf(df_pre)] = df_pre + + # For those technolofies with one value for each year + df_yr.loc[pd.isna(df_yr[yr])] = intpol( + df_pre, df_pp.shift(+1, axis=1), + year_pre, year_pp, yr).shift(+1, axis=1) + df_yr[pd.isna(df_yr)] = df_pre + + if extrapol_neg: + df_yr[(df_yr < 0) & (df_pre >= 0)] = df_pre * extrapol_neg + df_yr.loc[:, df_yr.columns < yr] = np.nan + + # c) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_next = min([x for x in horizon if x > yr]) + + df_pre = slice_df(df2, idx, year_ref, [year_pre], yr) + df_next = slice_df(df2, idx, year_ref, [year_next], yr) + df_yr = pd.concat(( + df_pre, + df_next.loc[df_next.index.isin(df_pre.index)]), + axis=0).groupby(level=idx).mean() + df_yr[yr] = df_yr[yr].fillna( + df_yr[[year_pre, year_next]].mean(axis=1)) + df_yr[np.isinf(df_pre)] = df_pre + + # Creating a mask to remove extra values + df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[ + df_next.index.isin(df_pre.index)].count(axis=1)}, + index=df_yr.index) + + for i in df_yr.index: + # Mainly for cases of two new consecutive years (like 2022 + # and 2024) + if ~np.isnan( + df_count['c_next'][i]) and df_count[ + 'c_pre'][i] >= df_count['c_next'][i] + 2: + df_yr[year_pre] = np.nan + + # For technologies phasing out before the end of horizon + # (like nuc_lc) + elif np.isnan(df_count['c_next'][i]): + df_yr.loc[i, :] = df_pre.loc[i, :].shift(+1) + if tec_list: + mask_df(df_yr, i, df_count['c_pre'][i] + 1, np.nan) + else: + mask_df(df_yr, i, df_count['c_pre'][i], np.nan) + + # For the rest + else: + df_yr[year_pre] = np.nan + mask_df(df_yr, i, df_count['c_pre'][i], np.nan) + + else: + continue + + df2 = df2.append(df_yr) + df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() + # --------------------------------------------------------------------------- + # Forth: final masking based on technical lifetime + if tec_list: + + df3 = df2.copy() + for y in sorted([x for x in list( + set(df2.index.get_level_values(year_ref)) + ) if x in df_dur.index]): + df3.loc[df3.index.get_level_values(year_ref).isin([y]), + df3.columns.isin(df_dur.columns)] = df_dur.loc[ + y, df_dur.columns.isin(df3.columns)].values + + df3 = df3.reset_index().set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + par_tec = par_tec.set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + + for i in [x for x in par_tec.index if x in df3.index]: + df3.loc[i, 'lifetime'] = par_tec.loc[i, 'value'].copy() + + df3 = df3.reset_index().set_index(idx).dropna( + subset=['lifetime']).sort_index() + for i in df3.index: + df2.loc[i, df3.loc[i, :] >= int( + df3.loc[i, 'lifetime'])] = np.nan + + # Removing extra values from non-lifetime technologies + for i in [x for x in df2.index if x not in df3.index]: + df2.loc[i, df2.columns > df2.loc[[i]].index.get_level_values( + year_ref)[0]] = np.nan + + df_par = pd.melt( + df2.reset_index(), + id_vars=idx, + value_vars=[ + x for x in df2.columns if x not in idx], + var_name=year_col, + value_name='value').dropna( + subset=['value']) + df_par = df_par.sort_values(idx).reset_index(drop=True) + return df_par From ead8656c91cd386e936f4f979b2c07a9287a5103 Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 2 May 2019 18:34:08 +0200 Subject: [PATCH 30/44] cleanup of main function --- message_ix/tools/add_year/__init__.py | 1677 ++++++++++++------------- 1 file changed, 780 insertions(+), 897 deletions(-) diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 17eb2e85a..ec8d25046 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -12,10 +12,10 @@ Sections of this code: I. Required python packages are imported and ixmp platform loaded. II. Generic utilities for dataframe manipulation - III. The main class called "addNewYear" - IV. Submodule "addSets" for adding and modifying the sets - V. Submodule "addPar" for copying and modifying each parameter - VI. The submodule "addPar" calls two utility functions ("df_interpolate" + III. The main class called "add_year" + IV. Submodule "add_year_set" for adding and modifying the sets + V. Submodule "add_year_par" for copying and modifying each parameter + VI. The submodule "add_year_par" calls two utility functions ("interpolate_1d" and "df_interpolate_2D") for calculating missing values. VII. Code for running the script as "main" @@ -24,12 +24,12 @@ This script can be used either: A) By running directly from the command line, example: --------------------------------------------------------------------------- - python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" + python f_add_year.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" --------------------------------------------------------------------------- (Other input arguments are optional. For more info see Section V below.) - B) By calling the class "addNewYear" from other python scripts + B) By calling the class "add_year" from other python scripts """ # %% I) Importing required packages and loading ixmp platform @@ -58,7 +58,7 @@ def intpol(y1, y2, x1, x2, x): # value: integer/string -def f_slice(df, idx, level, locator, value): +def slice_df(df, idx, level, locator, value): if locator: df = df.reset_index().loc[df.reset_index()[level].isin(locator)].copy() else: @@ -71,7 +71,7 @@ def f_slice(df, idx, level, locator, value): # dataframe -def f_mask(df, index, count, value): +def mask_df(df, index, count, value): df.loc[index, df.columns > (df.loc[[index]].notnull().cumsum( axis=1) == count).idxmax(axis=1).values[0]] = value @@ -93,10 +93,13 @@ def unit_uniform(df): # %% III) The main class -class addNewYear(object): +def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, + macro=False, baseyear_macro=None, parameter='all', region='all', + rewrite=True, unit_check=True, extrapol_neg=None, + bound_extend=True): ''' This class does the following: - A. calls function "addSets" to add and modify required sets. - B. calls function "addPar" to add new years and modifications + A. calls function "add_year_set" to add and modify required sets. + B. calls function "add_year_par" to add new years and modifications to each parameter if needed. Parameters: @@ -131,916 +134,796 @@ class addNewYear(object): (e.g., permitting the extension of a bound to 2025, when there is only one value in 2020) ''' - - def __init__(self, sc_ref, sc_new, years_new, firstyear_new=None, - lastyear_new=None, macro=False, baseyear_macro=None, - parameter=all, region=all, rewrite=True, unit_check=True, - extrapol_neg=None, bound_extend=True): - - # III.A) Adding sets and required modifications - years_new = sorted([x for x in years_new if str(x) - not in set(sc_ref.set('year'))]) - self.addSets( - sc_ref, - sc_new, - years_new, - firstyear_new, - lastyear_new, - baseyear_macro) - # -------------------------------------------------------------------------- - # III.B) Adding parameters and calculating the missing values for the - # additonal years - if parameter == all: - par_list = sorted(sc_ref.par_list()) - elif isinstance(parameter, list): - par_list = parameter - elif isinstance(parameter, str): - par_list = [parameter] + # III.A) Adding sets and required modifications + years_new = sorted([x for x in years_new if str(x) + not in set(sc_ref.set('year'))]) + add_year_set(sc_ref, sc_new, years_new, firstyear_new, lastyear_new, + baseyear_macro) + # ------------------------------------------------------------------------- + # III.B) Adding parameters and calculating the missing values for the + # additonal years + if parameter == 'all': + par_list = sorted(sc_ref.par_list()) + elif isinstance(parameter, list): + par_list = parameter + elif isinstance(parameter, str): + par_list = [parameter] + else: + print('Parameters should be defined in a list of strings or as' + ' a single string!') + + if 'technical_lifetime' in par_list: + par_list.insert(0, par_list.pop(par_list.index('technical_lifetime'))) + + if region == 'all': + reg_list = sc_ref.set('node').tolist() + elif isinstance(region, list): + reg_list = region + elif isinstance(region, str): + reg_list = [region] + else: + print('Regions should be defined in a list of strings or as' + ' a single string!') + + # List of parameters to be ignored (even not copied to the new + # scenario) + par_ignore = ['duration_period'] + par_list = [x for x in par_list if x not in par_ignore] + + if not macro: + par_macro = ['demand_MESSAGE', 'price_MESSAGE', 'cost_MESSAGE', + 'gdp_calibrate', 'historical_gdp', 'MERtoPPP', 'kgdp', + 'kpvs', 'depr', 'drate', 'esub', 'lotol', 'p_ref', 'lakl', + 'prfconst', 'grow', 'aeei', 'aeei_factor', 'gdp_rate'] + par_list = [x for x in par_list if x not in par_macro] + + firstyear = sc_new.set('cat_year', {'type_year': 'firstmodelyear'})['year'] + for parname in par_list: + # For historical parameters extrapolation permitted (e.g., from + # 2010 to 2015) + if 'historical' in parname: + extrapol = True + yrs_new = [x for x in years_new if x < int(firstyear)] else: - print( - 'Parameters should be defined in a list of strings or as' + - 'a single string!') - - if 'technical_lifetime' in par_list: - par_list.insert( - 0, par_list.pop( - par_list.index('technical_lifetime'))) - - if region == all: - reg_list = sc_ref.set('node').tolist() - elif isinstance(region, list): - reg_list = region - elif isinstance(region, str): - reg_list = [region] + extrapol = False + yrs_new = years_new + + if 'bound' in parname: + bound_ext = bound_extend else: - print('Regions should be defined in a list of strings or as' + - 'a single string!') - - # List of parameters to be ignored (even not copied to the new - # scenario) - par_ignore = ['duration_period'] - par_list = [x for x in par_list if x not in par_ignore] - - if not macro: - par_macro = [ - 'demand_MESSAGE', - 'price_MESSAGE', - 'cost_MESSAGE', - 'gdp_calibrate', - 'historical_gdp', - 'MERtoPPP', - 'kgdp', - 'kpvs', - 'depr', - 'drate', - 'esub', - 'lotol', - 'p_ref', - 'lakl', - 'prfconst', - 'grow', - 'aeei', - 'aeei_factor', - 'gdp_rate'] - par_list = [x for x in par_list if x not in par_macro] - - for parname in par_list: - # For historical parameters extrapolation permitted (e.g., from - # 2010 to 2015) - if 'historical' in parname: - extrapol = True - yrs_new = [ - x for x in years_new if x < int( - sc_new.set( - 'cat_year', { - 'type_year': 'firstmodelyear'})['year'])] - else: - extrapol = False - yrs_new = years_new + bound_ext = True - if 'bound' in parname: - bound_ext = bound_extend - else: - bound_ext = True - - year_list = [x for x in sc_ref.idx_sets(parname) if 'year' in x] - - if len(year_list) == 2 or parname in ['land_output']: - # The loop over "node" is only for reducing the size of tables - for node in reg_list: - self.addPar( - sc_ref, - sc_new, - yrs_new, - parname, - [node], - extrapolate=extrapol, - rewrite=rewrite, - unit_check=unit_check, - extrapol_neg=extrapol_neg, - bound_extend=bound_ext) - else: - self.addPar( - sc_ref, - sc_new, - yrs_new, - parname, - reg_list, - extrapolate=extrapol, - rewrite=rewrite, - unit_check=unit_check, - extrapol_neg=extrapol_neg, - bound_extend=bound_ext) - - sc_new.set_as_default() - print('> All required parameters were successfully ' + - 'added to the new scenario.') - - # %% Submodules needed for running the main function - # IV) Adding new years to sets - def addSets( - self, - sc_ref, - sc_new, - years_new, - firstyear_new=None, - lastyear_new=None, - baseyear_macro=None): - ''' - Description: - Adding required sets and relevant modifications: - This function adds additional years to an existing scenario, - by starting to make a new scenario from scratch. - After modification of the year-related sets, this function copeis - all other sets from the "reference" scenario - to the "new" scenario. - - Input arguments: - Please see the description for the input arguments under the main - function "addNewYear" - - Usage: - This module is called by function "addNewYear" - ''' - # IV.A) Treatment of the additional years in the year-related sets - - # A.1. Set - year - yrs_old = list(map(int, sc_ref.set('year'))) - horizon_new = sorted(yrs_old + years_new) - sc_new.add_set('year', [str(yr) for yr in horizon_new]) - - # A.2. Set _ type_year - yr_typ = sc_ref.set('type_year').tolist() - sc_new.add_set('type_year', sorted( - yr_typ + [str(yr) for yr in years_new])) - - # A.3. Set _ cat_year - yr_cat = sc_ref.set('cat_year') - - # A.4. Changing the first year if needed - if firstyear_new: - yr_cat.loc[yr_cat['type_year'] == - 'firstmodelyear', 'year'] = firstyear_new - if lastyear_new: - yr_cat.loc[yr_cat['type_year'] == - 'lastmodelyear', 'year'] = lastyear_new - - # A.5. Changing the base year and initialization year of macro if a new - # year specified - if not yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'].empty and baseyear_macro: - yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'] = baseyear_macro - if not yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year' - ].empty and baseyear_macro: - yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year'] = baseyear_macro - - yr_pair = [] - for yr in years_new: - yr_pair.append([yr, yr]) - yr_pair.append(['cumulative', yr]) - - yr_cat = yr_cat.append( - pd.DataFrame( - yr_pair, - columns=[ - 'type_year', - 'year']), - ignore_index=True).sort_values('year').reset_index( - drop=True) - - # A.6. Changing the cumulative years based on the new first model year - firstyear_new = int( - yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', 'year']) - yr_cat = yr_cat.drop(yr_cat.loc[ - (yr_cat['type_year'] == 'cumulative') & ( - yr_cat['year'] < firstyear_new)].index) - sc_new.add_set('cat_year', yr_cat) - - # IV.B) Copying all other sets - set_list = [s for s in sc_ref.set_list() if 'year' not in s] - # Sets with one index set - index_list = [ - x for x in set_list if not isinstance( - sc_ref.set(x), pd.DataFrame)] - for set_name in index_list: - if set_name not in sc_new.set_list(): - sc_new.init_set(set_name, idx_sets=None, idx_names=None) - sc_new.add_set(set_name, sc_ref.set(set_name).tolist()) - - # The rest of the sets - for set_name in [x for x in set_list if x not in index_list]: - if set_name not in sc_new.set_list() and not [ - x for x in sc_ref.idx_sets( - set_name) if x not in sc_ref.set_list()]: - sc_new.init_set(set_name, - idx_sets=sc_ref.idx_sets(set_name).tolist(), - idx_names=sc_ref.idx_names(set_name).tolist()) - sc_new.add_set(set_name, sc_ref.set(set_name)) - - sc_new.commit('sets added!') - print('> All the sets updated and added to the new scenario.') - - # %% V) Adding new years to parameters - - def addPar( - self, - sc_ref, - sc_new, - yrs_new, - parname, - region_list, - extrapolate=False, - rewrite=True, - unit_check=True, - extrapol_neg=None, - bound_extend=True): - ''' Adding required parameters and relevant modifications: - Description: - This function adds additional years to a parameter. - The value of the parameter for additional years is calculated - mainly by interpolating and extrapolating of data from existing - years. - - Input arguments: - Please see the description for the input arguments under the - main function "addNewYear" - - Usage: - This module is called by function "addNewYear" - ''' - - # V.A) Initialization and checks - - par_list_new = sc_new.par_list().tolist() - idx_names = sc_ref.idx_names(parname) - horizon = sorted([int(x) for x in list(set(sc_ref.set('year')))]) - node_col = [ - x for x in idx_names if x in [ - 'node', - 'node_loc', - 'node_rel']] - - if parname not in par_list_new: - sc_new.check_out() - sc_new.init_par( - parname, - idx_sets=sc_ref.idx_sets(parname).tolist(), - idx_names=sc_ref.idx_names(parname).tolist()) - sc_new.commit('New parameter initiated!') - - if node_col: - par_old = sc_ref.par(parname, {node_col[0]: region_list}) - par_new = sc_new.par(parname, {node_col[0]: region_list}) - sort_order = [ - node_col[0], - 'technology', - 'commodity', - 'year_vtg', - 'year_act'] - nodes = par_old[node_col[0]].unique().tolist() + year_list = [x for x in sc_ref.idx_sets(parname) if 'year' in x] + + if len(year_list) == 2 or parname in ['land_output']: + # The loop over "node" is only for reducing the size of tables + for node in reg_list: + add_year_par(sc_ref, sc_new, yrs_new, parname, [node], + extrapol, rewrite, unit_check, extrapol_neg, bound_ext) else: - par_old = sc_ref.par(parname) - par_new = sc_new.par(parname) - sort_order = ['technology', 'commodity', 'year_vtg', 'year_act'] - nodes = ['N/A'] - - if not par_new.empty and not rewrite: - print( - '> Parameter "' + parname + '" already has data ' - 'in new scenario and left unchanged for node/s: {}.'.format( - region_list)) - return - if par_old.empty: - print( - '> Parameter "' + parname + '" is empty in reference scenario ' - 'for node/s: {}!'.format(region_list)) - return - - # Sorting the data to make it ready for dataframe manupulation - sort_order = [x for x in sort_order if x in idx_names] - if sort_order: - par_old = par_old.sort_values(sort_order).reset_index(drop=True) + add_year_par(sc_ref, sc_new, yrs_new, parname, reg_list, + extrapol, rewrite, unit_check, extrapol_neg, bound_ext) + + sc_new.set_as_default() + print('> All required parameters were successfully ' + + 'added to the new scenario.') + +# %% Submodules needed for running the main function +# IV) Adding new years to sets +def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, + lastyear_new=None, baseyear_macro=None): + ''' + Description: + Adding required sets and relevant modifications: + This function adds additional years to an existing scenario, + by starting to make a new scenario from scratch. + After modification of the year-related sets, this function copeis + all other sets from the "reference" scenario + to the "new" scenario. + + Input arguments: + Please see the description for the input arguments under the main + function "add_year" + + Usage: + This module is called by function "add_year" + ''' +# IV.A) Treatment of the additional years in the year-related sets + + # A.1. Set - year + yrs_old = list(map(int, sc_ref.set('year'))) + horizon_new = sorted(yrs_old + years_new) + sc_new.add_set('year', [str(yr) for yr in horizon_new]) + + # A.2. Set _ type_year + yr_typ = sc_ref.set('type_year').tolist() + sc_new.add_set('type_year', sorted(yr_typ + [str(yr) for yr in years_new])) + + # A.3. Set _ cat_year + yr_cat = sc_ref.set('cat_year') + + # A.4. Changing the first year if needed + if firstyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'firstmodelyear', 'year'] = firstyear_new + if lastyear_new: + yr_cat.loc[yr_cat['type_year'] == + 'lastmodelyear', 'year'] = lastyear_new + + # A.5. Changing the base year and initialization year of macro if a new + # year specified + if not yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'] = baseyear_macro + if not yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'].empty and baseyear_macro: + yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'] = baseyear_macro + + yr_pair = [] + for yr in years_new: + yr_pair.append([yr, yr]) + yr_pair.append(['cumulative', yr]) + + yr_cat = yr_cat.append(pd.DataFrame(yr_pair, + columns=['type_year', 'year']), + ignore_index=True + ).sort_values('year' + ).reset_index(drop=True) + + # A.6. Changing the cumulative years based on the new first model year + firstyear_new = int(yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', + 'year']) + yr_cat = yr_cat.drop(yr_cat.loc[(yr_cat['type_year'] == 'cumulative') & ( + yr_cat['year'] < firstyear_new)].index) + sc_new.add_set('cat_year', yr_cat) + + # IV.B) Copying all other sets + set_list = [s for s in sc_ref.set_list() if 'year' not in s] + # Sets with one index set + index_list = [x for x in set_list if not isinstance(sc_ref.set(x), + pd.DataFrame)] + for set_name in index_list: + if set_name not in sc_new.set_list(): + sc_new.init_set(set_name, idx_sets=None, idx_names=None) + sc_new.add_set(set_name, sc_ref.set(set_name).tolist()) + + # The rest of the sets + + for set_name in [x for x in set_list if x not in index_list]: + new_set = [x for x in sc_ref.idx_sets(set_name + ) if x not in sc_ref.set_list()] + if set_name not in sc_new.set_list() and not new_set: + sc_new.init_set(set_name, + idx_sets=sc_ref.idx_sets(set_name).tolist(), + idx_names=sc_ref.idx_names(set_name).tolist()) + sc_new.add_set(set_name, sc_ref.set(set_name)) + + sc_new.commit('sets added!') + print('> All the sets updated and added to the new scenario.') + +# %% V) Adding new years to parameters + +def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, + extrapolate=False, rewrite=True, unit_check=True, + extrapol_neg=None, bound_extend=True): + ''' Adding required parameters and relevant modifications: + Description: + This function adds additional years to a parameter. + The value of the parameter for additional years is calculated + mainly by interpolating and extrapolating of data from existing + years. + + Input arguments: + Please see the description for the input arguments under the + main function "add_year" + + Usage: + This module is called by function "add_year" + ''' + +# V.A) Initialization and checks + par_list_new = sc_new.par_list().tolist() + idx_names = sc_ref.idx_names(parname) + horizon = sorted([int(x) for x in list(set(sc_ref.set('year')))]) + node_col = [x for x in idx_names if x in ['node', 'node_loc', 'node_rel']] + + if parname not in par_list_new: sc_new.check_out() - if not par_new.empty and rewrite: - print( - '> Parameter "' + parname + '" is being removed from new ' - 'scenario to be updated for node/s in {}...'.format(nodes)) - sc_new.remove_par(parname, par_new) - - col_list = sc_ref.idx_names(parname).tolist() - year_list = [ - c for c in col_list if c in [ - 'year', - 'year_vtg', - 'year_act', - 'year_rel']] - - # A uniform "unit" for values in different years to prevent mistakes in - # indexing and grouping - if 'unit' in col_list and unit_check: - par_old = unit_uniform(par_old) - # -------------------------------------------------------------------------------------------------------- - # V.B) Treatment of the new years in the specified parameter based on - # the time-related dimension of that parameter - # V.B.1) Parameters with no time component - if len(year_list) == 0: - sc_new.add_par(parname, par_old) - sc_new.commit(parname) - print( - '> Parameter "' + parname + '" just copied to new scenario ' - 'since has no time-related entries.') - - # V.B.2) Parameters with one dimension related to time - elif len(year_list) == 1: - year_col = year_list[0] - df = par_old.copy() - df_y = self.df_interpolate( - df, - yrs_new, - horizon, - year_col, - 'value', - extrapolate=extrapolate, - extrapol_neg=extrapol_neg, - bound_extend=bound_extend) - sc_new.add_par(parname, df_y) - sc_new.commit(' ') - print( - '> Parameter "' + parname + '" copied and new years ' - 'added for node/s: "{}".'.format(nodes)) - - # V.B.3) Parameters with two dimensions related to time (such as - # 'input','output', etc.) - elif len(year_list) == 2: - year_col = 'year_act' - node_col = 'node_loc' - year_ref = [x for x in year_list if x != year_col][0] - - year_diff = [x for x in horizon[1:- - 1] if horizon[ - horizon.index(x) + 1] - - horizon[horizon.index(x)] > horizon[ - horizon.index(x)] - horizon[ - horizon.index(x) - 1]] - print( - '> Parameter "{}" is being added for node/s "{}"...'.format( - parname, - nodes)) - - # Flagging technologies that have lifetime for adding new timesteps - firstyear_new = int( - sc_new.set( - 'cat_year', { - 'type_year': 'firstmodelyear'})['year']) - min_step = min(np.diff( - sorted([int(x) for x in set(sc_new.set('year')) if int( - x) > firstyear_new]))) - par_tec = sc_new.par( - 'technical_lifetime', { - 'node_loc': nodes}) - # Technologies with lifetime bigger than minimum time interval - par_tec = par_tec.loc[par_tec['value'] > min_step] - df = par_old.copy() - - if parname == 'relation_activity': - tec_list = [] + sc_new.init_par(parname, idx_sets=sc_ref.idx_sets(parname).tolist(), + idx_names=sc_ref.idx_names(parname).tolist()) + sc_new.commit('New parameter initiated!') + + if node_col: + par_old = sc_ref.par(parname, {node_col[0]: region_list}) + par_new = sc_new.par(parname, {node_col[0]: region_list}) + sort_order = [node_col[0], 'technology', 'commodity', 'year_vtg', + 'year_act'] + nodes = par_old[node_col[0]].unique().tolist() + else: + par_old = sc_ref.par(parname) + par_new = sc_new.par(parname) + sort_order = ['technology', 'commodity', 'year_vtg', 'year_act'] + nodes = ['N/A'] + + if not par_new.empty and not rewrite: + print('> Parameter "' + parname + '" already has data in new scenario' + ' and left unchanged for node/s: {}.'.format(region_list)) + return + if par_old.empty: + print('> Parameter "' + parname + '" is empty in reference scenario' + ' for node/s: {}!'.format(region_list)) + return + + # Sorting the data to make it ready for dataframe manupulation + sort_order = [x for x in sort_order if x in idx_names] + if sort_order: + par_old = par_old.sort_values(sort_order).reset_index(drop=True) + + sc_new.check_out() + if not par_new.empty and rewrite: + print('> Parameter "' + parname + '" is being removed from new' + ' scenario to be updated for node/s in {}...'.format(nodes)) + sc_new.remove_par(parname, par_new) + + col_list = sc_ref.idx_names(parname).tolist() + year_list = [c for c in col_list if c in ['year', 'year_vtg', 'year_act', + 'year_rel']] + + # A uniform "unit" for values in different years to prevent mistakes in + # indexing and grouping + if 'unit' in col_list and unit_check: + par_old = unit_uniform(par_old) +# -------------------------------------------------------------------------------------------------------- +# V.B) Treatment of the new years in the specified parameter based on +# the time-related dimension of that parameter +# V.B.1) Parameters with no time component + if len(year_list) == 0: + sc_new.add_par(parname, par_old) + sc_new.commit(parname) + print('> Parameter "' + parname + '" just copied to new scenario ' + 'since has no time-related entries.') + +# V.B.2) Parameters with one dimension related to time + elif len(year_list) == 1: + year_col = year_list[0] + df = par_old.copy() + df_y = interpolate_1d(df, yrs_new, horizon, year_col, 'value', + extrapolate, extrapol_neg, bound_extend) + sc_new.add_par(parname, df_y) + sc_new.commit(' ') + print('> Parameter "{}" copied and new years ' + 'added for node/s: "{}".'.format(parname, nodes)) + +# V.B.3) Parameters with two dimensions related to time (such as +# 'input','output', etc.) + elif len(year_list) == 2: + year_col = 'year_act' + node_col = 'node_loc' + year_ref = [x for x in year_list if x != year_col][0] + g = lambda li, i: li[i + 1] - li[i] > li[i] - li[i - 1] + + year_diff = [x for x in horizon[1:-1] if g(horizon, horizon.index(x))] + print('> Parameter "{}" is being added for node/s' + ' "{}"...'.format(parname, nodes)) + + # Flagging technologies that have lifetime for adding new timesteps + firstyear_new = sc_new.set('cat_year', + {'type_year': 'firstmodelyear'})['year'] + yr_list = [int(x) for x in set(sc_new.set('year') + ) if int(x) > int(firstyear_new)] + min_step = min(np.diff(sorted(yr_list))) + par_tec = sc_new.par('technical_lifetime', {'node_loc': nodes}) + # Technologies with lifetime bigger than minimum time interval + par_tec = par_tec.loc[par_tec['value'] > min_step] + df = par_old.copy() + + if parname == 'relation_activity': + tec_list = [] + else: + tec_list = [t for t in list(set(df[ + 'technology'])) if t in list(set(par_tec[ + 'technology']))] + + df_y = interpolate_2d(df, yrs_new, horizon, year_ref, year_col, + tec_list, par_tec, 'value', extrapolate, + extrapol_neg, year_diff) + sc_new.add_par(parname, df_y) + sc_new.commit(parname) + print( + '> Parameter "' + parname + '" copied and new years added ' + 'for node/s: "{}".'.format(nodes)) + +# %% VI) Required functions +# VI.A) A function to add new years to a datafarme by interpolation and +# (extrapolation if needed) + +def interpolate_1d( + df, + yrs_new, + horizon, + year_col, + value_col='value', + extrapolate=False, + extrapol_neg=None, + bound_extend=True): + ''' + Description: + This function receives a parameter data as a dataframe, and adds + new data for the additonal years by interpolation and + extrapolation. + + Input arguments: + df_par (dataframe): the dataframe of the parameter to which new + years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_col (string): the header of the column to which the new years + should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This function is called by function "add_year_par" + ''' + horizon_new = sorted(horizon + yrs_new) + idx = [x for x in df.columns if x not in [year_col, value_col]] + if not df.empty: + df2 = df.pivot_table(index=idx, columns=year_col, values=value_col) + + # To sort the new years smaller than the first year for + # extrapolation (e.g. 2025 values are calculated first; then + # values of 2015 based on 2020 and 2025) + year_before = sorted( + [x for x in yrs_new if x < min(df2.columns)], reverse=True) + if year_before and extrapolate: + for y in year_before: + yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y))) + + for yr in yrs_new: + if yr > max(horizon): + extrapol = True else: - tec_list = [t for t in list(set(df[ - 'technology'])) if t in list(set(par_tec[ - 'technology']))] - - df_y = self.df_interpolate_2d( - df, - yrs_new, - horizon, - year_ref, - year_col, - tec_list, - par_tec, - value_col='value', - extrapolate=extrapolate, - extrapol_neg=extrapol_neg, - year_diff=year_diff) - sc_new.add_par(parname, df_y) - sc_new.commit(parname) - print( - '> Parameter "' + parname + '" copied and new years added ' - 'for node/s: "{}".'.format(nodes)) - - # %% VI) Required functions - # VI.A) A function to add new years to a datafarme by interpolation and - # (extrapolation if needed) - - def df_interpolate( - self, - df, - yrs_new, - horizon, - year_col, - value_col='value', - extrapolate=False, - extrapol_neg=None, - bound_extend=True): - ''' - Description: - This function receives a parameter data as a dataframe, and adds - new data for the additonal years by interpolation and - extrapolation. - - Input arguments: - df_par (dataframe): the dataframe of the parameter to which new - years to be added - yrs_new (list of integers): new years to be added - horizon (list of integers): the horizon of the reference scenario - year_col (string): the header of the column to which the new years - should be added, for example, "year_act" - value_col (string): the header of the column containing values - extrapolate: if True, the extrapolation is allowed when a new year - is outside the parameter years - extrapol_neg: if True, negative values obtained by extrapolation - are allowed. - - Usage: - This function is called by function "addPar" - ''' - horizon_new = sorted(horizon + yrs_new) - idx = [x for x in df.columns if x not in [year_col, value_col]] - if not df.empty: - df2 = df.pivot_table(index=idx, columns=year_col, values=value_col) - - # To sort the new years smaller than the first year for - # extrapolation (e.g. 2025 values are calculated first; then - # values of 2015 based on 2020 and 2025) - year_before = sorted( - [x for x in yrs_new if x < min(df2.columns)], reverse=True) - if year_before and extrapolate: - for y in year_before: - yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y))) - - for yr in yrs_new: - if yr > max(horizon): - extrapol = True - else: - extrapol = extrapolate - - # a) If this new year is after the current range of modeled - # years, do extrapolation - if extrapol: - if yr == horizon_new[horizon_new.index( - max(df2.columns)) + 1]: - year_pre = max([x for x in df2.columns if x < yr]) - - if len([x for x in df2.columns if x < yr]) >= 2: - year_pp = max( - [x for x in df2.columns if x < year_pre]) - - df2[yr] = intpol(df2[year_pre], df2[year_pp], - year_pre, year_pp, yr) - - if bound_extend: - df2[yr] = df2[yr].fillna(df2[year_pre]) - - df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_pre] >= 0)].empty: - df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), - yr] = df2.loc[(df2[yr] < 0) & (df2[ - year_pre] >= 0), - year_pre] * extrapol_neg - else: - df2[yr] = df2[year_pre] - - # b) If this new year is before the current range of modeled - # years, do extrapolation - elif yr < min(df2.columns) and extrapol: - year_next = min([x for x in df2.columns if x > yr]) - - if len([x for x in df2.columns if x > yr]) >= 2: - year_nn = horizon[horizon.index(yr) + 2] - df2[yr] = intpol( - df2[year_next], df2[year_nn], - year_next, year_nn, yr) - df2[yr][np.isinf(df2[year_next])] = df2[year_next] - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_next] >= 0)].empty: - df2.loc[(df2[yr] < 0) & - (df2[year_next] >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_next] >= 0), - year_next] * extrapol_neg - - elif bound_extend: - df2[yr] = df2[year_next] - - # c) Otherise, do intrapolation - elif yr > min(df2.columns) and yr < max(df2.columns): + extrapol = extrapolate + + # a) If this new year is after the current range of modeled + # years, do extrapolation + if extrapol: + if yr == horizon_new[horizon_new.index( + max(df2.columns)) + 1]: year_pre = max([x for x in df2.columns if x < yr]) - year_next = min([x for x in df2.columns if x > yr]) - df2[yr] = intpol( - df2[year_pre], df2[year_next], year_pre, year_next, yr) - - # Extrapolate for new years if the value exists for the - # previous year but not for the next years - # TODO: here is the place that should be changed if the - # new year should go the time step before the existing one - if [x for x in df2.columns if x < year_pre]: - year_pp = max([x for x in df2.columns if x < year_pre]) - df2[yr] = df2[yr].fillna( - intpol( - df2[year_pre], - df2[year_pp], - year_pre, - year_pp, - yr)) + + if len([x for x in df2.columns if x < yr]) >= 2: + year_pp = max( + [x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol(df2[year_pre], df2[year_pp], + year_pre, year_pp, yr) + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) + + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] if extrapol_neg and not df2[yr].loc[( df2[yr] < 0) & (df2[year_pre] >= 0)].empty: df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & (df2[ + year_pre] >= 0), year_pre] * extrapol_neg - - if bound_extend: - df2[yr] = df2[yr].fillna(df2[year_pre]) - df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - - df2 = pd.melt( - df2.reset_index(), - id_vars=idx, - value_vars=[ - x for x in df2.columns if x not in idx], - var_name=year_col, - value_name=value_col).dropna( - subset=[value_col]).reset_index( - drop=True) - df2 = df2.sort_values(idx).reset_index(drop=True) - else: - print( - '+++ WARNING: The submitted dataframe is empty, so returned' + - 'empty results!!! +++') - df2 = df - return df2 - - # %% VI.B) A function to interpolate the data for new time steps in - # parameters with two dimensions related to time - - def df_interpolate_2d( - self, - df, - yrs_new, - horizon, - year_ref, - year_col, - tec_list, - par_tec, - value_col='value', - extrapolate=False, - extrapol_neg=None, - year_diff=None): - ''' - Description: - This function receives a dataframe that has 2 time-related columns - (e.g., "input" or "relation_activity"), and adds new data for the - additonal years in both time-related columns by interpolation - and extrapolation. - - Input arguments: - df (dataframe): the parameter to which new years to be added - yrs_new (list of integers): new years to be added - horizon (list of integers): the horizon of the reference scenario - year_ref (string): the header of the first column to which the new - years should be added, for example, "year_vtg" - year_col (string): the header of the second column to which the - new years should be added, for example, "year_act" - value_col (string): the header of the column containing values - extrapolate: if True, the extrapolation is allowed when a new year - is outside the parameter years - extrapol_neg: if True, negative values obtained by extrapolation - are allowed. - - Usage: - This utility is called by function "addPar" - ''' - def f_index(df1, df2): return df1.loc[df1.index.isin( - df2.index)] # For checking the index of two dataframes - - idx = [x for x in df.columns if x not in [year_col, value_col]] - if df.empty: - return df - print( - '+++ WARNING: The submitted dataframe is empty, so' + - 'returned empty results!!! +++') - - df_tec = df.loc[df['technology'].isin(tec_list)] - df2 = df.pivot_table(index=idx, columns=year_col, values='value') - df2_tec = df_tec.pivot_table( - index=idx, columns=year_col, values='value') - - # ------------------------------------------------------------------------------ - # First, changing the time interval for the transition period - # (e.g., year 2010 in old R11 model transits from 5 year to 10 year) - horizon_new = sorted(horizon + - [x for x in yrs_new if x not in horizon]) - yr_diff_new = [x for x in horizon_new[1:- - 1] if horizon_new[ - horizon_new.index(x) + - 1] - - horizon_new[horizon_new.index(x)] > horizon_new[ - horizon_new.index(x)] - horizon_new[ - horizon_new.index(x) - 1]] - - if year_diff and tec_list: - if isinstance(year_diff, list): - year_diff = year_diff[0] - - # Removing data from old transition year - if not yr_diff_new or year_diff not in yr_diff_new: - year_next = [x for x in df2.columns if x > year_diff][0] - - df_pre = f_slice( - df2_tec, idx, year_ref, [year_diff], year_diff) - df_next = f_slice( - df2_tec, idx, year_ref, [year_next], year_diff) - df_count = pd.DataFrame({ - 'c_pre': df_pre.count(axis=1), - 'c_next': df_next.loc[df_next.index.isin( - df_pre.index)].count(axis=1)}, - index=df_pre.index) - df_y = df_count.loc[df_count['c_pre'] == df_count[ - 'c_next'] + 1] - - for i in df_y.index: - df2.loc[i, df2.columns > (df2.loc[[i]].notnull().cumsum( - axis=1) == df_count[ - 'c_next'][i]).idxmax(axis=1).values[0]] = np.nan - - # Generating duration_period_sum matrix for masking - df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) - for i in df_dur.index: - for j in [x for x in df_dur.columns if x > i]: - df_dur.loc[i, j] = j - i - - # Adding data for new transition year - if yr_diff_new and tec_list and year_diff not in yr_diff_new: - yrs = [x for x in horizon if x <= yr_diff_new[0]] - year_next = min([x for x in df2.columns if x > yr_diff_new[0]]) - df_yrs = f_slice(df2_tec, idx, year_ref, yrs, []) - if yr_diff_new[0] in df2.columns: - df_yrs = df_yrs.loc[~pd.isna(df_yrs[yr_diff_new[0]]), :] - df_yrs = df_yrs.append( - f_slice( - df2_tec, - idx, - year_ref, - [year_next], - []), - ignore_index=False).reset_index( - ).sort_values(idx).set_index(idx) - - for yr in sorted( - [x for x in list(set(df_yrs.reset_index( - )[year_ref])) if x < year_next]): - yr_next = min([x for x in horizon_new if x > yr]) - d = f_slice(df_yrs, idx, year_ref, [yr], []) - d_n = f_slice(df_yrs, idx, year_ref, [yr_next], yr) - - if d_n[year_next].loc[~pd.isna(d_n[year_next])].empty: - if [x for x in horizon_new if x > yr_next]: - yr_nn = min([x for x in horizon_new if x > yr_next]) else: - yr_nn = yr_next - d_n = f_slice(df_yrs, idx, year_ref, [yr_nn], yr) - d_n = d_n.loc[d_n.index.isin(d.index), :] - d = d.loc[d.index.isin(d_n.index), :] - d[d.isnull() & d_n.notnull()] = d_n - df2.loc[df2.index.isin(d.index), :] = d - - df_dur.loc[df_dur.index <= yr_diff_new[0], - df_dur.columns >= year_next] = df_dur.loc[ - df_dur.index <= yr_diff_new[0], - df_dur.columns >= year_next] - ( - yr_diff_new[0] - horizon_new[horizon_new.index( - yr_diff_new[0]) - 1]) - # -------------------------------------------------------------------------- - # Second, adding year_act of new years when year_vtg is in the existing - # years - for yr in yrs_new: - if yr > max(horizon): - extrapol = True - else: - extrapol = extrapolate + df2[yr] = df2[year_pre] - # a) If this new year is greater than the current range of modeled + # b) If this new year is before the current range of modeled # years, do extrapolation - if yr > horizon_new[horizon_new.index( - max(df2.columns))] and extrapol: - year_pre = max([x for x in df2.columns if x < yr]) - year_pp = max([x for x in df2.columns if x < year_pre]) + elif yr < min(df2.columns) and extrapol: + year_next = min([x for x in df2.columns if x > yr]) - df2[yr] = intpol( - df2[year_pre], - df2[year_pp], - year_pre, - year_pp, - yr) - df2[yr][np.isinf(df2[year_pre].shift(+1)) - ] = df2[year_pre].shift(+1) - df2[yr] = df2[yr].fillna(df2[year_pre]) - - if yr - horizon_new[horizon_new.index(yr) - 1] >= horizon_new[ - horizon_new.index(yr) - 1] - horizon_new[ - horizon_new.index(yr) - 2]: - - df2[yr].loc[(pd.isna(df2[year_pre].shift(+1))) & ( - ~pd.isna(df2[year_pp].shift(+1)))] = np.nan - if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( - df2[year_pre].shift(+1) >= 0)].empty: - df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_pre].shift(+1) >= 0), - year_pre] * extrapol_neg - - # b) Otherise, do intrapolation + if len([x for x in df2.columns if x > yr]) >= 2: + year_nn = horizon[horizon.index(yr) + 2] + df2[yr] = intpol( + df2[year_next], df2[year_nn], + year_next, year_nn, yr) + df2[yr][np.isinf(df2[year_next])] = df2[year_next] + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_next] >= 0)].empty: + df2.loc[(df2[yr] < 0) & + (df2[year_next] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_next] >= 0), + year_next] * extrapol_neg + + elif bound_extend: + df2[yr] = df2[year_next] + + # c) Otherise, do intrapolation elif yr > min(df2.columns) and yr < max(df2.columns): year_pre = max([x for x in df2.columns if x < yr]) year_next = min([x for x in df2.columns if x > yr]) - df2[yr] = intpol( - df2[year_pre], - df2[year_next], - year_pre, - year_next, - yr) - df2_t = df2.loc[df2_tec.index, :].copy() - - # This part calculates the missing value if only the previous - # timestep has a value (and not the next) - if tec_list: - df2_t[yr].loc[(pd.isna(df2_t[yr])) & (~pd.isna(df2_t[ - year_pre]))] = intpol( - df2_t[year_pre], - df2_t[year_next].shift(-1), - year_pre, year_next, yr) - - # Treating technologies with phase-out in model years - if [x for x in df2.columns if x < year_pre]: - year_pp = max([x for x in df2.columns if x < year_pre]) - df2_t[yr].loc[(pd.isna(df2_t[yr])) & ( - pd.isna(df2_t[year_pre].shift(-1))) & ( - ~pd.isna(df2_t[year_pre]))] = intpol( - df2_t[year_pre], - df2_t[year_pp], - year_pre, year_pp, yr) - - if extrapol_neg and not df2_t[yr].loc[( - df2_t[yr] < 0) & (df2_t[year_pre] >= 0)].empty: - df2_t.loc[(df2_t[yr] < 0) & (df2_t[year_pre] >= 0), - yr] = df2_t.loc[(df2_t[yr] < 0) & ( - df2_t[year_pre] >= 0), - year_pre] * extrapol_neg - df2.loc[df2_tec.index, :] = df2_t + df2[year_pre], df2[year_next], year_pre, year_next, yr) + + # Extrapolate for new years if the value exists for the + # previous year but not for the next years + # TODO: here is the place that should be changed if the + # new year should go the time step before the existing one + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2[yr] = df2[yr].fillna( + intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr)) + if extrapol_neg and not df2[yr].loc[( + df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre] >= 0), + year_pre] * extrapol_neg + + if bound_extend: + df2[yr] = df2[yr].fillna(df2[year_pre]) df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - df2 = df2.reindex(sorted(df2.columns), axis=1) - # -------------------------------------------------------------------------- - # Third, adding year_vtg of new years and their respective year_act for - # both existing and new years - for yr in yrs_new: - # a) If this new year is after the current range of modeled years, - # do extrapolation - if yr > max(horizon): - year_pre = horizon_new[horizon_new.index(yr) - 1] - year_pp = horizon_new[horizon_new.index(yr) - 2] - df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) - df_pp = f_slice(df2, idx, year_ref, [year_pp], yr) - df_yr = intpol( - df_pre, - f_index( - df_pp, - df_pre), - year_pre, - year_pp, - yr) - df_yr[np.isinf(df_pre)] = df_pre - - # For those technolofies with one value for each year - df_yr.loc[pd.isna(df_yr[yr])] = intpol( - df_pre, df_pp.shift(+1, axis=1), - year_pre, year_pp, yr).shift(+1, axis=1) - df_yr[pd.isna(df_yr)] = df_pre - - if extrapol_neg: - df_yr[(df_yr < 0) & (df_pre >= 0)] = df_pre * extrapol_neg - df_yr.loc[:, df_yr.columns < yr] = np.nan - - # c) Otherise, do intrapolation - elif yr > min(df2.columns) and yr < max(horizon): - year_pre = horizon_new[horizon_new.index(yr) - 1] - year_next = min([x for x in horizon if x > yr]) - - df_pre = f_slice(df2, idx, year_ref, [year_pre], yr) - df_next = f_slice(df2, idx, year_ref, [year_next], yr) - df_yr = pd.concat(( - df_pre, - df_next.loc[df_next.index.isin(df_pre.index)]), - axis=0).groupby(level=idx).mean() - df_yr[yr] = df_yr[yr].fillna( - df_yr[[year_pre, year_next]].mean(axis=1)) - df_yr[np.isinf(df_pre)] = df_pre - - # Creating a mask to remove extra values - df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), - 'c_next': df_next.loc[ - df_next.index.isin(df_pre.index)].count(axis=1)}, - index=df_yr.index) - - for i in df_yr.index: - # Mainly for cases of two new consecutive years (like 2022 - # and 2024) - if ~np.isnan( - df_count['c_next'][i]) and df_count[ - 'c_pre'][i] >= df_count['c_next'][i] + 2: - df_yr[year_pre] = np.nan - - # For technologies phasing out before the end of horizon - # (like nuc_lc) - elif np.isnan(df_count['c_next'][i]): - df_yr.loc[i, :] = df_pre.loc[i, :].shift(+1) - if tec_list: - f_mask(df_yr, i, df_count['c_pre'][i] + 1, np.nan) - else: - f_mask(df_yr, i, df_count['c_pre'][i], np.nan) - - # For the rest - else: - df_yr[year_pre] = np.nan - f_mask(df_yr, i, df_count['c_pre'][i], np.nan) - else: - continue - - df2 = df2.append(df_yr) - df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() - # --------------------------------------------------------------------------- - # Forth: final masking based on technical lifetime - if tec_list: - - df3 = df2.copy() - for y in sorted([x for x in list( - set(df2.index.get_level_values(year_ref)) - ) if x in df_dur.index]): - df3.loc[df3.index.get_level_values(year_ref).isin([y]), - df3.columns.isin(df_dur.columns)] = df_dur.loc[ - y, df_dur.columns.isin(df3.columns)].values - - df3 = df3.reset_index().set_index( - ['node_loc', 'technology', year_ref]).sort_index(level=1) - par_tec = par_tec.set_index( - ['node_loc', 'technology', year_ref]).sort_index(level=1) - - for i in [x for x in par_tec.index if x in df3.index]: - df3.loc[i, 'lifetime'] = par_tec.loc[i, 'value'].copy() - - df3 = df3.reset_index().set_index(idx).dropna( - subset=['lifetime']).sort_index() - for i in df3.index: - df2.loc[i, df3.loc[i, :] >= int( - df3.loc[i, 'lifetime'])] = np.nan - - # Removing extra values from non-lifetime technologies - for i in [x for x in df2.index if x not in df3.index]: - df2.loc[i, df2.columns > df2.loc[[i]].index.get_level_values( - year_ref)[0]] = np.nan - - df_par = pd.melt( + df2 = pd.melt( df2.reset_index(), id_vars=idx, value_vars=[ x for x in df2.columns if x not in idx], var_name=year_col, - value_name='value').dropna( - subset=['value']) - df_par = df_par.sort_values(idx).reset_index(drop=True) - return df_par + value_name=value_col).dropna( + subset=[value_col]).reset_index( + drop=True) + df2 = df2.sort_values(idx).reset_index(drop=True) + else: + print( + '+++ WARNING: The submitted dataframe is empty, so returned' + + 'empty results!!! +++') + df2 = df + return df2 + +# %% VI.B) A function to interpolate the data for new time steps in +# parameters with two dimensions related to time + +def interpolate_2d( + df, + yrs_new, + horizon, + year_ref, + year_col, + tec_list, + par_tec, + value_col='value', + extrapolate=False, + extrapol_neg=None, + year_diff=None): + ''' + Description: + This function receives a dataframe that has 2 time-related columns + (e.g., "input" or "relation_activity"), and adds new data for the + additonal years in both time-related columns by interpolation + and extrapolation. + + Input arguments: + df (dataframe): the parameter to which new years to be added + yrs_new (list of integers): new years to be added + horizon (list of integers): the horizon of the reference scenario + year_ref (string): the header of the first column to which the new + years should be added, for example, "year_vtg" + year_col (string): the header of the second column to which the + new years should be added, for example, "year_act" + value_col (string): the header of the column containing values + extrapolate: if True, the extrapolation is allowed when a new year + is outside the parameter years + extrapol_neg: if True, negative values obtained by extrapolation + are allowed. + + Usage: + This utility is called by function "add_year_par" + ''' + def f_index(df1, df2): return df1.loc[df1.index.isin( + df2.index)] # For checking the index of two dataframes + + idx = [x for x in df.columns if x not in [year_col, value_col]] + if df.empty: + return df + print( + '+++ WARNING: The submitted dataframe is empty, so' + + 'returned empty results!!! +++') + + df_tec = df.loc[df['technology'].isin(tec_list)] + df2 = df.pivot_table(index=idx, columns=year_col, values='value') + df2_tec = df_tec.pivot_table( + index=idx, columns=year_col, values='value') + + # ------------------------------------------------------------------------------ + # First, changing the time interval for the transition period + # (e.g., year 2010 in old R11 model transits from 5 year to 10 year) + horizon_new = sorted(horizon + + [x for x in yrs_new if x not in horizon]) + yr_diff_new = [x for x in horizon_new[1:- + 1] if horizon_new[ + horizon_new.index(x) + + 1] - + horizon_new[horizon_new.index(x)] > horizon_new[ + horizon_new.index(x)] - horizon_new[ + horizon_new.index(x) - 1]] + + if year_diff and tec_list: + if isinstance(year_diff, list): + year_diff = year_diff[0] + + # Removing data from old transition year + if not yr_diff_new or year_diff not in yr_diff_new: + year_next = [x for x in df2.columns if x > year_diff][0] + + df_pre = slice_df( + df2_tec, idx, year_ref, [year_diff], year_diff) + df_next = slice_df( + df2_tec, idx, year_ref, [year_next], year_diff) + df_count = pd.DataFrame({ + 'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[df_next.index.isin( + df_pre.index)].count(axis=1)}, + index=df_pre.index) + df_y = df_count.loc[df_count['c_pre'] == df_count[ + 'c_next'] + 1] + + for i in df_y.index: + df2.loc[i, df2.columns > (df2.loc[[i]].notnull().cumsum( + axis=1) == df_count[ + 'c_next'][i]).idxmax(axis=1).values[0]] = np.nan + + # Generating duration_period_sum matrix for masking + df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) + for i in df_dur.index: + for j in [x for x in df_dur.columns if x > i]: + df_dur.loc[i, j] = j - i + + # Adding data for new transition year + if yr_diff_new and tec_list and year_diff not in yr_diff_new: + yrs = [x for x in horizon if x <= yr_diff_new[0]] + year_next = min([x for x in df2.columns if x > yr_diff_new[0]]) + df_yrs = slice_df(df2_tec, idx, year_ref, yrs, []) + if yr_diff_new[0] in df2.columns: + df_yrs = df_yrs.loc[~pd.isna(df_yrs[yr_diff_new[0]]), :] + df_yrs = df_yrs.append( + slice_df( + df2_tec, + idx, + year_ref, + [year_next], + []), + ignore_index=False).reset_index( + ).sort_values(idx).set_index(idx) + + for yr in sorted( + [x for x in list(set(df_yrs.reset_index( + )[year_ref])) if x < year_next]): + yr_next = min([x for x in horizon_new if x > yr]) + d = slice_df(df_yrs, idx, year_ref, [yr], []) + d_n = slice_df(df_yrs, idx, year_ref, [yr_next], yr) + + if d_n[year_next].loc[~pd.isna(d_n[year_next])].empty: + if [x for x in horizon_new if x > yr_next]: + yr_nn = min([x for x in horizon_new if x > yr_next]) + else: + yr_nn = yr_next + d_n = slice_df(df_yrs, idx, year_ref, [yr_nn], yr) + d_n = d_n.loc[d_n.index.isin(d.index), :] + d = d.loc[d.index.isin(d_n.index), :] + d[d.isnull() & d_n.notnull()] = d_n + df2.loc[df2.index.isin(d.index), :] = d + + df_dur.loc[df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] = df_dur.loc[ + df_dur.index <= yr_diff_new[0], + df_dur.columns >= year_next] - ( + yr_diff_new[0] - horizon_new[horizon_new.index( + yr_diff_new[0]) - 1]) + # -------------------------------------------------------------------------- + # Second, adding year_act of new years when year_vtg is in the existing + # years + for yr in yrs_new: + if yr > max(horizon): + extrapol = True + else: + extrapol = extrapolate + + # a) If this new year is greater than the current range of modeled + # years, do extrapolation + if yr > horizon_new[horizon_new.index( + max(df2.columns))] and extrapol: + year_pre = max([x for x in df2.columns if x < yr]) + year_pp = max([x for x in df2.columns if x < year_pre]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_pp], + year_pre, + year_pp, + yr) + df2[yr][np.isinf(df2[year_pre].shift(+1)) + ] = df2[year_pre].shift(+1) + df2[yr] = df2[yr].fillna(df2[year_pre]) + + if yr - horizon_new[horizon_new.index(yr) - 1] >= horizon_new[ + horizon_new.index(yr) - 1] - horizon_new[ + horizon_new.index(yr) - 2]: + + df2[yr].loc[(pd.isna(df2[year_pre].shift(+1))) & ( + ~pd.isna(df2[year_pp].shift(+1)))] = np.nan + if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0)].empty: + df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), + yr] = df2.loc[(df2[yr] < 0) & ( + df2[year_pre].shift(+1) >= 0), + year_pre] * extrapol_neg + + # b) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(df2.columns): + year_pre = max([x for x in df2.columns if x < yr]) + year_next = min([x for x in df2.columns if x > yr]) + + df2[yr] = intpol( + df2[year_pre], + df2[year_next], + year_pre, + year_next, + yr) + df2_t = df2.loc[df2_tec.index, :].copy() + + # This part calculates the missing value if only the previous + # timestep has a value (and not the next) + if tec_list: + df2_t[yr].loc[(pd.isna(df2_t[yr])) & (~pd.isna(df2_t[ + year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_next].shift(-1), + year_pre, year_next, yr) + + # Treating technologies with phase-out in model years + if [x for x in df2.columns if x < year_pre]: + year_pp = max([x for x in df2.columns if x < year_pre]) + df2_t[yr].loc[(pd.isna(df2_t[yr])) & ( + pd.isna(df2_t[year_pre].shift(-1))) & ( + ~pd.isna(df2_t[year_pre]))] = intpol( + df2_t[year_pre], + df2_t[year_pp], + year_pre, year_pp, yr) + + if extrapol_neg and not df2_t[yr].loc[( + df2_t[yr] < 0) & (df2_t[year_pre] >= 0)].empty: + df2_t.loc[(df2_t[yr] < 0) & (df2_t[year_pre] >= 0), + yr] = df2_t.loc[(df2_t[yr] < 0) & ( + df2_t[year_pre] >= 0), + year_pre] * extrapol_neg + df2.loc[df2_tec.index, :] = df2_t + df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] + df2 = df2.reindex(sorted(df2.columns), axis=1) + # -------------------------------------------------------------------------- + # Third, adding year_vtg of new years and their respective year_act for + # both existing and new years + for yr in yrs_new: + # a) If this new year is after the current range of modeled years, + # do extrapolation + if yr > max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_pp = horizon_new[horizon_new.index(yr) - 2] + df_pre = slice_df(df2, idx, year_ref, [year_pre], yr) + df_pp = slice_df(df2, idx, year_ref, [year_pp], yr) + df_yr = intpol( + df_pre, + f_index( + df_pp, + df_pre), + year_pre, + year_pp, + yr) + df_yr[np.isinf(df_pre)] = df_pre + + # For those technolofies with one value for each year + df_yr.loc[pd.isna(df_yr[yr])] = intpol( + df_pre, df_pp.shift(+1, axis=1), + year_pre, year_pp, yr).shift(+1, axis=1) + df_yr[pd.isna(df_yr)] = df_pre + + if extrapol_neg: + df_yr[(df_yr < 0) & (df_pre >= 0)] = df_pre * extrapol_neg + df_yr.loc[:, df_yr.columns < yr] = np.nan + + # c) Otherise, do intrapolation + elif yr > min(df2.columns) and yr < max(horizon): + year_pre = horizon_new[horizon_new.index(yr) - 1] + year_next = min([x for x in horizon if x > yr]) + + df_pre = slice_df(df2, idx, year_ref, [year_pre], yr) + df_next = slice_df(df2, idx, year_ref, [year_next], yr) + df_yr = pd.concat(( + df_pre, + df_next.loc[df_next.index.isin(df_pre.index)]), + axis=0).groupby(level=idx).mean() + df_yr[yr] = df_yr[yr].fillna( + df_yr[[year_pre, year_next]].mean(axis=1)) + df_yr[np.isinf(df_pre)] = df_pre + + # Creating a mask to remove extra values + df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), + 'c_next': df_next.loc[ + df_next.index.isin(df_pre.index)].count(axis=1)}, + index=df_yr.index) + + for i in df_yr.index: + # Mainly for cases of two new consecutive years (like 2022 + # and 2024) + if ~np.isnan( + df_count['c_next'][i]) and df_count[ + 'c_pre'][i] >= df_count['c_next'][i] + 2: + df_yr[year_pre] = np.nan + + # For technologies phasing out before the end of horizon + # (like nuc_lc) + elif np.isnan(df_count['c_next'][i]): + df_yr.loc[i, :] = df_pre.loc[i, :].shift(+1) + if tec_list: + mask_df(df_yr, i, df_count['c_pre'][i] + 1, np.nan) + else: + mask_df(df_yr, i, df_count['c_pre'][i], np.nan) + + # For the rest + else: + df_yr[year_pre] = np.nan + mask_df(df_yr, i, df_count['c_pre'][i], np.nan) + + else: + continue + + df2 = df2.append(df_yr) + df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() + # --------------------------------------------------------------------------- + # Forth: final masking based on technical lifetime + if tec_list: + + df3 = df2.copy() + for y in sorted([x for x in list( + set(df2.index.get_level_values(year_ref)) + ) if x in df_dur.index]): + df3.loc[df3.index.get_level_values(year_ref).isin([y]), + df3.columns.isin(df_dur.columns)] = df_dur.loc[ + y, df_dur.columns.isin(df3.columns)].values + + df3 = df3.reset_index().set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + par_tec = par_tec.set_index( + ['node_loc', 'technology', year_ref]).sort_index(level=1) + + for i in [x for x in par_tec.index if x in df3.index]: + df3.loc[i, 'lifetime'] = par_tec.loc[i, 'value'].copy() + + df3 = df3.reset_index().set_index(idx).dropna( + subset=['lifetime']).sort_index() + for i in df3.index: + df2.loc[i, df3.loc[i, :] >= int( + df3.loc[i, 'lifetime'])] = np.nan + + # Removing extra values from non-lifetime technologies + for i in [x for x in df2.index if x not in df3.index]: + df2.loc[i, df2.columns > df2.loc[[i]].index.get_level_values( + year_ref)[0]] = np.nan + + df_par = pd.melt( + df2.reset_index(), + id_vars=idx, + value_vars=[ + x for x in df2.columns if x not in idx], + var_name=year_col, + value_name='value').dropna( + subset=['value']) + df_par = df_par.sort_values(idx).reset_index(drop=True) + return df_par From 0376ea829c0e14f103a037c1edc95f76406aa2f7 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 6 May 2019 17:04:36 +0200 Subject: [PATCH 31/44] cleaned interface 1d --- message_ix/tools/add_year/__init__.py | 124 ++++++++++---------------- 1 file changed, 47 insertions(+), 77 deletions(-) diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index ec8d25046..77d25677f 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -15,8 +15,8 @@ III. The main class called "add_year" IV. Submodule "add_year_set" for adding and modifying the sets V. Submodule "add_year_par" for copying and modifying each parameter - VI. The submodule "add_year_par" calls two utility functions ("interpolate_1d" - and "df_interpolate_2D") for calculating missing values. + VI. Submodule "add_year_par" calls two utility functions ("interpolate_1d" + and "interpolate_2D") for calculating missing values. VII. Code for running the script as "main" @@ -423,32 +423,21 @@ def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, if parname == 'relation_activity': tec_list = [] else: - tec_list = [t for t in list(set(df[ - 'technology'])) if t in list(set(par_tec[ - 'technology']))] + tec_list = [t for t in (set(df['technology']) + ) if t in list(set(par_tec['technology']))] df_y = interpolate_2d(df, yrs_new, horizon, year_ref, year_col, tec_list, par_tec, 'value', extrapolate, extrapol_neg, year_diff) sc_new.add_par(parname, df_y) sc_new.commit(parname) - print( - '> Parameter "' + parname + '" copied and new years added ' - 'for node/s: "{}".'.format(nodes)) + print('> Parameter "{}" copied and new years added' + ' for node/s: "{}".'.format(parname, nodes)) -# %% VI) Required functions -# VI.A) A function to add new years to a datafarme by interpolation and -# (extrapolation if needed) -def interpolate_1d( - df, - yrs_new, - horizon, - year_col, - value_col='value', - extrapolate=False, - extrapol_neg=None, - bound_extend=True): +# %% VI) Required functions +def interpolate_1d(df, yrs_new, horizon, year_col, value_col='value', + extrapolate=False, extrapol_neg=None, bound_extend=True): ''' Description: This function receives a parameter data as a dataframe, and adds @@ -467,9 +456,7 @@ def interpolate_1d( is outside the parameter years extrapol_neg: if True, negative values obtained by extrapolation are allowed. - - Usage: - This function is called by function "add_year_par" + bound_extend: if True, allows extrapolation of bounds for new years ''' horizon_new = sorted(horizon + yrs_new) idx = [x for x in df.columns if x not in [year_col, value_col]] @@ -479,8 +466,8 @@ def interpolate_1d( # To sort the new years smaller than the first year for # extrapolation (e.g. 2025 values are calculated first; then # values of 2015 based on 2020 and 2025) - year_before = sorted( - [x for x in yrs_new if x < min(df2.columns)], reverse=True) + year_before = sorted([x for x in yrs_new if x < min(df2.columns + )], reverse=True) if year_before and extrapolate: for y in year_before: yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y))) @@ -491,17 +478,13 @@ def interpolate_1d( else: extrapol = extrapolate - # a) If this new year is after the current range of modeled - # years, do extrapolation + # a) If this new year greater than modeled years, do extrapolation if extrapol: - if yr == horizon_new[horizon_new.index( - max(df2.columns)) + 1]: + if yr == horizon_new[horizon_new.index(max(df2.columns)) + 1]: year_pre = max([x for x in df2.columns if x < yr]) if len([x for x in df2.columns if x < yr]) >= 2: - year_pp = max( - [x for x in df2.columns if x < year_pre]) - + year_pp = max([x for x in df2.columns if x < year_pre]) df2[yr] = intpol(df2[year_pre], df2[year_pp], year_pre, year_pp, yr) @@ -509,33 +492,30 @@ def interpolate_1d( df2[yr] = df2[yr].fillna(df2[year_pre]) df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_pre] >= 0)].empty: + if not df2[yr].loc[(df2[yr] < 0) & (df2[year_pre] >= 0) + ].empty and extrapol_neg: df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), - yr] = df2.loc[(df2[yr] < 0) & (df2[ - year_pre] >= 0), - year_pre] * extrapol_neg + yr] = df2.loc[(df2[yr] < 0 + ) & (df2[year_pre] >= 0), + year_pre] * extrapol_neg else: df2[yr] = df2[year_pre] - # b) If this new year is before the current range of modeled - # years, do extrapolation + # b) If the new year is smaller than modeled years, extrapolate elif yr < min(df2.columns) and extrapol: year_next = min([x for x in df2.columns if x > yr]) if len([x for x in df2.columns if x > yr]) >= 2: year_nn = horizon[horizon.index(yr) + 2] - df2[yr] = intpol( - df2[year_next], df2[year_nn], - year_next, year_nn, yr) + df2[yr] = intpol(df2[year_next], df2[year_nn], + year_next, year_nn, yr) df2[yr][np.isinf(df2[year_next])] = df2[year_next] - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_next] >= 0)].empty: - df2.loc[(df2[yr] < 0) & - (df2[year_next] >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_next] >= 0), - year_next] * extrapol_neg + if not df2[yr].loc[(df2[yr] < 0) & (df2[year_next] >= 0) + ].empty and extrapol_neg: + df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), yr + ] = df2.loc[(df2[yr] < 0 + ) & (df2[year_next] >= 0), + year_next] * extrapol_neg elif bound_extend: df2[yr] = df2[year_next] @@ -544,47 +524,37 @@ def interpolate_1d( elif yr > min(df2.columns) and yr < max(df2.columns): year_pre = max([x for x in df2.columns if x < yr]) year_next = min([x for x in df2.columns if x > yr]) - df2[yr] = intpol( - df2[year_pre], df2[year_next], year_pre, year_next, yr) + df2[yr] = intpol(df2[year_pre], df2[year_next], + year_pre, year_next, yr) # Extrapolate for new years if the value exists for the # previous year but not for the next years # TODO: here is the place that should be changed if the - # new year should go the time step before the existing one + # new year should go to the time step before the existing one if [x for x in df2.columns if x < year_pre]: year_pp = max([x for x in df2.columns if x < year_pre]) - df2[yr] = df2[yr].fillna( - intpol( - df2[year_pre], - df2[year_pp], - year_pre, - year_pp, - yr)) - if extrapol_neg and not df2[yr].loc[( - df2[yr] < 0) & (df2[year_pre] >= 0)].empty: - df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_pre] >= 0), - year_pre] * extrapol_neg + df2[yr] = df2[yr].fillna(intpol(df2[year_pre], + df2[year_pp], year_pre, + year_pp, yr)) + if not df2[yr].loc[(df2[yr] < 0) & (df2[year_pre] >= 0) + ].empty and extrapol_neg: + df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), yr + ] = df2.loc[(df2[yr] < 0 + ) & (df2[year_pre] >= 0), + year_pre] * extrapol_neg if bound_extend: df2[yr] = df2[yr].fillna(df2[year_pre]) df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] - df2 = pd.melt( - df2.reset_index(), - id_vars=idx, - value_vars=[ - x for x in df2.columns if x not in idx], - var_name=year_col, - value_name=value_col).dropna( - subset=[value_col]).reset_index( - drop=True) + df2 = pd.melt(df2.reset_index(), id_vars=idx, + value_vars=[x for x in df2.columns if x not in idx], + var_name=year_col, value_name=value_col + ).dropna(subset=[value_col]).reset_index(drop=True) df2 = df2.sort_values(idx).reset_index(drop=True) else: - print( - '+++ WARNING: The submitted dataframe is empty, so returned' + - 'empty results!!! +++') + print('+++ WARNING: The submitted dataframe is empty, so returned' + ' empty results!!! +++') df2 = df return df2 From 439d0622d85608f8b4ad53af7c653a02f2d255a3 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 6 May 2019 18:09:39 +0200 Subject: [PATCH 32/44] md file deleted --- message_ix/tools/add_year/README.md | 37 ----------------------------- 1 file changed, 37 deletions(-) delete mode 100644 message_ix/tools/add_year/README.md diff --git a/message_ix/tools/add_year/README.md b/message_ix/tools/add_year/README.md deleted file mode 100644 index e8df02917..000000000 --- a/message_ix/tools/add_year/README.md +++ /dev/null @@ -1,37 +0,0 @@ -## Description -This functionality adds new modeling years to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: -- Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") -- Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. - -## Main features -- It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. -- The new years can be consecutive, between existing years, and after the model horizon. -- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. - -## Main steps -1. An existing scenario is loaded and the desired new years is specified. -2. A new (empty) scenario is created for adding the new time steps. -3. The new years are added to the relevant sets: -- adding new years to the set "year" and "type_year" -- changing "firstmodelyear", "lastmodelyear", "baseyear_macro", and "initializeyear_macro" if needed. -- modifying the set "cat_year" for the new years. -4. The new years are added to the index sets of relevant parameters, and the missing data for the new years are calculated based on interpolation of adjacent data points. The following steps are applied: -- each non-empty parameter is loaded from the reference scenario -- the year-related indexes of the parameter are identified (either 0, 1, and 2 index under MESSAGE scheme) -- the new years are added to the parameter, and the missing data is calculated based on the number of year-related indexes. For example, the new years are added to index "year_vtg" in parameter "inv_cost", while these new years are added both to "year_vtg" and "year_act" in parameter "output". -- the missing data is calculated by interpolation. -- for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. -5. The changes are commited and saved to the new scenario. - -## Notice: -I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. - -## Usage: -This script can be used either: -- By running directly from the command line, example: - ``` - python f_addNewYear.py --model_ref "MESSAGE_Model" --scen_ref "baseline" --years_new "[2015,2025,2035,2045]" - ``` - -For the full list of input arguments see the explanation in the code. -- By calling the class "addNewYear" from another python script. From c88c1ad4177f445d1eb6692b45cf1431d3c76df5 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 6 May 2019 18:14:30 +0200 Subject: [PATCH 33/44] reformatted to function and clean-up --- message_ix/tools/add_year/__init__.py | 317 +++++++++++--------------- message_ix/tools/add_year/__main__.py | 7 +- tests/tools/test_add_year.py | 59 ++--- 3 files changed, 153 insertions(+), 230 deletions(-) diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 77d25677f..33ecaa314 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -90,9 +90,8 @@ def unit_uniform(df): df['unit'] = df['unit'].mode()[0] return df -# %% III) The main class - +# %% III) The main function def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, macro=False, baseyear_macro=None, parameter='all', region='all', rewrite=True, unit_check=True, extrapol_neg=None, @@ -150,7 +149,7 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, par_list = [parameter] else: print('Parameters should be defined in a list of strings or as' - ' a single string!') + ' a single string!') if 'technical_lifetime' in par_list: par_list.insert(0, par_list.pop(par_list.index('technical_lifetime'))) @@ -163,7 +162,7 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, reg_list = [region] else: print('Regions should be defined in a list of strings or as' - ' a single string!') + ' a single string!') # List of parameters to be ignored (even not copied to the new # scenario) @@ -199,15 +198,18 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, # The loop over "node" is only for reducing the size of tables for node in reg_list: add_year_par(sc_ref, sc_new, yrs_new, parname, [node], - extrapol, rewrite, unit_check, extrapol_neg, bound_ext) + extrapol, rewrite, unit_check, extrapol_neg, + bound_ext) else: add_year_par(sc_ref, sc_new, yrs_new, parname, reg_list, - extrapol, rewrite, unit_check, extrapol_neg, bound_ext) + extrapol, rewrite, unit_check, extrapol_neg, + bound_ext) sc_new.set_as_default() print('> All required parameters were successfully ' + 'added to the new scenario.') + # %% Submodules needed for running the main function # IV) Adding new years to sets def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, @@ -268,15 +270,15 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, yr_cat = yr_cat.append(pd.DataFrame(yr_pair, columns=['type_year', 'year']), - ignore_index=True - ).sort_values('year' - ).reset_index(drop=True) + ignore_index=True + ).sort_values('year').reset_index(drop=True) # A.6. Changing the cumulative years based on the new first model year firstyear_new = int(yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', 'year']) - yr_cat = yr_cat.drop(yr_cat.loc[(yr_cat['type_year'] == 'cumulative') & ( - yr_cat['year'] < firstyear_new)].index) + yr_cat = yr_cat.drop(yr_cat.loc[(yr_cat['type_year'] == 'cumulative' + ) & (yr_cat['year'] < firstyear_new) + ].index) sc_new.add_set('cat_year', yr_cat) # IV.B) Copying all other sets @@ -303,8 +305,8 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, sc_new.commit('sets added!') print('> All the sets updated and added to the new scenario.') -# %% V) Adding new years to parameters +# %% V) Adding new years to parameters def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, extrapolate=False, rewrite=True, unit_check=True, extrapol_neg=None, bound_extend=True): @@ -376,17 +378,16 @@ def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, # indexing and grouping if 'unit' in col_list and unit_check: par_old = unit_uniform(par_old) -# -------------------------------------------------------------------------------------------------------- -# V.B) Treatment of the new years in the specified parameter based on -# the time-related dimension of that parameter -# V.B.1) Parameters with no time component +# --------------------------------------------------------------------------- +# V.B) Adding new years to a parameter based on time-related indexes +# V.B.1) Parameters with no time index if len(year_list) == 0: sc_new.add_par(parname, par_old) sc_new.commit(parname) print('> Parameter "' + parname + '" just copied to new scenario ' 'since has no time-related entries.') -# V.B.2) Parameters with one dimension related to time +# V.B.2) Parameters with one index related to time elif len(year_list) == 1: year_col = year_list[0] df = par_old.copy() @@ -394,18 +395,19 @@ def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, extrapolate, extrapol_neg, bound_extend) sc_new.add_par(parname, df_y) sc_new.commit(' ') - print('> Parameter "{}" copied and new years ' - 'added for node/s: "{}".'.format(parname, nodes)) + print('> Parameter "{}" copied and new years' + ' added for node/s: "{}".'.format(parname, nodes)) -# V.B.3) Parameters with two dimensions related to time (such as -# 'input','output', etc.) +# V.B.3) Parameters with two indexes related to time (such as 'input') elif len(year_list) == 2: year_col = 'year_act' node_col = 'node_loc' year_ref = [x for x in year_list if x != year_col][0] - g = lambda li, i: li[i + 1] - li[i] > li[i] - li[i - 1] - year_diff = [x for x in horizon[1:-1] if g(horizon, horizon.index(x))] + def f(x, i): + return x[i + 1] - x[i] > x[i] - x[i - 1] + + year_diff = [x for x in horizon[1:-1] if f(horizon, horizon.index(x))] print('> Parameter "{}" is being added for node/s' ' "{}"...'.format(parname, nodes)) @@ -558,21 +560,11 @@ def interpolate_1d(df, yrs_new, horizon, year_col, value_col='value', df2 = df return df2 -# %% VI.B) A function to interpolate the data for new time steps in -# parameters with two dimensions related to time - -def interpolate_2d( - df, - yrs_new, - horizon, - year_ref, - year_col, - tec_list, - par_tec, - value_col='value', - extrapolate=False, - extrapol_neg=None, - year_diff=None): + +# %% VI.B) Interpolating parameters with two dimensions related to time +def interpolate_2d(df, yrs_new, horizon, year_ref, year_col, tec_list, par_tec, + value_col='value', extrapolate=False, extrapol_neg=None, + year_diff=None): ''' Description: This function receives a dataframe that has 2 time-related columns @@ -593,37 +585,29 @@ def interpolate_2d( is outside the parameter years extrapol_neg: if True, negative values obtained by extrapolation are allowed. - - Usage: - This utility is called by function "add_year_par" ''' - def f_index(df1, df2): return df1.loc[df1.index.isin( - df2.index)] # For checking the index of two dataframes + def idx_check(df1, df2): + return df1.loc[df1.index.isin(df2.index)] idx = [x for x in df.columns if x not in [year_col, value_col]] if df.empty: return df - print( - '+++ WARNING: The submitted dataframe is empty, so' + - 'returned empty results!!! +++') + print('+++ WARNING: The submitted dataframe is empty, so' + ' returned empty results!!! +++') df_tec = df.loc[df['technology'].isin(tec_list)] df2 = df.pivot_table(index=idx, columns=year_col, values='value') - df2_tec = df_tec.pivot_table( - index=idx, columns=year_col, values='value') + df2_tec = df_tec.pivot_table(index=idx, columns=year_col, values='value') - # ------------------------------------------------------------------------------ + # ------------------------------------------------------------------------- # First, changing the time interval for the transition period # (e.g., year 2010 in old R11 model transits from 5 year to 10 year) - horizon_new = sorted(horizon + - [x for x in yrs_new if x not in horizon]) - yr_diff_new = [x for x in horizon_new[1:- - 1] if horizon_new[ - horizon_new.index(x) + - 1] - - horizon_new[horizon_new.index(x)] > horizon_new[ - horizon_new.index(x)] - horizon_new[ - horizon_new.index(x) - 1]] + horizon_new = sorted(horizon + [x for x in yrs_new if x not in horizon]) + + def f(x, i): + return x[i + 1] - x[i] > x[i] - x[i - 1] + yr_diff_new = [x for x in horizon_new[1:-1] if f(horizon, + horizon.index(x))] if year_diff and tec_list: if isinstance(year_diff, list): @@ -633,22 +617,19 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( if not yr_diff_new or year_diff not in yr_diff_new: year_next = [x for x in df2.columns if x > year_diff][0] - df_pre = slice_df( - df2_tec, idx, year_ref, [year_diff], year_diff) - df_next = slice_df( - df2_tec, idx, year_ref, [year_next], year_diff) - df_count = pd.DataFrame({ - 'c_pre': df_pre.count(axis=1), - 'c_next': df_next.loc[df_next.index.isin( - df_pre.index)].count(axis=1)}, - index=df_pre.index) - df_y = df_count.loc[df_count['c_pre'] == df_count[ - 'c_next'] + 1] + df_pre = slice_df(df2_tec, idx, year_ref, [year_diff], year_diff) + df_next = slice_df(df2_tec, idx, year_ref, [year_next], year_diff) + df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), + 'c_next': idx_check(df_next, df_pre + ).count(axis=1)}, + index=df_pre.index) + df_y = df_count.loc[df_count['c_pre'] == df_count['c_next'] + 1] for i in df_y.index: - df2.loc[i, df2.columns > (df2.loc[[i]].notnull().cumsum( - axis=1) == df_count[ - 'c_next'][i]).idxmax(axis=1).values[0]] = np.nan + condition = ((df2.loc[[i]].notnull().cumsum(axis=1) + ) == df_count['c_next'][i]) + df2.loc[i, df2.columns > condition.idxmax(axis=1 + ).values[0]] = np.nan # Generating duration_period_sum matrix for masking df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) @@ -663,19 +644,13 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( df_yrs = slice_df(df2_tec, idx, year_ref, yrs, []) if yr_diff_new[0] in df2.columns: df_yrs = df_yrs.loc[~pd.isna(df_yrs[yr_diff_new[0]]), :] - df_yrs = df_yrs.append( - slice_df( - df2_tec, - idx, - year_ref, - [year_next], - []), - ignore_index=False).reset_index( - ).sort_values(idx).set_index(idx) - - for yr in sorted( - [x for x in list(set(df_yrs.reset_index( - )[year_ref])) if x < year_next]): + df_yrs = df_yrs.append(slice_df(df2_tec, idx, year_ref, + [year_next], []), + ignore_index=False).reset_index() + df_yrs = df_yrs.sort_values(idx).set_index(idx) + + for yr in sorted([x for x in list(set(df_yrs.reset_index()[year_ref]) + ) if x < year_next]): yr_next = min([x for x in horizon_new if x > yr]) d = slice_df(df_yrs, idx, year_ref, [yr], []) d_n = slice_df(df_yrs, idx, year_ref, [yr_next], yr) @@ -691,117 +666,91 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( d[d.isnull() & d_n.notnull()] = d_n df2.loc[df2.index.isin(d.index), :] = d - df_dur.loc[df_dur.index <= yr_diff_new[0], - df_dur.columns >= year_next] = df_dur.loc[ - df_dur.index <= yr_diff_new[0], - df_dur.columns >= year_next] - ( - yr_diff_new[0] - horizon_new[horizon_new.index( - yr_diff_new[0]) - 1]) - # -------------------------------------------------------------------------- - # Second, adding year_act of new years when year_vtg is in the existing - # years + cond1 = (df_dur.index <= yr_diff_new[0]) + cond2 = (df_dur.columns >= year_next) + subt = yr_diff_new[0] - horizon_new[horizon_new.index(yr_diff_new[0] + ) - 1] + df_dur.loc[cond1, cond2] = df_dur.loc[cond1, cond2] - subt + # ------------------------------------------------------------------------- + # Second, adding year_act of new years if year_vtg is in existing years for yr in yrs_new: if yr > max(horizon): extrapol = True else: extrapol = extrapolate - # a) If this new year is greater than the current range of modeled - # years, do extrapolation - if yr > horizon_new[horizon_new.index( - max(df2.columns))] and extrapol: + # a) If this new year is greater than modeled years, do extrapolation + if yr > horizon_new[horizon_new.index(max(df2.columns))] and extrapol: year_pre = max([x for x in df2.columns if x < yr]) year_pp = max([x for x in df2.columns if x < year_pre]) - df2[yr] = intpol( - df2[year_pre], - df2[year_pp], - year_pre, - year_pp, - yr) + df2[yr] = intpol(df2[year_pre], df2[year_pp], + year_pre, year_pp, yr) df2[yr][np.isinf(df2[year_pre].shift(+1)) ] = df2[year_pre].shift(+1) df2[yr] = df2[yr].fillna(df2[year_pre]) - if yr - horizon_new[horizon_new.index(yr) - 1] >= horizon_new[ - horizon_new.index(yr) - 1] - horizon_new[ - horizon_new.index(yr) - 2]: + j = horizon_new.index(yr) + if yr - horizon_new[j - 1] >= horizon_new[j - 1 + ] - horizon_new[j - 2]: - df2[yr].loc[(pd.isna(df2[year_pre].shift(+1))) & ( - ~pd.isna(df2[year_pp].shift(+1)))] = np.nan - if extrapol_neg and not df2[yr].loc[(df2[yr] < 0) & ( - df2[year_pre].shift(+1) >= 0)].empty: - df2.loc[(df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0), - yr] = df2.loc[(df2[yr] < 0) & ( - df2[year_pre].shift(+1) >= 0), - year_pre] * extrapol_neg + df2[yr].loc[(pd.isna(df2[year_pre].shift(+1)) + ) & (~pd.isna(df2[year_pp].shift(+1)))] = np.nan + cond = (df2[yr] < 0) & (df2[year_pre].shift(+1) >= 0) + if not df2[yr].loc[cond].empty and extrapol_neg: + df2.loc[cond, yr] = df2.loc[cond, year_pre] * extrapol_neg # b) Otherise, do intrapolation elif yr > min(df2.columns) and yr < max(df2.columns): year_pre = max([x for x in df2.columns if x < yr]) year_next = min([x for x in df2.columns if x > yr]) - df2[yr] = intpol( - df2[year_pre], - df2[year_next], - year_pre, - year_next, - yr) + df2[yr] = intpol(df2[year_pre], df2[year_next], + year_pre, year_next, yr) df2_t = df2.loc[df2_tec.index, :].copy() # This part calculates the missing value if only the previous # timestep has a value (and not the next) if tec_list: - df2_t[yr].loc[(pd.isna(df2_t[yr])) & (~pd.isna(df2_t[ - year_pre]))] = intpol( - df2_t[year_pre], - df2_t[year_next].shift(-1), - year_pre, year_next, yr) + df2_t[yr].loc[(pd.isna(df2_t[yr]) + ) & (~pd.isna(df2_t[year_pre])) + ] = intpol(df2_t[year_pre], + df2_t[year_next].shift(-1), + year_pre, year_next, yr) # Treating technologies with phase-out in model years if [x for x in df2.columns if x < year_pre]: year_pp = max([x for x in df2.columns if x < year_pre]) - df2_t[yr].loc[(pd.isna(df2_t[yr])) & ( - pd.isna(df2_t[year_pre].shift(-1))) & ( - ~pd.isna(df2_t[year_pre]))] = intpol( - df2_t[year_pre], - df2_t[year_pp], - year_pre, year_pp, yr) - - if extrapol_neg and not df2_t[yr].loc[( - df2_t[yr] < 0) & (df2_t[year_pre] >= 0)].empty: - df2_t.loc[(df2_t[yr] < 0) & (df2_t[year_pre] >= 0), - yr] = df2_t.loc[(df2_t[yr] < 0) & ( - df2_t[year_pre] >= 0), - year_pre] * extrapol_neg + df2_t[yr].loc[(pd.isna(df2_t[yr]) + ) & (pd.isna(df2_t[year_pre].shift(-1)) + ) & (~pd.isna(df2_t[year_pre])) + ] = intpol(df2_t[year_pre], df2_t[year_pp], + year_pre, year_pp, yr) + cond = (df2_t[yr] < 0) & (df2_t[year_pre] >= 0) + if not df2_t[yr].loc[cond].empty and extrapol_neg: + df2_t.loc[cond, yr] = df2_t.loc[cond, year_pre + ] * extrapol_neg df2.loc[df2_tec.index, :] = df2_t df2[yr][np.isinf(df2[year_pre])] = df2[year_pre] df2 = df2.reindex(sorted(df2.columns), axis=1) - # -------------------------------------------------------------------------- - # Third, adding year_vtg of new years and their respective year_act for - # both existing and new years + # ------------------------------------------------------------------------- + # Third, adding year_vtg of new years for yr in yrs_new: - # a) If this new year is after the current range of modeled years, - # do extrapolation + # a) If this new year is greater than modeled years, do extrapolation if yr > max(horizon): year_pre = horizon_new[horizon_new.index(yr) - 1] year_pp = horizon_new[horizon_new.index(yr) - 2] df_pre = slice_df(df2, idx, year_ref, [year_pre], yr) df_pp = slice_df(df2, idx, year_ref, [year_pp], yr) - df_yr = intpol( - df_pre, - f_index( - df_pp, - df_pre), - year_pre, - year_pp, - yr) + df_yr = intpol(df_pre, idx_check(df_pp, df_pre), + year_pre, year_pp, yr) df_yr[np.isinf(df_pre)] = df_pre # For those technolofies with one value for each year - df_yr.loc[pd.isna(df_yr[yr])] = intpol( - df_pre, df_pp.shift(+1, axis=1), - year_pre, year_pp, yr).shift(+1, axis=1) + df_yr.loc[pd.isna(df_yr[yr])] = intpol(df_pre, + df_pp.shift(+1, axis=1), + year_pre, year_pp, + yr).shift(+1, axis=1) df_yr[pd.isna(df_yr)] = df_pre if extrapol_neg: @@ -815,30 +764,25 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( df_pre = slice_df(df2, idx, year_ref, [year_pre], yr) df_next = slice_df(df2, idx, year_ref, [year_next], yr) - df_yr = pd.concat(( - df_pre, - df_next.loc[df_next.index.isin(df_pre.index)]), + df_yr = pd.concat((df_pre, idx_check(df_next, df_pre)), axis=0).groupby(level=idx).mean() - df_yr[yr] = df_yr[yr].fillna( - df_yr[[year_pre, year_next]].mean(axis=1)) + df_yr[yr] = df_yr[yr].fillna(df_yr[[year_pre, year_next] + ].mean(axis=1)) df_yr[np.isinf(df_pre)] = df_pre # Creating a mask to remove extra values df_count = pd.DataFrame({'c_pre': df_pre.count(axis=1), - 'c_next': df_next.loc[ - df_next.index.isin(df_pre.index)].count(axis=1)}, - index=df_yr.index) + 'c_next': idx_check(df_next, df_pre + ).count(axis=1)}, + index=df_yr.index) for i in df_yr.index: - # Mainly for cases of two new consecutive years (like 2022 - # and 2024) - if ~np.isnan( - df_count['c_next'][i]) and df_count[ - 'c_pre'][i] >= df_count['c_next'][i] + 2: - df_yr[year_pre] = np.nan + # Mainly for cases of two new consecutive new years + cond = df_count['c_pre'][i] >= df_count['c_next'][i] + 2 + if ~np.isnan(df_count['c_next'][i]) and cond: + df_yr[year_pre] = np.nan # For technologies phasing out before the end of horizon - # (like nuc_lc) elif np.isnan(df_count['c_next'][i]): df_yr.loc[i, :] = df_pre.loc[i, :].shift(+1) if tec_list: @@ -856,17 +800,16 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( df2 = df2.append(df_yr) df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() - # --------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # Forth: final masking based on technical lifetime if tec_list: df3 = df2.copy() - for y in sorted([x for x in list( - set(df2.index.get_level_values(year_ref)) - ) if x in df_dur.index]): + idx_list = list(set(df2.index.get_level_values(year_ref))) + for y in sorted([x for x in idx_list if x in df_dur.index]): df3.loc[df3.index.get_level_values(year_ref).isin([y]), - df3.columns.isin(df_dur.columns)] = df_dur.loc[ - y, df_dur.columns.isin(df3.columns)].values + df3.columns.isin(df_dur.columns) + ] = df_dur.loc[y, df_dur.columns.isin(df3.columns)].values df3 = df3.reset_index().set_index( ['node_loc', 'technology', year_ref]).sort_index(level=1) @@ -876,24 +819,20 @@ def f_index(df1, df2): return df1.loc[df1.index.isin( for i in [x for x in par_tec.index if x in df3.index]: df3.loc[i, 'lifetime'] = par_tec.loc[i, 'value'].copy() - df3 = df3.reset_index().set_index(idx).dropna( - subset=['lifetime']).sort_index() + df3 = df3.reset_index().set_index(idx).dropna(subset=['lifetime'] + ).sort_index() for i in df3.index: df2.loc[i, df3.loc[i, :] >= int( df3.loc[i, 'lifetime'])] = np.nan # Removing extra values from non-lifetime technologies for i in [x for x in df2.index if x not in df3.index]: - df2.loc[i, df2.columns > df2.loc[[i]].index.get_level_values( - year_ref)[0]] = np.nan - - df_par = pd.melt( - df2.reset_index(), - id_vars=idx, - value_vars=[ - x for x in df2.columns if x not in idx], - var_name=year_col, - value_name='value').dropna( - subset=['value']) + condition = df2.loc[[i]].index.get_level_values(year_ref)[0] + df2.loc[i, df2.columns > condition] = np.nan + + df_par = pd.melt(df2.reset_index(), id_vars=idx, + value_vars=[x for x in df2.columns if x not in idx], + var_name=year_col, + value_name='value').dropna(subset=['value']) df_par = df_par.sort_values(idx).reset_index(drop=True) return df_par diff --git a/message_ix/tools/add_year/__main__.py b/message_ix/tools/add_year/__main__.py index 166c3d2f8..6949c0910 100644 --- a/message_ix/tools/add_year/__main__.py +++ b/message_ix/tools/add_year/__main__.py @@ -24,7 +24,7 @@ import ixmp import message_ix -from . import addNewYear +from . import add_year def split_value(ctx, param, value, type=str): @@ -121,8 +121,7 @@ def main(model_ref, scen_ref, version_ref, model_new, scen_new, create_new, 'rewrite:', rewrite, 'unit_check:', unit_check, 'extrapol_neg:', extrapol_neg, - 'bound_extend:', bound_extend, - sep='\n') + 'bound_extend:', bound_extend) return # Load the ixmp Platform @@ -141,7 +140,7 @@ def main(model_ref, scen_ref, version_ref, model_new, scen_new, create_new, sc_new.check_out() # Calling the main function - addNewYear( + add_year( sc_ref=sc_ref, sc_new=sc_new, years_new=years_new, diff --git a/tests/tools/test_add_year.py b/tests/tools/test_add_year.py index 0a422098f..285cb8f99 100644 --- a/tests/tools/test_add_year.py +++ b/tests/tools/test_add_year.py @@ -6,7 +6,7 @@ import message_ix from conftest import here -from message_ix.tools.add_year import addNewYear +from message_ix.tools.add_year import add_year db_dir = os.path.join(here, 'testdb') test_db = os.path.join(db_dir, 'ixmptest') @@ -16,48 +16,37 @@ def test_addNewYear(test_mp): msg_multiyear_args = ('canning problem (MESSAGE scheme)', 'multi-year') sc_ref = message_ix.Scenario(test_mp, *msg_multiyear_args) - assert sc_ref.par('technical_lifetime')[ - 'technology'].isin(['canning_plant']).any() + assert sc_ref.par('technical_lifetime')['technology' + ].isin(['canning_plant']).any() sc_ref.solve() # Building a new scenario and adding new years - sc_new = message_ix.Scenario(test_mp, - model='foo', - scenario='bar', - version='new', - annonation=' ') + sc_new = message_ix.Scenario(test_mp, model='foo', scenario='bar', + version='new', annonation=' ') # Using the function addNewYear years_new = [2015, 2025] - addNewYear( - sc_ref, - sc_new, - years_new, - ) + add_year(sc_ref, sc_new, years_new) # 1. Testing the set "year" for the new years horizon_old = sorted([int(x) for x in sc_ref.set('year')]) horizon_test = sorted(horizon_old + years_new) - horizon_new = sorted([int(x) for x in sc_new.set('year')]) # Asserting if the lists are equal assert (horizon_test == horizon_new) # 2. Testing parameter "technical_lifetime" - df_tec = sc_ref.par( - 'technical_lifetime', { - 'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_vtg': [2020, 2030]}) + df_tec = sc_ref.par('technical_lifetime', {'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_vtg': [2020, 2030]}) value_ref = df_tec['value'].mean() - - df_tec_new = sc_new.par( - 'technical_lifetime', { - 'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_vtg': 2025}) + df_tec_new = sc_new.par('technical_lifetime', + {'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_vtg': 2025} + ) value_new = float(df_tec_new['value']) @@ -66,21 +55,17 @@ def test_addNewYear(test_mp): assert math.isclose(value_ref, value_new, rel_tol=1e-04) # 3. Testing parameter "output" - df_tec = sc_ref.par( - 'output', { - 'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_act': 2030, - 'year_vtg': [2020, 2030]}) + df_tec = sc_ref.par('output', {'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_act': 2030, + 'year_vtg': [2020, 2030]}) value_ref = df_tec['value'].mean() - df_tec_new = sc_new.par( - 'output', { - 'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_act': 2025, - 'year_vtg': 2025}) + df_tec_new = sc_new.par('output', {'node_loc': 'seattle', + 'technology': 'canning_plant', + 'year_act': 2025, + 'year_vtg': 2025}) # Asserting if the missing data is generated or not assert df_tec_new['value'].tolist() From ba9c4ce0b49f732c521f2924442fd80d6275c3ba Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 7 May 2019 10:17:53 +0200 Subject: [PATCH 34/44] Use find_packages() in setup.py --- setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 2960b66a3..abd314d03 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import re import shutil -from setuptools import setup +from setuptools import find_packages, setup from setuptools.command.install import install fname = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -47,9 +47,6 @@ def all_gams_files(path, strip=None): def main(): - packages = [ - 'message_ix' - ] pack_dir = { 'message_ix': 'message_ix', } @@ -74,7 +71,7 @@ def main(): "url": 'http://github.com/iiasa/message_ix', "install_requires": INSTALL_REQUIRES, "extras_require": EXTRAS_REQUIRE, - "packages": packages, + "packages": find_packages(), "package_dir": pack_dir, "package_data": pack_data, "entry_points": entry_points, From 7094bbe5784badcefe90ab9ec7ca20c2865dcf36 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 7 May 2019 11:01:59 +0200 Subject: [PATCH 35/44] Update docstrings --- message_ix/tools/add_year/README.rst | 33 +-- message_ix/tools/add_year/__init__.py | 296 +++++++++++++------------- 2 files changed, 163 insertions(+), 166 deletions(-) diff --git a/message_ix/tools/add_year/README.rst b/message_ix/tools/add_year/README.rst index 0b67c2109..1c92304e6 100644 --- a/message_ix/tools/add_year/README.rst +++ b/message_ix/tools/add_year/README.rst @@ -3,19 +3,22 @@ Add model years to an existing Scenario Description ----------- -This functionality adds new modeling years to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: + +This tool adds new modeling years to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: - Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. -Main features -------------- +Features +-------- + - It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. - The new years can be consecutive, between existing years, and after the model horizon. - The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. Main steps ---------- + 1. An existing scenario is loaded and the desired new years is specified. 2. A new (empty) scenario is created for adding the new time steps. 3. The new years are added to the relevant sets: @@ -32,32 +35,34 @@ Main steps - the missing data is calculated by interpolation. - for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. -5. The changes are commited and saved to the new scenario. +5. The changes are committed and saved to the new scenario. -Notice ------- -I. This functionality in the current format does not ensure that the new scenario will solve after adding the new years. The user needs to load the new scenario, check some key parameters (like bounds) and solve the new scenario. +.. warning:: + The tool does not ensure that the new scenario will solve after adding the + new years. The user needs to load the new scenario, check some key + parameters (like bounds) and solve the new scenario. Usage ----- -This script can be used either: -- By running directly from the command line, example:: - +The tool can be used either: + +1. Directly from the command line:: + $ python -m message_ix.tools.add_year \ --model_ref MESSAGE_Model \ --scen_ref baseline \ --years_new 2015,2025,2035,2045 - - For the full list of input arguments, run:: + + For the full list of input arguments, run:: $ python -m message_ix.tools.add_year --help -- By calling the class "addNewYear" from another python script. +2. By calling the :meth:`message_ix.tools.add_year.add_year` from a Python + script. API --- .. automodule:: message_ix.tools.add_year :members: - diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 33ecaa314..d7f8281c4 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -1,47 +1,30 @@ # -*- coding: utf-8 -*- -""" -Description: - This functionality adds new time steps to an existing MESSAGE scenario - (hereafter "reference scenario"). This is done by creating a new empty - scenario (hereafter "new scenario") and: - - Copying all sets from reference scenario and adding new time steps to - relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") - - Copying all parameters from reference scenario, adding new time steps to - relevant parameters, calculating missing values for the added time steps. - -Sections of this code: - I. Required python packages are imported and ixmp platform loaded. - II. Generic utilities for dataframe manipulation - III. The main class called "add_year" - IV. Submodule "add_year_set" for adding and modifying the sets - V. Submodule "add_year_par" for copying and modifying each parameter - VI. Submodule "add_year_par" calls two utility functions ("interpolate_1d" - and "interpolate_2D") for calculating missing values. - VII. Code for running the script as "main" - - -Usage: - This script can be used either: - A) By running directly from the command line, example: - --------------------------------------------------------------------------- - python f_add_year.py --model_ref "MESSAGE_Model" --scen_ref "baseline" - --years_new "[2015,2025,2035,2045]" - --------------------------------------------------------------------------- - (Other input arguments are optional. For more info see Section V below.) - - B) By calling the class "add_year" from other python scripts -""" -# %% I) Importing required packages and loading ixmp platform +"""Add model years to an existing Scenario.""" +# Sections of the code: +# +# I. Required python packages are imported +# II. Generic utilities for dataframe manipulation +# III. The main function, add_year() +# IV. Function add_year_set() for adding and modifying the sets +# V. Function add_year_par() for copying and modifying each parameter +# VI. Two utility functions, interpolate_1d() and interpolate_2d(), for +# calculating missing values + +# %% I) Importing required packages import numpy as np import pandas as pd -# %% II) Required utility functions for dataframe manupulation -# II.A) Utility function for interpolation/extrapolation of two numbers, -# lists or series (x: time steps, y: data points) - +# %% II) Utility functions for dataframe manupulation def intpol(y1, y2, x1, x2, x): + """Interpolate between (*x1*, *y1*) and (*x2*, *y2*) at *x*. + + Parameters + ---------- + y1, y2 : float or pd.Series + x1, x2, x : int + """ if x2 == x1 and y2 != y1: print('>>> Warning <<<: No difference between x1 and x2,' + 'returned empty!!!') @@ -53,12 +36,20 @@ def intpol(y1, y2, x1, x2, x): return y # II.B) Utility function for slicing a MultiIndex dataframe and -# setting a value to a specific level -# df: dataframe, idx: list of indexes, level: string, locator: list, -# value: integer/string - +# setting a value to a specific level def slice_df(df, idx, level, locator, value): + """Slice a MultiIndex DataFrame and set a value to a specific level. + + Parameters + ---------- + df : pd.DataFrame + idx : list of indices + level: str + locator : list + value : int or str + + """ if locator: df = df.reset_index().loc[df.reset_index()[level].isin(locator)].copy() else: @@ -67,19 +58,15 @@ def slice_df(df, idx, level, locator, value): df[level] = value return df.set_index(idx) -# II.C) A function for creating a mask for removing extra values from a -# dataframe - def mask_df(df, index, count, value): + """Create a mask for removing extra values from *df*.""" df.loc[index, df.columns > (df.loc[[index]].notnull().cumsum( axis=1) == count).idxmax(axis=1).values[0]] = value -# II.D) Function for unifroming the "unit" in different years to prevent -# mistakes in indexing and grouping - def unit_uniform(df): + """Make units in *df* uniform.""" column = [x for x in df.columns if x in ['commodity', 'emission']] if column: com_list = set(df[column[0]]) @@ -96,43 +83,44 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, macro=False, baseyear_macro=None, parameter='all', region='all', rewrite=True, unit_check=True, extrapol_neg=None, bound_extend=True): - ''' This class does the following: - A. calls function "add_year_set" to add and modify required sets. - B. calls function "add_year_par" to add new years and modifications - to each parameter if needed. + """Add years to *sc_ref* to produce *sc_new*. + + :meth:`add_year` does the following: + + 1. calls :meth:`add_year_set` to add and modify required sets. + 2. calls :meth:`add_year_par` to add new years and modifications to each + parameter if needed. - Parameters: + Parameters ----------- - sc_ref : string - reference scenario (MESSAGE model/scenario instance) - sc_new : string - new scenario for adding new years and required modifications - (MESSAGE model/scenario instance) - yrs_new : list of integers - new years to be added - firstyear_new : integer, default None - a new first model year for new scenario (optional) - macro : boolean, default False - a flag to add new years to macro parameters (True) or not (False) - baseyear_macro : integer, default None - a new base year for macro - parameter: list of strings, default all - parameters for adding new years - rewrite: boolean, default True - a flag to permit rewriting a parameter in new scenario when adding - new years (True) or not (False) - check_unit: boolean, default False - a flag to uniform the units under each commodity (if there is - inconsistency between units in two model years) - extrapol_neg: float, default None - a number to multiply by the data of the previous timestep, - when extrapolation is negative - bound_extend: boolean, default True - a flag to permit the duplication of the data from the previous - timestep, when there is only one data point for interpolation - (e.g., permitting the extension of a bound to 2025, when there - is only one value in 2020) - ''' + sc_ref : str + Name of reference scenario (MESSAGE model/scenario instance) + sc_new : str + Name of new scenario for adding new years and required modifications + (MESSAGE model/scenario instance) + yrs_new : list of int + New years to be added. + firstyear_new : int, optional + New first model year for new scenario. + macro : bool + Add new years to parameters of the MACRO model. + baseyear_macro : int + New base year for the MACRO model. + parameter: list of str or 'all' + Parameters for adding new years. + rewrite: bool + Permit rewriting a parameter in new scenario when adding new years. + check_unit: bool + Harmonize the units for each commodity, if there is inconsistency + across model years. + extrapol_neg: float + When extrapolation produces negative values, replace with a multiple of + the value for the previous timestep. + bound_extend: bool + Duplicate data from the previous timestep when there is only one data + point for interpolation (e.g., permitting the extension of a bound to + 2025, when there is only one value in 2020). + """ # III.A) Adding sets and required modifications years_new = sorted([x for x in years_new if str(x) not in set(sc_ref.set('year'))]) @@ -214,22 +202,15 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, # IV) Adding new years to sets def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, baseyear_macro=None): - ''' - Description: - Adding required sets and relevant modifications: - This function adds additional years to an existing scenario, - by starting to make a new scenario from scratch. - After modification of the year-related sets, this function copeis - all other sets from the "reference" scenario - to the "new" scenario. - - Input arguments: - Please see the description for the input arguments under the main - function "add_year" - - Usage: - This module is called by function "add_year" - ''' + """Add new years to sets. + + :meth:`add_year_set` adds additional years to an existing scenario, by + starting to make a new scenario from scratch. After modification of the + year-related sets, all other sets are copied from *sc_ref* to *sc_new*. + + See :meth:`add_year` for parameter descriptions. + + """ # IV.A) Treatment of the additional years in the year-related sets # A.1. Set - year @@ -310,22 +291,16 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, extrapolate=False, rewrite=True, unit_check=True, extrapol_neg=None, bound_extend=True): - ''' Adding required parameters and relevant modifications: - Description: - This function adds additional years to a parameter. - The value of the parameter for additional years is calculated - mainly by interpolating and extrapolating of data from existing - years. + """Add new years to parameters. - Input arguments: - Please see the description for the input arguments under the - main function "add_year" + This function adds additional years to a parameter. The value of the + parameter for additional years is calculated mainly by interpolating and + extrapolating data from existing years. - Usage: - This module is called by function "add_year" - ''' + See :meth:`add_year` for parameter descriptions. -# V.A) Initialization and checks + """ + # V.A) Initialization and checks par_list_new = sc_new.par_list().tolist() idx_names = sc_ref.idx_names(parname) @@ -440,26 +415,31 @@ def f(x, i): # %% VI) Required functions def interpolate_1d(df, yrs_new, horizon, year_col, value_col='value', extrapolate=False, extrapol_neg=None, bound_extend=True): - ''' - Description: - This function receives a parameter data as a dataframe, and adds - new data for the additonal years by interpolation and - extrapolation. - - Input arguments: - df_par (dataframe): the dataframe of the parameter to which new - years to be added - yrs_new (list of integers): new years to be added - horizon (list of integers): the horizon of the reference scenario - year_col (string): the header of the column to which the new years - should be added, for example, "year_act" - value_col (string): the header of the column containing values - extrapolate: if True, the extrapolation is allowed when a new year - is outside the parameter years - extrapol_neg: if True, negative values obtained by extrapolation - are allowed. - bound_extend: if True, allows extrapolation of bounds for new years - ''' + """Interpolate data with one year dimension. + + This function receives a parameter data as a dataframe, and adds new data + for the additonal years by interpolation and extrapolation. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe of the parameter to which new years to be added. + yrs_new : list of int + New years to be added. + horizon: list of int + The horizon of the reference scenario. + year_col : str + The header of the column to which the new years should be added, e.g. + `'year_act'`. + value_col : str + The header of the column containing values. + extrapolate : bool + Allow extrapolation when a new year is outside the parameter years. + extrapol_neg : bool + Allow negative values obtained by extrapolation. + bound_extend : bool + Allow extrapolation of bounds for new years + """ horizon_new = sorted(horizon + yrs_new) idx = [x for x in df.columns if x not in [year_col, value_col]] if not df.empty: @@ -565,27 +545,39 @@ def interpolate_1d(df, yrs_new, horizon, year_col, value_col='value', def interpolate_2d(df, yrs_new, horizon, year_ref, year_col, tec_list, par_tec, value_col='value', extrapolate=False, extrapol_neg=None, year_diff=None): - ''' - Description: - This function receives a dataframe that has 2 time-related columns - (e.g., "input" or "relation_activity"), and adds new data for the - additonal years in both time-related columns by interpolation - and extrapolation. - - Input arguments: - df (dataframe): the parameter to which new years to be added - yrs_new (list of integers): new years to be added - horizon (list of integers): the horizon of the reference scenario - year_ref (string): the header of the first column to which the new - years should be added, for example, "year_vtg" - year_col (string): the header of the second column to which the - new years should be added, for example, "year_act" - value_col (string): the header of the column containing values - extrapolate: if True, the extrapolation is allowed when a new year - is outside the parameter years - extrapol_neg: if True, negative values obtained by extrapolation - are allowed. - ''' + """Interpolate parameters with two dimensions related year. + + This function receives a dataframe that has 2 time-related columns (e.g., + "input" or "relation_activity"), and adds new data for the additonal years + in both time-related columns by interpolation and extrapolation. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe of the parameter to which new years to be added. + yrs_new : list of int + New years to be added. + horizon: list of int + The horizon of the reference scenario. + year_ref : str + The header of the first column to which the new years should be added, + e.g. `'year_vtg'`. + year_col : str + The header of the column to which the new years should be added, e.g. + `'year_act'`. + tec_list : ??? + ??? + par_tec : ??? + ??? + value_col : str + The header of the column containing values. + extrapolate : bool + Allow extrapolation when a new year is outside the parameter years. + extrapol_neg : bool + Allow negative values obtained by extrapolation. + year_diff : ??? + ??? + """ def idx_check(df1, df2): return df1.loc[df1.index.isin(df2.index)] From 2e40f88684f2f29d5facb757b2fb76aecd706b4d Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 8 May 2019 10:48:07 +0200 Subject: [PATCH 36/44] Revamped the test to explicitly test each function --- tests/tools/test_add_year.py | 111 ++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/tests/tools/test_add_year.py b/tests/tools/test_add_year.py index 285cb8f99..b4e2c5177 100644 --- a/tests/tools/test_add_year.py +++ b/tests/tools/test_add_year.py @@ -1,75 +1,78 @@ # -*- coding: utf-8 -*- -# Importing required packages -import os -import math -import ixmp as ix -import message_ix -from conftest import here - +from message_ix import Scenario from message_ix.tools.add_year import add_year +import math -db_dir = os.path.join(here, 'testdb') -test_db = os.path.join(db_dir, 'ixmptest') -test_mp = ix.Platform(dbprops=test_db, dbtype='HSQLDB') +def model_setup(scen, data): + years = sorted(list(set(data.keys()))) + scen.add_set('node', 'node') + scen.add_set('commodity', 'comm') + scen.add_set('level', 'level') + scen.add_set('year', years) + scen.add_set('technology', 'tec') + scen.add_set('mode', 'mode') + output_specs = ['node', 'comm', 'level', 'year', 'year'] -def test_addNewYear(test_mp): - msg_multiyear_args = ('canning problem (MESSAGE scheme)', 'multi-year') - sc_ref = message_ix.Scenario(test_mp, *msg_multiyear_args) - assert sc_ref.par('technical_lifetime')['technology' - ].isin(['canning_plant']).any() - sc_ref.solve() + for (yr, value) in data.items(): + scen.add_par('demand', ['node', 'comm', 'level', yr, 'year'], 1, 'GWa') + scen.add_par('technical_lifetime', ['node', 'tec', yr], 10, 'y') + tec_specs = ['node', 'tec', yr, yr, 'mode'] + scen.add_par('output', tec_specs + output_specs, 1, '-') + scen.add_par('var_cost', tec_specs + ['year'], value, 'USD/GWa') - # Building a new scenario and adding new years - sc_new = message_ix.Scenario(test_mp, model='foo', scenario='bar', - version='new', annonation=' ') - # Using the function addNewYear - years_new = [2015, 2025] +def adding_years(test_mp, scen_ref, years_new): + scen_new = Scenario(test_mp, model='add_year', scenario='standard', + version='new', annonation=' ') + add_year(scen_ref, scen_new, years_new) + return scen_new - add_year(sc_ref, sc_new, years_new) +def assert_function(scen_ref, scen_new, years_new, yr_test): # 1. Testing the set "year" for the new years - horizon_old = sorted([int(x) for x in sc_ref.set('year')]) + horizon_old = sorted([int(x) for x in scen_ref.set('year')]) horizon_test = sorted(horizon_old + years_new) - horizon_new = sorted([int(x) for x in sc_new.set('year')]) - - # Asserting if the lists are equal + horizon_new = sorted([int(x) for x in scen_new.set('year')]) assert (horizon_test == horizon_new) - # 2. Testing parameter "technical_lifetime" - df_tec = sc_ref.par('technical_lifetime', {'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_vtg': [2020, 2030]}) - value_ref = df_tec['value'].mean() - df_tec_new = sc_new.par('technical_lifetime', - {'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_vtg': 2025} - ) + # 2. Testing parameter "technical_lifetime" (function interpolate_1d) + yr_next = min([x for x in horizon_old if x > yr_test]) + yr_pre = max([x for x in horizon_old if x < yr_test]) + parname = 'technical_lifetime' + ref = scen_ref.par(parname, {'technology': 'tec', + 'year_vtg': [yr_pre, yr_next]}) + value_ref = ref['value'].mean() + new = scen_new.par(parname, {'technology': 'tec', 'year_vtg': yr_test}) + value_new = float(new['value']) + assert math.isclose(value_ref, value_new, rel_tol=1e-04) + + # 3. Testing parameter "var_cost" (function interpolate_2d) + ref = scen_ref.par('var_cost', {'technology': 'tec', + 'year_act': [yr_pre, yr_next], + 'year_vtg': [yr_pre, yr_next]}) + + value_ref = ref['value'].mean() - value_new = float(df_tec_new['value']) + new = scen_new.par('var_cost', {'technology': 'tec', + 'year_act': yr_test, + 'year_vtg': yr_test}) # Asserting if the missing data is generated accurately by interpolation - # of adjacent data points + value_new = float(new['value']) assert math.isclose(value_ref, value_new, rel_tol=1e-04) - # 3. Testing parameter "output" - df_tec = sc_ref.par('output', {'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_act': 2030, - 'year_vtg': [2020, 2030]}) - - value_ref = df_tec['value'].mean() - df_tec_new = sc_new.par('output', {'node_loc': 'seattle', - 'technology': 'canning_plant', - 'year_act': 2025, - 'year_vtg': 2025}) +def test_add_year(test_mp): + scen_ref = Scenario(test_mp, 'model', 'standard', version='new') + model_setup(scen_ref, {2020: 1, 2030: 2, 2040: 3}) + scen_ref.commit('initialize test model') + scen_ref.solve(case='original_years') - # Asserting if the missing data is generated or not - assert df_tec_new['value'].tolist() + # Adding new years + years_new = [2025, 2035] + scen_new = adding_years(test_mp, scen_ref, years_new) + scen_new.solve(case='new_years') - # Asserting if the missing data is generated accurately by interpolation - value_new = float(df_tec_new['value']) - assert math.isclose(value_ref, value_new, rel_tol=1e-04) + # Running the tests + assert_function(scen_ref, scen_new, years_new, yr_test=2025) From afa507149ac2627cd63d026808130dc16bd13bb5 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 8 May 2019 10:48:50 +0200 Subject: [PATCH 37/44] Improve init to account for models with no first year specified --- message_ix/tools/add_year/__init__.py | 107 ++++++++++++++------------ 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index d7f8281c4..81494513a 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -15,8 +15,8 @@ import numpy as np import pandas as pd -# %% II) Utility functions for dataframe manupulation +# %% II) Utility functions for dataframe manupulation def intpol(y1, y2, x1, x2, x): """Interpolate between (*x1*, *y1*) and (*x2*, *y2*) at *x*. @@ -35,8 +35,6 @@ def intpol(y1, y2, x1, x2, x): y = y1 + ((y2 - y1) / (x2 - x1)) * (x - x1) return y -# II.B) Utility function for slicing a MultiIndex dataframe and -# setting a value to a specific level def slice_df(df, idx, level, locator, value): """Slice a MultiIndex DataFrame and set a value to a specific level. @@ -164,13 +162,17 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, 'prfconst', 'grow', 'aeei', 'aeei_factor', 'gdp_rate'] par_list = [x for x in par_list if x not in par_macro] - firstyear = sc_new.set('cat_year', {'type_year': 'firstmodelyear'})['year'] + if not sc_new.set('cat_year', {'type_year': 'firstmodelyear'}).empty: + firstyear_new = sc_new.set('cat_year', + {'type_year': 'firstmodelyear'})['year'] + else: + firstyear_new = min([int(x) for x in sc_new.set('year').tolist()]) for parname in par_list: # For historical parameters extrapolation permitted (e.g., from # 2010 to 2015) if 'historical' in parname: extrapol = True - yrs_new = [x for x in years_new if x < int(firstyear)] + yrs_new = [x for x in years_new if x < int(firstyear_new)] else: extrapol = False yrs_new = years_new @@ -186,12 +188,12 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, # The loop over "node" is only for reducing the size of tables for node in reg_list: add_year_par(sc_ref, sc_new, yrs_new, parname, [node], - extrapol, rewrite, unit_check, extrapol_neg, - bound_ext) + firstyear_new, extrapol, rewrite, unit_check, + extrapol_neg, bound_ext) else: add_year_par(sc_ref, sc_new, yrs_new, parname, reg_list, - extrapol, rewrite, unit_check, extrapol_neg, - bound_ext) + firstyear_new, extrapol, rewrite, unit_check, + extrapol_neg, bound_ext) sc_new.set_as_default() print('> All required parameters were successfully ' + @@ -212,7 +214,6 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, """ # IV.A) Treatment of the additional years in the year-related sets - # A.1. Set - year yrs_old = list(map(int, sc_ref.set('year'))) horizon_new = sorted(yrs_old + years_new) @@ -235,14 +236,15 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, # A.5. Changing the base year and initialization year of macro if a new # year specified - if not yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'].empty and baseyear_macro: - yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'] = baseyear_macro - if not yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year'].empty and baseyear_macro: - yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year'] = baseyear_macro + if baseyear_macro: + if not yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'].empty: + yr_cat.loc[yr_cat['type_year'] == + 'baseyear_macro', 'year'] = baseyear_macro + if not yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'].empty: + yr_cat.loc[yr_cat['type_year'] == + 'initializeyear_macro', 'year'] = baseyear_macro yr_pair = [] for yr in years_new: @@ -255,11 +257,12 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, ).sort_values('year').reset_index(drop=True) # A.6. Changing the cumulative years based on the new first model year - firstyear_new = int(yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', - 'year']) - yr_cat = yr_cat.drop(yr_cat.loc[(yr_cat['type_year'] == 'cumulative' - ) & (yr_cat['year'] < firstyear_new) - ].index) + if 'firstmodelyear' in set(yr_cat['type_year']): + firstyear_new = int(yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', + 'year']) + yr_cat = yr_cat.drop(yr_cat.loc[(yr_cat['type_year'] == 'cumulative' + ) & (yr_cat['year'] < firstyear_new) + ].index) sc_new.add_set('cat_year', yr_cat) # IV.B) Copying all other sets @@ -288,7 +291,7 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, # %% V) Adding new years to parameters -def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, +def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, firstyear_new, extrapolate=False, rewrite=True, unit_check=True, extrapol_neg=None, bound_extend=True): """Add new years to parameters. @@ -301,7 +304,6 @@ def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, """ # V.A) Initialization and checks - par_list_new = sc_new.par_list().tolist() idx_names = sc_ref.idx_names(parname) horizon = sorted([int(x) for x in list(set(sc_ref.set('year')))]) @@ -349,8 +351,7 @@ def add_year_par(sc_ref, sc_new, yrs_new, parname, region_list, year_list = [c for c in col_list if c in ['year', 'year_vtg', 'year_act', 'year_rel']] - # A uniform "unit" for values in different years to prevent mistakes in - # indexing and grouping + # A uniform "unit" for values in different years if 'unit' in col_list and unit_check: par_old = unit_uniform(par_old) # --------------------------------------------------------------------------- @@ -387,8 +388,6 @@ def f(x, i): ' "{}"...'.format(parname, nodes)) # Flagging technologies that have lifetime for adding new timesteps - firstyear_new = sc_new.set('cat_year', - {'type_year': 'firstmodelyear'})['year'] yr_list = [int(x) for x in set(sc_new.set('year') ) if int(x) > int(firstyear_new)] min_step = min(np.diff(sorted(yr_list))) @@ -405,7 +404,7 @@ def f(x, i): df_y = interpolate_2d(df, yrs_new, horizon, year_ref, year_col, tec_list, par_tec, 'value', extrapolate, - extrapol_neg, year_diff) + extrapol_neg, year_diff, bound_extend) sc_new.add_par(parname, df_y) sc_new.commit(parname) print('> Parameter "{}" copied and new years added' @@ -544,7 +543,7 @@ def interpolate_1d(df, yrs_new, horizon, year_col, value_col='value', # %% VI.B) Interpolating parameters with two dimensions related to time def interpolate_2d(df, yrs_new, horizon, year_ref, year_col, tec_list, par_tec, value_col='value', extrapolate=False, extrapol_neg=None, - year_diff=None): + year_diff=None, bound_extend=True): """Interpolate parameters with two dimensions related year. This function receives a dataframe that has 2 time-related columns (e.g., @@ -565,18 +564,20 @@ def interpolate_2d(df, yrs_new, horizon, year_ref, year_col, tec_list, par_tec, year_col : str The header of the column to which the new years should be added, e.g. `'year_act'`. - tec_list : ??? - ??? - par_tec : ??? - ??? + tec_list : list of str + List of technologies in the parameter `'technical_lifetime'` + par_tec : pandas.DataFrame + Parameter `'technical_lifetime'` value_col : str The header of the column containing values. extrapolate : bool Allow extrapolation when a new year is outside the parameter years. extrapol_neg : bool Allow negative values obtained by extrapolation. - year_diff : ??? - ??? + year_diff : list of int + List of model years with different time intervals before and after them + bound_extend : bool + Allow extrapolation of bounds for new years """ def idx_check(df1, df2): return df1.loc[df1.index.isin(df2.index)] @@ -598,8 +599,8 @@ def idx_check(df1, df2): def f(x, i): return x[i + 1] - x[i] > x[i] - x[i - 1] - yr_diff_new = [x for x in horizon_new[1:-1] if f(horizon, - horizon.index(x))] + yr_diff_new = [x for x in horizon_new[1:-1] if f(horizon_new, + horizon_new.index(x))] if year_diff and tec_list: if isinstance(year_diff, list): @@ -618,10 +619,7 @@ def f(x, i): df_y = df_count.loc[df_count['c_pre'] == df_count['c_next'] + 1] for i in df_y.index: - condition = ((df2.loc[[i]].notnull().cumsum(axis=1) - ) == df_count['c_next'][i]) - df2.loc[i, df2.columns > condition.idxmax(axis=1 - ).values[0]] = np.nan + mask_df(df2, i, df_count['c_next'][i], np.nan) # Generating duration_period_sum matrix for masking df_dur = pd.DataFrame(index=horizon_new[:-1], columns=horizon_new[1:]) @@ -758,8 +756,20 @@ def f(x, i): df_next = slice_df(df2, idx, year_ref, [year_next], yr) df_yr = pd.concat((df_pre, idx_check(df_next, df_pre)), axis=0).groupby(level=idx).mean() - df_yr[yr] = df_yr[yr].fillna(df_yr[[year_pre, year_next] - ].mean(axis=1)) + if df_yr.empty: + continue + if not df_yr.empty and yr not in df_yr.columns: + df_yr[yr] = np.nan + # TODO: here is the place that should be changed if the + # new year should go to the time step before the existing one + if bound_extend and not df_next.empty: + df_yr[yr] = df_yr[year_next] + elif bound_extend and not df_pre.empty: + df_yr[yr] = df_yr[year_pre] + + else: + df_yr[yr] = df_yr[yr].fillna(df_yr[[year_pre, year_next] + ].mean(axis=1)) df_yr[np.isinf(df_pre)] = df_pre # Creating a mask to remove extra values @@ -794,10 +804,11 @@ def f(x, i): df2 = df2.reindex(sorted(df2.columns), axis=1).sort_index() # ------------------------------------------------------------------------- # Forth: final masking based on technical lifetime - if tec_list: + if tec_list and not df_dur.empty: - df3 = df2.copy() - idx_list = list(set(df2.index.get_level_values(year_ref))) + df3 = df2.loc[df2.index.get_level_values('technology').isin(tec_list), + :].copy().dropna(how='all') + idx_list = list(set(df3.index.get_level_values(year_ref))) for y in sorted([x for x in idx_list if x in df_dur.index]): df3.loc[df3.index.get_level_values(year_ref).isin([y]), df3.columns.isin(df_dur.columns) From cdd7b1de79931d5219b19c2c66eda98b2262ab0b Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 8 May 2019 11:14:01 +0200 Subject: [PATCH 38/44] replaced math.isclose with pytest.approx in the test --- tests/tools/test_add_year.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tools/test_add_year.py b/tests/tools/test_add_year.py index b4e2c5177..e51e533ad 100644 --- a/tests/tools/test_add_year.py +++ b/tests/tools/test_add_year.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from message_ix import Scenario from message_ix.tools.add_year import add_year -import math +import pytest def model_setup(scen, data): @@ -45,7 +45,7 @@ def assert_function(scen_ref, scen_new, years_new, yr_test): value_ref = ref['value'].mean() new = scen_new.par(parname, {'technology': 'tec', 'year_vtg': yr_test}) value_new = float(new['value']) - assert math.isclose(value_ref, value_new, rel_tol=1e-04) + assert value_new == pytest.approx(value_ref, rel=1e-04) # 3. Testing parameter "var_cost" (function interpolate_2d) ref = scen_ref.par('var_cost', {'technology': 'tec', @@ -60,7 +60,7 @@ def assert_function(scen_ref, scen_new, years_new, yr_test): # Asserting if the missing data is generated accurately by interpolation value_new = float(new['value']) - assert math.isclose(value_ref, value_new, rel_tol=1e-04) + assert value_new == pytest.approx(value_ref, rel=1e-04) def test_add_year(test_mp): From 58d4f37ebc665716b4bae85310d7470fdd594383 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 8 May 2019 14:44:58 +0200 Subject: [PATCH 39/44] Copyedit documentation --- message_ix/tools/add_year/README.rst | 82 ++++++++++++++++----------- message_ix/tools/add_year/__init__.py | 13 ++--- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/message_ix/tools/add_year/README.rst b/message_ix/tools/add_year/README.rst index 1c92304e6..f93b8f363 100644 --- a/message_ix/tools/add_year/README.rst +++ b/message_ix/tools/add_year/README.rst @@ -4,43 +4,29 @@ Add model years to an existing Scenario Description ----------- -This tool adds new modeling years to an existing scenario (hereafter "reference scenario"). This will be done by creating a new empty scenario (hereafter "new scenario") and: +This tool adds new modeling years to an existing :class:`ixmp.Scenario` (hereafter "reference scenario"). For instance, in a scenario define with:: -- Copying all sets from reference scenario and adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set "year") -- Copying all parameters from reference scenario, adding new time steps to relevant parameters, and calculating missing values for the added time steps. + history = [690] + model_horizon = [700, 710, 720] + scenario.add_horizon({'year': history + model_horizon, + 'firstmodelyear': model_horizon[0]}) -Features --------- - -- It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. -- The new years can be consecutive, between existing years, and after the model horizon. -- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters had been successfully added to the new scenario previously. +…additional years can be added:: -Main steps ----------- + sc_new = scenario.clone() + add_year(scenario, sc_new, [705, 712, 718, 725]) -1. An existing scenario is loaded and the desired new years is specified. -2. A new (empty) scenario is created for adding the new time steps. -3. The new years are added to the relevant sets: +The tool operates by creating a new empty Scenario (hereafter "new scenario") and: - - adding new years to the set "year" and "type_year" - - changing "firstmodelyear", "lastmodelyear", "baseyear_macro", and "initializeyear_macro" if needed. - - modifying the set "cat_year" for the new years. +- Copying all **sets** from the reference scenario, adding new time steps to relevant sets (e.g., adding 2025 between 2020 and 2030 in the set ``year``) +- Copying all **parameters** from the reference scenario, adding new years to relevant parameters, and calculating missing values for the added years. -4. The new years are added to the index sets of relevant parameters, and the missing data for the new years are calculated based on interpolation of adjacent data points. The following steps are applied: - - - each non-empty parameter is loaded from the reference scenario - - the year-related indexes of the parameter are identified (either 0, 1, and 2 index under MESSAGE scheme) - - the new years are added to the parameter, and the missing data is calculated based on the number of year-related indexes. For example, the new years are added to index "year_vtg" in parameter "inv_cost", while these new years are added both to "year_vtg" and "year_act" in parameter "output". - - the missing data is calculated by interpolation. - - for the parameters with 2 year-related index (such as "output"), a final check is applied so ensure that the vintaging is correct. This step is done based on lifetime of each technology. - -5. The changes are committed and saved to the new scenario. +Features +~~~~~~~~ -.. warning:: - The tool does not ensure that the new scenario will solve after adding the - new years. The user needs to load the new scenario, check some key - parameters (like bounds) and solve the new scenario. +- It can be used for any MESSAGE scenario, from tutorials, country-level, and global models. +- The new years can be consecutive, between existing years, and/or after the model horizon. +- The user can define for what regions and parameters the new years should be added. This saves time when adding the new years to only one parameter of the reference scenario, when other parameters have previously been successfully added to the new scenario. Usage ----- @@ -58,11 +44,39 @@ The tool can be used either: $ python -m message_ix.tools.add_year --help -2. By calling the :meth:`message_ix.tools.add_year.add_year` from a Python - script. +2. By calling the function :meth:`message_ix.tools.add_year.add_year` from a Python script. + + +Technical details +----------------- + +1. An existing scenario is loaded and the desired new years are specified. +2. A new (empty) scenario is created for adding the new years. +3. The new years are added to the relevant sets, ``year`` and ``type_year``. + + - The sets ``firstmodelyear``, ``lastmodelyear``, ``baseyear_macro``, and ``initializeyear_macro`` are modified, if needed. + - The set ``cat_year`` is modified for the new years. + +4. The new years are added to the index sets of relevant parameters, and the missing data for the new years are calculated based on interpolation of adjacent data points. The following steps are applied: + + a. Each non-empty parameter is loaded from the reference scenario. + b. The year-related indexes (0, 1, or 2) of the parameter are identified. + c. The new years are added to the parameter, and the missing data is calculated based on the number of year-related indexes. For example: + + - The parameter ``inv_cost`` has index ``year_vtg``, to which the new years are added. + - The parameter ``output`` has indices ``year_act`` and ``year_vtg``. The new years are added to *both* of these dimensions. + d. Missing data is calculated by interpolation. + e. For parameters with 2 year-related indices (e.g. ``output``), a final check is applied so ensure that the vintaging is correct. This step is done based on the lifetime of each technology. + +5. The changes are committed and saved to the new scenario. + +.. warning:: + The tool does not ensure that the new scenario will solve after adding the + new years. The user needs to load the new scenario, check some key + parameters (like bounds) and solve the new scenario. -API ---- +API reference +------------- .. automodule:: message_ix.tools.add_year :members: diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 81494513a..36f9d2717 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -91,11 +91,10 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, Parameters ----------- - sc_ref : str - Name of reference scenario (MESSAGE model/scenario instance) - sc_new : str - Name of new scenario for adding new years and required modifications - (MESSAGE model/scenario instance) + sc_ref : ixmp.Scenario + Reference scenario. + sc_new : ixmp.Scenario + New scenario. yrs_new : list of int New years to be added. firstyear_new : int, optional @@ -565,9 +564,9 @@ def interpolate_2d(df, yrs_new, horizon, year_ref, year_col, tec_list, par_tec, The header of the column to which the new years should be added, e.g. `'year_act'`. tec_list : list of str - List of technologies in the parameter `'technical_lifetime'` + List of technologies in the parameter ``technical_lifetime``. par_tec : pandas.DataFrame - Parameter `'technical_lifetime'` + Parameter ``technical_lifetime``. value_col : str The header of the column containing values. extrapolate : bool From 94e9a98efadc5190d3cc3c4675b1d46a8c9145fd Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 8 May 2019 15:29:20 +0200 Subject: [PATCH 40/44] Fix sc_new creation in demo code --- message_ix/tools/add_year/README.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/message_ix/tools/add_year/README.rst b/message_ix/tools/add_year/README.rst index f93b8f363..cb6d3d94c 100644 --- a/message_ix/tools/add_year/README.rst +++ b/message_ix/tools/add_year/README.rst @@ -8,13 +8,17 @@ This tool adds new modeling years to an existing :class:`ixmp.Scenario` (hereaft history = [690] model_horizon = [700, 710, 720] - scenario.add_horizon({'year': history + model_horizon, + sc_ref.add_horizon({'year': history + model_horizon, 'firstmodelyear': model_horizon[0]}) …additional years can be added:: - sc_new = scenario.clone() - add_year(scenario, sc_new, [705, 712, 718, 725]) + sc_new = message_ix.Scenario(mp, sc_ref.model, sc_ref.scenario, + version='new') + add_year(sc_ref, sc_new, [705, 712, 718, 725]) + +At this point, ``sc_new`` will have the years [700, 705, 710, 712, 718, 720, 725], and original or interpolated data for all these years in all parameters. + The tool operates by creating a new empty Scenario (hereafter "new scenario") and: From 55dc30e2ead0424bd9af2e5779ec6239a1160130 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 8 May 2019 17:25:38 +0200 Subject: [PATCH 41/44] adding check for first model year --- message_ix/tools/add_year/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 36f9d2717..22cf0af18 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -227,11 +227,17 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, # A.4. Changing the first year if needed if firstyear_new: - yr_cat.loc[yr_cat['type_year'] == - 'firstmodelyear', 'year'] = firstyear_new + if not yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear'].empty: + yr_cat.loc[yr_cat['type_year'] == 'firstmodelyear', + 'year'] = firstyear_new + else: + yr_cat.loc[len(yr_cat.index)] = ['firstmodelyear', firstyear_new] if lastyear_new: - yr_cat.loc[yr_cat['type_year'] == - 'lastmodelyear', 'year'] = lastyear_new + if not yr_cat.loc[yr_cat['type_year'] == 'lastmodelyear'].empty: + yr_cat.loc[yr_cat['type_year'] == 'lastmodelyear', + 'year'] = lastyear_new + else: + yr_cat.loc[len(yr_cat.index)] = ['lastmodelyear', lastyear_new] # A.5. Changing the base year and initialization year of macro if a new # year specified From 00f14bb163dbbc50d5ea88e90a0004d2dc5142e3 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 17 Jun 2019 15:25:10 +0200 Subject: [PATCH 42/44] release notes updated --- RELEASE_NOTES.md | 5 +++-- message_ix/tools/add_year/__init__.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ccea0cce1..a6c88a7c2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,7 +1,9 @@ # Next Release -- [#161](https://github.com/iiasa/message_ix/pull/161): A feature for adding new time steps to a scenario +- [#161](https://github.com/iiasa/message_ix/pull/161): A feature for adding new periods to a scenario +- [#196](https://github.com/iiasa/message_ix/pull/196): Improve testing by re-using :mod:`ixmp` apparatus. +- [#187](https://github.com/iiasa/message_ix/pull/187): Test for cumulative bound on emissions. - [#182](https://github.com/iiasa/message_ix/pull/182): Fix cross-platform cloning. - [#173](https://github.com/iiasa/message_ix/pull/173): The `solve` command now takes additional arguments when solving with CPLEX. The `cplex.opt` file is now generated on the fly during the solve command and removed after successfully solving. - [#154](https://github.com/iiasa/message_ix/pull/154): Enable documentation build on ReadTheDocs. @@ -120,4 +122,3 @@ jdbc.pwd = ixmp - [#65](https://github.com/iiasa/message_ix/pull/65): Bugfix for downloading tutorials. Now downloads current installed version by default. - [#60](https://github.com/iiasa/message_ix/pull/60): Add basic ability to write and read model input to/from Excel - [#59](https://github.com/iiasa/message_ix/pull/59): Added MacOSX CI support -- [#187](https://github.com/iiasa/message_ix/pull/187): Test for cumulative bound on emissions diff --git a/message_ix/tools/add_year/__init__.py b/message_ix/tools/add_year/__init__.py index 22cf0af18..c115b5c0c 100644 --- a/message_ix/tools/add_year/__init__.py +++ b/message_ix/tools/add_year/__init__.py @@ -26,7 +26,7 @@ def intpol(y1, y2, x1, x2, x): x1, x2, x : int """ if x2 == x1 and y2 != y1: - print('>>> Warning <<<: No difference between x1 and x2,' + + print('>>> Warning <<<: No difference between x1 and x2,' 'returned empty!!!') return [] elif x2 == x1 and y2 == y1: @@ -195,7 +195,7 @@ def add_year(sc_ref, sc_new, years_new, firstyear_new=None, lastyear_new=None, extrapol_neg, bound_ext) sc_new.set_as_default() - print('> All required parameters were successfully ' + + print('> All required parameters were successfully ' 'added to the new scenario.') @@ -242,14 +242,14 @@ def add_year_set(sc_ref, sc_new, years_new, firstyear_new=None, # A.5. Changing the base year and initialization year of macro if a new # year specified if baseyear_macro: - if not yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'].empty: - yr_cat.loc[yr_cat['type_year'] == - 'baseyear_macro', 'year'] = baseyear_macro - if not yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year'].empty: - yr_cat.loc[yr_cat['type_year'] == - 'initializeyear_macro', 'year'] = baseyear_macro + if not yr_cat.loc[yr_cat['type_year'] == 'baseyear_macro', + 'year'].empty: + yr_cat.loc[yr_cat['type_year'] == 'baseyear_macro', + 'year'] = baseyear_macro + if not yr_cat.loc[yr_cat['type_year'] == 'initializeyear_macro', + 'year'].empty: + yr_cat.loc[yr_cat['type_year'] == 'initializeyear_macro', + 'year'] = baseyear_macro yr_pair = [] for yr in years_new: From 61d06ac114a5332bc61fef7835179987f763f994 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 21 Jun 2019 10:56:31 +0200 Subject: [PATCH 43/44] Remove references to f_addYear.py per @gidden --- message_ix/tools/add_year/__main__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/message_ix/tools/add_year/__main__.py b/message_ix/tools/add_year/__main__.py index 6949c0910..3e3064c73 100644 --- a/message_ix/tools/add_year/__main__.py +++ b/message_ix/tools/add_year/__main__.py @@ -2,10 +2,12 @@ \b Examples: -$ python f_addYear.py --model_ref Austria_tutorial --scen_ref test_core \ - --scen_new test_5y --years_new 2015,2025,2035,2045 -$ python f_addNewYear.py --model_ref CD_Links_SSP2 --scen_ref baseline \ - --years_new "[2015,2025,2035,2045,2055,2120]" +$ python -m message_ix.tools.add_year \ + --model_ref Austria_tutorial --scen_ref test_core \ + --scen_new test_5y --years_new 2015,2025,2035,2045 +$ python -m message_ix.tools.add_year \ + --model_ref CD_Links_SSP2 --scen_ref baseline \ + --years_new "[2015,2025,2035,2045,2055,2120]" If --create_new=False is given, the target Scenario must already exist. @@ -81,7 +83,7 @@ def main(model_ref, scen_ref, version_ref, model_new, scen_new, create_new, dry_run): start = timer() - print('>> Running the script f_addYears.py...') + print('>> message_ix.tools.add_year...') # Handle default arguments ref_kw = dict(model=model_ref, scen=scen_ref) From 1bcb147969b1cb5eb63983327e83fb59a72ab6bd Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 21 Jun 2019 11:10:35 +0200 Subject: [PATCH 44/44] Fix typo in test_add_year.py --- tests/tools/test_add_year.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tools/test_add_year.py b/tests/tools/test_add_year.py index e51e533ad..ada76fa68 100644 --- a/tests/tools/test_add_year.py +++ b/tests/tools/test_add_year.py @@ -24,7 +24,7 @@ def model_setup(scen, data): def adding_years(test_mp, scen_ref, years_new): scen_new = Scenario(test_mp, model='add_year', scenario='standard', - version='new', annonation=' ') + version='new', annotation=' ') add_year(scen_ref, scen_new, years_new) return scen_new