From b2ec28537bb68397a6f349a6ff1612c495b18bd1 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:05:55 +0100 Subject: [PATCH 01/10] Add .report.legacy.advanced_hydro_tables Copied from iiasa/message_data branch `dev`. --- .../report/legacy/advanced_hydro_tables.py | 4033 +++++++++++++++++ 1 file changed, 4033 insertions(+) create mode 100644 message_ix_models/report/legacy/advanced_hydro_tables.py diff --git a/message_ix_models/report/legacy/advanced_hydro_tables.py b/message_ix_models/report/legacy/advanced_hydro_tables.py new file mode 100644 index 0000000000..ddda88e984 --- /dev/null +++ b/message_ix_models/report/legacy/advanced_hydro_tables.py @@ -0,0 +1,4033 @@ +import pandas as pd + +import message_data.tools.post_processing.pp_utils as pp_utils + +pp = None +mu = None +run_history = None +urban_perc_data = None +kyoto_hist_data = None +lu_hist_data = None + +# Dictionary where all functions defined in the script are stored. +func_dict = {} # type: dict + + +def return_func_dict(): + return func_dict + + +def _register(func): + """Function to register reporting functions. + + Parameters + ---------- + + func : str + Function name + """ + + func_dict[func.__name__] = func + return func + + +def _pe_wCCSretro(tec, scrub_tec, group, inpfilter, units, share=1): + """Calculates primary energy use of technologies with scrubbers. + + Parameters + ---------- + + tec : str + Technology name + scrub_tec : str + Name of CO2 scrubbing technology linked to `tec`. + group : list + Indexes by which results are to be grouped. + inpfilter : dict + `level` and/or `commodity` used for retrieving the input. + units : str + Units to which variables should be converted. + share : number or dataframe (default: 1) + Share of `tec` activity if multiple technologies share the same scrubber. + """ + + # The multiplication of `share` is required to determine in which years + # the powerplant is active, because the scrubber activity + # is not necessarily equal to the powerplant activity if + # there are two powerplants using the same scrubber. + + df = ( + pp.out(scrub_tec, units) + * share + / pp_utils.ppgroup( + (pp.act(tec, group=group) / pp.act(tec)).fillna(0) + * pp.eff( + tec, + inpfilter=inpfilter, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + group=group, + ) + ) + ).fillna(0) + + return df + + +def _pe_elec_woCCSretro(tec, scrub_tec, group, inpfilter, units, _Frac, share=1): + """Calculates primary energy electricity generation equivalent. + + This applies to technologies WITHOUT scrubbers. + + Parameters + ---------- + + tec : str + Technology name + scrub_tec : str + Name of CO2 scrubbing technology linked to `tec`. + inpfilter : dict + `level` and/or `commodity` used for retrieving the input. + units : str + Units to which variables should be converted. + _Frac : dataframe + Regional share of actual cogeneration (po_turbine). + share : number or dataframe (default: 1) + Share of `tec` activity if multiple technologies share the same scrubber. + """ + + df = ( + ( + pp.out(tec, units) + * (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + - (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + * pp.out(scrub_tec, units) + * share + ) + / pp_utils.ppgroup( + (pp.act(tec, group=group) / pp.act(tec)).fillna(0) + * pp.eff( + tec, + inpfilter=inpfilter, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + group=group, + ) + ) + ).fillna(0) + + return df + + +def _pe_elec_wCCSretro(tec, scrub_tec, group, inpfilter, units, _Frac, share=1): + """Calculates primary energy electricity generation equivalent. + + This applies to technologies WITH scrubbers. + + Parameters + ---------- + + tec : str + Technology name + scrub_tec : str + Name of CO2 scrubbing technology linked to `tec`. + group : list + Indexes by which resultsa re to be grouped. + inpfilter : dict + `level` and/or `commodity` used for retrieving the input. + units : str + Units to which variables should be converted. + _Frac : dataframe + Regional share of actual cogeneration (po_turbine). + share : number or dataframe (default: 1) + Share of `tec` activity if multiple technologies share the same scrubber. + """ + + df = ( + ( + (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + * pp.out(scrub_tec, units) + * share + ) + / pp_utils.ppgroup( + (pp.act(tec, group=group) / pp.act(tec)).fillna(0) + * pp.eff( + tec, + inpfilter=inpfilter, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + group=group, + ) + ) + ).fillna(0) + + return df + + +def _se_elec_woCCSretro(tec, scrub_tec, units, _Frac, share=1): + """Calculates secondary energy electricity generation. + + This applies to technologies WITHOUT scrubbers. + + Parameters + ---------- + + tec : str + Technology name + scrub_tec : str + Name of CO2 scrubbing technology linked to `tec`. + units : str + Units to which variables should be converted. + _Frac : dataframe + Regional share of actual cogeneration (po_turbine). + share : number or dataframe (default: 1) + Share of `tec` activity if multiple technologies share the same scrubber. + """ + + df = ( + pp.out(tec, units) + * (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + - (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + * pp.out(scrub_tec, units) + * share + ) + + return df + + +def _se_elec_wCCSretro(tec, scrub_tec, units, _Frac, share=1): + """Calculates secondary energy electricity generation. + + This applies to technologies WITH scrubbers. + + Parameters + ---------- + + tec : str + Technology name + scrub_tec : str + Name of CO2 scrubbing technology linked to `tec`. + units : str + Units to which variables should be converted. + _Frac : dataframe + Regional share of actual cogeneration (po_turbine). + share : number or dataframe (default: 1) + Share of `tec` activity if multiple technologies share the same scrubber. + """ + + df = ( + (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + * pp.out(scrub_tec, units) + * share + ) + + return df + + +def _pe_elec_po_turb(tec, group, units, _Frac, inpfilter): + """Calculates primary energy electricity equivalent generation. + + This calcualtes the amount of electricity used in primary energy equivalent used + for cogeneration (po-turbine). + + Parameters + ---------- + + tec : str + Technology name + group : list + Indexes by which resultsa re to be grouped. + units : str + Units to which variables should be converted. + _Frac : dataframe + Regional share of actual cogeneration (po_turbine). + inpfilter : dict + `level` and/or `commodity` used for retrieving the input. + """ + + df = pp_utils.ppgroup( + ( + ( + pp.out(tec, units, group=group) + * (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + ) + / pp.eff(tec, inpfilter=inpfilter, group=group) + ).fillna(0) + ) + + return df + + +def _se_elec_po_turb(tec, units, _Frac, outfilter=None): + """Calculates secondary energy electricity generation. + + This calcualtes the amount of electricity used for cogeneration (po-turbine). + + Parameters + ---------- + + tec : str + Technology name + units : str + Units to which variables should be converted. + _Frac : dataframe + Regional share of actual cogeneration (po_turbine). + outfilter : dict + `level` and/or `commodity` used for retrieving the output. + """ + + df = pp.out(tec, units, outfilter=outfilter) * ( + 1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac + ) + + return df + + +def _se_heat_po_turb(tec, units, _Frac, outfilter=None): + """Calculates secondary energy heat generation. + + This calcualtes the amount of heat produced from cogeneration (po-turbine) + for a specific technology. + + Parameters + ---------- + + tec : str + Technology name + units : str + Units to which variables should be converted. + _Frac : dataframe + Regional share of actual cogeneration (po_turbine). + outfilter : dict + `level` and/or `commodity` used for retrieving the output. + """ + + df = pp.out(tec, units, outfilter=outfilter) * ( + pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac + ) + + return df + + +def _out_div_eff(tec, group, inpfilter, outfilter): + """Calculates input based on output. + + This calculates the amount of input required based on the output. + Mainly this is used for water related reporting. + + Parameters + ---------- + + tec : str + Technology name + group : list + Indexes by which resultsa re to be grouped. + inpfilter : dict + `level` and/or `commodity` used for retrieving the input. + outfilter : dict + `level` and/or `commodity` used for retrieving the output. + """ + + tec = [tec] if type(tec) == str else tec + + dfs = [] + for t in tec: + dfs.append( + pp_utils.ppgroup( + (pp.out(t, outfilter=outfilter, group=group)) + / pp.eff(t, inpfilter=inpfilter, group=group) + ) + ) + df = pd.concat(dfs, sort=True) + + return df.groupby(df.index.name).sum() + + +# ------------------- +# Reporting functions +# ------------------- + + +@_register +def retr_SE_elecgen(units): + """Energy: Secondary Energy electricity generation. + + Parameters + ---------- + + units : str + Units to which variables should be converted. + """ + + vars = {} + + # -------------------------------- + # Calculation of helping variables + # -------------------------------- + + _Cogen = pp.inp("po_turbine", units, inpfilter={"commodity": ["electr"]}) + + # Calculate the possible potential for po-turbine usage + _Potential = pp.act_rel( + [ + "coal_ppl_u", + "coal_ppl", + "coal_adv", + "coal_adv_ccs", + "foil_ppl", + "loil_ppl", + "loil_cc", + "gas_ppl", + "gas_ct", + "gas_cc", + "gas_cc_ccs", + "gas_htfc", + "bio_ppl", + "bio_istig", + "bio_istig_ccs", + "igcc", + "igcc_ccs", + "nuc_lc", + "nuc_hc", + "geo_ppl", + ], + relfilter={"relation": ["pass_out_trb"]}, + units=units, + ) + + _Biogas = pp.out("gas_bio", units) + + _gas_inp_tecs = [ + "gas_ppl", + "gas_cc", + "gas_cc_ccs", + "gas_ct", + "gas_htfc", + "gas_hpl", + "meth_ng", + "meth_ng_ccs", + "h2_smr", + "h2_smr_ccs", + "gas_t_d", + "gas_t_d_ch4", + ] + + _totgas = pp.inp(_gas_inp_tecs, units, inpfilter={"commodity": ["gas"]}) + + _Frac = (_Cogen / _Potential).fillna(0) + + _BGas_share = (_Biogas / _totgas).fillna(0) + + # Calculate shares for ppl feeding into g_ppl_co2scr (gas_cc and gas_ppl) + _gas_cc_shr = (pp.out("gas_cc") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) + + _gas_ppl_shr = (pp.out("gas_ppl") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) + + # -------------------------------- + # Electricity generation from coal + # -------------------------------- + + vars["Coal|w/o CCS"] = ( + _se_elec_woCCSretro("coal_ppl", "c_ppl_co2scr", units, _Frac) + + _se_elec_po_turb("coal_adv", units, _Frac) + + _se_elec_po_turb("coal_ppl_u", units, _Frac) + + _se_elec_woCCSretro("igcc", "igcc_co2scr", units, _Frac) + + pp.out( + ["meth_coal", "h2_coal", "syn_liq"], + units, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + ) + + vars["Coal|w/ CCS"] = ( + _se_elec_wCCSretro("coal_ppl", "c_ppl_co2scr", units, _Frac) + + _se_elec_po_turb("coal_adv_ccs", units, _Frac) + + _se_elec_wCCSretro("igcc", "igcc_co2scr", units, _Frac) + + _se_elec_po_turb("igcc_ccs", units, _Frac) + + pp.out( + ["meth_coal_ccs", "h2_coal_ccs", "syn_liq_ccs"], + units, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + ) + + # ------------------------------- + # Electricity generation from gas + # ------------------------------- + + _Gas_woCCS = ( + _se_elec_woCCSretro("gas_ppl", "g_ppl_co2scr", units, _Frac, share=_gas_ppl_shr) + + _se_elec_po_turb("gas_ct", units, _Frac) + + _se_elec_woCCSretro("gas_cc", "g_ppl_co2scr", units, _Frac, share=_gas_cc_shr) + + _se_elec_woCCSretro("gas_htfc", "gfc_co2scr", units, _Frac) + + pp.out( + ["h2_smr"], + units, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + ) + + vars["Gas|w/o CCS"] = _Gas_woCCS * (1 - _BGas_share) + + _Gas_wCCS = ( + _se_elec_wCCSretro("gas_ppl", "g_ppl_co2scr", units, _Frac, share=_gas_ppl_shr) + + _se_elec_wCCSretro("gas_cc", "g_ppl_co2scr", units, _Frac, share=_gas_cc_shr) + + _se_elec_wCCSretro("gas_htfc", "gfc_co2scr", units, _Frac) + + _se_elec_po_turb("gas_cc_ccs", units, _Frac) + + pp.out( + ["h2_smr_ccs"], + units, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + ) + + vars["Gas|w/ CCS"] = _Gas_wCCS * (1 - _BGas_share) + + # ------------------------------- + # Electricity generation from oil + # ------------------------------- + + vars["Oil|w/o CCS"] = ( + _se_elec_po_turb("foil_ppl", units, _Frac) + + _se_elec_po_turb("loil_ppl", units, _Frac) + + _se_elec_po_turb("oil_ppl", units, _Frac) + + _se_elec_po_turb("loil_cc", units, _Frac) + ) + + # ----------------------------------- + # Electricity generation from biomass + # ----------------------------------- + + vars["Biomass|w/o CCS"] = ( + _se_elec_woCCSretro("bio_ppl", "bio_ppl_co2scr", units, _Frac) + + _se_elec_po_turb("bio_istig", units, _Frac) + + pp.out( + ["h2_bio", "eth_bio", "liq_bio"], + units, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + + _Gas_woCCS * _BGas_share + ) + + # eth bio CCS is set to 0 because OFR doesnt know what to do with eff = 0. + # normally it is replaced by 1, but this doesnt work in this case! + vars["Biomass|w/ CCS"] = ( + _se_elec_wCCSretro("bio_ppl", "bio_ppl_co2scr", units, _Frac) + + _se_elec_po_turb("bio_istig_ccs", units, _Frac) + + pp.out( + ["h2_bio_ccs", "eth_bio_ccs", "liq_bio_ccs"], + units, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + + _Gas_wCCS * _BGas_share + ) + + # ------------------- + # Electricity storage + # ------------------- + + # _Storage_output = pp.out("stor_ppl", units) + # _Storage_input = (pp.inp("stor_ppl", units) / _Storage_output).fillna(0) + + # ------------------------------------ + # Operating reserve for solar and wind + # ------------------------------------ + + # _SolarPV_OR = pp.act_rel(["solar_cv1", "solar_cv2", + # "solar_cv3", "solar_cv4"], + # relfilter={"relation": ["oper_res"]}, units=units) + + # _Wind_OR = pp.act_rel(["wind_cv1", "wind_cv2", + # "wind_cv3", "wind_cv4"], + # relfilter={"relation": ["oper_res"]}, units=units) + + # -------------------------- + # Solar PV related variables + # -------------------------- + + _SolarPV_curt_inp = pp.inp( + ["solar_curtailment1", "solar_curtailment2", "solar_curtailment3"], + units, + inpfilter={"commodity": ["electr"]}, + ) + + _SolarPV_raw = ( + pp.out( + [ + "solar_res1", + "solar_res2", + "solar_res3", + "solar_res4", + "solar_res5", + "solar_res6", + "solar_res7", + "solar_res8", + "solar_res_hist_2005", + "solar_res_hist_2010", + "solar_res_hist_2015", + "solar_res_hist_2020", + ], + units, + ) + - _SolarPV_curt_inp + ) + + _SolarPV_onsite = pp.out(["solar_pv_I", "solar_pv_RC"], units) + + SolarPV = _SolarPV_raw + _SolarPV_onsite + + # _SolarPV_woStorage_total = _SolarPV_raw - _Storage_input *\ + # (_SolarPV_OR / (_Wind_OR + _SolarPV_OR)).fillna(0) + + # _SolarPV_woStorage = _SolarPV_woStorage_total + _SolarPV_onsite + # _SolarPV_wStorage = _Storage_output *\ + # (_SolarPV_OR / (_Wind_OR + _SolarPV_OR)).fillna(0) + + vars["Solar|PV"] = SolarPV + vars["Solar|PV|Curtailment"] = _SolarPV_curt_inp + + # --------------------- + # CSP related variables + # --------------------- + + CSP = pp.out( + [ + "csp_sm1_res", + "csp_sm1_res1", + "csp_sm1_res2", + "csp_sm1_res3", + "csp_sm1_res4", + "csp_sm1_res5", + "csp_sm1_res6", + "csp_sm1_res7", + "csp_res_hist_2005", + "csp_res_hist_2010", + "csp_res_hist_2015", + "csp_res_hist_2020", + "csp_sm3_res", + "csp_sm3_res1", + "csp_sm3_res2", + "csp_sm3_res3", + "csp_sm3_res4", + "csp_sm3_res5", + "csp_sm3_res6", + "csp_sm3_res7", + ], + units, + ) + + vars["Solar|CSP"] = CSP + + # ---------------------- + # Wind related variables + # ---------------------- + + _Wind_res_act = pp.out( + [ + "wind_res1", + "wind_res2", + "wind_res3", + "wind_res4", + "wind_res_hist_2005", + "wind_res_hist_2010", + "wind_res_hist_2015", + "wind_res_hist_2020", + ], + units, + ) + + _Wind_ref_act = pp.out( + [ + "wind_ref1", + "wind_ref2", + "wind_ref3", + "wind_ref4", + "wind_ref5", + "wind_ref_hist_2005", + "wind_ref_hist_2010", + "wind_ref_hist_2015", + "wind_ref_hist_2020", + ], + units, + ) + + _Wind_curt_inp = pp.inp( + ["wind_curtailment1", "wind_curtailment2", "wind_curtailment3"], + units, + inpfilter={"commodity": ["electr"]}, + ) + + _Wind_onshore = _Wind_res_act - _Wind_curt_inp * ( + _Wind_res_act / (_Wind_res_act + _Wind_ref_act) + ).fillna(0) + + _Wind_offshore = _Wind_ref_act - _Wind_curt_inp * ( + _Wind_ref_act / (_Wind_res_act + _Wind_ref_act) + ).fillna(0) + + # _Wind = _Wind_onshore + _Wind_offshore + + # _Wind_woStorage = _Wind - (_Storage_input * _Wind_OR / + # (_Wind_OR + _SolarPV_OR)).fillna(0) + # _Wind_woStorage_onshore = _Wind_woStorage *\ + # (_Wind_onshore / _Wind).fillna(0)\ + # _Wind_woStorage_offshore = _Wind_woStorage *\ + # (_Wind_offshore / _Wind).fillna(0) + # _Wind_wStorage = _Storage_output *\ + # (_Wind_OR / (_Wind_OR + _SolarPV_OR)).fillna(0) + # _Wind_wStorage_onshore = _Wind_wStorage * (_Wind_onshore / _Wind).fillna(0) + # _Wind_wStorage_offshore = _Wind_wStorage *\ + # (_Wind_offshore / _Wind).fillna(0) + + vars["Wind|Onshore"] = _Wind_onshore + vars["Wind|Offshore"] = _Wind_offshore + vars["Wind|Curtailment"] = _Wind_curt_inp + + vars["Hydro"] = pp.out( + [ + "hydro_1", + "hydro_2", + "hydro_3", + "hydro_4", + "hydro_5", + "hydro_6", + "hydro_7", + "hydro_8", + ], + units, + ) + + vars["Geothermal"] = _se_elec_po_turb("geo_ppl", units, _Frac) + + vars["Nuclear"] = ( + _se_elec_po_turb( + "nuc_lc", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + + _se_elec_po_turb( + "nuc_hc", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + + _se_elec_po_turb( + "nuc_fbr", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + ) + + # comment["Other"] = "includes electricity production from fuel" + # " cells on the final energy level" + + vars["Other"] = ( + pp.inp( + "h2_fc_trp", units, inpfilter={"level": ["final"], "commodity": ["electr"]} + ) + + pp.inp( + "h2_fc_I", units, inpfilter={"level": ["final"], "commodity": ["electr"]} + ) + + pp.inp( + "h2_fc_RC", units, inpfilter={"level": ["final"], "commodity": ["electr"]} + ) + ) + + # ------------------------------------------------ + # Additonal reporting for GAINS diagnostic linkage + # ------------------------------------------------ + + vars["Storage Losses"] = pp.inp( + "stor_ppl", units, inpfilter={"commodity": ["electr"]} + ) + + vars["Transmission Losses"] = pp.inp( + "elec_t_d", units, inpfilter={"commodity": ["electr"]} + ) - pp.out("elec_t_d", units) + + df = pp_utils.make_outputdf(vars, units) + return df + + +@_register +def retr_pe(units, method=None): + """Energy: Primary Energy. + + Parameters + ---------- + + units : str + Units to which variables should be converted. + method : str + If supplied, the direct equivalent will be calculated. + Otherwise "substiution" will apply IEA conversion factors + for heat/electricity for renewables and nuclear. + """ + + vars = {} + + if run_history != "True": + group = ["Region", "Mode", "Vintage"] + else: + group = ["Region"] + + if method == "substitution": + elec_factor = 0.35 + # hydrogen_factor = 0.6 + heat_factor = 0.8 + + # -------------------------------- + # Calculation of helping variables + # -------------------------------- + + _Cogen = pp.inp("po_turbine", units, inpfilter={"commodity": ["electr"]}) + + _Potential = pp.act_rel( + [ + "coal_ppl_u", + "coal_ppl", + "coal_adv", + "coal_adv_ccs", + "foil_ppl", + "loil_ppl", + "loil_cc", + "gas_ppl", + "gas_ct", + "gas_cc", + "gas_cc_ccs", + "gas_htfc", + "bio_ppl", + "bio_istig", + "bio_istig_ccs", + "igcc", + "igcc_ccs", + "nuc_lc", + "nuc_hc", + "geo_ppl", + ], + relfilter={"relation": ["pass_out_trb"]}, + units=units, + ) + + _Biogas = pp.out("gas_bio", units) + _H2gas = pp.out("h2_mix", units) + _Coalgas = pp.out("coal_gas", units) + + # Calculate shares for ppl feeding into g_ppl_co2scr (gas_cc and gas_ppl) + _gas_cc_shr = (pp.out("gas_cc") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) + + _gas_ppl_shr = (pp.out("gas_ppl") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) + + # Example of a set + _gas_inp_tecs = [ + "gas_ppl", + "gas_cc", + "gas_cc_ccs", + "gas_ct", + "gas_htfc", + "gas_hpl", + "meth_ng", + "meth_ng_ccs", + "h2_smr", + "h2_smr_ccs", + "gas_t_d", + "gas_t_d_ch4", + ] + + _totgas = pp.inp(_gas_inp_tecs, units, inpfilter={"commodity": ["gas"]}) + _Frac = (_Cogen / _Potential).fillna(0) + _BGas_share = (_Biogas / _totgas).fillna(0) + _SynGas_share = ((_Biogas + _H2gas + _Coalgas) / _totgas).fillna(0) + + # ------------------- + # Primary Energy Coal + # ------------------- + + vars["Coal"] = ( + pp.inp( + ["coal_extr_ch4", "coal_extr", "lignite_extr", "coal_imp"], + units, + inpfilter={"commodity": ["coal", "lignite"]}, + ) + - pp.inp("coal_exp", units) + + pp.inp(["meth_bunker"], units) + ) + + # Note OFR 20180412: Correction inserted. scrubber output per vintage cannot + # be divided by powerplant eff per vintage. In most cases this does work except + # if the powerplant doesnt exist anymore, or if vintaging is switched on. + # In either case, the efficiency with which the scrubber output is divided + # is a weighted average based on the powerplant activity + vars["Coal|w/ CCS"] = ( + _pe_wCCSretro( + "coal_ppl", + "c_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + ) + + _pe_wCCSretro( + "igcc", + "igcc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + ) + + pp.inp( + ["igcc_ccs", "coal_adv_ccs", "meth_coal_ccs", "syn_liq_ccs", "h2_coal_ccs"], + units, + inpfilter={"commodity": ["coal"]}, + ) + ) + + vars["Coal|w/o CCS"] = vars["Coal"] - vars["Coal|w/ CCS"] + + # ------------------ + # Primary Energy Oil + # ------------------ + + vars["Oil|w/o CCS"] = ( + pp.inp( + [ + "oil_extr_1_ch4", + "oil_extr_2_ch4", + "oil_extr_3_ch4", + "oil_extr_1", + "oil_extr_2", + "oil_extr_3", + ], + units, + inpfilter={"level": ["resource"]}, + ) + + ( + pp.out( + [ + "oil_extr_4_ch4", + "oil_extr_4", + "oil_extr_5", + "oil_extr_6", + "oil_extr_7", + "oil_extr_8", + ], + units, + ) + / 0.9 + ).fillna(0) + + pp.inp(["oil_imp", "foil_imp", "loil_imp"], units) + - pp.inp(["oil_exp", "foil_exp", "loil_exp"], units) + + pp.inp(["foil_bunker", "loil_bunker"], units) + ) + + # ------------------ + # Primary Energy Gas + # ------------------ + + vars["Gas"] = ( + pp.inp( + [ + "gas_extr_1", + "gas_extr_2", + "gas_extr_3", + "gas_extr_4", + "gas_extr_5", + "gas_extr_6", + "gas_extr_7", + "gas_extr_8", + ], + units, + inpfilter={"level": ["resource"]}, + ) + + pp.inp(["LNG_imp", "gas_imp"], units) + - pp.inp( + [ + "LNG_exp", + "gas_exp_nam", + "gas_exp_weu", + "gas_exp_eeu", + "gas_exp_pao", + "gas_exp_cpa", + "gas_exp_afr", + "gas_exp_sas", + "gas_exp_pas", + "gas_exp_scs", + "gas_exp_cas", + "gas_exp_ubm", + "gas_exp_rus", + ], + units, + ) + + pp.inp(["LNG_bunker"], units) + ) + + # Note OFR 20180412: Correction inserted. scrubber output per vintage cannot + # be divided by powerplant eff per vintage. In most cases this does work except + # if the powerplant doesnt exist anymore, or if vintaging is switched on. + # In either case, the efficiency with which the scrubber output is divided + # is a weighted average based on the powerplant activity + _Gas_wCCS = ( + _pe_wCCSretro( + "gas_cc", + "g_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + share=_gas_cc_shr, + ) + + _pe_wCCSretro( + "gas_ppl", + "g_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + share=_gas_ppl_shr, + ) + + _pe_wCCSretro( + "gas_htfc", + "gfc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + ) + + pp.inp(["gas_cc_ccs", "h2_smr_ccs"], units, inpfilter={"commodity": ["gas"]}) + ) + + vars["Gas|w/ CCS"] = _Gas_wCCS * (1 - _BGas_share) + + vars["Gas|w/o CCS"] = vars["Gas"] - vars["Gas|w/ CCS"] + + # ---------------------- + # Primary Energy Nuclear + # ---------------------- + + if method == "substitution": + vars["Nuclear"] = ( + _se_elec_po_turb( + "nuc_lc", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + / elec_factor + + _se_elec_po_turb( + "nuc_hc", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + / elec_factor + + _se_elec_po_turb( + "nuc_fbr", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + / elec_factor + + _se_heat_po_turb( + "nuc_lc", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + / heat_factor + + _se_heat_po_turb( + "nuc_hc", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + / heat_factor + + _se_heat_po_turb( + "nuc_fbr", + units, + _Frac, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + / heat_factor + ) + else: + vars["Nuclear"] = pp.out( + ["nuc_lc", "nuc_hc", "nuc_fbr"], + units, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + ) + + # ---------------------- + # Primary Energy Biomass + # ---------------------- + + vars["Biomass"] = pp.land_out( + lu_out_filter={"level": ["land_use"], "commodity": ["bioenergy"]}, units=units + ) + + # Note OFR 20180412: Correction inserted. scrubber output per vintage cannot + # be divided by powerplant eff per vintage. In most cases this does work except + # if the powerplant doesnt exist anymore, or if vintaging is switched on. + # In either case, the efficiency with which the scrubber output is divided + # is a weighted average based on the powerplant activity + vars["Biomass|w/ CCS"] = ( + _pe_wCCSretro( + "bio_ppl", + "bio_ppl_co2scr", + group, + inpfilter={"level": ["primary"], "commodity": ["biomass"]}, + units=units, + ) + + pp.inp("bio_istig_ccs", units, inpfilter={"commodity": ["biomass"]}) + + pp.inp( + ["h2_bio_ccs", "eth_bio_ccs", "liq_bio_ccs"], + units, + inpfilter={"commodity": ["biomass"]}, + ) + + _Gas_wCCS * _BGas_share + ) + + # Note OFR: eth_bunker should be excluded because biomass from GLOBIOM + # already includes the total biomass production per region. + vars["Biomass|w/o CCS"] = vars["Biomass"] - vars["Biomass|w/ CCS"] + # + pp.inp(["eth_bunker"], units) + + # ------------------- + # Primary Energy Wind + # ------------------- + + vars["Wind"] = pp.out( + [ + "wind_res1", + "wind_res2", + "wind_res3", + "wind_res4", + "wind_ref1", + "wind_ref2", + "wind_ref3", + "wind_ref4", + "wind_ref5", + "wind_res_hist_2005", + "wind_res_hist_2010", + "wind_res_hist_2015", + "wind_res_hist_2020", + "wind_ref_hist_2005", + "wind_ref_hist_2010", + "wind_ref_hist_2015", + "wind_ref_hist_2020", + ], + units, + ) - pp.inp( + ["wind_curtailment1", "wind_curtailment2", "wind_curtailment3"], + units, + inpfilter={"commodity": ["electr"]}, + ) + + if method == "substitution": + vars["Wind"] /= elec_factor + + # -------------------- + # Primary Energy Hydro + # -------------------- + + vars["Hydro"] = pp.out( + [ + "hydro_1", + "hydro_2", + "hydro_3", + "hydro_4", + "hydro_5", + "hydro_6", + "hydro_7", + "hydro_8", + ], + units, + ) + + if method == "substitution": + vars["Hydro"] /= elec_factor + + # -------------------- + # Primary Energy Ocean + # -------------------- + + vars["Ocean"] = pp_utils._make_zero() + + # --------------------------------- + # Primary Energy Solar (PV and CSP) + # --------------------------------- + + _pv_elec = pp.out( + [ + "solar_res1", + "solar_res2", + "solar_res3", + "solar_res4", + "solar_res5", + "solar_res6", + "solar_res7", + "solar_res8", + "solar_pv_RC", + "solar_pv_I", + "solar_res_hist_2005", + "solar_res_hist_2010", + "solar_res_hist_2015", + "solar_res_hist_2020", + ], + units, + ) + + _solar_curtailment_elec = pp.inp( + ["solar_curtailment1", "solar_curtailment2", "solar_curtailment3"], + units, + inpfilter={"commodity": ["electr"]}, + ) + + _solar_heat = pp.out(["solar_rc", "solar_i"], units) + + _csp_elec = pp.out( + [ + "csp_sm1_res", + "csp_sm1_res1", + "csp_sm1_res2", + "csp_sm1_res3", + "csp_sm1_res4", + "csp_sm1_res5", + "csp_sm1_res6", + "csp_sm1_res7", + "csp_res_hist_2005", + "csp_res_hist_2010", + "csp_res_hist_2015", + "csp_res_hist_2020", + "csp_sm3_res", + "csp_sm3_res1", + "csp_sm3_res2", + "csp_sm3_res3", + "csp_sm3_res4", + "csp_sm3_res5", + "csp_sm3_res6", + "csp_sm3_res7", + ], + units, + ) + + if method == "substitution": + vars["Solar"] = ( + (_pv_elec - _solar_curtailment_elec + _csp_elec) / elec_factor + ).fillna(0) + (_solar_heat / heat_factor).fillna(0) + else: + vars["Solar"] = _pv_elec - _solar_curtailment_elec + _csp_elec + _solar_heat + + # ------------------------- + # Primary Energy Geothermal + # ------------------------- + + _geothermal_elec = _se_elec_po_turb("geo_ppl", units, _Frac) + _geothermal_heat = _se_heat_po_turb("geo_ppl", units, _Frac) + pp.out( + ["geo_hpl"], units + ) + + if method == "substitution": + vars["Geothermal"] = (_geothermal_elec / elec_factor).fillna(0) + ( + _geothermal_heat / heat_factor + ).fillna(0) + else: + vars["Geothermal"] = _geothermal_elec + _geothermal_heat + + # Primary Energy - SE Trade + vars["Secondary Energy Trade"] = pp.out( + [ + "meth_imp", + "lh2_imp", + "eth_imp", + "elec_imp", + "elec_imp_africa", + "elec_imp_america", + "elec_imp_asia", + "elec_imp_eurasia", + "elec_imp_eur_afr", + "elec_imp_asia_afr", + ], + units, + ) - pp.inp( + [ + "meth_exp", + "lh2_exp", + "eth_exp", + "elec_exp", + "elec_exp_africa", + "elec_exp_america", + "elec_exp_asia", + "elec_exp_eurasia", + "elec_exp_eur_afr", + "elec_exp_asia_afr", + ], + units, + ) + + # -------------------- + # Priamry Energy Other + # -------------------- + + vars["Other"] = pp.inp("LH2_bunker", units) + + if method != "substitution": + + # ------------------------------------ + # Primary Energy Electricity from coal + # ------------------------------------ + + vars["Coal|Electricity|w/o CCS"] = ( + _pe_elec_woCCSretro( + "coal_ppl", + "c_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_woCCSretro( + "igcc", + "igcc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "coal_adv", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) + + _pe_elec_po_turb( + "coal_ppl_u", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) + ) + + vars["Coal|Electricity|w/ CCS"] = ( + _pe_elec_wCCSretro( + "coal_ppl", + "c_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_wCCSretro( + "igcc", + "igcc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "coal_adv_ccs", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) + + _pe_elec_po_turb( + "igcc_ccs", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) + ) + + # ----------------------------------- + # Primary Energy Electricity from gas + # ----------------------------------- + + _ElecGas_woCCS = ( + _pe_elec_woCCSretro( + "gas_ppl", + "g_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + _Frac=_Frac, + share=_gas_ppl_shr, + ) + + _pe_elec_woCCSretro( + "gas_cc", + "g_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + _Frac=_Frac, + share=_gas_cc_shr, + ) + + _pe_elec_woCCSretro( + "gas_htfc", + "gfc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "gas_ct", group, units, _Frac, inpfilter={"commodity": ["gas"]} + ) + ) + + vars["Gas|Electricity|w/o CCS"] = _ElecGas_woCCS * (1 - _BGas_share) + + _ElecGas_wCCS = ( + _pe_elec_wCCSretro( + "gas_cc", + "g_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + _Frac=_Frac, + share=_gas_cc_shr, + ) + + _pe_elec_wCCSretro( + "gas_ppl", + "g_ppl_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + _Frac=_Frac, + share=_gas_ppl_shr, + ) + + _pe_elec_wCCSretro( + "gas_htfc", + "gfc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "gas_cc_ccs", group, units, _Frac, inpfilter={"commodity": ["gas"]} + ) + ) + + vars["Gas|Electricity|w/ CCS"] = _ElecGas_wCCS * (1 - _BGas_share) + + # ----------------------------------- + # Primary Energy Electricity from oil + # ----------------------------------- + + vars["Oil|Electricity|w/o CCS"] = ( + _pe_elec_po_turb( + "foil_ppl", group, units, _Frac, inpfilter={"commodity": ["fueloil"]} + ) + + _pe_elec_po_turb( + "loil_ppl", group, units, _Frac, inpfilter={"commodity": ["lightoil"]} + ) + + _pe_elec_po_turb( + "oil_ppl", group, units, _Frac, inpfilter={"commodity": ["crudeoil"]} + ) + + _pe_elec_po_turb( + "loil_cc", group, units, _Frac, inpfilter={"commodity": ["lightoil"]} + ) + ) + + vars["Oil|Electricity|w/ CCS"] = pp_utils._make_zero() + + # --------------------------------------- + # Primary Energy Electricity from biomass + # --------------------------------------- + + vars["Biomass|Electricity|w/o CCS"] = ( + _pe_elec_woCCSretro( + "bio_ppl", + "bio_ppl_co2scr", + group, + inpfilter={"level": ["primary"], "commodity": ["biomass"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "bio_istig", group, units, _Frac, inpfilter={"commodity": ["biomass"]} + ) + + _ElecGas_woCCS * _BGas_share + ) + + vars["Biomass|Electricity|w/ CCS"] = ( + _pe_elec_wCCSretro( + "bio_ppl", + "bio_ppl_co2scr", + group, + inpfilter={"level": ["primary"], "commodity": ["biomass"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "bio_istig_ccs", + group, + units, + _Frac, + inpfilter={"commodity": ["biomass"]}, + ) + + _ElecGas_wCCS * _BGas_share + ) + + # ----------------------------------- + # Primary Energy from biomass (other) + # ----------------------------------- + + vars["Biomass|1st Generation"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|" "1st Generation"], + } + ) + + vars["Biomass|1st Generation|Biodiesel"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|" "1st Generation|Biodiesel"], + } + ) + + vars["Biomass|1st Generation|Bioethanol"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|" "1st Generation|Bioethanol"], + } + ) + + vars["Biomass|Energy Crops"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|Energy Crops"], + } + ) + + vars["Biomass|Fuelwood"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|Fuelwood"], + } + ) + + vars["Biomass|Modern"] = pp.land_out( + lu_out_filter={"level": ["land_use"], "commodity": ["bioenergy"]}, + units=units, + ) - pp.inp("biomass_nc", units) + + vars["Biomass|Other"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|Other"], + } + ) + + vars["Biomass|Residues"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|Residues"], + } + ) + + vars["Biomass|Residues|Forest industry"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|Residues|" "Forest industry"], + } + ) + + vars["Biomass|Residues|Logging"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Primary Energy|Biomass|Residues|" "Logging"], + } + ) + + vars["Biomass|Traditional"] = pp.inp("biomass_nc", units) + + # ---------------------------------------------------- + # Additonal reporting for GAINS diagnostic for biomass + # ---------------------------------------------------- + + vars["Biomass|Gases"] = pp.inp( + ["gas_bio"], units, inpfilter={"commodity": ["biomass"]} + ) + + vars["Biomass|Hydrogen"] = pp.inp( + ["h2_bio", "h2_bio_ccs"], units, inpfilter={"commodity": ["biomass"]} + ) + + vars["Biomass|Liquids"] = pp.inp( + ["eth_bio", "eth_bio_ccs", "liq_bio", "liq_bio_ccs"], + units, + inpfilter={"commodity": ["biomass"]}, + ) + + vars["Biomass|Solids"] = pp.inp( + ["biomass_t_d"], units, inpfilter={"commodity": ["biomass"]} + ) + + # ------------------------------------------------- + # Additonal reporting for GAINS diagnostic for coal + # ------------------------------------------------- + + vars["Coal|Gases"] = pp.inp( + ["coal_gas"], units, inpfilter={"commodity": ["coal"]} + ) + + vars["Coal|Hydrogen"] = pp.inp( + ["h2_coal", "h2_coal_ccs"], units, inpfilter={"commodity": ["coal"]} + ) + + vars["Coal|Liquids"] = pp.inp( + ["syn_liq", "syn_liq_ccs", "meth_coal", "meth_coal_ccs"], + units, + inpfilter={"commodity": ["coal"]}, + ) + + vars["Coal|Solids"] = pp.inp( + [ + "coal_t_d-rc-06p", + "coal_t_d-in-06p", + "coal_t_d-in-SO2", + "coal_t_d-rc-SO2", + "coal_t_d", + ], + units, + inpfilter={"commodity": ["coal"]}, + ) + + # ------------------------------------------------ + # Additonal reporting for GAINS diagnostic for gas + # ------------------------------------------------ + + vars["Gas|Gases"] = pp.inp( + ["gas_t_d", "gas_t_d_ch4"], units, inpfilter={"commodity": ["gas"]} + ) * (1 - _SynGas_share) + + vars["Gas|Hydrogen"] = pp.inp( + ["h2_smr", "h2_smr_ccs"], units, inpfilter={"commodity": ["gas"]} + ) + + vars["Gas|Liquids"] = pp.inp( + ["meth_ng", "meth_ng_ccs"], units, inpfilter={"commodity": ["gas"]} + ) + + vars["Gas|Solids"] = pp_utils._make_zero() + + # ------------------------------------------------ + # Additonal reporting for GAINS diagnostic for oil + # ------------------------------------------------ + + vars["Oil|Gases"] = pp_utils._make_zero() + vars["Oil|Hydrogen"] = pp_utils._make_zero() + vars["Oil|Liquids"] = pp.inp( + ["loil_t_d", "foil_t_d"], + units, + inpfilter={"commodity": ["fueloil", "lightoil"]}, + ) + + vars["Oil|Solids"] = pp_utils._make_zero() + + df = pp_utils.make_outputdf(vars, units) + return df + + +@_register +def retr_ppl_capparameters(prmfunc, units): + """Technology results: Capacity related. + + Parameters + ---------- + + prmfunc : str + Name of function in `postprocess.py` to be used. + Options + - `pp.tic` to retrieve total installed capacity. + - `pp.nic` to retrieve new installed capacity. + - `pp.cumcap` to retrieve cumualtive installed capacity. + units : str + Units to which variables should be converted. + """ + + prmfunc = eval(prmfunc) + vars = {} + + # ["Note": "IGCC with CCS"] + vars["Electricity|Biomass|w/ CCS|1"] = prmfunc("bio_istig_ccs") + + # ["Note": "IGCC"] + vars["Electricity|Biomass|w/o CCS|1"] = prmfunc("bio_istig") + + # ["Note": "Steam cycle"] + vars["Electricity|Biomass|w/o CCS|2"] = prmfunc("bio_ppl") + + # ["Note": "IGCC with CCS"] + vars["Electricity|Coal|w/ CCS|1"] = prmfunc("igcc_ccs") + + # ["Note": "Steam cycle super critical with CCS"] + vars["Electricity|Coal|w/ CCS|2"] = prmfunc("coal_adv_ccs") + + # ["Note": "IGCC"] + vars["Electricity|Coal|w/o CCS|1"] = prmfunc("igcc") + + # ["Note": "Steam cycle super critical"] + vars["Electricity|Coal|w/o CCS|2"] = prmfunc("coal_adv") + + # ["Note": "Steam cycle sub critical Filtered"] + vars["Electricity|Coal|w/o CCS|3"] = prmfunc("coal_ppl") + + # ["Note": "Steam cycle sub critical unfiltered"] + vars["Electricity|Coal|w/o CCS|4"] = prmfunc("coal_ppl_u") + + # ["Note": "Combined cycle with CCS"] + vars["Electricity|Gas|w/ CCS|1"] = prmfunc("gas_cc_ccs") + + # ["Note": "Combined cycle"] + vars["Electricity|Gas|w/o CCS|1"] = prmfunc("gas_cc") + + # ["Note": "Steam cycle sub cricitcal"] + vars["Electricity|Gas|w/o CCS|2"] = prmfunc("gas_ppl") + + # ["Note": "Combustion turbine"] + vars["Electricity|Gas|w/o CCS|3"] = prmfunc("gas_ct") + vars["Electricity|Geothermal"] = prmfunc("geo_ppl") + + # ["Note": "Cost hydropower 1-8"] + vars["Electricity|Hydro|1"] = prmfunc("hydro_1") + vars["Electricity|Hydro|2"] = prmfunc("hydro_2") + vars["Electricity|Hydro|3"] = prmfunc("hydro_3") + vars["Electricity|Hydro|4"] = prmfunc("hydro_4") + vars["Electricity|Hydro|5"] = prmfunc("hydro_5") + vars["Electricity|Hydro|6"] = prmfunc("hydro_6") + vars["Electricity|Hydro|7"] = prmfunc("hydro_7") + vars["Electricity|Hydro|8"] = prmfunc("hydro_8") + + # ["Note": "Generation I.-II."] + vars["Electricity|Nuclear|1"] = prmfunc("nuc_lc") + + # ["Note": "Generation III.-IV."] + vars["Electricity|Nuclear|2"] = prmfunc("nuc_hc") + vars["Electricity|Oil|w/ CCS"] = pp_utils._make_zero() + + # ["Note": "Combined cycle light oil"] + vars["Electricity|Oil|w/o CCS|1"] = prmfunc("loil_cc") + + # ["Note": "Steam cycle light oil"] + vars["Electricity|Oil|w/o CCS|2"] = prmfunc("loil_ppl") + + # ["Note": "Steam cycle fuel oil"] + vars["Electricity|Oil|w/o CCS|3"] = prmfunc("foil_ppl") + + vars["Electricity|Solar|CSP|1"] = prmfunc("csp_sm1_ppl") + vars["Electricity|Solar|CSP|2"] = prmfunc("csp_sm3_ppl") + vars["Electricity|Solar|PV"] = prmfunc("solar_pv_ppl") + vars["Electricity|Wind|Offshore"] = prmfunc("wind_ppf") + vars["Electricity|Wind|Onshore"] = prmfunc("wind_ppl") + vars["Electricity|Storage"] = prmfunc("stor_ppl") + vars["Gases|Biomass|w/o CCS"] = prmfunc("gas_bio") + vars["Gases|Coal|w/o CCS"] = prmfunc("coal_gas") + vars["Hydrogen|Biomass|w/ CCS"] = prmfunc("h2_bio_ccs") + vars["Hydrogen|Biomass|w/o CCS"] = prmfunc("h2_bio") + vars["Hydrogen|Coal|w/ CCS"] = prmfunc("h2_coal_ccs") + vars["Hydrogen|Coal|w/o CCS"] = prmfunc("h2_coal") + vars["Hydrogen|Electricity"] = prmfunc("h2_elec") + vars["Hydrogen|Gas|w/ CCS"] = prmfunc("h2_smr_ccs") + vars["Hydrogen|Gas|w/o CCS"] = prmfunc("h2_smr") + + # ["Note": "Ethanol synthesis via biomass gasification with CCS"] + vars["Liquids|Biomass|w/ CCS|1"] = prmfunc("eth_bio_ccs") + + # ["Note": "FTL (Fischer Tropsch Liquids) with CCS"] + vars["Liquids|Biomass|w/ CCS|2"] = prmfunc("liq_bio_ccs") + + # ["Note": "Ethanol synthesis via biomass gasification"] + vars["Liquids|Biomass|w/o CCS|1"] = prmfunc("eth_bio") + + # ["Note": "FTL (Fischer Tropsch Liquids)"] + vars["Liquids|Biomass|w/o CCS|2"] = prmfunc("liq_bio") + + # ["Note": "Methanol synthesis via coal gasification with CCS"] + vars["Liquids|Coal|w/ CCS|1"] = prmfunc("meth_coal_ccs") + + # ["Note": "FTL (Fischer Tropsch Liquids) with CCS"] + vars["Liquids|Coal|w/ CCS|2"] = prmfunc("syn_liq_ccs") + + # ["Note": "Methanol synthesis via coal gasification"] + vars["Liquids|Coal|w/o CCS|1"] = prmfunc("meth_coal") + + # ["Note": "FTL (Fischer Tropsch Liquids)"] + vars["Liquids|Coal|w/o CCS|2"] = prmfunc("syn_liq") + vars["Liquids|Gas|w/ CCS"] = prmfunc("meth_ng_ccs") + vars["Liquids|Gas|w/o CCS"] = prmfunc("meth_ng") + + # ["Note": "Refinery low yield"] + vars["Liquids|Oil|w/o CCS|1"] = prmfunc("ref_lol") + + # ["Note": "Refinery high yield"] + vars["Liquids|Oil|w/o CCS|2"] = prmfunc("ref_hil") + + df = pp_utils.make_outputdf(vars, units) + return df + + +@_register +def retr_ppl_parameters(prmfunc, units): + """Technology Inputs: Investment costs and technical lifetime. + + Parameters + ---------- + + prmfunc : str + Name of function in `postprocess.py` to be used. + Options + - `pp.inv_cost` to retrieve technology investment costs. + - `pp.pll` to retrieve technical lifetime of technologies. + units : str + Units to which variables should be converted. + """ + + prmfunc = eval(prmfunc) + vars = {} + + # ["Note": "IGCC with CCS"] + vars["Electricity|Biomass|w/ CCS|1"] = prmfunc("bio_istig_ccs", units=units) + + # ["Note": "IGCC"] + vars["Electricity|Biomass|w/o CCS|1"] = prmfunc("bio_istig", units=units) + + # ["Note": "Steam cycle"] + vars["Electricity|Biomass|w/o CCS|2"] = prmfunc("bio_ppl", units=units) + + # ["Note": "IGCC with CCS"] + vars["Electricity|Coal|w/ CCS|1"] = prmfunc("igcc_ccs", units=units) + + # ["Note": "Steam cycle super critical with CCS"] + vars["Electricity|Coal|w/ CCS|2"] = prmfunc("coal_adv_ccs", units=units) + + # ["Note": "IGCC"] + vars["Electricity|Coal|w/o CCS|1"] = prmfunc("igcc", units=units) + + # ["Note": "Steam cycle super critical"] + vars["Electricity|Coal|w/o CCS|2"] = prmfunc("coal_adv", units=units) + + # ["Note": "Steam cycle sub critical Filtered"] + vars["Electricity|Coal|w/o CCS|3"] = prmfunc("coal_ppl", units=units) + + # ["Note": "Steam cycle sub critical unfiltered"] + vars["Electricity|Coal|w/o CCS|4"] = prmfunc("coal_ppl_u", units=units) + + # ["Note": "Combined cycle with CCS"] + vars["Electricity|Gas|w/ CCS|1"] = prmfunc("gas_cc_ccs", units=units) + + # ["Note": "Combined cycle"] + vars["Electricity|Gas|w/o CCS|1"] = prmfunc("gas_cc", units=units) + + # ["Note": "Steam cycle sub cricitcal"] + vars["Electricity|Gas|w/o CCS|2"] = prmfunc("gas_ppl", units=units) + + # ["Note": "Combustion turbine"] + vars["Electricity|Gas|w/o CCS|3"] = prmfunc("gas_ct", units=units) + vars["Electricity|Geothermal"] = prmfunc("geo_ppl", units=units) + + # ["Note": "Cst hydropower 1-8"] + vars["Electricity|Hydro|1"] = prmfunc("hydro_1", units=units) + vars["Electricity|Hydro|2"] = prmfunc("hydro_2", units=units) + vars["Electricity|Hydro"] = vars["Electricity|Hydro|3"] = prmfunc( + "hydro_3", units=units + ) + vars["Electricity|Hydro|4"] = prmfunc("hydro_4", units=units) + vars["Electricity|Hydro|5"] = prmfunc("hydro_5", units=units) + vars["Electricity|Hydro|6"] = prmfunc("hydro_6", units=units) + vars["Electricity|Hydro|7"] = prmfunc("hydro_7", units=units) + vars["Electricity|Hydro|8"] = prmfunc("hydro_8", units=units) + + # ["Note": "Generation I.-II."] + vars["Electricity|Nuclear|1"] = prmfunc("nuc_lc", units=units) + + # ["Note": "Generation III.-IV."] + vars["Electricity|Nuclear"] = vars["Electricity|Nuclear|2"] = prmfunc( + "nuc_hc", units=units + ) + + # ["Note": "Combined cycle light oil"] + vars["Electricity|Oil|w/o CCS"] = vars["Electricity|Oil|w/o CCS|1"] = prmfunc( + "loil_cc", units=units + ) + + # ["Note": "Steam cycle light oil"] + vars["Electricity|Oil|w/o CCS|2"] = prmfunc("loil_ppl", units=units) + + # ["Note": "Steam cycle fuel oil"] + vars["Electricity|Oil|w/o CCS|3"] = prmfunc("foil_ppl", units=units) + vars["Electricity|Solar|CSP|1"] = prmfunc("csp_sm1_ppl", units=units) + vars["Electricity|Solar|CSP|2"] = prmfunc("csp_sm3_ppl", units=units) + vars["Electricity|Solar|PV"] = prmfunc("solar_pv_ppl", units=units) + vars["Electricity|Wind|Offshore"] = prmfunc("wind_ppf", units=units) + vars["Electricity|Wind|Onshore"] = prmfunc("wind_ppl", units=units) + vars["Electricity|Storage"] = prmfunc("stor_ppl", units=units) + vars["Gases|Biomass|w/o CCS"] = prmfunc("gas_bio", units=units) + vars["Gases|Coal|w/o CCS"] = prmfunc("coal_gas", units=units) + vars["Hydrogen|Biomass|w/ CCS"] = prmfunc("h2_bio_ccs", units=units) + vars["Hydrogen|Biomass|w/o CCS"] = prmfunc("h2_bio", units=units) + vars["Hydrogen|Coal|w/ CCS"] = prmfunc("h2_coal_ccs", units=units) + vars["Hydrogen|Coal|w/o CCS"] = prmfunc("h2_coal", units=units) + vars["Hydrogen|Electricity"] = prmfunc("h2_elec", units=units) + vars["Hydrogen|Gas|w/ CCS"] = prmfunc("h2_smr_ccs", units=units) + vars["Hydrogen|Gas|w/o CCS"] = prmfunc("h2_smr", units=units) + + # ["Note": "Ethanol synthesis via biomass gasification with CCS"] + vars["Liquids|Biomass|w/ CCS|1"] = prmfunc("eth_bio_ccs", units=units) + + # ["Note": "FTL (Fischer Tropsch Liquids) with CCS"] + vars["Liquids|Biomass|w/ CCS"] = vars["Liquids|Biomass|w/ CCS|2"] = prmfunc( + "liq_bio_ccs", units=units + ) + + # ["Note": "Ethanol synthesis via biomass gasification"] + vars["Liquids|Biomass|w/o CCS|1"] = prmfunc("eth_bio", units=units) + + # ["Note": "FTL (Fischer Tropsch Liquids)"] + vars["Liquids|Biomass|w/o CCS"] = vars["Liquids|Biomass|w/o CCS|2"] = prmfunc( + "liq_bio", units=units + ) + + # ["Note": "Methanol synthesis via coal gasification with CCS"] + vars["Liquids|Coal|w/ CCS|1"] = prmfunc("meth_coal_ccs", units=units) + + # ["Note": "FTL (Fischer Tropsch Liquids) with CCS"] + vars["Liquids|Coal|w/ CCS"] = vars["Liquids|Coal|w/ CCS|2"] = prmfunc( + "syn_liq_ccs", units=units + ) + + # ["Note": "Methanol synthesis via coal gasification"] + vars["Liquids|Coal|w/o CCS|1"] = prmfunc("meth_coal", units=units) + + # ["Note": "FTL (Fischer Tropsch Liquids)"] + vars["Liquids|Coal|w/o CCS"] = vars["Liquids|Coal|w/o CCS|2"] = prmfunc( + "syn_liq", units=units + ) + + vars["Liquids|Gas|w/ CCS"] = prmfunc("meth_ng_ccs", units=units) + vars["Liquids|Gas|w/o CCS"] = prmfunc("meth_ng", units=units) + + # ["Note": "Refinery low yield"] + vars["Liquids|Oil|w/o CCS|1"] = prmfunc("ref_lol", units=units) + + # ["Note": "Refinery high yield"] + vars["Liquids|Oil|w/o CCS"] = vars["Liquids|Oil|w/o CCS|2"] = prmfunc( + "ref_hil", units=units + ) + + df = pp_utils.make_outputdf(vars, units, glb=False) + return df + + +@_register +def retr_ppl_opcost_parameters(prmfunc, units): + """Technology Inputs: Fixed and variable costs. + + Parameters + ---------- + + prmfunc : str + Name of function in `postprocess.py` to be used. + Options + - `pp.fom` to retrieve technology fixed operating and maintenance costs. + - `pp.vom` to retrieve technology variable operating and maintenance costs. + units : str + Units to which variables should be converted. + """ + + prmfunc = eval(prmfunc) + + vars = {} + + group = ["Region"] + formatting = "reporting" + + # ["Note": "IGCC with CCS"] + vars["Electricity|Biomass|w/ CCS|1"] = prmfunc( + "bio_istig_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "IGCC"] + vars["Electricity|Biomass|w/o CCS|1"] = prmfunc( + "bio_istig", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle"] + vars["Electricity|Biomass|w/o CCS|2"] = prmfunc( + "bio_ppl", units=units, group=group, formatting=formatting + ) + + # ["Note": "IGCC with CCS"] + vars["Electricity|Coal|w/ CCS|1"] = prmfunc( + "igcc_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle super critical with CCS"] + vars["Electricity|Coal|w/ CCS|2"] = prmfunc( + "coal_adv_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "IGCC"] + vars["Electricity|Coal|w/o CCS|1"] = prmfunc( + "igcc", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle super critical"] + vars["Electricity|Coal|w/o CCS|2"] = prmfunc( + "coal_adv", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle sub critical Filtered"] + vars["Electricity|Coal|w/o CCS|3"] = prmfunc( + "coal_ppl", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle sub critical unfiltered"] + vars["Electricity|Coal|w/o CCS|4"] = prmfunc( + "coal_ppl_u", units=units, group=group, formatting=formatting + ) + + # ["Note": "Combined cycle with CCS"] + vars["Electricity|Gas|w/ CCS|1"] = prmfunc( + "gas_cc_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "Combined cycle"] + vars["Electricity|Gas|w/o CCS|1"] = prmfunc( + "gas_cc", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle sub cricitcal"] + vars["Electricity|Gas|w/o CCS|2"] = prmfunc( + "gas_ppl", units=units, group=group, formatting=formatting + ) + + # ["Note": "Combustion turbine"] + vars["Electricity|Gas|w/o CCS|3"] = prmfunc( + "gas_ct", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Geothermal"] = prmfunc( + "geo_ppl", units=units, group=group, formatting=formatting + ) + + # ["Note": "Cst hydropower 1-8"] + vars["Electricity|Hydro|1"] = prmfunc( + "hydro_1", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Hydro|2"] = prmfunc( + "hydro_2", units=units, group=group, formatting=formatting + ) + vars["Electricity|Hydro"] = vars["Electricity|Hydro|3"] = prmfunc( + "hydro_3", units=units, group=group, formatting=formatting + ) + vars["Electricity|Hydro|4"] = prmfunc( + "hydro_4", units=units, group=group, formatting=formatting + ) + vars["Electricity|Hydro|5"] = prmfunc( + "hydro_5", units=units, group=group, formatting=formatting + ) + vars["Electricity|Hydro|6"] = prmfunc( + "hydro_6", units=units, group=group, formatting=formatting + ) + vars["Electricity|Hydro|7"] = prmfunc( + "hydro_7", units=units, group=group, formatting=formatting + ) + vars["Electricity|Hydro|8"] = prmfunc( + "hydro_8", units=units, group=group, formatting=formatting + ) + + # ["Note": "Generation I.-II."] + vars["Electricity|Nuclear|1"] = prmfunc( + "nuc_lc", units=units, group=group, formatting=formatting + ) + + # ["Note": "Generation III.-IV."] + vars["Electricity|Nuclear"] = vars["Electricity|Nuclear|2"] = prmfunc( + "nuc_hc", units=units, group=group, formatting=formatting + ) + + # ["Note": "Combined cycle light oil"] + vars["Electricity|Oil|w/o CCS"] = vars["Electricity|Oil|w/o CCS|1"] = prmfunc( + "loil_cc", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle light oil"] + vars["Electricity|Oil|w/o CCS|2"] = prmfunc( + "loil_ppl", units=units, group=group, formatting=formatting + ) + + # ["Note": "Steam cycle fuel oil"] + vars["Electricity|Oil|w/o CCS|3"] = prmfunc( + "foil_ppl", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Solar|CSP|1"] = prmfunc( + "csp_sm1_ppl", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Solar|CSP|2"] = prmfunc( + "csp_sm3_ppl", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Solar|PV"] = prmfunc( + "solar_pv_ppl", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Wind|Offshore"] = prmfunc( + "wind_ppf", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Wind|Onshore"] = prmfunc( + "wind_ppl", units=units, group=group, formatting=formatting + ) + + vars["Electricity|Storage"] = prmfunc( + "stor_ppl", units=units, group=group, formatting=formatting + ) + + vars["Gases|Biomass|w/o CCS"] = prmfunc( + "gas_bio", units=units, group=group, formatting=formatting + ) + + vars["Gases|Coal|w/o CCS"] = prmfunc( + "coal_gas", units=units, group=group, formatting=formatting + ) + + vars["Hydrogen|Biomass|w/ CCS"] = prmfunc( + "h2_bio_ccs", units=units, group=group, formatting=formatting + ) + + vars["Hydrogen|Biomass|w/o CCS"] = prmfunc( + "h2_bio", units=units, group=group, formatting=formatting + ) + + vars["Hydrogen|Coal|w/ CCS"] = prmfunc( + "h2_coal_ccs", units=units, group=group, formatting=formatting + ) + + vars["Hydrogen|Coal|w/o CCS"] = prmfunc( + "h2_coal", units=units, group=group, formatting=formatting + ) + + vars["Hydrogen|Electricity"] = prmfunc( + "h2_elec", units=units, group=group, formatting=formatting + ) + + vars["Hydrogen|Gas|w/ CCS"] = prmfunc( + "h2_smr_ccs", units=units, group=group, formatting=formatting + ) + + vars["Hydrogen|Gas|w/o CCS"] = prmfunc( + "h2_smr", units=units, group=group, formatting=formatting + ) + + # ["Note": "Ethanol synthesis via biomass gasification with CCS"] + vars["Liquids|Biomass|w/ CCS|1"] = prmfunc( + "eth_bio_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "FTL (Fischer Tropsch Liquids) with CCS"] + vars["Liquids|Biomass|w/ CCS"] = vars["Liquids|Biomass|w/ CCS|2"] = prmfunc( + "liq_bio_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "Ethanol synthesis via biomass gasification"] + vars["Liquids|Biomass|w/o CCS|1"] = prmfunc( + "eth_bio", units=units, group=group, formatting=formatting + ) + + # ["Note": "FTL (Fischer Tropsch Liquids)"] + vars["Liquids|Biomass|w/o CCS"] = vars["Liquids|Biomass|w/o CCS|2"] = prmfunc( + "liq_bio", units=units, group=group, formatting=formatting + ) + + # ["Note": "Methanol synthesis via coal gasification with CCS"] + vars["Liquids|Coal|w/ CCS|1"] = prmfunc( + "meth_coal_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "FTL (Fischer Tropsch Liquids) with CCS"] + vars["Liquids|Coal|w/ CCS"] = vars["Liquids|Coal|w/ CCS|2"] = prmfunc( + "syn_liq_ccs", units=units, group=group, formatting=formatting + ) + + # ["Note": "Methanol synthesis via coal gasification"] + vars["Liquids|Coal|w/o CCS|1"] = prmfunc( + "meth_coal", units=units, group=group, formatting=formatting + ) + + # ["Note": "FTL (Fischer Tropsch Liquids)"] + vars["Liquids|Coal|w/o CCS"] = vars["Liquids|Coal|w/o CCS|2"] = prmfunc( + "syn_liq", units=units, group=group, formatting=formatting + ) + + vars["Liquids|Gas|w/ CCS"] = prmfunc( + "meth_ng_ccs", units=units, group=group, formatting=formatting + ) + + vars["Liquids|Gas|w/o CCS"] = prmfunc( + "meth_ng", units=units, group=group, formatting=formatting + ) + + # ["Note": "Refinery low yield"] + vars["Liquids|Oil|w/o CCS|1"] = prmfunc( + "ref_lol", units=units, group=group, formatting=formatting + ) + + # ["Note": "Refinery high yield"] + vars["Liquids|Oil|w/o CCS"] = vars["Liquids|Oil|w/o CCS|2"] = prmfunc( + "ref_hil", units=units, group=group, formatting=formatting + ) + + df = pp_utils.make_outputdf(vars, units, glb=False) + return df + + +@_register +def retr_supply_inv(units_energy, units_emi, units_ene_mdl): + """Technology results: Investments. + + Investments into technologies. + + Note OFR - 20.04.2017: The following has been checked between Volker + Krey, David McCollum and Oliver Fricko. + + There are fixed factors by which ccs technologies are multiplied which + equate to the share of the powerplant costs which split investments into the + share associated with the standard powerplant and the share associated + with those investments related to CCS. + + For some extraction and synfuel technologies, a certain share of the voms and + foms are attributed to the investments which is based on the GEA-Study + where the derived investment costs were partly attributed to the + voms/foms. + + Parameters + ---------- + units_energy : str + Units to which energy variables should be converted. + units_emi : str + Units to which emission variables should be converted. + units_ene_mdl : str + Native units of energy in the model. + """ + + vars = {} + + # ---------- + # Extraction + # ---------- + + # Note OFR 25.04.2017: All non-extraction costs for Coal, Gas and Oil + # have been moved to "Energy|Other" + + vars["Extraction|Coal"] = pp.investment( + ["coal_extr_ch4", "coal_extr", "lignite_extr"], units=units_energy + ) + + vars["Extraction|Gas|Conventional"] = ( + pp.investment( + ["gas_extr_1", "gas_extr_2", "gas_extr_3", "gas_extr_4"], units=units_energy + ) + + pp.act_vom( + ["gas_extr_1", "gas_extr_2", "gas_extr_3", "gas_extr_4"], units=units_energy + ) + * 0.5 + ) + + vars["Extraction|Gas|Unconventional"] = ( + pp.investment( + ["gas_extr_5", "gas_extr_6", "gas_extr_7", "gas_extr_8"], units=units_energy + ) + + pp.act_vom( + ["gas_extr_5", "gas_extr_6", "gas_extr_7", "gas_extr_8"], units=units_energy + ) + * 0.5 + ) + + # Note OFR 25.04.2017: Any costs relating to refineries have been + # removed (compared to GEA) as these are reported under "Liquids|Oil" + + vars["Extraction|Oil|Conventional"] = ( + pp.investment( + [ + "oil_extr_1", + "oil_extr_2", + "oil_extr_3", + "oil_extr_1_ch4", + "oil_extr_2_ch4", + "oil_extr_3_ch4", + ], + units=units_energy, + ) + + pp.act_vom( + [ + "oil_extr_1", + "oil_extr_2", + "oil_extr_3", + "oil_extr_1_ch4", + "oil_extr_2_ch4", + "oil_extr_3_ch4", + ], + units=units_energy, + ) + * 0.5 + ) + + vars["Extraction|Oil|Unconventional"] = ( + pp.investment( + [ + "oil_extr_4", + "oil_extr_4_ch4", + "oil_extr_5", + "oil_extr_6", + "oil_extr_7", + "oil_extr_8", + ], + units=units_energy, + ) + + pp.act_vom( + [ + "oil_extr_4", + "oil_extr_4_ch4", + "oil_extr_5", + "oil_extr_6", + "oil_extr_7", + "oil_extr_8", + ], + units=units_energy, + ) + * 0.5 + ) + + # As no mode is specified, u5-reproc will account for all 3 modes. + vars["Extraction|Uranium"] = ( + pp.investment( + ["uran2u5", "Uran_extr", "u5-reproc", "plutonium_prod"], units=units_energy + ) + + pp.act_vom(["uran2u5", "Uran_extr"], units=units_energy) + + pp.act_vom(["u5-reproc"], actfilter={"mode": ["M1"]}, units=units_energy) + + pp.tic_fom(["uran2u5", "Uran_extr", "u5-reproc"], units=units_energy) + ) + + # --------------------- + # Electricity - Fossils + # --------------------- + + vars["Electricity|Coal|w/ CCS"] = ( + pp.investment(["c_ppl_co2scr", "cfc_co2scr"], units=units_energy) + + pp.investment("coal_adv_ccs", units=units_energy) * 0.25 + + pp.investment("igcc_ccs", units=units_energy) * 0.31 + ) + + vars["Electricity|Coal|w/o CCS"] = ( + pp.investment(["coal_ppl", "coal_ppl_u", "coal_adv"], units=units_energy) + + pp.investment("coal_adv_ccs", units=units_energy) * 0.75 + + pp.investment("igcc", units=units_energy) + + pp.investment("igcc_ccs", units=units_energy) * 0.69 + ) + + vars["Electricity|Gas|w/ CCS"] = ( + pp.investment(["g_ppl_co2scr", "gfc_co2scr"], units=units_energy) + + pp.investment("gas_cc_ccs", units=units_energy) * 0.53 + ) + + vars["Electricity|Gas|w/o CCS"] = ( + pp.investment(["gas_cc", "gas_ct", "gas_ppl"], units=units_energy) + + pp.investment("gas_cc_ccs", units=units_energy) * 0.47 + ) + + vars["Electricity|Oil|w/o CCS"] = pp.investment( + ["foil_ppl", "loil_ppl", "oil_ppl", "loil_cc"], units=units_energy + ) + + # ------------------------ + # Electricity - Renewables + # ------------------------ + + vars["Electricity|Biomass|w/ CCS"] = ( + pp.investment("bio_ppl_co2scr", units=units_energy) + + pp.investment("bio_istig_ccs", units=units_energy) * 0.31 + ) + + vars["Electricity|Biomass|w/o CCS"] = ( + pp.investment(["bio_ppl", "bio_istig"], units=units_energy) + + pp.investment("bio_istig_ccs", units=units_energy) * 0.69 + ) + + vars["Electricity|Geothermal"] = pp.investment("geo_ppl", units=units_energy) + + vars["Electricity|Hydro"] = pp.investment( + [ + "hydro_1", + "hydro_2", + "hydro_3", + "hydro_4", + "hydro_5", + "hydro_6", + "hydro_7", + "hydro_8", + ], + units=units_energy, + ) + vars["Electricity|Other"] = pp.investment( + ["h2_fc_I", "h2_fc_RC"], units=units_energy + ) + + _solar_pv_elec = pp.investment( + ["solar_pv_ppl", "solar_pv_I", "solar_pv_RC"], units=units_energy + ) + + _solar_th_elec = pp.investment(["csp_sm1_ppl", "csp_sm3_ppl"], units=units_energy) + + vars["Electricity|Solar|PV"] = _solar_pv_elec + vars["Electricity|Solar|CSP"] = _solar_th_elec + vars["Electricity|Wind|Onshore"] = pp.investment(["wind_ppl"], units=units_energy) + vars["Electricity|Wind|Offshore"] = pp.investment(["wind_ppf"], units=units_energy) + + # ------------------- + # Electricity Nuclear + # ------------------- + + vars["Electricity|Nuclear"] = pp.investment( + ["nuc_hc", "nuc_lc"], units=units_energy + ) + + # -------------------------------------------------- + # Electricity Storage, transmission and distribution + # -------------------------------------------------- + + vars["Electricity|Electricity Storage"] = pp.investment( + "stor_ppl", units=units_energy + ) + + vars["Electricity|Transmission and Distribution"] = ( + pp.investment( + [ + "elec_t_d", + "elec_exp", + "elec_exp_africa", + "elec_exp_america", + "elec_exp_asia", + "elec_exp_eurasia", + "elec_exp_eur_afr", + "elec_exp_asia_afr", + "elec_imp", + "elec_imp_africa", + "elec_imp_america", + "elec_imp_asia", + "elec_imp_eurasia", + "elec_imp_eur_afr", + "elec_imp_asia_afr", + ], + units=units_energy, + ) + + pp.act_vom( + [ + "elec_t_d", + "elec_exp", + "elec_exp_africa", + "elec_exp_america", + "elec_exp_asia", + "elec_exp_eurasia", + "elec_exp_eur_afr", + "elec_exp_asia_afr", + "elec_imp", + "elec_imp_africa", + "elec_imp_america", + "elec_imp_asia", + "elec_imp_eurasia", + "elec_imp_eur_afr", + "elec_imp_asia_afr", + ], + units=units_energy, + ) + * 0.5 + + pp.tic_fom( + [ + "elec_t_d", + "elec_exp", + "elec_exp_africa", + "elec_exp_america", + "elec_exp_asia", + "elec_exp_eurasia", + "elec_exp_eur_afr", + "elec_exp_asia_afr", + "elec_imp", + "elec_imp_africa", + "elec_imp_america", + "elec_imp_asia", + "elec_imp_eurasia", + "elec_imp_eur_afr", + "elec_imp_asia_afr", + ], + units=units_energy, + ) + * 0.5 + ) + + # ------------------------------------------ + # CO2 Storage, transmission and distribution + # ------------------------------------------ + + _CCS_coal_elec = -1 * pp.emi( + ["c_ppl_co2scr", "coal_adv_ccs", "igcc_ccs", "cement_co2scr"], + units_ene_mdl, + emifilter={"relation": ["CO2_Emission"]}, + emission_units=units_emi, + ) + + _CCS_coal_synf = -1 * pp.emi( + ["syn_liq_ccs", "h2_coal_ccs", "meth_coal_ccs"], + units_ene_mdl, + emifilter={"relation": ["CO2_Emission"]}, + emission_units=units_emi, + ) + + _CCS_gas_elec = -1 * pp.emi( + ["g_ppl_co2scr", "gas_cc_ccs"], + units_ene_mdl, + emifilter={"relation": ["CO2_Emission"]}, + emission_units=units_emi, + ) + + _CCS_gas_synf = -1 * pp.emi( + ["h2_smr_ccs", "meth_ng_ccs"], + units_ene_mdl, + emifilter={"relation": ["CO2_Emission"]}, + emission_units=units_emi, + ) + + _CCS_bio_elec = -1 * pp.emi( + ["bio_ppl_co2scr", "bio_istig_ccs"], + units_ene_mdl, + emifilter={"relation": ["CO2_Emission"]}, + emission_units=units_emi, + ) + + _CCS_bio_synf = -1 * pp.emi( + ["eth_bio_ccs", "liq_bio_ccs", "h2_bio_ccs"], + units_ene_mdl, + emifilter={"relation": ["CO2_Emission"]}, + emission_units=units_emi, + ) + + _Biogas_use_tot = pp.out("gas_bio") + + _Gas_use_tot = pp.inp( + [ + "gas_ppl", + "gas_cc", + "gas_cc_ccs", + "gas_ct", + "gas_htfc", + "gas_hpl", + "meth_ng", + "meth_ng_ccs", + "h2_smr", + "h2_smr_ccs", + "gas_rc", + "hp_gas_rc", + "gas_i", + "hp_gas_i", + "gas_trp", + "gas_fs", + ] + ) + + _Biogas_share = (_Biogas_use_tot / _Gas_use_tot).fillna(0) + + _CCS_Foss = ( + _CCS_coal_elec + + _CCS_coal_synf + + _CCS_gas_elec * (1 - _Biogas_share) + + _CCS_gas_synf * (1 - _Biogas_share) + ) + + _CCS_Bio = ( + _CCS_bio_elec + _CCS_bio_synf - (_CCS_gas_elec + _CCS_gas_synf) * _Biogas_share + ) + + _CCS_coal_elec_shr = (_CCS_coal_elec / _CCS_Foss).fillna(0) + _CCS_coal_synf_shr = (_CCS_coal_synf / _CCS_Foss).fillna(0) + _CCS_gas_elec_shr = (_CCS_gas_elec / _CCS_Foss).fillna(0) + _CCS_gas_synf_shr = (_CCS_gas_synf / _CCS_Foss).fillna(0) + _CCS_bio_elec_shr = (_CCS_bio_elec / _CCS_Bio).fillna(0) + _CCS_bio_synf_shr = (_CCS_bio_synf / _CCS_Bio).fillna(0) + + CO2_trans_dist_elec = ( + pp.act_vom("co2_tr_dis", units=units_energy) * 0.5 * _CCS_coal_elec_shr + + pp.act_vom("co2_tr_dis", units=units_energy) * 0.5 * _CCS_gas_elec_shr + + pp.act_vom("bco2_tr_dis", units=units_energy) * 0.5 * _CCS_bio_elec_shr + ) + + CO2_trans_dist_synf = ( + pp.act_vom("co2_tr_dis", units=units_energy) * 0.5 * _CCS_coal_synf_shr + + pp.act_vom("co2_tr_dis", units=units_energy) * 0.5 * _CCS_gas_synf_shr + + pp.act_vom("bco2_tr_dis", units=units_energy) * 0.5 * _CCS_bio_synf_shr + ) + + vars["CO2 Transport and Storage"] = CO2_trans_dist_elec + CO2_trans_dist_synf + + # ---- + # Heat + # ---- + + vars["Heat"] = pp.investment( + ["coal_hpl", "foil_hpl", "gas_hpl", "bio_hpl", "heat_t_d", "po_turbine"], + units=units_energy, + ) + + # ------------------------- + # Synthetic fuel production + # ------------------------- + + # Note OFR 25.04.2017: XXX_synf_ccs has been split into hydrogen and + # liquids. The shares then add up to 1, but the variables are kept + # separate in order to preserve the split between CCS and non-CCS + + _Coal_synf_ccs_liq = ( + pp.investment("meth_coal_ccs", units=units_energy) * 0.02 + + pp.investment("syn_liq_ccs", units=units_energy) * 0.01 + ) + + _Gas_synf_ccs_liq = pp.investment("meth_ng_ccs", units=units_energy) * 0.08 + + _Bio_synf_ccs_liq = ( + pp.investment("eth_bio_ccs", units=units_energy) * 0.34 + + pp.investment("liq_bio_ccs", units=units_energy) * 0.02 + ) + + _Coal_synf_ccs_h2 = pp.investment("h2_coal_ccs", units=units_energy) * 0.03 + _Gas_synf_ccs_h2 = pp.investment("h2_smr_ccs", units=units_energy) * 0.17 + _Bio_synf_ccs_h2 = pp.investment("h2_bio_ccs", units=units_energy) * 0.02 + + # Note OFR 25.04.2017: "coal_gas" have been moved to "other" + vars["Liquids|Coal and Gas"] = ( + pp.investment(["meth_coal", "syn_liq", "meth_ng"], units=units_energy) + + pp.investment("meth_ng_ccs", units=units_energy) * 0.92 + + pp.investment("meth_coal_ccs", units=units_energy) * 0.98 + + pp.investment("syn_liq_ccs", units=units_energy) * 0.99 + + _Coal_synf_ccs_liq + + _Gas_synf_ccs_liq + ) + + # Note OFR 25.04.2017: "gas_bio" has been moved to "other" + vars["Liquids|Biomass"] = ( + pp.investment(["eth_bio", "liq_bio"], units=units_energy) + + pp.investment("liq_bio_ccs", units=units_energy) * 0.98 + + pp.investment("eth_bio_ccs", units=units_energy) * 0.66 + + _Bio_synf_ccs_liq + ) + + # Note OFR 25.04.2017: "transport, import and exports costs related to + # liquids are only included in the total" + _Synfuel_other = pp.investment( + [ + "meth_exp", + "meth_imp", + "meth_t_d", + "meth_bal", + "eth_exp", + "eth_imp", + "eth_t_d", + "eth_bal", + "SO2_scrub_synf", + ], + units=units_energy, + ) + + vars["Liquids|Oil"] = pp.investment( + ["ref_lol", "ref_hil"], units=units_energy + ) + pp.tic_fom(["ref_hil", "ref_lol"], units=units_energy) + + vars["Liquids"] = ( + vars["Liquids|Coal and Gas"] + + vars["Liquids|Biomass"] + + vars["Liquids|Oil"] + + _Synfuel_other + ) + + # -------- + # Hydrogen + # -------- + + vars["Hydrogen|Fossil"] = ( + pp.investment(["h2_coal", "h2_smr"], units=units_energy) + + pp.investment("h2_coal_ccs", units=units_energy) * 0.97 + + pp.investment("h2_smr_ccs", units=units_energy) * 0.83 + + _Coal_synf_ccs_h2 + + _Gas_synf_ccs_h2 + ) + + vars["Hydrogen|Renewable"] = ( + pp.investment("h2_bio", units=units_energy) + + pp.investment("h2_bio_ccs", units=units_energy) * 0.98 + + _Bio_synf_ccs_h2 + ) + + vars["Hydrogen|Other"] = ( + pp.investment( + [ + "h2_elec", + "h2_liq", + "h2_t_d", + "lh2_exp", + "lh2_imp", + "lh2_bal", + "lh2_regas", + "lh2_t_d", + ], + units=units_energy, + ) + + pp.act_vom("h2_mix", units=units_energy) * 0.5 + ) + + # ----- + # Other + # ----- + + # All questionable variables from extraction that are not related directly + # to extraction should be moved to Other + # Note OFR 25.04.2017: Any costs relating to refineries have been + # removed (compared to GEA) as these are reported under "Liquids|Oil" + + vars["Other|Liquids|Oil|Transmission and Distribution"] = pp.investment( + ["foil_t_d", "loil_t_d"], units=units_energy + ) + + vars["Other|Liquids|Oil|Other"] = pp.investment( + [ + "foil_exp", + "loil_exp", + "oil_exp", + "oil_imp", + "foil_imp", + "loil_imp", + "loil_std", + "oil_bal", + "loil_sto", + ], + units=units_energy, + ) + + vars["Other|Gases|Transmission and Distribution"] = pp.investment( + ["gas_t_d", "gas_t_d_ch4"], units=units_energy + ) + + vars["Other|Gases|Production"] = pp.investment( + ["gas_bio", "coal_gas"], units=units_energy + ) + + vars["Other|Gases|Other"] = pp.investment( + [ + "LNG_bal", + "LNG_prod", + "LNG_regas", + "LNG_exp", + "LNG_imp", + "gas_bal", + "gas_std", + "gas_sto", + "gas_exp_eeu", + "gas_exp_nam", + "gas_exp_pao", + "gas_exp_weu", + "gas_exp_cpa", + "gas_exp_afr", + "gas_exp_sas", + "gas_exp_scs", + "gas_exp_cas", + "gas_exp_ubm", + "gas_exp_rus", + "gas_imp", + ], + units=units_energy, + ) + + vars["Other|Solids|Coal|Transmission and Distribution"] = ( + pp.investment( + [ + "coal_t_d", + "coal_t_d-rc-SO2", + "coal_t_d-rc-06p", + "coal_t_d-in-SO2", + "coal_t_d-in-06p", + ], + units=units_energy, + ) + + pp.act_vom( + [ + "coal_t_d-rc-SO2", + "coal_t_d-rc-06p", + "coal_t_d-in-SO2", + "coal_t_d-in-06p", + "coal_t_d", + ], + units=units_energy, + ) + * 0.5 + ) + + vars["Other|Solids|Coal|Other"] = pp.investment( + ["coal_exp", "coal_imp", "coal_bal", "coal_std"], units=units_energy + ) + + vars["Other|Solids|Biomass|Transmission and Distribution"] = pp.investment( + "biomass_t_d", units=units_energy + ) + + vars["Other|Other"] = pp.investment( + ["SO2_scrub_ref"], units=units_energy + ) * 0.5 + pp.investment(["SO2_scrub_ind"], units=units_energy) + + df = pp_utils.make_outputdf(vars, units_energy) + return df + + +@_register +def retr_water_use(units, method): + """Water: Withdrawal or Consumption. + + Parameters + ---------- + units : str + Units to which variables should be converted. + method : str + `withdrawal` will calculate the water withdrawn, while + `consumption` will calculate the `withdrawal` and subtrace the + water quantities returned. + """ + + vars = {} + + if run_history != "True": + group = ["Region", "Mode", "Vintage"] + else: + group = ["Region"] + + # -------------------------------- + # Calculation of helping variables + # -------------------------------- + + _Cogen = pp.inp("po_turbine", inpfilter={"commodity": ["electr"]}) + + _Potential = pp.act_rel( + [ + "coal_ppl_u", + "coal_ppl", + "coal_adv", + "coal_adv_ccs", + "foil_ppl", + "loil_ppl", + "loil_cc", + "gas_ppl", + "gas_ct", + "gas_cc", + "gas_cc_ccs", + "gas_htfc", + "bio_ppl", + "bio_istig", + "bio_istig_ccs", + "igcc", + "igcc_ccs", + "nuc_lc", + "nuc_hc", + "geo_ppl", + ], + relfilter={"relation": ["pass_out_trb"]}, + ) + + _Biogas = pp.out("gas_bio") + + _gas_inp_tecs = [ + "gas_ppl", + "gas_cc", + "gas_cc_ccs", + "gas_ct", + "gas_htfc", + "gas_hpl", + "meth_ng", + "meth_ng_ccs", + "h2_smr", + "h2_smr_ccs", + "gas_t_d", + "gas_t_d_ch4", + ] + + _totgas = pp.inp(_gas_inp_tecs, inpfilter={"commodity": ["gas"]}) + _Frac = (_Cogen / _Potential).fillna(0) + _BGas_share = (_Biogas / _totgas).fillna(0) + + # Calculate shares for ppl feeding into g_ppl_co2scr (gas_cc and gas_ppl) + _gas_cc_shr = (pp.out("gas_cc") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) + + _gas_ppl_shr = (pp.out("gas_ppl") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) + + inpfilter = {"level": ["water_supply"], "commodity": ["freshwater_supply"]} + outfilter = {"level": ["secondary"], "commodity": ["electr"]} + emiffilter = {"emission": ["fresh_wastewater"]} + + # -------------------- + # Once Through Cooling + # -------------------- + + _gas_ot_fresh_wo_CCS = pp.inp( + ["gas_cc__ot_fresh", "gas_ppl__ot_fresh"], units, inpfilter=inpfilter + ) + + _gas_ot_fresh_w_CCS = pp.inp(["gas_cc_ccs__ot_fresh"], units, inpfilter=inpfilter) + + _coal_ot_fresh_wo_CCS = pp.inp( + [ + "coal_adv__ot_fresh", + "coal_ppl__ot_fresh", + "coal_ppl_u__ot_fresh", + "igcc__ot_fresh", + ], + units, + inpfilter=inpfilter, + ) + + _coal_ot_fresh_w_CCS = pp.inp( + ["coal_adv_ccs__ot_fresh", "igcc_ccs__ot_fresh"], units, inpfilter=inpfilter + ) + + _oil_ot_fresh_wo_CCS = pp.inp( + ["foil_ppl__ot_fresh", "loil_cc__ot_fresh", "loil_ppl__ot_fresh"], + units, + inpfilter=inpfilter, + ) + + _bio_ot_fresh_wo_CCS = pp.inp( + ["bio_istig__ot_fresh", "bio_ppl__ot_fresh"], units, inpfilter=inpfilter + ) + + _bio_ot_fresh_w_CCS = pp.inp( + ["bio_istig_ccs__ot_fresh"], units, inpfilter=inpfilter + ) + + _geo_ot_fresh_wo_CCS = pp.inp(["geo_ppl__ot_fresh"], units, inpfilter=inpfilter) + + _nuc_ot_fresh_wo_CCS = pp.inp( + ["nuc_hc__ot_fresh", "nuc_lc__ot_fresh"], units, inpfilter=inpfilter + ) + + _solar_ot_fresh_wo_CCS = pp.inp( + "solar_th_ppl__ot_fresh", units, inpfilter=inpfilter + ) + + if method == "consumption": + _gas_ot_fresh_wo_CCS -= pp.act_emif( + ["gas_cc__ot_fresh", "gas_ppl__ot_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _gas_ot_fresh_w_CCS -= pp.act_emif( + ["gas_cc_ccs__ot_fresh"], units=units, emiffilter=emiffilter + ) + + _coal_ot_fresh_wo_CCS -= pp.act_emif( + [ + "coal_adv__ot_fresh", + "coal_ppl__ot_fresh", + "coal_ppl_u__ot_fresh", + "igcc__ot_fresh", + ], + units=units, + emiffilter=emiffilter, + ) + + _coal_ot_fresh_w_CCS -= pp.act_emif( + ["coal_adv_ccs__ot_fresh", "igcc_ccs__ot_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _oil_ot_fresh_wo_CCS -= pp.act_emif( + ["foil_ppl__ot_fresh", "loil_cc__ot_fresh", "loil_ppl__ot_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _bio_ot_fresh_wo_CCS -= pp.act_emif( + ["bio_istig__ot_fresh", "bio_ppl__ot_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _bio_ot_fresh_w_CCS -= pp.act_emif( + ["bio_istig_ccs__ot_fresh"], units=units, emiffilter=emiffilter + ) + + _geo_ot_fresh_wo_CCS -= pp.act_emif( + ["geo_ppl__ot_fresh"], units=units, emiffilter=emiffilter + ) + + _nuc_ot_fresh_wo_CCS -= pp.act_emif( + ["nuc_hc__ot_fresh", "nuc_lc__ot_fresh"], units=units, emiffilter=emiffilter + ) + + _solar_ot_fresh_wo_CCS -= pp.act_emif( + ["solar_th_ppl__ot_fresh"], units=units, emiffilter=emiffilter + ) + + # ----------- + # Closed Loop + # ----------- + + _gas_cl_fresh_wo_CCS = pp.inp( + ["gas_cc__cl_fresh", "gas_ppl__cl_fresh"], units, inpfilter=inpfilter + ) + + _gas_cl_fresh_w_CCS = pp.inp(["gas_cc_ccs__cl_fresh"], units, inpfilter=inpfilter) + + _coal_cl_fresh_wo_CCS = pp.inp( + [ + "coal_adv__cl_fresh", + "coal_ppl__cl_fresh", + "coal_ppl_u__cl_fresh", + "igcc__cl_fresh", + ], + units, + inpfilter=inpfilter, + ) + + _coal_cl_fresh_w_CCS = pp.inp( + ["coal_adv_ccs__cl_fresh", "igcc_ccs__cl_fresh"], units, inpfilter=inpfilter + ) + + _oil_cl_fresh_wo_CCS = pp.inp( + ["foil_ppl__cl_fresh", "loil_cc__cl_fresh", "loil_ppl__cl_fresh"], + units, + inpfilter=inpfilter, + ) + + _bio_cl_fresh_wo_CCS = pp.inp( + ["bio_istig__cl_fresh", "bio_ppl__cl_fresh"], units, inpfilter=inpfilter + ) + + _bio_cl_fresh_w_CCS = pp.inp( + ["bio_istig_ccs__cl_fresh"], units, inpfilter=inpfilter + ) + + _geo_cl_fresh_wo_CCS = pp.inp(["geo_ppl__cl_fresh"], units, inpfilter=inpfilter) + + _nuc_cl_fresh_wo_CCS = pp.inp( + ["nuc_hc__cl_fresh", "nuc_lc__cl_fresh"], units, inpfilter=inpfilter + ) + + _solar_cl_fresh_wo_CCS = pp.inp( + "solar_th_ppl__cl_fresh", units, inpfilter=inpfilter + ) + + if method == "consumption": + _gas_cl_fresh_wo_CCS -= pp.act_emif( + ["gas_cc__cl_fresh", "gas_ppl__cl_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _gas_cl_fresh_w_CCS -= pp.act_emif( + ["gas_cc_ccs__cl_fresh"], units=units, emiffilter=emiffilter + ) + + _coal_cl_fresh_wo_CCS -= pp.act_emif( + [ + "coal_adv__cl_fresh", + "coal_ppl__cl_fresh", + "coal_ppl_u__cl_fresh", + "igcc__cl_fresh", + ], + units=units, + emiffilter=emiffilter, + ) + + _coal_cl_fresh_w_CCS -= pp.act_emif( + ["coal_adv_ccs__cl_fresh", "igcc_ccs__cl_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _oil_cl_fresh_wo_CCS -= pp.act_emif( + ["foil_ppl__cl_fresh", "loil_cc__cl_fresh", "loil_ppl__cl_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _bio_cl_fresh_wo_CCS -= pp.act_emif( + ["bio_istig__cl_fresh", "bio_ppl__cl_fresh"], + units=units, + emiffilter=emiffilter, + ) + + _bio_cl_fresh_w_CCS -= pp.act_emif( + ["bio_istig_ccs__cl_fresh"], units=units, emiffilter=emiffilter + ) + + _geo_cl_fresh_wo_CCS -= pp.act_emif( + ["geo_ppl__cl_fresh"], units=units, emiffilter=emiffilter + ) + + _nuc_cl_fresh_wo_CCS -= pp.act_emif( + ["nuc_hc__cl_fresh", "nuc_lc__cl_fresh"], units=units, emiffilter=emiffilter + ) + + _solar_cl_fresh_wo_CCS -= pp.act_emif( + ["solar_th_ppl__cl_fresh"], units=units, emiffilter=emiffilter + ) + + # ----------------------------- + # Once Through Cooling (SALINE) + # ----------------------------- + sinpfilter = {"level": ["water_supply"], "commodity": ["saline_supply_ppl"]} + + _gas_ot_saline_wo_CCS = pp.inp( + ["gas_cc__ot_saline", "gas_ppl__ot_saline"], units, inpfilter=sinpfilter + ) + + _gas_ot_saline_w_CCS = pp.inp( + ["gas_cc_ccs__ot_saline"], units, inpfilter=sinpfilter + ) + + _coal_ot_saline_wo_CCS = pp.inp( + [ + "coal_adv__ot_saline", + "coal_ppl__ot_saline", + "coal_ppl_u__ot_saline", + "igcc__ot_saline", + ], + units, + inpfilter=sinpfilter, + ) + + _coal_ot_saline_w_CCS = pp.inp( + ["coal_adv_ccs__ot_saline", "igcc_ccs__ot_saline"], units, inpfilter=sinpfilter + ) + + _oil_ot_saline_wo_CCS = pp.inp( + ["foil_ppl__ot_saline", "loil_cc__ot_saline", "loil_ppl__ot_saline"], + units, + inpfilter=sinpfilter, + ) + + _bio_ot_saline_wo_CCS = pp.inp( + ["bio_istig__ot_saline", "bio_ppl__ot_saline"], units, inpfilter=sinpfilter + ) + + _bio_ot_saline_w_CCS = pp.inp( + ["bio_istig_ccs__ot_saline"], units, inpfilter=sinpfilter + ) + + _geo_ot_saline_wo_CCS = pp.inp(["geo_ppl__ot_saline"], units, inpfilter=sinpfilter) + + _nuc_ot_saline_wo_CCS = pp.inp( + ["nuc_hc__ot_saline", "nuc_lc__ot_saline"], units, inpfilter=sinpfilter + ) + + _solar_ot_saline_wo_CCS = pp.inp( + "solar_th_ppl__ot_saline", units, inpfilter=sinpfilter + ) + + if method == "consumption": + _gas_ot_saline_wo_CCS -= pp.act_emif( + ["gas_cc__ot_saline", "gas_ppl__ot_saline"], + units=units, + emiffilter=emiffilter, + ) + + _gas_ot_saline_w_CCS -= pp.act_emif( + ["gas_cc_ccs__ot_saline"], units=units, emiffilter=emiffilter + ) + + _coal_ot_saline_wo_CCS -= pp.act_emif( + [ + "coal_adv__ot_saline", + "coal_ppl__ot_saline", + "coal_ppl_u__ot_saline", + "igcc__ot_saline", + ], + units=units, + emiffilter=emiffilter, + ) + + _coal_ot_saline_w_CCS -= pp.act_emif( + ["coal_adv_ccs__ot_saline", "igcc_ccs__ot_saline"], + units=units, + emiffilter=emiffilter, + ) + + _oil_ot_saline_wo_CCS -= pp.act_emif( + ["foil_ppl__ot_saline", "loil_cc__ot_saline", "loil_ppl__ot_saline"], + units=units, + emiffilter=emiffilter, + ) + + _bio_ot_saline_wo_CCS -= pp.act_emif( + ["bio_istig__ot_saline", "bio_ppl__ot_saline"], + units=units, + emiffilter=emiffilter, + ) + + _bio_ot_saline_w_CCS -= pp.act_emif( + ["bio_istig_ccs__ot_saline"], units=units, emiffilter=emiffilter + ) + + _geo_ot_saline_wo_CCS -= pp.act_emif( + ["geo_ppl__ot_saline"], units=units, emiffilter=emiffilter + ) + + _nuc_ot_saline_wo_CCS -= pp.act_emif( + ["nuc_hc__ot_saline", "nuc_lc__ot_saline"], + units=units, + emiffilter=emiffilter, + ) + + _solar_ot_saline_wo_CCS -= pp.act_emif( + ["solar_th_ppl__ot_saline"], units=units, emiffilter=emiffilter + ) + + # ----------- + # Dry cooling + # ----------- + + _gas_air = pp.inp(["gas_cc__air", "gas_ppl__air"], units, inpfilter=inpfilter) + + _coal_air = pp.inp( + ["coal_adv__air", "coal_ppl__air", "coal_ppl_u__air", "igcc__air"], + units, + inpfilter=inpfilter, + ) + + _oil_air = pp.inp( + ["foil_ppl__air", "loil_cc__air", "loil_ppl__air"], units, inpfilter=inpfilter + ) + + _bio_air = pp.inp(["bio_istig__air", "bio_ppl__air"], units, inpfilter=inpfilter) + + _geo_air = pp.inp(["geo_ppl__air"], units, inpfilter=inpfilter) + _solar_air = pp.inp("solar_th_ppl__air", units, inpfilter=inpfilter) + + # ----------------------- + # Water use for coal elec + # ----------------------- + + _Cooling_ele_coal_wCCS = ( + _coal_ot_fresh_w_CCS + _coal_cl_fresh_w_CCS + _coal_ot_saline_w_CCS + ) + + _Direct_ele_coal_wCCS = ( + _pe_elec_wCCSretro( + "coal_ppl", + "c_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + ) + + _pe_elec_wCCSretro( + "igcc", "igcc_co2scr", group, inpfilter=inpfilter, units=units, _Frac=_Frac + ) + + _pe_elec_po_turb("coal_adv_ccs", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("igcc_ccs", group, units, _Frac, inpfilter=inpfilter) + + _out_div_eff( + ["meth_coal_ccs", "h2_coal_ccs", "syn_liq_ccs"], group, inpfilter, outfilter + ) + ) + + vars["Electricity|Coal|w/ CCS"] = _Direct_ele_coal_wCCS + _Cooling_ele_coal_wCCS + + _Cooling_ele_coal_woCCS = ( + _coal_ot_fresh_wo_CCS + + _coal_cl_fresh_wo_CCS + + _coal_ot_saline_wo_CCS + + _coal_air + ) + + _Direct_ele_coal_woCCS = ( + _pe_elec_woCCSretro( + "coal_ppl", + "c_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + ) + + _pe_elec_woCCSretro( + "igcc", "igcc_co2scr", group, inpfilter=inpfilter, units=units, _Frac=_Frac + ) + + _pe_elec_po_turb("coal_adv", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("coal_ppl_u", group, units, _Frac, inpfilter=inpfilter) + + _out_div_eff(["meth_coal", "h2_coal", "syn_liq"], group, inpfilter, outfilter) + ) + + vars["Electricity|Coal|w/o CCS"] = _Direct_ele_coal_woCCS + _Cooling_ele_coal_woCCS + + # ----------------------------- + # Water use for electricity Gas + # ----------------------------- + + _Cooling_ele_gas_wCCS = ( + _gas_ot_fresh_w_CCS + _gas_cl_fresh_w_CCS + _gas_ot_saline_w_CCS + ) + + _Direct_ele_gas_wCCS = ( + _pe_elec_wCCSretro( + "gas_cc", + "g_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + share=_gas_cc_shr, + ) + + _pe_elec_wCCSretro( + "gas_ppl", + "g_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + share=_gas_ppl_shr, + ) + + _pe_elec_wCCSretro( + "gas_htfc", + "gfc_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb("gas_cc_ccs", group, units, _Frac, inpfilter=inpfilter) + + _out_div_eff(["h2_smr_ccs"], group, inpfilter, outfilter) + ) + + vars["Electricity|Gas|w/ CCS"] = (_Direct_ele_gas_wCCS + _Cooling_ele_gas_wCCS) * ( + 1 - _BGas_share + ) + + _Cooling_ele_gas_woCCS = ( + _gas_ot_fresh_wo_CCS + _gas_cl_fresh_wo_CCS + _gas_ot_saline_wo_CCS + _gas_air + ) + + _Direct_ele_gas_woCCS = ( + _pe_elec_woCCSretro( + "gas_ppl", + "g_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + share=_gas_ppl_shr, + ) + + _pe_elec_woCCSretro( + "gas_cc", + "g_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + share=_gas_cc_shr, + ) + + _pe_elec_woCCSretro( + "gas_htfc", + "gfc_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb("gas_ct", group, units, _Frac, inpfilter=inpfilter) + + _out_div_eff(["h2_smr"], group, inpfilter, outfilter) + ) + + vars["Electricity|Gas|w/o CCS"] = ( + _Direct_ele_gas_woCCS + _Cooling_ele_gas_woCCS + ) * (1 - _BGas_share) + + # ----------------------------- + # Water use for electricity bio + # ----------------------------- + + _Cooling_ele_bio_wCCS = ( + _bio_ot_fresh_w_CCS + _bio_cl_fresh_w_CCS + _bio_ot_saline_w_CCS + ) + + _Direct_ele_bio_wCCS = ( + _pe_elec_wCCSretro( + "bio_ppl", + "bio_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb("bio_istig_ccs", group, units, _Frac, inpfilter=inpfilter) + + _out_div_eff( + ["h2_bio_ccs", "eth_bio_ccs", "liq_bio_ccs"], group, inpfilter, outfilter + ) + ) + + vars["Electricity|Biomass|w/ CCS"] = ( + _Direct_ele_bio_wCCS + + _Cooling_ele_bio_wCCS + + (_Direct_ele_gas_wCCS + _Cooling_ele_gas_wCCS) * _BGas_share + ) + + _Cooling_ele_bio_woCCS = ( + _bio_ot_fresh_wo_CCS + _bio_cl_fresh_wo_CCS + _bio_ot_saline_wo_CCS + _bio_air + ) + + _Direct_ele_bio_woCCS = ( + _pe_elec_woCCSretro( + "bio_ppl", + "bio_ppl_co2scr", + group, + inpfilter=inpfilter, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb("bio_istig", group, units, _Frac, inpfilter=inpfilter) + + _out_div_eff(["h2_bio", "eth_bio", "liq_bio"], group, inpfilter, outfilter) + ) + + vars["Electricity|Biomass|w/o CCS"] = ( + _Direct_ele_bio_woCCS + + _Cooling_ele_bio_woCCS + + (_Direct_ele_gas_woCCS + _Cooling_ele_gas_woCCS) * _BGas_share + ) + + # ----------------------------- + # Water use for electricity oil + # ----------------------------- + + _Cooling_ele_oil = ( + _oil_ot_fresh_wo_CCS + _oil_cl_fresh_wo_CCS + _oil_ot_saline_wo_CCS + _oil_air + ) + + _Direct_ele_oil = ( + _pe_elec_po_turb("foil_ppl", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("loil_ppl", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("oil_ppl", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("loil_cc", group, units, _Frac, inpfilter=inpfilter) + ) + + vars["Electricity|Oil"] = _Direct_ele_oil + _Cooling_ele_oil + + # ------------------------------------ + # Water use for electricity geothermal + # ------------------------------------ + + _Cooling_ele_geo = ( + _geo_ot_fresh_wo_CCS + _geo_cl_fresh_wo_CCS + _geo_ot_saline_wo_CCS + _geo_air + ) + + _Direct_ele_geo = _pe_elec_po_turb( + "geo_ppl", group, units, _Frac, inpfilter=inpfilter + ) + + vars["Electricity|Geothermal"] = _Direct_ele_geo + _Cooling_ele_geo + + # ------------------------------------ + # Water use for electricity hydropower + # ------------------------------------ + + # Note Wenji 03.05.2017: Hydropower uses a different commodity than the + # other technologies. + + vars["Electricity|Hydro"] = pp.inp( + [ + "hydro_1", + "hydro_2", + "hydro_3", + "hydro_4", + "hydro_5", + "hydro_6", + "hydro_7", + "hydro_8", + ], + units, + inpfilter={"level": ["water_supply"], "commodity": ["freshwater_instream"]}, + ) + + # --------------------------------- + # Water use for electricity nuclear + # --------------------------------- + + _Cooling_ele_nuc = ( + _nuc_ot_fresh_wo_CCS + _nuc_cl_fresh_wo_CCS + _nuc_ot_saline_wo_CCS + ) + + _Direct_ele_nuc = ( + _pe_elec_po_turb("nuc_lc", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("nuc_hc", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("nuc_fbr", group, units, _Frac, inpfilter=inpfilter) + ) + + vars["Electricity|Nuclear"] = _Direct_ele_nuc + _Cooling_ele_nuc + + # ------------------------------ + # Water use for electricity wind + # ------------------------------ + + # Note Wenji 02.05.2017: In the GDX file it seems that these technologies + # have no input + + Wind_onshore = pp.inp( + [ + "wind_res1", + "wind_res2", + "wind_res3", + "wind_res4", + "wind_res_hist_2005", + "wind_res_hist_2010", + "wind_res_hist_2015", + "wind_res_hist_2020", + ], + units, + inpfilter=inpfilter, + ) + + Wind_offshore = pp.inp( + [ + "wind_ref1", + "wind_ref2", + "wind_ref3", + "wind_ref4", + "wind_ref5", + "wind_ref_hist_2005", + "wind_ref_hist_2010", + "wind_ref_hist_2015", + "wind_ref_hist_2020", + ], + units, + inpfilter=inpfilter, + ) + + vars["Electricity|Wind"] = Wind_onshore + Wind_offshore + + # ------------------------------- + # Water use for electricity solar + # ------------------------------- + + _Cooling_ele_solar_th = ( + _solar_ot_fresh_wo_CCS + + _solar_cl_fresh_wo_CCS + + _solar_ot_saline_wo_CCS + + _solar_air + ) + + _Direct_ele_solar_th = pp.inp( + [ + "solar_th_ppl", + "csp_sm1_res", + "csp_sm1_res1", + "csp_sm1_res2", + "csp_sm1_res3", + "csp_sm1_res4", + "csp_sm1_res5", + "csp_sm1_res6", + "csp_sm1_res7", + "csp_res_hist_2005", + "csp_res_hist_2010", + "csp_res_hist_2015", + "csp_res_hist_2020", + "csp_sm3_res", + "csp_sm3_res1", + "csp_sm3_res2", + "csp_sm3_res3", + "csp_sm3_res4", + "csp_sm3_res5", + "csp_sm3_res6", + "csp_sm3_res7", + ], + units, + inpfilter=inpfilter, + ) + + vars["Electricity|Solar|CSP"] = _Direct_ele_solar_th + _Cooling_ele_solar_th + + # Note Wenji 02.05.2017: In the GDX file it seems that these technologies + # have no input + + vars["Electricity|Solar|PV"] = pp.inp( + [ + "solar_res1", + "solar_res2", + "solar_res3", + "solar_res4", + "solar_res5", + "solar_res6", + "solar_res7", + "solar_res8", + "solar_pv_I", + "solar_pv_RC", + "solar_res_hist_2005", + "solar_res_hist_2010", + "solar_res_hist_2015", + "solar_res_hist_2020", + ], + units, + inpfilter=inpfilter, + ) + + vars["Electricity|Other"] = pp.inp( + ["h2_fc_trp", "h2_fc_I", "h2_fc_RC"], units, inpfilter=inpfilter + ) + + vars["Electricity|Dry Cooling"] = ( + _gas_air + _coal_air + _oil_air + _bio_air + _geo_air + _solar_air + ) + + vars["Electricity|Once Through"] = ( + _gas_ot_fresh_wo_CCS + + _gas_ot_fresh_w_CCS + + _coal_ot_fresh_wo_CCS + + _coal_ot_fresh_w_CCS + + _oil_ot_fresh_wo_CCS + + _bio_ot_fresh_wo_CCS + + _bio_ot_fresh_w_CCS + + _geo_ot_fresh_wo_CCS + + _nuc_ot_fresh_wo_CCS + + _solar_ot_fresh_wo_CCS + ) + + vars["Electricity|Sea Cooling"] = ( + _gas_ot_saline_wo_CCS + + _gas_ot_saline_w_CCS + + _coal_ot_saline_wo_CCS + + _coal_ot_saline_w_CCS + + _oil_ot_saline_wo_CCS + + _bio_ot_saline_wo_CCS + + _bio_ot_saline_w_CCS + + _geo_ot_saline_wo_CCS + + _nuc_ot_saline_wo_CCS + + _solar_ot_saline_wo_CCS + ) + + vars["Electricity|Wet Tower"] = ( + _gas_cl_fresh_wo_CCS + + _gas_cl_fresh_w_CCS + + _coal_cl_fresh_wo_CCS + + _coal_cl_fresh_w_CCS + + _oil_cl_fresh_wo_CCS + + _bio_cl_fresh_wo_CCS + + _bio_cl_fresh_w_CCS + + _geo_cl_fresh_wo_CCS + + _nuc_cl_fresh_wo_CCS + + _solar_cl_fresh_wo_CCS + ) + + # ---------- + # Extraction + # ---------- + + vars["Extraction|Coal"] = pp.inp( + ["coal_extr", "coal_extr_ch4", "lignite_extr"], units, inpfilter=inpfilter + ) + + vars["Extraction|Gas"] = pp.inp( + [ + "gas_extr_1", + "gas_extr_2", + "gas_extr_3", + "gas_extr_4", + "gas_extr_5", + "gas_extr_6", + "gas_extr_7", + "gas_extr_8", + ], + units, + inpfilter=inpfilter, + ) + + vars["Extraction|Oil"] = pp.inp( + [ + "oil_extr_1", + "oil_extr_2", + "oil_extr_3", + "oil_extr_4", + "oil_extr_5", + "oil_extr_6", + "oil_extr_7", + "oil_extr_8", + "oil_extr_1_ch4", + "oil_extr_2_ch4", + "oil_extr_3_ch4", + "oil_extr_4_ch4", + ], + units, + inpfilter=inpfilter, + ) + + vars["Extraction|Uranium"] = pp.inp("Uran_extr", units, inpfilter=inpfilter) + + if method == "consumption": + vars["Extraction|Coal"] -= pp.act_emif( + ["coal_extr", "coal_extr_ch4", "lignite_extr"], + units=units, + emiffilter=emiffilter, + ) + + vars["Extraction|Gas"] -= pp.act_emif( + [ + "gas_extr_1", + "gas_extr_2", + "gas_extr_3", + "gas_extr_4", + "gas_extr_5", + "gas_extr_6", + "gas_extr_7", + "gas_extr_8", + ], + units=units, + emiffilter=emiffilter, + ) + + vars["Extraction|Oil"] -= pp.act_emif( + [ + "oil_extr_1", + "oil_extr_2", + "oil_extr_3", + "oil_extr_4", + "oil_extr_5", + "oil_extr_6", + "oil_extr_7", + "oil_extr_8", + "oil_extr_1_ch4", + "oil_extr_2_ch4", + "oil_extr_3_ch4", + "oil_extr_4_ch4", + ], + units=units, + emiffilter=emiffilter, + ) + + vars["Extraction|Uranium"] -= pp.act_emif( + "Uran_extr", units=units, emiffilter=emiffilter + ) + + # ----- + # Gases + # ----- + + vars["Gases|Biomass"] = pp.inp("gas_bio", units, inpfilter=inpfilter) + + vars["Gases|Coal"] = pp.inp("coal_gas", units, inpfilter=inpfilter) + + if method == "consumption": + vars["Gases|Biomass"] -= pp.act_emif( + "gas_bio", units=units, emiffilter=emiffilter + ) + + vars["Gases|Coal"] -= pp.act_emif( + "coal_gas", units=units, emiffilter=emiffilter + ) + + # ---- + # Heat + # ---- + + _Cooling_heat_gas = pp.inp( + [ + "gas_hpl__ot_fresh", + "gas_hpl__cl_fresh", + "gas_hpl__ot_saline", + "gas_hpl__air", + ], + units, + inpfilter=inpfilter, + ) + + _Direct_heat_gas = pp.inp("gas_hpl", units, inpfilter=inpfilter) + + vars["Heat|Gas"] = _Cooling_heat_gas + _Direct_heat_gas + + _Cooling_heat_geo = pp.inp( + [ + "geo_hpl__ot_fresh", + "geo_hpl__cl_fresh", + "geo_hpl__ot_saline", + "geo_hpl__air", + ], + units, + inpfilter=inpfilter, + ) + + _Direct_heat_geo = pp.inp("geo_hpl", units, inpfilter=inpfilter) + + vars["Heat|Geothermal"] = _Cooling_heat_geo + _Direct_heat_geo + + _Cooling_heat_bio = pp.inp( + [ + "bio_hpl__ot_fresh", + "bio_hpl__cl_fresh", + "bio_hpl__ot_saline", + "bio_hpl__air", + ], + units, + inpfilter=inpfilter, + ) + + _Direct_heat_bio = pp.inp("bio_hpl", units, inpfilter=inpfilter) + + vars["Heat|Biomass"] = _Cooling_heat_bio + _Direct_heat_bio + + vars["Heat|Coal"] = pp.inp("coal_hpl", units, inpfilter=inpfilter) + + _Cooling_heat_oil = pp.inp( + [ + "foil_hpl__ot_fresh", + "foil_hpl__cl_fresh", + "foil_hpl__ot_saline", + "foil_hpl__air", + ], + units, + inpfilter=inpfilter, + ) + + _Direct_heat_oil = pp.inp("foil_hpl", units, inpfilter=inpfilter) + + vars["Heat|Oil"] = _Cooling_heat_oil + _Direct_heat_oil + + vars["Heat|Other"] = pp.inp("po_turbine", units, inpfilter=inpfilter) + + if method == "consumption": + vars["Heat|Gas"] = vars["Heat|Gas"] - pp.act_emif( + [ + "gas_hpl__ot_fresh", + "gas_hpl__cl_fresh", + "gas_hpl__ot_saline", + "gas_hpl__air", + ], + units=units, + emiffilter=emiffilter, + ) + + vars["Heat|Geothermal"] -= pp.act_emif( + [ + "geo_hpl__ot_fresh", + "geo_hpl__cl_fresh", + "geo_hpl__ot_saline", + "geo_hpl__air", + ], + units=units, + emiffilter=emiffilter, + ) + + vars["Heat|Biomass"] -= pp.act_emif( + [ + "bio_hpl__ot_fresh", + "bio_hpl__cl_fresh", + "bio_hpl__ot_saline", + "bio_hpl__air", + ], + units=units, + emiffilter=emiffilter, + ) + + vars["Heat|Coal"] -= pp.act_emif("coal_hpl", units=units, emiffilter=emiffilter) + + vars["Heat|Oil"] -= pp.act_emif( + [ + "foil_hpl__ot_fresh", + "foil_hpl__cl_fresh", + "foil_hpl__ot_saline", + "foil_hpl__air", + ], + units=units, + emiffilter=emiffilter, + ) + + vars["Heat|Other"] -= pp.act_emif( + "po_turbine", units=units, emiffilter=emiffilter + ) + + # -------- + # Hydrogen + # -------- + + hydrogen_outputfilter = {"level": ["secondary"], "commodity": ["hydrogen"]} + + vars["Hydrogen|Biomass|w/ CCS"] = _out_div_eff( + "h2_bio_ccs", group, inpfilter, hydrogen_outputfilter + ) + + vars["Hydrogen|Biomass|w/o CCS"] = _out_div_eff( + "h2_bio", group, inpfilter, hydrogen_outputfilter + ) + + vars["Hydrogen|Coal|w/ CCS"] = _out_div_eff( + "h2_coal_ccs", group, inpfilter, hydrogen_outputfilter + ) + + vars["Hydrogen|Coal|w/o CCS"] = _out_div_eff( + "h2_coal", group, inpfilter, hydrogen_outputfilter + ) + + vars["Hydrogen|Gas|w/ CCS"] = _out_div_eff( + "h2_smr_ccs", group, inpfilter, hydrogen_outputfilter + ) + + vars["Hydrogen|Gas|w/o CCS"] = _out_div_eff( + "h2_smr", group, inpfilter, hydrogen_outputfilter + ) + + vars["Hydrogen|Electricity"] = _out_div_eff( + "h2_elec", group, inpfilter, hydrogen_outputfilter + ) + + if method == "consumption": + vars["Hydrogen|Biomass|w/ CCS"] -= pp.act_emif( + "h2_bio_ccs", units=units, emiffilter=emiffilter + ) + + vars["Hydrogen|Biomass|w/o CCS"] -= pp.act_emif( + "h2_bio", units=units, emiffilter=emiffilter + ) + + vars["Hydrogen|Coal|w/ CCS"] -= pp.act_emif( + "h2_coal_ccs", units=units, emiffilter=emiffilter + ) + + vars["Hydrogen|Coal|w/o CCS"] -= pp.act_emif( + "h2_coal", units=units, emiffilter=emiffilter + ) + + vars["Hydrogen|Gas|w/ CCS"] -= pp.act_emif( + "h2_smr_ccs", units=units, emiffilter=emiffilter + ) + + vars["Hydrogen|Gas|w/o CCS"] -= pp.act_emif( + "h2_smr", units=units, emiffilter=emiffilter + ) + + vars["Hydrogen|Electricity"] -= pp.act_emif( + "h2_elec", units=units, emiffilter=emiffilter + ) + + # -------------------- + # Irrigation water use + # -------------------- + + vars["Irrigation"] = ( + pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Water|Withdrawal|Irrigation"], + } + ) + * 0.001 + ) + + vars["Irrigation|Cereals"] = ( + pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Water|Withdrawal|Irrigation|Cereals"], + } + ) + * 0.001 + ) + + vars["Irrigation|Oilcrops"] = ( + pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Water|Withdrawal|Irrigation|Oilcrops"], + } + ) + * 0.001 + ) + + vars["Irrigation|Sugarcrops"] = ( + pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Water|Withdrawal|Irrigation|Sugarcrops"], + } + ) + * 0.001 + ) + + if method == "consumption": + vars["Irrigation"] = pp_utils._make_zero() + vars["Irrigation|Cereals"] = pp_utils._make_zero() + vars["Irrigation|Oilcrops"] = pp_utils._make_zero() + vars["Irrigation|Sugarcrops"] = pp_utils._make_zero() + + # ----------- + # Bio ethanol + # ----------- + + bio_liquid_outfilter = {"level": ["primary"], "commodity": ["ethanol"]} + + vars["Liquids|Biomass|w/ CCS"] = _out_div_eff( + ["eth_bio_ccs", "liq_bio_ccs"], group, inpfilter, bio_liquid_outfilter + ) + + vars["Liquids|Biomass|w/o CCS"] = _out_div_eff( + ["eth_bio", "liq_bio"], group, inpfilter, bio_liquid_outfilter + ) + + if method == "consumption": + vars["Liquids|Biomass|w/ CCS"] -= pp.act_emif( + ["eth_bio_ccs", "liq_bio_ccs"], units=units, emiffilter=emiffilter + ) + + vars["Liquids|Biomass|w/o CCS"] -= pp.act_emif( + ["eth_bio", "liq_bio"], units=units, emiffilter=emiffilter + ) + + # --------------------------- + # Synthetic liquids from coal + # --------------------------- + + # Note OFR 20210731: The filter below seems to be incorrect and hence was + # corrected. + # syn_liquid_outfilter = {"level": ["primary"], + # "commodity": ["methanol"]} + + syn_liquid_outfilter = { + "level": ["secondary"], + "commodity": ["methanol", "lightoil"], + } + + vars["Liquids|Coal|w/ CCS"] = _out_div_eff( + ["meth_coal_ccs", "syn_liq_ccs"], group, inpfilter, syn_liquid_outfilter + ) + + vars["Liquids|Coal|w/o CCS"] = _out_div_eff( + ["meth_coal", "syn_liq"], group, inpfilter, syn_liquid_outfilter + ) + + vars["Liquids|Gas|w/ CCS"] = _out_div_eff( + ["meth_ng_ccs"], group, inpfilter, syn_liquid_outfilter + ) + + vars["Liquids|Gas|w/o CCS"] = _out_div_eff( + ["meth_ng"], group, inpfilter, syn_liquid_outfilter + ) + + if method == "consumption": + vars["Liquids|Coal|w/ CCS"] -= pp.act_emif( + ["meth_coal_ccs", "syn_liq_ccs"], units=units, emiffilter=emiffilter + ) + + vars["Liquids|Coal|w/o CCS"] -= pp.act_emif( + ["meth_coal", "syn_liq"], units=units, emiffilter=emiffilter + ) + + vars["Liquids|Gas|w/ CCS"] -= pp.act_emif( + "meth_ng_ccs", units=units, emiffilter=emiffilter + ) + + vars["Liquids|Gas|w/o CCS"] -= pp.act_emif( + "meth_ng", units=units, emiffilter=emiffilter + ) + + # ---------- + # Refineries + # ---------- + + loil_filter = {"level": ["secondary"], "commodity": ["lightoil"]} + foil_filter = {"level": ["secondary"], "commodity": ["fueloil"]} + + vars["Liquids|Oil"] = _out_div_eff( + ["ref_lol", "ref_hil"], group, inpfilter, loil_filter + ) + _out_div_eff(["ref_lol", "ref_hil"], group, inpfilter, foil_filter) + + if method == "consumption": + vars["Liquids|Oil"] -= pp.act_emif( + ["ref_lol", "ref_hil"], units=units, emiffilter=emiffilter + ) + + df = pp_utils.make_outputdf(vars, units) + return df From 73051107b6f31c03f96ece13734e68b35a0f9716 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:06:56 +0100 Subject: [PATCH 02/10] Update .report.legacy.postprocess Overwrite with version from iiasa/message_data branch `dev`. --- .../report/legacy/postprocess.py | 109 ++++++------------ 1 file changed, 34 insertions(+), 75 deletions(-) diff --git a/message_ix_models/report/legacy/postprocess.py b/message_ix_models/report/legacy/postprocess.py index e0a2bbd186..f803ebc62f 100644 --- a/message_ix_models/report/legacy/postprocess.py +++ b/message_ix_models/report/legacy/postprocess.py @@ -1,4 +1,4 @@ -from . import pp_utils +import message_data.tools.post_processing.pp_utils as pp_utils class PostProcess(object): @@ -28,7 +28,7 @@ def capf(self, tec, group=["Region"]): self.ds, self.ix, "capacity_factor", tec ).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def fom(self, tec, units, group=["Region", "Vintage"], formatting="standard"): @@ -59,7 +59,7 @@ def fom(self, tec, units, group=["Region", "Vintage"], formatting="standard"): self.ds, self.ix, "fix_cost", tec, units, formatting=formatting ).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def inv_cost(self, tec, units, group=["Region"]): @@ -132,7 +132,7 @@ def rel(self, tec, relfilter, group=["Region"]): self.ds, self.ix, "relation_activity", relfilter, tec ).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def vom(self, tec, units, group=["Region", "Vintage"], formatting="standard"): @@ -164,7 +164,7 @@ def vom(self, tec, units, group=["Region", "Vintage"], formatting="standard"): self.ds, self.ix, "var_cost", tec, units, formatting=formatting ).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) # Functions which retrieve mutliple parameters and perform a mathematical @@ -223,7 +223,7 @@ def eff( df = df_out.divide(df_inp).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") df = df.groupby(group).sum(numeric_only=True) return df @@ -258,7 +258,7 @@ def act(self, tec, units=None, actfilter=None, group=["Region"]): self.ds, self.ix, label, actflt, units ).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def cumcap(self, tec, units="GW", group=["Region"]): @@ -295,7 +295,7 @@ def cumcap(self, tec, units="GW", group=["Region"]): if self.ix: df_hist = df_hist.unstack(level=df_hist.index.names).reset_index() df_hist = df_hist.rename(columns={"level_0": "year", 0: "value"}) - df_hist["Vintage"] = df_hist["year"] + df_hist.loc[:, "Vintage"] = df_hist.loc[:, "year"] df_hist = ( df_hist.pivot_table( values="value", @@ -308,14 +308,14 @@ def cumcap(self, tec, units="GW", group=["Region"]): else: df_hist = df_hist.reset_index() if "Vintage" in df_hist.columns: - df_hist["Vintage"] = df_hist["Vintage"].astype("object") + df_hist.loc[:, "Vintage"] = df_hist.loc[:, "Vintage"].astype("object") df_hist = df_hist.groupby(group).sum(numeric_only=True) label = "CAP_NEW" if self.ix else "historical_new_capacity" df = pp_utils._retr_nic_data(self.ds, self.ix, label, tec, units) df = df.reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") df = df.groupby(group).sum(numeric_only=True) df = df[[int(y) for y in df.columns if int(y) >= fyear]] df = df_hist.add(df, fill_value=0) @@ -364,7 +364,7 @@ def nic(self, tec, units="GW/yr", group=["Region"]): if self.ix: df_hist = df_hist.unstack(level=df_hist.index.names).reset_index() df_hist = df_hist.rename(columns={"level_0": "year", 0: "value"}) - df_hist["Vintage"] = df_hist["year"] + df_hist.loc[:, "Vintage"] = df_hist.loc[:, "year"] df_hist = ( df_hist.pivot_table( values="value", @@ -377,14 +377,14 @@ def nic(self, tec, units="GW/yr", group=["Region"]): else: df_hist = df_hist.reset_index() if "Vintage" in df_hist.columns: - df_hist["Vintage"] = df_hist["Vintage"].astype("object") + df_hist.loc[:, "Vintage"] = df_hist.loc[:, "Vintage"].astype("object") df_hist = df_hist.groupby(group).sum(numeric_only=True) label = "CAP_NEW" if self.ix else "historical_new_capacity" df = pp_utils._retr_nic_data(self.ds, self.ix, label, tec, units) df = df.reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") df = df.groupby(group).sum(numeric_only=True) df = df[[int(y) for y in df.columns if int(y) >= fyear]] df = df_hist.add(df, fill_value=0) @@ -413,7 +413,7 @@ def tic(self, tec, units="GW", group=["Region"]): df = pp_utils._retr_tic_data(self.ds, self.ix, label, tec, units) df = df.reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") df = df.groupby(group).sum(numeric_only=True) return df @@ -440,7 +440,7 @@ def extr(self, tec, units, group=["Region"]): if self.ix: df_extr = pp_utils._retr_extr_data(self.ds, self.ix, label, tec, units) for yr in range(1990, 2015, 5): - df_extr[yr] = 0.0 + df_extr.loc[:, yr] = 0.0 df_hist = pp_utils._retr_histextr_data( self.ds, self.ix, "historical_extraction", tec, units ) @@ -450,7 +450,7 @@ def extr(self, tec, units, group=["Region"]): self.ds, self.ix, label, tec, units ).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def extrc(self, tec, units, group=["Region"]): @@ -477,18 +477,18 @@ def extrc(self, tec, units, group=["Region"]): if self.ix: df_extr = pp_utils._retr_extr_data(self.ds, self.ix, label, tec, units) for yr in range(1990, 2015, 5): - df_extr[yr] = 0.0 + df_extr.loc[:, yr] = 0.0 df_hist = pp_utils._retr_histextr_data( self.ds, self.ix, "historical_extraction", tec, units ) for yr in range(1990, 2005, 5): - df_hist[yr] = 0.0 + df_hist.loc[:, yr] = 0.0 df = df_extr.add(df_hist, fill_value=0) else: df = pp_utils._retr_extr_data(self.ds, self.ix, label, tec, units) df = df.reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def cprc(self, units=None, group=["Region"]): @@ -683,7 +683,7 @@ def out(self, tec, units=None, actfilter=None, outfilter=None, group=["Region"]) df_out = pp_utils._retr_io_data(self.ds, self.ix, "output", outflt) df = df_act.multiply(df_out).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def inp( @@ -733,7 +733,7 @@ def inp( df_inp = pp_utils._retr_io_data(self.ds, self.ix, "input", inpflt) df = df_act.multiply(df_inp).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def emi( @@ -785,7 +785,7 @@ def emi( ) df = df_act.multiply(df_emi, fill_value=0).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def act_rel(self, tec, relfilter, units=None, actfilter=None, group=["Region"]): @@ -827,18 +827,10 @@ def act_rel(self, tec, relfilter, units=None, actfilter=None, group=["Region"]): ) df = df_act.multiply(df_rel).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) - def act_emif( - self, - tec, - emiffilter, - units=None, - actfilter=None, - group=["Region"], - emi_units=None, - ): + def act_emif(self, tec, emiffilter, units=None, actfilter=None, group=["Region"]): """Wrapper function retrieving activity * relation coefficient for a given technology or list of technologies @@ -846,7 +838,7 @@ def act_emif( ---------- tec : string or list name of emission relation - emiffilter : dictionary + wtrfilter : dictionary filters specific to emission_factor tables units : string see unit doc @@ -854,8 +846,6 @@ def act_emif( filters specific to 'reference_activity' or 'ACT' tables group : list (optional, standard = ['Region']) defines for which indices the data should returned - emi_units : string - see unit doc Returns ------- @@ -874,11 +864,11 @@ def act_emif( ) df_act = pp_utils._retr_act_data(self.ds, self.ix, label, actflt, units) df_emif = pp_utils._retr_emif_data( - self.ds, self.ix, "emission_factor", emiflt, tec, emi_units + self.ds, self.ix, "emission_factor", emiflt, tec ) df = df_act.multiply(df_emif).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def investment(self, tec, units, group=["Region"]): @@ -905,18 +895,18 @@ def investment(self, tec, units, group=["Region"]): self.ds, self.ix, label, tec, "GWa" ).reset_index() if "Vintage" in df_nic.columns: - df_nic["Vintage"] = df_nic["Vintage"].astype("object") + df_nic.loc[:, "Vintage"] = df_nic.loc[:, "Vintage"].astype("object") df_nic = df_nic.groupby(["Region", "Technology"]).sum(numeric_only=True) df_capcost = pp_utils._retr_capcost_data( self.ds, "inv_cost", tec, units ).reset_index() if "Vintage" in df_capcost.columns: - df_capcost["Vintage"] = df_capcost["Vintage"].astype("object") + df_capcost.loc[:, "Vintage"] = df_capcost.loc[:, "Vintage"].astype("object") df_capcost = df_capcost.groupby(["Region", "Technology"]).sum(numeric_only=True) df = df_nic.multiply(df_capcost).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def tic_fom(self, tec, units, group=["Region"]): @@ -953,7 +943,7 @@ def tic_fom(self, tec, units, group=["Region"]): ) df = df_tic.multiply(df_fom).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) def act_vom(self, tec, units, actfilter=None, group=["Region"]): @@ -987,7 +977,7 @@ def act_vom(self, tec, units, actfilter=None, group=["Region"]): ) df = df_act.multiply(df_vom).reset_index() if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].astype("object") + df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") return df.groupby(group).sum(numeric_only=True) # Functions that retrieve Land-use specific results @@ -1035,7 +1025,7 @@ def land_use(self, lu_use_filter, group=["Region"], units=None): df_land = pp_utils._retr_land_act(self.ds) df_use = pp_utils._retr_land_use(self.ds, lu_use_filter, units) df = df_land.multiply(df_use, fill_value=0).reset_index() - return df.groupby(group).sum() + return df.groupby(group).sum(numeric_only=True) def land_emi(self, tec, group=["Region"], units=None): """Wrapper function to retrieve land-use technology specific emissions @@ -1058,35 +1048,4 @@ def land_emi(self, tec, group=["Region"], units=None): df_land = pp_utils._retr_land_act(self.ds) df_emi = pp_utils._retr_land_emission(self.ds, tec, units) df = df_land.multiply(df_emi, fill_value=0).reset_index() - return df.groupby(group).sum() - - def retrieve_lu_price(self, tec, scale_tec, y0=2005): - """Wrapper function to retrieve indexed-prices. - - Indexed prices have a set of three formuals inorder to derive the sum across - regions. Therefore this wrapper function will retrieve the regional prices - and the quantity information, which is the processed together in a last step - in order to derive the World value. - - Parameters - ---------- - tec : string - `land_output` price commodity name - scale_tec : string - `land_output` quantity commodity name - y0 : int (default=2005) - initial indexing year ie. values are 1. - - Returns - ------- - df : dataframe - index: Region - """ - - price = self.land_out( - lu_out_filter={"level": ["land_use_reporting"], "commodity": [tec]} - ) - quantity = self.land_out( - lu_out_filter={"level": ["land_use_reporting"], "commodity": [scale_tec]} - ) - return pp_utils.globiom_glb_priceindex(self.ds, price, quantity, scale_tec, y0) + return df.groupby(group).sum(numeric_only=True) From 54723b5e3cc1fd1d8027866e7a4ae7ac8a031f27 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:08:13 +0100 Subject: [PATCH 03/10] Update .report.legacy.pp_utils Overwrite with version from iiasa/message_data branch `dev`. --- message_ix_models/report/legacy/pp_utils.py | 663 ++++++-------------- 1 file changed, 195 insertions(+), 468 deletions(-) mode change 100644 => 100755 message_ix_models/report/legacy/pp_utils.py diff --git a/message_ix_models/report/legacy/pp_utils.py b/message_ix_models/report/legacy/pp_utils.py old mode 100644 new mode 100755 index bb165b1289..b9147bb374 --- a/message_ix_models/report/legacy/pp_utils.py +++ b/message_ix_models/report/legacy/pp_utils.py @@ -1,19 +1,20 @@ # -*- coding: utf-8 -*- -import glob import inspect import itertools -import os +import logging import sys from functools import cmp_to_key +from itertools import product import numpy as np import pandas as pd +from message_ix_models.util import private_data_path from pandas.api.types import is_numeric_dtype -from message_ix_models.util import package_data_path -from message_ix_models.util.compat.message_data import utilities +import message_data.tools.post_processing.iamc_tree as iamc_tree +import message_data.tools.utilities.utilities as utilities -from . import iamc_tree +log = logging.getLogger(__name__) all_years = None years = None @@ -41,6 +42,14 @@ ] +def _vintage_dtype(df: pd.DataFrame) -> pd.DataFrame: + """Ensure "Vintage" column has integer values and :class:`object` dtype.""" + try: + return df.assign(Vintage=lambda _df: _df["Vintage"].astype(int).astype(object)) + except KeyError: + return df + + def combineDict(*args): result = {} for dic in args: @@ -55,17 +64,19 @@ def combineDict(*args): def fil(df, fil, factor, unit_out=None): """Uses predefined values from a fil file""" - inf = os.path.join( - package_data_path(), "report", "legacy", "fil_files", "*-{}.fil".format(fil) - ) - files = glob.glob(inf) + fil_path = private_data_path("report", "fil_files") + files = fil_path.glob("*-HFC_fac.fil") dfs = [] + + # Set of short region IDs matching file names to be included + match = set(regions.values()) | {"GLB"} + for f in files: - reg = os.path.basename(f).split("-")[0] - # Ensure that fil-files are only read for regions contained in the scenario. - if reg in [regions[r] for r in regions.keys()]: - dftmp = pd.read_csv(f) - dftmp["Region"] = reg + reg = f.name.split("-")[0] + if reg in match: + infil = fil_path / f.name + dftmp = pd.read_csv(infil, comment="#") + dftmp.loc[:, "Region"] = reg dfs.append(dftmp) df_fil = pd.concat(dfs, sort=True) df_fil.Region = df_fil.Region.map( @@ -86,10 +97,10 @@ def fil(df, fil, factor, unit_out=None): df_fil.loc[reg] = df_fil.loc[reg].interpolate(method="index") df = df_fil.fillna(0) * df.fillna(0) - df["Unit"] = "???" + df.loc[:, "Unit"] = "???" if unit_out: for col in numcols(df): - df[col] = df.apply( + df.loc[:, col] = df.apply( lambda row: row[col] * unit_conversion[row.Unit][unit_out], axis=1 ) df = df.drop("Unit", axis=1) @@ -181,79 +192,6 @@ def rem_reg(df): return df.sort_index() -def sum_reg(df): - """Overwrites glb values with the sum over regions - - Parameters - ---------- - df : dataframe - - Returns - ------- - df : dataframe - """ - df_tmp = df.copy().reset_index() - vals = df_tmp[df_tmp.Region != "World"].set_index(["Region"]).sum().values - df.loc["World"] = vals - return df - - -def globiom_glb_priceindex(ds, price, quantity, quanity_variable, y0=2005): - """Calculate the global price-index. - - The following formulat is used and is based on the approach used in GLOBIOM. - - Price-Index values are recalcualted using the following formula: - Laspeyres index = sum(r, p1*q0) / sum(r, q0) - Paasche index = sum(r, p1*q1) / sum(r, q1) - Fisher index = (laaspreyes * paasche) ** 0.5 - where, p=price, q=quantity produced, r=region - - Parameters - ---------- - ds : :class:`message_ix.Scenario` - price : :class:`pandas.DataFrame` - Regional price information. - quantity : :class:`pandas.DataFrame` - Regional quantity information. - quantity_variable : string - Name of the `land_output` commodity used to derive quantities. - y0 : int (default=2005) - initial indexing year ie. values are 1. - """ - - # Retrive values for reference index year. - q0 = quantity[y0] - - # Ensure that there are values for the year "y0", otherwise retrive these from - # the parameter land-output. - if q0.sum() == 0: - # Retrieve regional data from parameter - q0 = ds.par( - "land_output", - filters={ - "commodity": quanity_variable, - "land_scenario": ds.set("land_scenario")[0], - "year": y0, - }, - ) - # Re-Format - q0 = q0.drop(["land_scenario", "commodity", "level", "time", "unit"], axis=1) - q0 = q0.rename(columns={"node": "Region"}) - q0.Region = q0.Region.map(regions) - q0 = q0.pivot(index="Region", columns="year", values="value")[y0] - - las = price.multiply(q0, axis=0).sum().divide(q0.sum()) - - paas = price.multiply(quantity).sum().divide(quantity.sum()).fillna(0) - - fischer = (las * paas) ** 0.5 - - quantity.loc["World"] = fischer - - return quantity - - def numcols(df): """Retrieves numeric columns of a dataframe. @@ -309,19 +247,21 @@ def _convert_units(df, unit_out): df : dataframe """ - cols = [c for c in numcols(df) if c != "Vintage"] - if cols: - try: - df[cols] = df[cols].multiply( - df["Unit"].apply(lambda x: unit_conversion[x][unit_out]), axis=0 - ) - except Exception: - print( - f"No unit conversion factor found to convert {df['Unit'].unique()[0]} to {unit_out}" - ) - df.Unit = unit_out + dfs = pd.DataFrame() + units = df["Unit"].unique() + idx = [i for i in df.columns if i in index_order] + df = df.set_index(idx) - return df.sort_index() + # If there are no values to convert, return original dataframe + if df.empty: + return df.reset_index() + + # Itterate over the units and apply conversion + for u in units: + tmp = df.xs(u, level="Unit", drop_level=False) * unit_conversion[u][unit_out] + dfs = pd.concat([dfs, tmp.reset_index().assign(Unit=unit_out)]) + + return dfs def cum_vals(df): @@ -434,23 +374,66 @@ def gen_GLB(df, param, weighted_by): """ df = df.fillna(value=0).reset_index() - idx = ["Model", "Scenario", "Variable", "Unit"] if param == "sum": - df_tmp = df.groupby(idx).sum(numeric_only=True).reset_index() + df_tmp = ( + df.groupby( + [ + "Model", + "Scenario", + "Variable", + "Unit", + ] + ) + .sum(numeric_only=True) + .reset_index() + ) elif param == "max": - df_tmp = df.groupby(idx).max().reset_index() + df_tmp = ( + df.groupby( + [ + "Model", + "Scenario", + "Variable", + "Unit", + ] + ) + .max(numeric_only=True) + .reset_index() + ) elif param == "mean": # Global region needs to be dropped or else it is used for calcualting # the mean df_tmp = df[df.Region != "World"] - df_tmp = df_tmp.groupby(idx).mean(numeric_only=True).reset_index() + df_tmp = ( + df_tmp.groupby( + [ + "Model", + "Scenario", + "Variable", + "Unit", + ] + ) + .mean(numeric_only=True) + .reset_index() + ) elif param == "weighted_avg": weighted_by = weighted_by.reset_index() weighted_by = weighted_by[weighted_by.Region != "World"].set_index("Region") df_tmp = df[df.Region != "World"] df_tmp = df_tmp.set_index(iamc_idx) df_tmp = df_tmp * weighted_by - df_tmp = df_tmp.reset_index().groupby(idx).sum(numeric_only=True) + df_tmp = ( + df_tmp.reset_index() + .groupby( + [ + "Model", + "Scenario", + "Variable", + "Unit", + ] + ) + .sum(numeric_only=True) + ) df_tmp = (df_tmp / weighted_by.sum()).fillna(0).reset_index() df_tmp.loc[:, "Region"] = globalname @@ -521,42 +504,35 @@ def _make_emptydf(tec, vintage=None, grade=None, units="GWa"): index: Region, Technology, Vintage (optional), Mode, Unit, Grade (optional) """ + # Columns of the resulting data frame + columns = ["Region", "Unit"] + if vintage: + columns.extend(["Technology", "Vintage", "Mode"]) + elif grade: + columns.extend(["Commodity", "Grade"]) + else: + columns.extend(["Technology", "Mode"]) + + # Common values for all rows + data = { + "Region": list(regions.keys()), + "Grade": "a", + "Mode": "M1", + "Unit": units, + "Vintage": firstmodelyear, + } dfs = [] for t in tec: - if vintage: - df = pd.DataFrame( - data={ - "Region": list(regions.keys()), - "Technology": t, - "Vintage": firstmodelyear, - "Mode": "M1", - "Unit": units, - } - ) - elif grade: - df = pd.DataFrame( - data={ - "Region": list(regions.keys()), - "Commodity": t, - "Unit": units, - "Grade": "a", - } - ) - else: - df = pd.DataFrame( - data={ - "Region": list(regions.keys()), - "Technology": t, - "Mode": "M1", - "Unit": units, - } - ) - dfs.append(df) - df = pd.concat(dfs, sort=True) - if "Vintage" in df.columns: - df.loc[:, "Vintage"] = df.loc[:, "Vintage"].astype("object") - return df.sort_index() + data.update(Commodity=t, Technology=t) + dfs.append(pd.DataFrame({k: v for k, v in data.items() if k in columns})) + + # Concatenate `dfs`, or create an empty data frame + return ( + (pd.concat(dfs, sort=True) if len(dfs) else pd.DataFrame([], columns=columns)) + .pipe(_vintage_dtype) + .sort_index() + ) def _make_zero(): @@ -577,9 +553,8 @@ def _make_zero(): return df.sort_index() -def _clean_up_regions(df, units=None): - """Converts region names from the database format into desired - reporting region names. +def _clean_up_regions(df: pd.DataFrame, units=None) -> pd.DataFrame: + """Convert region names in `df` using `regions`. Should a region be missing, it is appended to the dataframe. values = 0. @@ -588,318 +563,79 @@ def _clean_up_regions(df, units=None): Parameters ---------- df : dataframe - units : string(optional, default=None) - inserts units into unit column which may be required for unit - conversion + units : str, optional + Value for "Unit" column e.g. as required for unit conversion. Returns ------- - df : dataframe + pandas.DataFrame index: Region, Technology(optional) or Commodity(optional), Unit, Vintage(optional), Mode, Grade(optional) """ - if not units: - if "Unit" in df.columns: - df_unit = df.Unit.unique() - if len(df_unit) > 1: - if verbose: - print( - ( - inspect.stack()[0][3], - ": there are more than 1", - "unit in the dataframe:", - df_unit, - ) - ) - units = "GWa" - df.Unit = "GWa" - idx = [i for i in index_order if i in df.columns.tolist()] - df = df.groupby(idx).sum(numeric_only=True).reset_index() - else: - units = df.Unit.unique()[0] - else: + if not len(df): # Empty data frame + return df + + # Placeholder for columns not present + missing = pd.Series([None], dtype=object) + + # Sorted, unique elements of certain columns; or `missing` if not present + technology = sorted(df.get("Technology", missing).unique()) + commodity = sorted(df.get("Commodity", missing).unique()) + df_regions = set(df["Region"].unique()) + df_units = sorted(df.get("Unit", missing).unique()) + + if units is None: + if len(df_units) > 1: + if verbose: + print( + f"{inspect.stack()[0][3]}: there are more than 1 unit in the " + f"dataframe: {df_units}" + ) units = "GWa" - # Makes exception if only a world value is available for the carbon price - # In MESSAGE_ix_legacy, the carbon price was returned for all regions when - # a budget is applied. In the January, 2018 version of MESSAGEix, only a - # global value is returned. - # Therefore the value needs to be replicated for all other regions. This is - # only done when there is only a single carbon price entry for the region - # 'World'. - # If there are multiple `emission`s for which a price is returned, then the - # order of the list below defines the order of presidence applied ot the - # dataframe `emission`. - if "Technology" in df.columns: - # List of different `emission`s for which a carbon price is retruned. - cprice_list = [ - "TCE", - "TCO2", - "TCE_CO2", - "TCE_non-CO2", - "TCE_FFI", - "TCE_LU", - "TCE_CO2_trade", - ] - # List of "technologies" (which are in fact the `emissions`) - teclist = df.Technology.unique().tolist() - - # Check if there is a match of the two lists - matchlist = [c for c in teclist if c in cprice_list] - if matchlist: - # Filter out priority from cprice_list - df = df.loc[df.Technology == matchlist[0]] - - if df.Region.unique().tolist() == ["World"]: - # Filter out a single carbon_price - dfs = [] - dfs.append(df) - for reg in regions.keys(): - tmp = df.copy() - tmp.Region = reg - dfs.append(tmp) - df = pd.concat(dfs, sort=True) - - # Removes aggregate region 'WORLD' - df = df[df["Region"] != "World"] + # - Assign common units. + # - Sum rows with identical indices but distinct units. + idx = list(filter(df.columns.__contains__, index_order)) + df = df.assign(Unit=units).groupby(idx).sum().reset_index() + else: + units = df_units[0] or "GWa" - # for reg in list(regions.keys()): - for reg in regions.keys(): - if reg not in df.Region.unique(): - if "Technology" in df.columns: - for tec in df.Technology.unique(): - if "Mode" in df.columns: - if "Vintage" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array( - [[reg, tec, units, firstmodelyear, "M1"]] - ), - columns=[ - "Region", - "Technology", - "Unit", - "Vintage", - "Mode", - ], - ), - df, - ], - sort=True, - ) - else: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units, "M1"]]), - columns=[ - "Region", - "Technology", - "Unit", - "Mode", - ], - ), - df, - ], - sort=True, - ) - elif "Unit" in df.columns: - if "Vintage" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units, firstmodelyear]]), - columns=[ - "Region", - "Technology", - "Unit", - "Vintage", - ], - ), - df, - ], - sort=True, - ) - elif "Grade" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units, "a"]]), - columns=[ - "Region", - "Technology", - "Unit", - "Grade", - ], - ), - df, - ], - sort=True, - ) - else: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units]]), - columns=["Region", "Technology", "Unit"], - ), - df, - ], - sort=True, - ) - else: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec]]), - columns=["Region", "Technology"], - ), - df, - ], - sort=True, - ) + # Makes exception if only a world value is available for the carbon price. In + # MESSAGE_ix_legacy, the carbon price was returned for all regions when a budget is + # applied. In the January 2018 version of MESSAGEix, only a global value is + # returned. Therefore the value needs to be replicated for all other regions. This + # is only done when there is only a single carbon price entry for the region + # 'World'. + tce_techs = {"TCE", "TCE_FFI", "TCE_LU", "TCO2", "TCE_CO2", "TCE_non-CO2"} + if set(technology) & tce_techs and set(df.Region.unique()) == {"World"}: + df = pd.concat([df] + [df.copy().assign(Region=r) for r in regions], sort=True) - elif "Commodity" in df.columns: - for tec in df.Commodity.unique(): - if "Mode" in df.columns: - if "Vintage" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array( - [[reg, tec, units, firstmodelyear, "M1"]] - ), - columns=[ - "Region", - "Commodity", - "Unit", - "Vintage", - "Mode", - ], - ), - df, - ], - sort=True, - ) - else: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units, "M1"]]), - columns=["Region", "Commodity", "Unit", "Mode"], - ), - df, - ], - sort=True, - ) - else: - if "Vintage" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units, firstmodelyear]]), - columns=[ - "Region", - "Commodity", - "Unit", - "Vintage", - ], - ), - df, - ], - sort=True, - ) - elif "Grade" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units, "a"]]), - columns=[ - "Region", - "Commodity", - "Unit", - "Grade", - ], - ), - df, - ], - sort=True, - ) - else: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, tec, units]]), - columns=["Region", "Commodity", "Unit"], - ), - df, - ], - sort=True, - ) - else: - if "Mode" in df.columns: - if "Vintage" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, units, firstmodelyear, "M1"]]), - columns=["Region", "Unit", "Vintage", "Mode"], - ), - df, - ], - sort=True, - ) - else: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, units, "M1"]]), - columns=["Region", "Unit", "Mode"], - ), - df, - ], - sort=True, - ) - else: - if "Vintage" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, units, firstmodelyear]]), - columns=["Region", "Unit", "Vintage"], - ), - df, - ], - sort=True, - ) - elif "Grade" in df.columns: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, units, "a"]]), - columns=["Region", "Unit", "Grade"], - ), - df, - ], - sort=True, - ) - else: - df = pd.concat( - [ - pd.DataFrame( - np.array([[reg, units]]), columns=["Region", "Unit"] - ), - df, - ], - sort=True, - ) + # Discard all data for aggregate region "World" + df = df[df["Region"] != "World"] - df.Region = df.Region.map(regions) - if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].apply(np.int64) - df["Vintage"] = df["Vintage"].astype("object") - return df.sort_index() + # Common data for infill + data = dict(Grade="a", Mode="M1", Unit=units, Vintage=firstmodelyear) + + # Create empty rows for Regions missing from `df`. Iterate over values for the + # Commodity and Region dimensions; if `df` does not have these, then c or t will be + # None. + new_rows = [] + for c, r, t in product( + commodity, + filter(lambda _r: _r not in df_regions, regions), + technology, + ): + # Construct and store a new row + data.update(Commodity=c, Region=r, Technology=t) + new_rows.append(pd.DataFrame(data, index=[0]).reindex(columns=df.columns)) + + # - Combine new rows with existing data. + # - Map region names for all data, including original `df`. + return ( + pd.concat(new_rows + [df], ignore_index=True) + .assign(Region=lambda _df: _df.Region.map(regions)) + .pipe(_vintage_dtype) + ) def _clean_up_vintage(ds, df, units=None): @@ -1068,9 +804,7 @@ def _clean_up_formatting(df): exception of 'Vintage' which will also be part of the index structure. the column units is dropped """ - if "Vintage" in df.columns: - df["Vintage"] = df["Vintage"].apply(np.int64) - df["Vintage"] = df["Vintage"].astype("object") + df = df.pipe(_vintage_dtype) yrs = [int(x) for x in numcols(df)] @@ -1440,7 +1174,7 @@ def _retr_tic_data(ds, ix, param, tec, units): ix : string 'True' or 'False' param : string - 'historical_new_capacity' or 'CAP' (IX only) + 'ref_capacity' or 'CAP' (IX only) tec : string or list technology name units : string @@ -1491,7 +1225,7 @@ def _retr_extr_data(ds, ix, param, tec, units): ix : string 'True' or 'False' param: string - 'historical_extraction' or 'EXT' (IX only) + 'ref_extraction' or 'EXT' (IX only) tec: string or list extraction technology name units: string @@ -1624,7 +1358,7 @@ def _retr_nic_data(ds, ix, param, tec, units): ix : string 'True' or 'False' param : string - 'historical_new_capacity' or 'CAP_NEW' (IX only) + 'ref_new_capacity' or 'CAP_NEW' (IX only) tec : string or list technology name units : string @@ -1640,7 +1374,7 @@ def _retr_nic_data(ds, ix, param, tec, units): if ix: df = ds.var(param, filter) # add a column "year_act" - df["year_act"] = df["year_vtg"] + df.loc[:, "year_act"] = df.loc[:, "year_vtg"] if df.empty: if verbose: print((inspect.stack()[0][3], ": technology", tec, "dataframe empty")) @@ -1865,20 +1599,16 @@ def _retr_crb_prc(ds, units): """ # Retrieve VAR - PRICE_EMISSION - # TODO iiasa/message_ix#726 is reworking PRICE_EMISSION_NEW to become the new PRICE_EMISSION - # Adjust the name here according to what you're running on - var = "PRICE_EMISSION" - df = ds.var(var, {"type_tec": ["all"]}) + df = ds.var("PRICE_EMISSION", {"type_tec": ["all"]}) if df.empty: - if var == "PRICE_EMISSION": - df = ds.par("tax_emission", {"type_tec": ["all"]}) - type_emi = df.type_emission.unique().tolist() - if len(type_emi) > 1: - df = df.loc[df.type_emission == type_emi[0]] - print("Reporting Carbon Price for", type_emi[0]) + df = ds.par("tax_emission", {"type_tec": ["all"]}) + type_emi = df.type_emission.unique().tolist() + if len(type_emi) > 1: + df = df.loc[df.type_emission == type_emi[0]] + print("Reporting Carbon Price for", type_emi[0]) if df.empty: if verbose: - print((inspect.stack()[0][3], f": {var} dataframe empty")) + print((inspect.stack()[0][3], ": PRICE_EMISSION dataframe empty")) df = _make_emptydf("TCE", units="US$2005/tC") else: @@ -2029,7 +1759,7 @@ def _retr_demands(ds, ix, commodity, level, units): return df.sort_index() -def _retr_emif_data(ds, ix, param, emiflt, tec, units=None): +def _retr_emif_data(ds, ix, param, emiflt, tec): """Retrieves coefficient with which a technology writes into a given relation for a single or set of technolgies. @@ -2044,8 +1774,6 @@ def _retr_emif_data(ds, ix, param, emiflt, tec, units=None): filters specific to emission_factor tables tec: string or list technology name - unit : string - see unit doc Returns ------- @@ -2072,8 +1800,6 @@ def _retr_emif_data(ds, ix, param, emiflt, tec, units=None): df = _drap(df, ["value", "year_act"], drop=drop) # Clean up operations - if units: - df = _convert_units(df, units) df = _clean_up_regions(df) if ix: df = _clean_up_vintage(ds, df, units="-") @@ -2224,6 +1950,7 @@ def _retr_var_cost(ds): " check the GAMS postproccesing in MESSAGE_run!", ) ) + else: drop = ["mrg"] add = {"Technology": "COST_NODAL_NET"} @@ -2308,7 +2035,7 @@ def _retr_land_emission(ds, tec, units, convert=1): " the land emulator is being used", ) ) - df = _make_emptydf(tec) + sys.exit(1) else: drop = ["emission"] df = _drap(df, ["value", "year"], drop=drop) @@ -2353,7 +2080,7 @@ def _retr_land_output(ds, filter, units, convert=1): " the land emulator is being used", ) ) - df = _make_emptydf(filter["commodity"]) + sys.exit(1) else: drop = ["commodity", "level", "time"] df = _drap(df, ["value", "year"], drop=drop) From 0ecdecb60f0e54bb971bafc2a6d67c6ae74f8ba3 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:08:47 +0100 Subject: [PATCH 04/10] Update .report.legacy.iamc_tree Overwrite with version from iiasa/message_data branch `dev`. --- message_ix_models/report/legacy/iamc_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/report/legacy/iamc_tree.py b/message_ix_models/report/legacy/iamc_tree.py index 2da75afa33..19ffb5eaea 100644 --- a/message_ix_models/report/legacy/iamc_tree.py +++ b/message_ix_models/report/legacy/iamc_tree.py @@ -2,7 +2,7 @@ import pandas as pd -from message_ix_models.util.compat.message_data import utilities +from message_data.tools.utilities import utilities _col = "Variable" From 7db8a769364726cff6e5751c1558ad9739baf4ca Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:09:10 +0100 Subject: [PATCH 05/10] Update .report.legacy.iamc_report_hackathon Overwrite with version from iiasa/message_data branch `dev`. --- .../report/legacy/iamc_report_hackathon.py | 141 +++++++++++------- 1 file changed, 85 insertions(+), 56 deletions(-) mode change 100644 => 100755 message_ix_models/report/legacy/iamc_report_hackathon.py diff --git a/message_ix_models/report/legacy/iamc_report_hackathon.py b/message_ix_models/report/legacy/iamc_report_hackathon.py old mode 100644 new mode 100755 index 4aa26004a1..8c2d5d98ee --- a/message_ix_models/report/legacy/iamc_report_hackathon.py +++ b/message_ix_models/report/legacy/iamc_report_hackathon.py @@ -1,19 +1,19 @@ import logging from pathlib import Path -from typing import Optional import pandas as pd import yaml -from yaml.loader import SafeLoader from message_ix_models import Context -from message_ix_models.util import package_data_path -from message_ix_models.util.compat.message_data.get_historical_years import main as get_historical_years -from message_ix_models.util.compat.message_data.get_nodes import get_nodes -from message_ix_models.util.compat.message_data.get_optimization_years import main as get_optimization_years -from message_ix_models.util.compat.message_data.utilities import retrieve_region_mapping +from message_ix_models.util import MESSAGE_DATA_PATH, private_data_path +from yaml.loader import SafeLoader -from . import postprocess -from . import pp_utils +import message_data.tools.post_processing.postprocess as postprocess +import message_data.tools.post_processing.pp_utils as pp_utils +from message_data.tools.utilities import ( + get_historical_years, + get_nodes, + get_optimization_years, +) log = logging.getLogger(__name__) @@ -36,22 +36,22 @@ def report( lu_hist=None, verbose=False, *, - context: Optional[Context] = None, + context: Context = None, ): """Main reporting function. - This function will run reporting for specific "tables" as specified in the - configuration file `run_config`. + This function will run reporting for specific "tables" as specified in the YAML + format configuration file given by `run_config`. Outputs will be stored as an xlsx file in IAMC format for upload to a scenario database/explorer instance. - IMPORTANT!! If extending the variable template, please ensure - NOT to overwrite the existing file as this is used for global model - intercomparison projects. + Within the `run_config` file, the key ``report_config: / var_def:`` refers to a CSV- + format file containing a “variable template”. Only variables defined in the variable + template are reported. All other variables are excluded. - Only variables defined in the variable template are reported. All other - variables will be excluded. + .. warning:: If extending the variable template, **do not** overwrite the existing + file as this is used for some global model intercomparison projects. Parameters ---------- @@ -66,12 +66,16 @@ def report( Model name of the scenario in the output file. scenario_out : str (default: None) Scenario name of the scenario in the output file. - out_dir : str (default: None) - Directory where the result file should be written to. + out_dir : str or Path + Directory in which to write the Excel output file. Default: a directory named + :file:`…/reporting_output/` within the :mod:`.message_data` source tree, i.e. + directly below :data:`message_ix_models.util.MESSAGE_DATA_PATH`. :func:`report` + does not respond to the ``message local data`` configuration key for + :ref:`local data `. merge_hist : boolean (default: False) Switch to determine whether the reporting results should be merged with - already processed histroical results, which are then, additionally, stored - as Timeseries with the scenario object. + already processed historical results, which are then, additionally, stored + as time series data with the scenario object. merge_ts : boolean (default: False) Switch to use data stored as TS to overwrite results from reporting. var_def : str (default: None) @@ -90,14 +94,14 @@ def report( lu_hist : str (default: None) Historic land-use GHG emissions for regions. verbose : str (default: False) - Option whther to print onscreen messages. + Option whether to print onscreen messages. context : .Context Only the ``dry_run`` setting is respected. If :data:`True`, configuration is read, but nothing is done. """ nds0 = get_nodes(scen) nds = [n for n in nds0 if "|" not in n] - region_id, reg_ts = retrieve_region_mapping(scen, mp, include_region_id=False) + region_id = list(set([x.split("_")[0] for x in get_nodes(scen)]))[0] # -------------------- # Set global variables @@ -106,7 +110,7 @@ def report( model_nm = model_out if model_out else scen.model scen_nm = scenario_out if scenario_out else scen.scenario run_history = ref_sol - path = package_data_path("report", "legacy") + path = private_data_path() / "report" # ---------------------------- # Read reporting configuration @@ -119,7 +123,7 @@ def report( # Alternative config # If an alternative config has been defined, then ONLY # those items specified will replace or be added to the - # default config + # default condig if run_config: with open(path / run_config) as f: tmp_config = yaml.load(f, Loader=SafeLoader) @@ -136,11 +140,9 @@ def report( # Config: urban/rural share # The standard file corresponds to data for R11-SSP2 # taken from the SSP2-database. Values in %. - if urban_perc is None: - if config["report_config"]["urban_perc"] is not False: - urban_perc_data = path / config["report_config"]["urban_perc"] - else: - urban_perc_data = False + urban_perc_data = ( + path / config["report_config"]["urban_perc"] if not urban_perc else urban_perc + ) # Config: historic emissions # The standard file corresponds to data for R11-SSP1/2/3 @@ -149,17 +151,15 @@ def report( # Historic land-use emissions (CO2) correspond to # R11-SSP1/2/3 if run_history == "True": - if kyoto_hist is None: - if config["report_config"]["kyoto_hist"] is not False: - kyoto_hist_data = path / config["report_config"]["kyoto_hist"] - else: - kyoto_hist_data = False - - if lu_hist is None: - if config["report_config"]["lu_hist"] is not False: - lu_hist_data = path / config["report_config"]["lu_hist"] - else: - lu_hist_data = False + kyoto_hist_data = ( + path / config["report_config"]["kyoto_hist"] + if not kyoto_hist + else kyoto_hist + ) + + lu_hist_data = ( + path / config["report_config"]["lu_hist"] if not lu_hist else lu_hist + ) # Config: Define unit conversion factors unit_yaml = ( @@ -175,7 +175,9 @@ def report( # Config: Define which variable definition should be used var_def = path / config["report_config"]["var_def"] if not var_def else var_def - # If dry_run is requested, leave here + # Directory for output + out_dir = Path(out_dir) if out_dir else MESSAGE_DATA_PATH / "reporting_output" + if context and context.dry_run: log.info(f"(DRY RUN) Would write to {out_dir}") return @@ -245,7 +247,7 @@ def report( # Based on the default config, populate func_dict, which # has all the function required for running the reporting. - DEFAULT_table_def = "message_ix_models.report.legacy.default_tables" + DEFAULT_table_def = "message_data.tools.post_processing.default_tables" dflt_tbl = __import__(DEFAULT_table_def, fromlist=[None]) dflt_tbl.pp = pp dflt_tbl.mu = mu @@ -259,6 +261,7 @@ def report( if config["report_config"]["table_def"] != DEFAULT_table_def: tmp_tbl = __import__(config["report_config"]["table_def"], fromlist=[None]) + log.info(f"Replacement tables from {tmp_tbl!r}:") tmp_tbl.pp = pp tmp_tbl.mu = mu tmp_tbl.run_history = run_history @@ -273,6 +276,7 @@ def report( for f in tmp_func_dict: if f in func_dict: func_dict.pop(f) + log.info(f"{f} → {tmp_func_dict[f]}") func_dict[f] = tmp_func_dict[f] # -------------------- @@ -298,6 +302,20 @@ def report( # Convert dataframes to IAMC-format # --------------------------------- + if merge_hist or merge_ts: + # Create mapping for regions + # {database name: output name} + # e.g. {"Subsaharan Africa (R11)": "AFR"} + reg_ts = mp.regions() + reg_ts = reg_ts[ + reg_ts.region.isin([r for r in scen.set("node") if r != "World"]) + ] + reg_ts["region"] = reg_ts["region"].str.replace(f"{region_id}_", "") + reg_ts["region"] = reg_ts["region"].str.replace("GLB", "World") + reg_ts = ( + reg_ts[["region", "mapped_to"]].set_index("mapped_to").to_dict()["region"] + ) + mapping = pd.read_csv(aggr_def) allowed_var = pd.read_csv(var_def)["Variable"].unique().tolist() df = [] @@ -331,8 +349,9 @@ def report( if merge_ts: # Filter out timeseries entries which exist for a certain variable var = config["run_tables"][i]["root"] - tmp = ts[ts.Variable.str.find(var) >= 0] - tmp.Variable = tmp.Variable.str.replace(f"{var}|".replace("|", "\|"), "") + tmp = ts[ts.Variable.str.contains(var, regex=False)].assign( + Variable=lambda df: df.Variable.str.replace(f"{var}|", "", regex=False) + ) if not tmp.empty: dfs[i] = ( tmp.set_index(iamc_index) @@ -349,14 +368,15 @@ def report( ) else: df.append(pp_utils.iamc_it(dfs[i], run_tables[i]["root"], mapping)) + df = pd.concat(df, sort=True) # -------------- # Process output # -------------- - # Ensure that only variables included in the template are included - # in the final output + # Ensure that only variables included in the template are included in the final + # output df = df.loc[df.Variable.isin(allowed_var)] # ------------------------------- @@ -386,13 +406,25 @@ def report( int(yr) for yr in col_yr if yr >= model_year ] ix_upload = ix_upload[cols] + # ix_mp._jobj.unlockRunid(11473) + + # NB could use scen.transact() here, if it accepted timeseries_only scen.check_out(timeseries_only=True) print("Starting to upload timeseries") print(ix_upload.head()) - scen.add_timeseries(ix_upload) - print("Finished uploading timeseries") - scen.commit("Reporting uploaded as timeseries") + + ix_upload.to_csv("debug.csv") # DEBUG + + try: + scen.add_timeseries(ix_upload) + except Exception as e: + print(f"Failed: {repr(e)}") + scen.discard_changes() # Don't leave scenario in a locked state + raise + else: + print("Finished uploading timeseries") + scen.commit("Reporting uploaded as timeseries") df = scen.timeseries(iamc=True) df = df.rename( @@ -414,10 +446,7 @@ def report( if "subannual" in df.columns: df = df.drop("subannual", axis=1) - if not out_dir: - out_dir = package_data_path("report", "legacy", "reporting_output") - else: - out_dir = Path(out_dir) - if not out_dir.exists(): - out_dir.mkdir() + # Write to an Excel file in the configured output directory + out_dir.mkdir(parents=True, exist_ok=True) + pp_utils.write_xlsx(df, out_dir) From dd4182f73b1e2740a70dbc35dfe8b27bc1686415 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:09:41 +0100 Subject: [PATCH 06/10] Update .report.legacy.default_tables Overwrite with version from iiasa/message_data branch `dev`. --- .../report/legacy/default_tables.py | 1853 +++++------------ 1 file changed, 544 insertions(+), 1309 deletions(-) diff --git a/message_ix_models/report/legacy/default_tables.py b/message_ix_models/report/legacy/default_tables.py index cb1f7711d7..974b912e1f 100644 --- a/message_ix_models/report/legacy/default_tables.py +++ b/message_ix_models/report/legacy/default_tables.py @@ -1,7 +1,15 @@ +import logging +from functools import reduce +from itertools import chain +from operator import add +from typing import List, Mapping, Sequence, cast + import numpy as np import pandas as pd -from . import pp_utils +import message_data.tools.post_processing.pp_utils as pp_utils + +log = logging.getLogger(__name__) pp = None mu = None @@ -33,6 +41,91 @@ def _register(func): return func +#: Mapping from commodity IDs to shorthands used in `TECHS` and elsewhere. +COMMODITY = { + "coal": "coal", + "d_heat": "heat", + "electr": "elec", + "ethanol": "eth", + "fueloil": "foil", + "gas": "gas", + "hydrogen": "h2", + "lightoil": "loil", + "methanol": "meth", + "solar_pv": "solar", +} + + +#: Lists of technologies. +#: +#: These are used below instead of, or in addition to, lists embedded in functions. +#: Modify these lists in order to control the lists of technologies handled by those +#: functions; this avoids the need to duplicate the entire function for small +#: modifications. +#: +#: .. todo:: store these on :class:`.Context`; read them from technology codelists. +TECHS = { + "cement with ccs": ["cement_co2scr"], + "industry with ccs": [], + "gas extra": [], + # Residential and commercial + "rc back": ["back_rc"], + "rc biomass": ["biomass_rc"], + "rc coal": ["coal_rc"], + "rc heat": ["heat_rc"], + "rc elec": ["elec_rc", "hp_el_rc"], + "rc eth": ["eth_rc"], + "rc foil": ["foil_rc"], + "rc gas": ["gas_rc", "hp_gas_rc"], + "rc h2": ["h2_rc", "h2_fc_rc"], + "rc loil": ["loil_rc"], + "rc meth": ["meth_rc"], + "rc solar": ["solar_rc"], + "refining": ["ref_hil", "ref_lol"], + "se solids biomass extra": ["biomass_rc"], + "se solids coal extra": ["coal_rc"], + "synfuels oil": [ + # (technology; inp or out; filters) + ( + "ref_lol", + "out", + dict(level=["secondary"], commodity=["lightoil", "fueloil"]), + ), + ( + "ref_hil", + "out", + dict(level=["secondary"], commodity=["lightoil", "fueloil"]), + ), + ], + # Transport + "trp back": ["back_trp"], + "trp coal": ["coal_trp"], + "trp elec": ["elec_trp"], + "trp eth": ["eth_fc_trp", "eth_ic_trp"], + "trp foil": ["foil_trp"], + "trp gas": ["gas_trp"], + "trp h2": ["h2_fc_trp"], + "trp loil": ["loil_trp"], + "trp meth": ["meth_fc_trp", "meth_ic_trp"], +} + + +def get_techs(prefix: str, kinds: str) -> List[str]: + """Return a list of technologies. + + The list is assembled from entries in :data:`TECHS` with the keys + "{prefix} {value}", with one `value` for each space-separated item in `kinds`. + """ + return list( + chain( + *[ + cast(Sequence[str], TECHS.get(f"{prefix} {kind}", [])) + for kind in kinds.split() + ] + ) + ) + + def _pe_wCCSretro(tec, scrub_tec, group, inpfilter, units, share=1): """Calculates primary energy use of technologies with scrubbers. @@ -99,30 +192,22 @@ def _pe_elec_woCCSretro(tec, scrub_tec, group, inpfilter, units, _Frac, share=1) df = ( ( - ( - pp.out( - tec, - units, - outfilter={"level": ["secondary"], "commodity": ["electr"]}, - ) - * (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) - - (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) - * pp.out(scrub_tec, units) - * share - ) - / pp_utils.ppgroup( - (pp.act(tec, group=group) / pp.act(tec)).fillna(0) - * pp.eff( - tec, - inpfilter=inpfilter, - outfilter={"level": ["secondary"], "commodity": ["electr"]}, - group=group, - ) + pp.out(tec, units) + * (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + - (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + * pp.out(scrub_tec, units) + * share + ) + / pp_utils.ppgroup( + (pp.act(tec, group=group) / pp.act(tec)).fillna(0) + * pp.eff( + tec, + inpfilter=inpfilter, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + group=group, ) ) - .fillna(0) - .replace([np.inf, -np.inf], 0) - ) + ).fillna(0) return df @@ -153,29 +238,25 @@ def _pe_elec_wCCSretro(tec, scrub_tec, group, inpfilter, units, _Frac, share=1): df = ( ( - ( - (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) - * pp.out(scrub_tec, units) - * share - ) - / pp_utils.ppgroup( - (pp.act(tec, group=group) / pp.act(tec)).fillna(0) - * pp.eff( - tec, - inpfilter=inpfilter, - outfilter={"level": ["secondary"], "commodity": ["electr"]}, - group=group, - ) + (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) + * pp.out(scrub_tec, units) + * share + ) + / pp_utils.ppgroup( + (pp.act(tec, group=group) / pp.act(tec)).fillna(0) + * pp.eff( + tec, + inpfilter=inpfilter, + outfilter={"level": ["secondary"], "commodity": ["electr"]}, + group=group, ) ) - .fillna(0) - .replace([np.inf, -np.inf], 0) - ) + ).fillna(0) return df -def _se_elec_woCCSretro(tec, scrub_tec, units, _Frac, share=1, outfilter=None): +def _se_elec_woCCSretro(tec, scrub_tec, units, _Frac, share=1): """Calculates secondary energy electricity generation. This applies to technologies WITHOUT scrubbers. @@ -193,12 +274,10 @@ def _se_elec_woCCSretro(tec, scrub_tec, units, _Frac, share=1, outfilter=None): Regional share of actual cogeneration (po_turbine). share : number or dataframe (default: 1) Share of `tec` activity if multiple technologies share the same scrubber. - outfilter : dict - `level` and/or `commodity` used for retrieving the output. """ df = ( - pp.out(tec, units, outfilter=outfilter) + pp.out(tec, units) * (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) - (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) * pp.out(scrub_tec, units) @@ -237,7 +316,7 @@ def _se_elec_wCCSretro(tec, scrub_tec, units, _Frac, share=1): return df -def _pe_elec_po_turb(tec, group, units, _Frac, inpfilter, outfilter): +def _pe_elec_po_turb(tec, group, units, _Frac, inpfilter): """Calculates primary energy electricity equivalent generation. This calcualtes the amount of electricity used in primary energy equivalent used @@ -256,17 +335,15 @@ def _pe_elec_po_turb(tec, group, units, _Frac, inpfilter, outfilter): Regional share of actual cogeneration (po_turbine). inpfilter : dict `level` and/or `commodity` used for retrieving the input. - outfilter : dict - `level` and/or `commodity` used for retrieving the output. """ df = pp_utils.ppgroup( ( ( - pp.out(tec, units, outfilter=outfilter, group=group) + pp.out(tec, units, group=group) * (1.0 - pp.rel(tec, relfilter={"relation": ["pass_out_trb"]}) * _Frac) ) - / pp.eff(tec, inpfilter=inpfilter, outfilter=outfilter, group=group) + / pp.eff(tec, inpfilter=inpfilter, group=group) ).fillna(0) ) @@ -350,7 +427,7 @@ def _out_div_eff(tec, group, inpfilter, outfilter): dfs.append( pp_utils.ppgroup( (pp.out(t, outfilter=outfilter, group=group)) - / pp.eff(t, inpfilter=inpfilter, outfilter=outfilter, group=group) + / pp.eff(t, inpfilter=inpfilter, group=group) ) ) df = pd.concat(dfs, sort=True) @@ -358,51 +435,6 @@ def _out_div_eff(tec, group, inpfilter, outfilter): return df.groupby(df.index.name).sum() -def _calc_scrubber_capacity( - prmfunc, tec, group, share=1, cumulative=False, efficiency=True -): - """Calculate the capacity of CO2-retrofit scrubbers. - - Parameters - ---------- - prmfunc : pp_utils.py function - Function which retrieves data for `tec` - tec : string - CO2-scrubber technology name - group : list - List of index names by which to group various prameters - share : :class:`pandas.DataFrame()` - Share values to be applied to data; must be indexed by `group`. - cumulative : boolean - Switch to indicate if values are being cumulated over time or not. - efficiency : boolemna - Switch whether the data should be corrected by the efficiency factor of `tec`. - """ - - # Retrieve vintage specific technology parameter - df = prmfunc(tec, group=group) - - # Retrieve efficiency - eff = ( - 1 - if efficiency is False - else (1 - 1 / pp.eff(tec, group=group)).replace([np.inf, -np.inf], 0) - ) - - # When cumulative values are being calculated, then it is necessary - # to extend vintage specific parameters for all activity years, independent - # if the technology is active or not, becuse the parameter e.g. CAP-Additions - # are also returned in the same way. - if cumulative and type(eff) != int: - eff = eff.replace(0, np.nan).ffill(axis=1).fillna(0) - - df = df * share * eff - - df = df.groupby("Region").sum(numeric_only=True) - - return df - - # ------------------- # Reporting functions # ------------------- @@ -536,63 +568,49 @@ def retr_agri_dem(units): vars["Energy|Crops|1st generation"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Energy|Crops|1st generation"], + "commodity": ["Agricultural Demand|Energy|Crops|" "1st generation"], } ) vars["Energy|Crops|2nd generation"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Energy|Crops|2nd generation"], + "commodity": ["Agricultural Demand|Energy|Crops|" "2nd generation"], } ) vars["Non-Energy|Crops|Feed"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Non-Energy|Crops|Feed"], + "commodity": ["Agricultural Demand|Non-Energy|" "Crops|Feed"], } ) vars["Non-Energy|Crops|Food"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Non-Energy|Crops|Food"], + "commodity": ["Agricultural Demand|Non-Energy|" "Crops|Food"], } ) vars["Non-Energy|Crops|Other"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Non-Energy|Crops|Other"], - } - ) - - vars["Non-Energy|Crops|Other|Waste"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Non-Energy|Crops|Other|Waste"], + "commodity": ["Agricultural Demand|Non-Energy|" "Crops|Other"], } ) vars["Non-Energy|Livestock|Food"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Non-Energy|Livestock|Food"], + "commodity": ["Agricultural Demand|Non-Energy|" "Livestock|Food"], } ) vars["Non-Energy|Livestock|Other"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Non-Energy|Livestock|Other"], - } - ) - - vars["Non-Energy|Livestock|Other|Waste"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Agricultural Demand|Non-Energy|Livestock|Other|Waste"], + "commodity": ["Agricultural Demand|Non-Energy|" "Livestock|Other"], } ) @@ -619,35 +637,30 @@ def retr_agri_prd(units): vars["Energy|Crops|1st generation"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Production|Energy|Crops|1st generation"], + "commodity": ["Agricultural Production|Energy|" "Crops|1st generation"], } ) vars["Energy|Crops|2nd generation"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Production|Energy|Crops|2nd generation"], - } - ) - - vars["Non-Energy|Crops"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Agricultural Production|Non-Energy|Crops"], + "commodity": ["Agricultural Production|Energy|" "Crops|2nd generation"], } ) vars["Non-Energy|Crops|Cereals"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Production|Non-Energy|Crops|Cereals"], + "commodity": ["Agricultural Production|Non-Energy|" "Crops|Cereals"], } ) + # "commodity": ["Agricultural Production|Non-Energy|" + # "Crops|Food|Cereals"]}) vars["Non-Energy|Livestock"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Agricultural Production|Non-Energy|Livestock"], + "commodity": ["Agricultural Production|Non-Energy|" "Livestock"], } ) @@ -734,7 +747,7 @@ def retr_lnd_cvr(units): vars["Forest|Afforestation and Reforestation"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Land Cover|Forest|Afforestation and Reforestation"], + "commodity": ["Land Cover|Forest|Afforestation and" " Reforestation"], } ) @@ -780,13 +793,6 @@ def retr_lnd_cvr(units): } ) - vars["Protected"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Land Cover|Protected"], - } - ) - df = pp_utils.make_outputdf(vars, units) return df @@ -805,29 +811,12 @@ def retr_yield(units): Units to which variables should be converted. """ - dfs = [] - - # Process and weight cereals vars = {} + vars["Cereal"] = pp.land_out( lu_out_filter={"level": ["land_use_reporting"], "commodity": ["Yield|Cereals"]} ) - cropland = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Land Cover|Cropland|Cereals"], - } - ) - - dfs.append( - pp_utils.make_outputdf( - vars, units, param="weighted_avg", weighted_by=pp_utils.sum_reg(cropland) - ) - ) - - # Process and weight Energy Crops - vars = {} vars["Energy Crops"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -835,21 +824,6 @@ def retr_yield(units): } ) - cropland = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Land Cover|Cropland|Energy Crops"], - } - ) - - dfs.append( - pp_utils.make_outputdf( - vars, units, param="weighted_avg", weighted_by=pp_utils.sum_reg(cropland) - ) - ) - - # Process and weight Non-Energy Crops - vars = {} vars["Non-Energy Crops"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -857,39 +831,10 @@ def retr_yield(units): } ) - cropland = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Land Cover|Cropland|Non-Energy Crops"], - } - ) - dfs.append( - pp_utils.make_outputdf( - vars, units, param="weighted_avg", weighted_by=pp_utils.sum_reg(cropland) - ) - ) - - # Process and weight Oilcrops - vars = {} vars["Oilcrops"] = pp.land_out( lu_out_filter={"level": ["land_use_reporting"], "commodity": ["Yield|Oilcrops"]} ) - cropland = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Land Cover|Cropland|Oilcrops"], - } - ) - - dfs.append( - pp_utils.make_outputdf( - vars, units, param="weighted_avg", weighted_by=pp_utils.sum_reg(cropland) - ) - ) - - # Process and weight Sugarcrops - vars = {} vars["Sugarcrops"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -897,20 +842,8 @@ def retr_yield(units): } ) - cropland = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Land Cover|Cropland|Sugarcrops"], - } - ) - - dfs.append( - pp_utils.make_outputdf( - vars, units, param="weighted_avg", weighted_by=pp_utils.sum_reg(cropland) - ) - ) - - return pd.concat(dfs, sort=True) + df = pp_utils.make_outputdf(vars, units, param="mean") + return df @_register @@ -968,15 +901,6 @@ def retr_fertilizer_int(units_nitrogen, units_phosphorus): Units to which phosphorus should be converted. """ - # This variable is only required for deriving the global value via weighted - # average. - cropland = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Land Cover|Cropland"], - } - ) - dfs = [] vars = {} @@ -986,11 +910,7 @@ def retr_fertilizer_int(units_nitrogen, units_phosphorus): "commodity": ["Fertilizer|Nitrogen|Intensity"], } ) - dfs.append( - pp_utils.make_outputdf( - vars, units_nitrogen, param="weighted_avg", weighted_by=cropland - ) - ) + dfs.append(pp_utils.make_outputdf(vars, units_nitrogen)) vars = {} vars["Phosphorus|Intensity"] = pp.land_out( @@ -1000,11 +920,7 @@ def retr_fertilizer_int(units_nitrogen, units_phosphorus): } ) - dfs.append( - pp_utils.make_outputdf( - vars, units_phosphorus, param="weighted_avg", weighted_by=cropland - ) - ) + dfs.append(pp_utils.make_outputdf(vars, units_phosphorus)) return pd.concat(dfs, sort=True) @@ -1058,13 +974,6 @@ def retr_food_dem(units): vars = {} - vars["Total"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Food Demand"], - } - ) - vars["Crops"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -1079,8 +988,6 @@ def retr_food_dem(units): } ) - # This variable is only required for deriving the global value via weighted - # average. pop = pp.act("Population") df = pp_utils.make_outputdf(vars, units, param="weighted_avg", weighted_by=pop) return df @@ -1128,7 +1035,7 @@ def retr_frst_dem(units): @_register -def retr_frst_prd(units_residues, units_roundwood): +def retr_frst_prd(units): """Landuse: Forestry product production. Land-use related forestry production. @@ -1141,18 +1048,15 @@ def retr_frst_prd(units_residues, units_roundwood): Units to which variables should be converted. """ - dfs = [] - vars = {} + vars["Forest Residues"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], "commodity": ["Forestry Production|Forest Residues"], } ) - dfs.append(pp_utils.make_outputdf(vars, units_residues)) - vars = {} vars["Roundwood"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -1163,7 +1067,7 @@ def retr_frst_prd(units_residues, units_roundwood): vars["Roundwood|Industrial Roundwood"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Forestry Production|Roundwood|Industrial Roundwood"], + "commodity": ["Forestry Production|Roundwood|Industrial" " Roundwood"], } ) @@ -1174,8 +1078,8 @@ def retr_frst_prd(units_residues, units_roundwood): } ) - dfs.append(pp_utils.make_outputdf(vars, units_roundwood)) - return pd.concat(dfs, sort=True) + df = pp_utils.make_outputdf(vars, units) + return df @_register @@ -1248,9 +1152,12 @@ def retr_pop(units): } ) - # Popultion entry into SolWaPOPLink is equal to share of urban population - # hence the act_rel will return absolue urban population. - vars["Urban"] = pp.act_rel("Population", {"relation": "SolWaPOPLink"}) + df_urban_perc = pd.read_csv(urban_perc_data).set_index("Region") / 100 + df_urban_perc = df_urban_perc.rename( + columns={i: int(i) for i in df_urban_perc.columns} + ) + + vars["Urban"] = vars["Total"].multiply(df_urban_perc, fill_value=0) vars["Rural"] = vars["Total"] - vars["Urban"] @@ -1299,22 +1206,10 @@ def retr_demands_input(units): ) + pp.out("solar_pv_I", units) vars["Input|RC Thermal"] = pp.inp( - [ - "foil_rc", - "loil_rc", - "meth_rc", - "eth_rc", - "gas_rc", - "coal_rc", - "biomass_rc", - "elec_rc", - "hp_el_rc", - "hp_gas_rc", - "heat_rc", - "h2_rc", - ], + # NB(PNK) h2_fc_rc is omitted + get_techs("rc", "biomass coal heat elec eth foil gas loil meth") + ["h2_rc"], units, - ) + pp.out("solar_rc", units) + ) + pp.out(TECHS["rc solar"], units) vars["Input|RC Specific"] = ( pp.inp(["sp_el_RC", "h2_fc_RC"], units) @@ -1322,7 +1217,7 @@ def retr_demands_input(units): + ( -1.0 * pp.out( - "h2_fc_trp", + TECHS["trp h2"], units, outfilter={"level": ["final"], "commodity": ["electr"]}, ) @@ -1330,19 +1225,7 @@ def retr_demands_input(units): ) vars["Input|Transport"] = pp.inp( - [ - "foil_trp", - "loil_trp", - "meth_ic_trp", - "meth_fc_trp", - "eth_ic_trp", - "eth_fc_trp", - "gas_trp", - "coal_trp", - "elec_trp", - "h2_fc_trp", - ], - units, + get_techs("trp", "coal elec eth foil gas h2 loil meth"), units ) vars["Input|Shipping"] = pp.inp( @@ -1720,23 +1603,21 @@ def retr_price( dfs.append(pp_utils.make_outputdf(vars, units_energy_outp, glb=False)) - # ---------------------------------------------------- - # Calculate Agricultural Prices where the global price - # is derived using a formula provided by GLOBIOM - # ---------------------------------------------------- - vars = {} - - # Define the technology which represents the quantities used to derive - # sums accross regions values. - scale_tec = "Agricultural Production|Non-Energy|Crops" - - vars["Agriculture|Non-Energy Crops and Livestock|Index"] = pp.retrieve_lu_price( - "Price|Agriculture|Non-Energy Crops and Livestock|Index", scale_tec + vars["Agriculture|Non-Energy Crops and Livestock|Index"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Price|Agriculture|Non-Energy" " Crops and Livestock|Index"], + } ) - vars["Agriculture|Non-Energy Crops|Index"] = pp.retrieve_lu_price( - "Price|Agriculture|Non-Energy Crops|Index", scale_tec + + vars["Agriculture|Non-Energy Crops|Index"] = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Price|Agriculture|Non-Energy" " Crops|Index"], + } ) + dfs.append(pp_utils.make_outputdf(vars, units_agri, glb=False)) return pd.concat(dfs, sort=True) @@ -1748,11 +1629,11 @@ def retr_globiom_feedback( units_emi_CO2, units_emi_N2O, units_ene, - units_ene_prc, units_CPrc_co2, units_CPrc_co2_outp, units_gdp, ): + dfs = [] vars = {} vars["Emissions|CH4|AFOLU"] = pp.land_out( @@ -1779,7 +1660,7 @@ def retr_globiom_feedback( "commodity": ["Emissions|N2O|AFOLU"], } ) - dfs.append(pp_utils.make_outputdf(vars, units_emi_N2O)) + dfs.append(pp_utils.make_outputdf(vars, units_emi_N2O, glb=False)) vars = {} vars["Primary Energy|Biomass"] = pp.land_out( @@ -1794,21 +1675,6 @@ def retr_globiom_feedback( "commodity": ["Primary Energy|Biomass|Energy Crops"], } ) - - vars["Primary Energy|Biomass|1st Generation"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Primary Energy|Biomass|1st Generation"], - } - ) - - vars["Primary Energy|Biomass|Fuelwood"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Primary Energy|Biomass|Fuelwood"], - } - ) - dfs.append(pp_utils.make_outputdf(vars, units_ene)) # Both the carbon price and the GDP should be reported in US$2005 @@ -1826,17 +1692,6 @@ def retr_globiom_feedback( vars["GDP|PPP"] = pp.var_gdp() * MERtoPPP dfs.append(pp_utils.make_outputdf(vars, units_gdp)) - # Provide "original" biomass price, without any of the additional adjustments - # too include the abatement of lu-emissions - vars = {} - vars["Price|Primary Energy|Biomass"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Price|Primary Energy|Biomass"], - } - ) - dfs.append(pp_utils.make_outputdf(vars, units_ene_prc, glb=False)) - return pd.concat(dfs, sort=True) @@ -1862,52 +1717,19 @@ def retr_kyoto(units): vars["Kyoto Gases"] = pp.emiss("TCE", "all") * mu["conv_c2co2"] if run_history == "True": - # Read historic data from file - if kyoto_hist_data is not False: - df_hist = pd.read_csv(kyoto_hist_data).set_index("Region") - df_hist = df_hist.rename(columns={i: int(i) for i in df_hist.columns}) - - # There is no need to add the historic emissions for other lu related - # emissions, as these remain unchanged in the historic time-period. - - df_new_lu_hist = pp.land_out( - lu_out_filter={"level": ["land_use_reporting"], "commodity": ["TCE"]}, - units="Mt CO2eq/yr", - ) + df_hist = pd.read_csv(kyoto_hist_data).set_index("Region") + df_hist = df_hist.rename(columns={i: int(i) for i in df_hist.columns}) - df_hist = df_new_lu_hist.add(df_hist, fill_value=0) - vars["Kyoto Gases"] = vars["Kyoto Gases"] + df_hist - # Derive from historic activity - else: - # Retrieve FFI related CO2 emissions (convert from MtC to MtCO2) - co2 = pp.act(["CO2_TCE", "CO2s_TCE", "CO2t_TCE"]) * mu["conv_c2co2"] - # Retrieve FFI related CH4 emissions - ch4 = ( - pp.act(["CH4_TCE", "CH4n_TCE", "CH4o_TCE", "CH4s_TCE"]) - * 6.82 - * mu["conv_c2co2"] - ) - # Retrieve FFI related N2O emissions - n2o = ( - pp.act(["N2O_TCE", "N2On_TCE", "N2Oo_TCE", "N2Os_TCE"]) - * 81.27 - * mu["conv_c2co2"] - ) + # There is no need to add the historic emissions for other lu related + # emissions, as these remain unchanged in the historic time-period. - # Retrieve F-gases: CF4+SF6+HFCs (convert to MtCO2) - fgases = ( - pp.act("CF4_TCE") * 2015.45 - + pp.act("SF6_TCE") * 6218.0 - + (pp.act("HFC_TCE") + pp.act("HFCo_TCE")) * 390.0 - ) * mu["conv_c2co2"] - - # Retrieve land-use related GHGs: CO2+CH4+N2O - land_use = pp.land_out( - lu_out_filter={"level": ["land_use_reporting"], "commodity": ["TCE"]}, - units="Mt CO2eq/yr", - ) + df_new_lu_hist = pp.land_out( + lu_out_filter={"level": ["land_use_reporting"], "commodity": ["TCE"]}, + units="Mt CO2eq/yr", + ) - vars["Kyoto Gases"] += co2 + ch4 + n2o + fgases + land_use + df_hist = df_new_lu_hist.add(df_hist, fill_value=0) + vars["Kyoto Gases"] = vars["Kyoto Gases"] + df_hist df = pp_utils.make_outputdf(vars, units) return df @@ -2117,14 +1939,14 @@ def retr_othemi(var, units): vars["AFOLU|Land|Grassland Pastures"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Emissions|N2O|AFOLU|Land|Grassland Pastures"], + "commodity": ["Emissions|N2O|AFOLU|" "Land|Grassland Pastures"], } ) vars["AFOLU|Agriculture|Managed Soils"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Emissions|N2O|AFOLU|Agriculture|Managed Soils"], + "commodity": ["Emissions|N2O|AFOLU|" "Agriculture|Managed Soils"], } ) @@ -2613,21 +2435,9 @@ def retr_othemi(var, units): ) ResComTherm = pp.emi( - [ - "coal_rc", - "foil_rc", - "loil_rc", - "gas_rc", - "elec_rc", - "biomass_rc", - "heat_rc", - "meth_rc", - "eth_rc", - "h2_rc", - "hp_el_rc", - "hp_gas_rc", - "solar_rc", - ], + # NB(PNK) h2_fc_rc is omitted + get_techs("rc", "biomass coal elec eth foil gas heat loil meth solar") + + ["h2_rc"], "GWa", emifilter={"relation": [f"{var}_Emission"]}, emission_units=units, @@ -2666,18 +2476,7 @@ def retr_othemi(var, units): # ------------------------------ Transport = pp.emi( - [ - "coal_trp", - "foil_trp", - "loil_trp", - "gas_trp", - "elec_trp", - "meth_ic_trp", - "eth_ic_trp", - "meth_fc_trp", - "eth_fc_trp", - "h2_fc_trp", - ], + get_techs("trp", "coal elec eth foil gas h2 loil meth"), "GWa", emifilter={"relation": [f"{var}_Emission"]}, emission_units=units, @@ -3000,8 +2799,11 @@ def retr_CO2_CCS(units_emi, units_ene): emission_units=units_emi, ) + # NB although the name here is "_cement", TECHS["industry with ccs"] may include + # other technologies; e.g. .model.material.report.tables includes ammonia + # production technologies. _CCS_cement = -1.0 * pp.emi( - "cement_co2scr", + TECHS["cement with ccs"] + TECHS["industry with ccs"], "GWa", emifilter={"relation": ["CO2_Emission"]}, emission_units=units_emi, @@ -3129,40 +2931,13 @@ def retr_hfc(hfc_lst): ) _HFC_mobVac = pp.act_rel( - [ - "gas_trp", - "elec_trp", - "foil_trp", - "loil_trp", - "meth_ic_trp", - "meth_fc_trp", - "eth_ic_trp", - "mvac_co2", - "eth_fc_trp", - "h2_fc_trp", - ], + ["mvac_co2"] + get_techs("trp", "gas elec eth foil h2 loil meth"), relfilter={"relation": ["HFC_Emission"]}, ) _HFC_foam = pp.act_rel( - [ - "coal_rc", - "foil_rc", - "loil_rc", - "gas_rc", - "biomass_rc", - "elec_rc", - "heat_rc", - "meth_rc", - "h2_rc", - "hp_el_rc", - "hp_gas_rc", - "h2_fc_RC", - "solar_rc", - "back_rc", - "eth_rc", - "repl_hc", - ], + get_techs("rc", "back biomass coal elec eth foil gas h2 heat loil meth solar") + + ["repl_hc"], relfilter={"relation": ["HFC_Emission"]}, ) @@ -3209,6 +2984,14 @@ def retr_hfc(hfc_lst): + pp_utils.fil(_HFC_Solvent, "HFC_fac", f"Solv{fil}", units) ) + # Check for rows with missing Region label + mask = vars[hfc].index.to_frame()["Region"].isna() + if mask.any(): + log.warning( + f"{mask.sum()} row(s) for {hfc!r} have missing 'Region' → dropped" + ) + vars[hfc] = vars[hfc][~mask] + dfs.append(pp_utils.make_outputdf(vars, units)) return pd.concat(dfs, sort=True) @@ -3243,18 +3026,18 @@ def retr_CO2emi(units_emi, units_ene_mdl): _inp_nonccs_gas_tecs = ( pp.inp( [ - "gas_rc", - "hp_gas_rc", "gas_i", "hp_gas_i", - "gas_trp", "gas_fs", "gas_ppl", "gas_ct", "gas_cc", "gas_htfc", "gas_hpl", - ], + ] + + TECHS["rc gas"] + + TECHS["trp gas"] + + TECHS["gas extra"], units_ene_mdl, inpfilter={"commodity": ["gas"]}, ) @@ -3336,7 +3119,15 @@ def retr_CO2emi(units_emi, units_ene_mdl): ).fillna(0) _Biogas_ind = _Biogas_tot * ( - pp.inp(["gas_i", "hp_gas_i"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) + pp.inp( + [ + "gas_i", + "hp_gas_i", + ] + + TECHS["gas extra"], + units_ene_mdl, + inpfilter={"commodity": ["gas"]}, + ) / _inp_all_gas_tecs ).fillna(0) @@ -3346,12 +3137,12 @@ def retr_CO2emi(units_emi, units_ene_mdl): ).fillna(0) _Biogas_res = _Biogas_tot * ( - pp.inp(["gas_rc", "hp_gas_rc"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) + pp.inp(TECHS["rc gas"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) / _inp_all_gas_tecs ).fillna(0) _Biogas_trp = _Biogas_tot * ( - pp.inp("gas_trp", units_ene_mdl, inpfilter={"commodity": ["gas"]}) + pp.inp(TECHS["trp gas"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) / _inp_all_gas_tecs ).fillna(0) @@ -3371,16 +3162,6 @@ def retr_CO2emi(units_emi, units_ene_mdl): / _inp_all_gas_tecs ).fillna(0) - # For deriving the distribution of hydrogen, the total gas use must exclude hydrogen - # gas use in feedstocks. The relation `gas_mix_lim` ensures that blended hydrogen - # can only make up 50% of the gas used by `gas_t_d` and `gas_t_d_ch4`. `gas_fs` - # is excluded by also writing into the relation. - _H2_inp_nonccs_gas_tecs_wo_CCSRETRO = _inp_nonccs_gas_tecs_wo_CCSRETRO - pp.inp( - ["gas_fs"], - units_ene_mdl, - inpfilter={"commodity": ["gas"]}, - ) - _Hydrogen_tot = pp.emi( "h2_mix", units_ene_mdl, @@ -3419,33 +3200,40 @@ def retr_CO2emi(units_emi, units_ene_mdl): units=units_ene_mdl, ) ) - / _H2_inp_nonccs_gas_tecs_wo_CCSRETRO + / _inp_nonccs_gas_tecs_wo_CCSRETRO ).fillna(0) _Hydrogen_heat = _Hydrogen_tot * ( pp.inp("gas_hpl", units_ene_mdl, inpfilter={"commodity": ["gas"]}) - / _H2_inp_nonccs_gas_tecs_wo_CCSRETRO + / _inp_nonccs_gas_tecs_wo_CCSRETRO ).fillna(0) _Hydrogen_ind = _Hydrogen_tot * ( - pp.inp(["gas_i", "hp_gas_i"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) - / _H2_inp_nonccs_gas_tecs_wo_CCSRETRO + pp.inp( + [ + "gas_i", + "hp_gas_i", + ] + + TECHS["gas extra"], + units_ene_mdl, + inpfilter={"commodity": ["gas"]}, + ) + / _inp_nonccs_gas_tecs_wo_CCSRETRO ).fillna(0) - # gas_fs is not able to use hydrogen, hence this is removed. - # _Hydrogen_fs = _Hydrogen_tot * ( - # pp.inp("gas_fs", units_ene_mdl, inpfilter={"commodity": ["gas"]}) - # / _H2_inp_nonccs_gas_tecs_wo_CCSRETRO - # ).fillna(0) + _Hydrogen_fs = _Hydrogen_tot * ( + pp.inp("gas_fs", units_ene_mdl, inpfilter={"commodity": ["gas"]}) + / _inp_nonccs_gas_tecs_wo_CCSRETRO + ).fillna(0) _Hydrogen_res = _Hydrogen_tot * ( - pp.inp(["gas_rc", "hp_gas_rc"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) - / _H2_inp_nonccs_gas_tecs_wo_CCSRETRO + pp.inp(TECHS["rc gas"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) + / _inp_nonccs_gas_tecs_wo_CCSRETRO ).fillna(0) _Hydrogen_trp = _Hydrogen_tot * ( - pp.inp("gas_trp", units_ene_mdl, inpfilter={"commodity": ["gas"]}) - / _H2_inp_nonccs_gas_tecs_wo_CCSRETRO + pp.inp(TECHS["trp gas"], units_ene_mdl, inpfilter={"commodity": ["gas"]}) + / _inp_nonccs_gas_tecs_wo_CCSRETRO ).fillna(0) _Hydrogen_td = _Hydrogen_tot * ( @@ -3461,7 +3249,7 @@ def retr_CO2emi(units_emi, units_ene_mdl): outfilter={"commodity": ["gas"]}, ) ) - / _H2_inp_nonccs_gas_tecs_wo_CCSRETRO + / _inp_nonccs_gas_tecs_wo_CCSRETRO ).fillna(0) _SE_Elec_gen = pp.emi( @@ -3500,13 +3288,13 @@ def retr_CO2emi(units_emi, units_ene_mdl): - pp.emi( ["bio_istig_ccs", "bio_ppl_co2scr"], units_ene_mdl, - emifilter={"relation": ["CO2_cc"]}, + emifilter={"relation": ["CO2_Emission"]}, emission_units=units_emi, ) - pp.emi( ["g_ppl_co2scr", "gas_cc_ccs"], units_ene_mdl, - emifilter={"relation": ["CO2_cc"]}, + emifilter={"relation": ["CO2_Emission"]}, emission_units=units_emi, ) * (_Biogas_tot_abs / _inp_all_gas_tecs) @@ -3520,14 +3308,20 @@ def retr_CO2emi(units_emi, units_ene_mdl): ) _FE_Feedstocks = pp.emi( - ["coal_fs", "foil_fs", "loil_fs", "gas_fs", "methanol_fs"], + [ + "coal_fs", + "foil_fs", + "loil_fs", + "gas_fs", + "methanol_fs", + ], units_ene_mdl, emifilter={"relation": ["CO2_feedstocks"]}, emission_units=units_emi, ) _FE_Res_com = pp.emi( - ["coal_rc", "foil_rc", "loil_rc", "gas_rc", "meth_rc", "hp_gas_rc"], + get_techs("rc", "coal foil gas loil meth"), units_ene_mdl, emifilter={"relation": ["CO2_r_c"]}, emission_units=units_emi, @@ -3550,7 +3344,7 @@ def retr_CO2emi(units_emi, units_ene_mdl): ) _FE_Transport = pp.emi( - ["gas_trp", "loil_trp", "meth_fc_trp", "meth_ic_trp", "coal_trp", "foil_trp"], + get_techs("trp", "coal foil gas loil meth"), units_ene_mdl, emifilter={"relation": ["CO2_trp"]}, emission_units=units_emi, @@ -3577,23 +3371,14 @@ def retr_CO2emi(units_emi, units_ene_mdl): _Other_gases_extr_fug = pp.emi( "flaring_CO2", units_ene_mdl, - emifilter={"relation": ["CO2_cc"]}, + emifilter={"relation": ["CO2_Emission"]}, emission_units=units_emi, ) # Note that this is not included in the total because - # the diff is only calculated from CO2_TCE and doesnt include trade + # the diff is only calcualted from CO2_TCE and doesnt include trade _Other_gases_trans_comb_trade = pp.emi( - [ - "LNG_trd", - "gas_exp_afr", - "gas_exp_cpa", - "gas_exp_eeu", - "gas_exp_nam", - "gas_exp_pao", - "gas_exp_sas", - "gas_exp_weu", - ], + ["LNG_trd"], units_ene_mdl, emifilter={"relation": ["CO2_trade"]}, emission_units=units_emi, @@ -3681,21 +3466,24 @@ def retr_CO2emi(units_emi, units_ene_mdl): # Note that this is not included in the total because # the diff is only calcualted from CO2_TCE and doesnt include trade _Other_liquids_trans_comb_trade = pp.emi( - ["foil_trd", "loil_trd", "oil_trd", "meth_trd"], + ["foil_trd", "loil_trd", "oil_trd", "meth_trd", "eth_trd"], units_ene_mdl, emifilter={"relation": ["CO2_trade"]}, emission_units=units_emi, ) _Other_liquids_trans_comb = pp.emi( - ["foil_t_d", "loil_t_d", "meth_t_d"], + ["foil_t_d", "loil_t_d", "meth_t_d", "eth_t_d"], units_ene_mdl, emifilter={"relation": ["CO2_cc"]}, emission_units=units_emi, ) _Other_liquids_oil_comb = pp.emi( - ["ref_lol", "ref_hil"], + [ + "ref_lol", + "ref_hil", + ], units_ene_mdl, emifilter={"relation": ["CO2_cc"]}, emission_units=units_emi, @@ -3764,9 +3552,9 @@ def retr_CO2emi(units_emi, units_ene_mdl): _Other_solids_total = _Other_solids_coal_trans_comb _Cement1 = pp.emi( - ["cement_CO2", "cement_co2scr"], + ["cement_CO2"] + TECHS["cement with ccs"], units_ene_mdl, - emifilter={"relation": ["CO2_cc"]}, + emifilter={"relation": ["CO2_Emission"]}, emission_units=units_emi, ) @@ -3780,12 +3568,11 @@ def retr_CO2emi(units_emi, units_ene_mdl): - _Biogas_tot + _Hydrogen_tot + _Cement1 - # + _Trade_losses ) # GLOBIOM with the new lu implementation, LU_CO2 no longer writes # into _CO2_tce1 (CO2_TCE), as these have emission factors only, # and therefore do not write into CO2_TCE - # + Landuse AFOLU) + # + _CO2_GLOBIOM) _Total_wo_BECCS = ( abs(_SE_District_heat - _Biogas_heat + _Hydrogen_heat) @@ -3807,25 +3594,45 @@ def retr_CO2emi(units_emi, units_ene_mdl): emission_units=units_emi, ) + _CO2_GLOBIOM = pp.land_out( + lu_out_filter={ + "level": ["land_use_reporting"], + "commodity": ["Emissions|CO2|AFOLU"], + }, + units=units_emi, + ) + _Diff1 = _CO2_tce1 - _Total - if run_history == "True" and lu_hist_data is not False: + if run_history == "True": df_hist = pd.read_csv(lu_hist_data).set_index("Region") df_hist = df_hist.rename(columns={i: int(i) for i in df_hist.columns}) _Diff1 = _Diff1 - df_hist - # ----- - # AFOLU - # ----- + # --------------------- + # Agriculture (Table 1) + # --------------------- - vars["AFOLU"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Emissions|CO2|AFOLU"], - }, - units=units_emi, - ) + AgricultureWasteBurning = pp_utils._make_zero() + vars["AFOLU|Biomass Burning"] = AgricultureWasteBurning + Agriculture = pp_utils._make_zero() + vars["AFOLU|Agriculture"] = Agriculture + + # --------------------------- + # Grassland Burning (Table 2) + # --------------------------- + GrasslandBurning = pp_utils._make_zero() + vars["AFOLU|Land|Grassland Burning"] = GrasslandBurning + + # ------------------------ + # Forest Burning (Table 3) + # ------------------------ + + ForestBurning = pp_utils._make_zero() + vars["AFOLU|Land|Forest Burning"] = ForestBurning + + vars["AFOLU"] = _CO2_GLOBIOM vars["AFOLU|Afforestation"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -3834,14 +3641,6 @@ def retr_CO2emi(units_emi, units_ene_mdl): units=units_emi, ) - vars["AFOLU|Agriculture"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Emissions|CO2|AFOLU|Agriculture"], - }, - units=units_emi, - ) - vars["AFOLU|Deforestation"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -3866,15 +3665,6 @@ def retr_CO2emi(units_emi, units_ene_mdl): units=units_emi, ) - # This variable is used to determine the compensation quantity of GHGs - # from land-use to ensure that the there are non non-convexities. - vars["AFOLU|Other"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Emissions|CO2|AFOLU|Other"], - }, - units=units_emi, - ) vars["AFOLU|Other LUC"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], @@ -3936,9 +3726,12 @@ def retr_CO2emi(units_emi, units_ene_mdl): vars["Energy|Supply|Gases|Coal|Fugitive"] = pp_utils._make_zero() - vars["Energy|Supply|Gases|Extraction|Combustion"] = ( - _Other_gases_extr_comb - + _Diff1 * (abs(_Other_gases_extr_comb) / _Total_wo_BECCS).fillna(0) + vars[ + "Energy|Supply|Gases|Extraction|Combustion" + ] = _Other_gases_extr_comb + _Diff1 * ( + abs(_Other_gases_extr_comb) / _Total_wo_BECCS + ).fillna( + 0 ) # _Diff1 is not disctributed across the variable @@ -3969,24 +3762,32 @@ def retr_CO2emi(units_emi, units_ene_mdl): vars["Energy|Supply|Gases|Transportation|Fugitive"] = pp_utils._make_zero() - vars["Energy|Supply|Liquids|Biomass|Combustion"] = ( - _Other_liquids_biomass_comb - + _Diff1 - * (abs(_Other_liquids_biomass_comb_woBECCS) / _Total_wo_BECCS).fillna(0) + vars[ + "Energy|Supply|Liquids|Biomass|Combustion" + ] = _Other_liquids_biomass_comb + _Diff1 * ( + abs(_Other_liquids_biomass_comb_woBECCS) / _Total_wo_BECCS + ).fillna( + 0 ) vars["Energy|Supply|Liquids|Biomass|Fugitive"] = pp_utils._make_zero() - vars["Energy|Supply|Liquids|Coal|Combustion"] = ( - _Other_liquids_coal_comb - + _Diff1 * (abs(_Other_liquids_coal_comb) / _Total_wo_BECCS).fillna(0) + vars[ + "Energy|Supply|Liquids|Coal|Combustion" + ] = _Other_liquids_coal_comb + _Diff1 * ( + abs(_Other_liquids_coal_comb) / _Total_wo_BECCS + ).fillna( + 0 ) vars["Energy|Supply|Liquids|Coal|Fugitive"] = pp_utils._make_zero() - vars["Energy|Supply|Liquids|Extraction|Combustion"] = ( - _Other_liquids_extr_comb - + _Diff1 * (abs(_Other_liquids_extr_comb) / _Total_wo_BECCS).fillna(0) + vars[ + "Energy|Supply|Liquids|Extraction|Combustion" + ] = _Other_liquids_extr_comb + _Diff1 * ( + abs(_Other_liquids_extr_comb) / _Total_wo_BECCS + ).fillna( + 0 ) vars["Energy|Supply|Liquids|Extraction|Fugitive"] = pp_utils._make_zero() @@ -4023,9 +3824,12 @@ def retr_CO2emi(units_emi, units_ene_mdl): vars["Energy|Supply|Solids|Coal|Combustion"] = pp_utils._make_zero() vars["Energy|Supply|Solids|Coal|Fugitive"] = pp_utils._make_zero() - vars["Energy|Supply|Solids|Extraction|Combustion"] = ( - _Other_solids_coal_trans_comb - + _Diff1 * (abs(_Other_solids_coal_trans_comb) / _Total_wo_BECCS).fillna(0) + vars[ + "Energy|Supply|Solids|Extraction|Combustion" + ] = _Other_solids_coal_trans_comb + _Diff1 * ( + abs(_Other_solids_coal_trans_comb) / _Total_wo_BECCS + ).fillna( + 0 ) vars["Energy|Supply|Solids|Extraction|Fugitive"] = pp_utils._make_zero() @@ -4044,7 +3848,7 @@ def retr_CO2emi(units_emi, units_ene_mdl): # Industrial Feedstocks # --------------------- - vars["Energy|Demand|Other Sector"] = _FE_Feedstocks - _Biogas_fs # + _Hydrogen_fs + vars["Energy|Demand|Other Sector"] = _FE_Feedstocks - _Biogas_fs + _Hydrogen_fs # -------------------------------------------- # Industrial process and product use (Table 8) @@ -4404,28 +4208,14 @@ def retr_SE_synfuels(units): vars = {} - vars["Liquids|Oil"] = ( - pp.out( - "ref_lol", - units, - outfilter={"level": ["secondary"], "commodity": ["lightoil"]}, - ) - + pp.out( - "ref_lol", - units, - outfilter={"level": ["secondary"], "commodity": ["fueloil"]}, - ) - + pp.out( - "ref_hil", - units, - outfilter={"level": ["secondary"], "commodity": ["lightoil"]}, - ) - + pp.out( - "ref_hil", - units, - outfilter={"level": ["secondary"], "commodity": ["fueloil"]}, - ) - ) + # Retrieve data for Liquids|Oil + oil_data = [] + for t, kind, filters in TECHS["synfuels oil"]: + func = getattr(pp, kind) # Either pp.inp() or pp.out() + args = {f"{kind}filter": filters} # Either inpfilter or outfilter + oil_data.append(func(t, units, **args)) + + vars["Liquids|Oil"] = reduce(add, oil_data) vars["Liquids|Biomass|w/o CCS"] = pp.out( ["eth_bio", "liq_bio"], @@ -4534,15 +4324,22 @@ def retr_SE_solids(units): vars = {} - BiomassIND = pp.inp("biomass_i", units) - BiomassNC = pp.inp("biomass_nc", units) - BiomassRC = pp.inp("biomass_rc", units) - vars["Biomass"] = BiomassNC + BiomassIND + BiomassRC + # Add the entries together + vars["Biomass"] = pp.inp( + ["biomass_i", "biomass_nc"] + + TECHS["rc biomass"] + + TECHS["se solids biomass extra"], + units=units, + ) - CoalIND = pp.inp(["coal_i", "coal_fs"], units) - CoalRC = pp.inp("coal_rc", units) - CoalTRP = pp.inp("coal_trp", units) - vars["Coal"] = CoalIND + CoalRC + CoalTRP + # Same for coal + vars["Coal"] = pp.inp( + ["coal_i", "coal_fs"] + + TECHS["rc coal"] + + TECHS["trp coal"] + + TECHS["se solids coal extra"], + units=units, + ) df = pp_utils.make_outputdf(vars, units) return df @@ -4628,16 +4425,10 @@ def retr_SE_elecgen(units): # -------------------------------- vars["Coal|w/o CCS"] = ( - _se_elec_woCCSretro( - "coal_ppl", "c_ppl_co2scr", units, _Frac, outfilter={"commodity": "electr"} - ) - + _se_elec_po_turb("coal_adv", units, _Frac, outfilter={"commodity": "electr"}) - + _se_elec_po_turb( - "coal_ppl_u", units, _Frac, outfilter={"commodity": "electr"} - ) - + _se_elec_woCCSretro( - "igcc", "igcc_co2scr", units, _Frac, outfilter={"commodity": "electr"} - ) + _se_elec_woCCSretro("coal_ppl", "c_ppl_co2scr", units, _Frac) + + _se_elec_po_turb("coal_adv", units, _Frac) + + _se_elec_po_turb("coal_ppl_u", units, _Frac) + + _se_elec_woCCSretro("igcc", "igcc_co2scr", units, _Frac) + pp.out( ["meth_coal", "h2_coal", "syn_liq"], units, @@ -4647,11 +4438,9 @@ def retr_SE_elecgen(units): vars["Coal|w/ CCS"] = ( _se_elec_wCCSretro("coal_ppl", "c_ppl_co2scr", units, _Frac) - + _se_elec_po_turb( - "coal_adv_ccs", units, _Frac, outfilter={"commodity": "electr"} - ) + + _se_elec_po_turb("coal_adv_ccs", units, _Frac) + _se_elec_wCCSretro("igcc", "igcc_co2scr", units, _Frac) - + _se_elec_po_turb("igcc_ccs", units, _Frac, outfilter={"commodity": "electr"}) + + _se_elec_po_turb("igcc_ccs", units, _Frac) + pp.out( ["meth_coal_ccs", "h2_coal_ccs", "syn_liq_ccs"], units, @@ -4664,26 +4453,10 @@ def retr_SE_elecgen(units): # ------------------------------- _Gas_woCCS = ( - _se_elec_woCCSretro( - "gas_ppl", - "g_ppl_co2scr", - units, - _Frac, - share=_gas_ppl_shr, - outfilter={"commodity": "electr"}, - ) - + _se_elec_po_turb("gas_ct", units, _Frac, outfilter={"commodity": "electr"}) - + _se_elec_woCCSretro( - "gas_cc", - "g_ppl_co2scr", - units, - _Frac, - share=_gas_cc_shr, - outfilter={"commodity": "electr"}, - ) - + _se_elec_woCCSretro( - "gas_htfc", "gfc_co2scr", units, _Frac, outfilter={"commodity": "electr"} - ) + _se_elec_woCCSretro("gas_ppl", "g_ppl_co2scr", units, _Frac, share=_gas_ppl_shr) + + _se_elec_po_turb("gas_ct", units, _Frac) + + _se_elec_woCCSretro("gas_cc", "g_ppl_co2scr", units, _Frac, share=_gas_cc_shr) + + _se_elec_woCCSretro("gas_htfc", "gfc_co2scr", units, _Frac) + pp.out( ["h2_smr"], units, @@ -4697,9 +4470,7 @@ def retr_SE_elecgen(units): _se_elec_wCCSretro("gas_ppl", "g_ppl_co2scr", units, _Frac, share=_gas_ppl_shr) + _se_elec_wCCSretro("gas_cc", "g_ppl_co2scr", units, _Frac, share=_gas_cc_shr) + _se_elec_wCCSretro("gas_htfc", "gfc_co2scr", units, _Frac) - + _se_elec_po_turb( - "gas_cc_ccs", units, _Frac, outfilter={"commodity": "electr"} - ) + + _se_elec_po_turb("gas_cc_ccs", units, _Frac) + pp.out( ["h2_smr_ccs"], units, @@ -4714,10 +4485,10 @@ def retr_SE_elecgen(units): # ------------------------------- vars["Oil|w/o CCS"] = ( - _se_elec_po_turb("foil_ppl", units, _Frac, outfilter={"commodity": "electr"}) - + _se_elec_po_turb("loil_ppl", units, _Frac, outfilter={"commodity": "electr"}) - + _se_elec_po_turb("oil_ppl", units, _Frac, outfilter={"commodity": "electr"}) - + _se_elec_po_turb("loil_cc", units, _Frac, outfilter={"commodity": "electr"}) + _se_elec_po_turb("foil_ppl", units, _Frac) + + _se_elec_po_turb("loil_ppl", units, _Frac) + + _se_elec_po_turb("oil_ppl", units, _Frac) + + _se_elec_po_turb("loil_cc", units, _Frac) ) # ----------------------------------- @@ -4725,10 +4496,8 @@ def retr_SE_elecgen(units): # ----------------------------------- vars["Biomass|w/o CCS"] = ( - _se_elec_woCCSretro( - "bio_ppl", "bio_ppl_co2scr", units, _Frac, outfilter={"commodity": "electr"} - ) - + _se_elec_po_turb("bio_istig", units, _Frac, outfilter={"commodity": "electr"}) + _se_elec_woCCSretro("bio_ppl", "bio_ppl_co2scr", units, _Frac) + + _se_elec_po_turb("bio_istig", units, _Frac) + pp.out( ["h2_bio", "eth_bio", "liq_bio"], units, @@ -4741,9 +4510,7 @@ def retr_SE_elecgen(units): # normally it is replaced by 1, but this doesnt work in this case! vars["Biomass|w/ CCS"] = ( _se_elec_wCCSretro("bio_ppl", "bio_ppl_co2scr", units, _Frac) - + _se_elec_po_turb( - "bio_istig_ccs", units, _Frac, outfilter={"commodity": "electr"} - ) + + _se_elec_po_turb("bio_istig_ccs", units, _Frac) + pp.out( ["h2_bio_ccs", "eth_bio_ccs", "liq_bio_ccs"], units, @@ -4915,9 +4682,7 @@ def retr_SE_elecgen(units): vars["Hydro"] = pp.out(["hydro_lc", "hydro_hc"], units) - vars["Geothermal"] = _se_elec_po_turb( - "geo_ppl", units, _Frac, outfilter={"commodity": "electr"} - ) + vars["Geothermal"] = _se_elec_po_turb("geo_ppl", units, _Frac) vars["Nuclear"] = ( _se_elec_po_turb( @@ -4945,7 +4710,9 @@ def retr_SE_elecgen(units): vars["Other"] = ( pp.inp( - "h2_fc_trp", units, inpfilter={"level": ["final"], "commodity": ["electr"]} + TECHS["trp h2"], + units, + inpfilter={"level": ["final"], "commodity": ["electr"]}, ) + pp.inp( "h2_fc_I", units, inpfilter={"level": ["final"], "commodity": ["electr"]} @@ -5277,30 +5044,9 @@ def retr_pe(units, method=None): # Primary Energy Biomass # ---------------------- - # The individual technologies need to be listed instead of using the output - # of "land_use_biomass, because other wise the historical activty is not - # trackable. - vars["Biomass"] = pp.out( - ["biomass_imp"], units, outfilter={"commodity": ["biomass"]} - ) + pp.inp( - [ - "bio_hpl", - "bio_istig", - "bio_istig_ccs", - "bio_ppl", - "biomass_nc", - "biomass_t_d", - "eth_bio", - "eth_bio_ccs", - "gas_bio", - "h2_bio", - "h2_bio_ccs", - "liq_bio", - "liq_bio_ccs", - ], - units, - inpfilter={"commodity": ["biomass"]}, - ) # - pp.inp(['biomass_exp'], units, inpfilter={"commodity": ["biomass"]}) + vars["Biomass"] = pp.land_out( + lu_out_filter={"level": ["land_use"], "commodity": ["bioenergy"]}, units=units + ) # Note OFR 20180412: Correction inserted. scrubber output per vintage cannot # be divided by powerplant eff per vintage. In most cases this does work except @@ -5408,7 +5154,7 @@ def retr_pe(units, method=None): inpfilter={"commodity": ["electr"]}, ) - _solar_heat = pp.out(["solar_rc", "solar_i"], units) + _solar_heat = pp.out(TECHS["rc solar"] + ["solar_i"], units) _csp_elec = pp.out( [ @@ -5447,21 +5193,17 @@ def retr_pe(units, method=None): # Primary Energy Geothermal # ------------------------- - _geothermal_elec = _se_elec_po_turb( - "geo_ppl", units, _Frac, outfilter={"commodity": "electr"} + _geothermal_elec = _se_elec_po_turb("geo_ppl", units, _Frac) + _geothermal_heat = _se_heat_po_turb("geo_ppl", units, _Frac) + pp.out( + ["geo_hpl"], units ) - _geothermal_heat = _se_heat_po_turb( - "geo_ppl", units, _Frac, outfilter={"commodity": "electr"} - ) + pp.out(["geo_hpl"], units) if method == "substitution": vars["Geothermal"] = (_geothermal_elec / elec_factor).fillna(0) + ( _geothermal_heat / heat_factor ).fillna(0) else: - vars["Geothermal|Electricity|w/o CCS"] = _geothermal_elec - - vars["Geothermal|Heat"] = _geothermal_heat + vars["Geothermal"] = _geothermal_elec + _geothermal_heat # Primary Energy - SE Trade vars["Secondary Energy Trade"] = pp.out( @@ -5501,22 +5243,12 @@ def retr_pe(units, method=None): vars["Other"] = pp.inp("LH2_bunker", units) if method != "substitution": - # ------------------------------------------------ - # Primary Energy Electricity from coal without CCS - # ------------------------------------------------ - vars["Coal|Electricity|w/o CCS|Hardcoal Subcritical w/o FGD/DeNOx"] = ( - _pe_elec_po_turb( - "coal_ppl_u", - group, - units, - _Frac, - inpfilter={"commodity": ["coal"]}, - outfilter={"commodity": "electr"}, - ) - ) + # ------------------------------------ + # Primary Energy Electricity from coal + # ------------------------------------ - vars["Coal|Electricity|w/o CCS|Hardcoal Subcritical w/ FGD/DeNOx"] = ( + vars["Coal|Electricity|w/o CCS"] = ( _pe_elec_woCCSretro( "coal_ppl", "c_ppl_co2scr", @@ -5525,31 +5257,23 @@ def retr_pe(units, method=None): units=units, _Frac=_Frac, ) + + _pe_elec_woCCSretro( + "igcc", + "igcc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "coal_adv", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) + + _pe_elec_po_turb( + "coal_ppl_u", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) ) - vars["Coal|Electricity|w/o CCS|Hardcoal Supercritical"] = _pe_elec_po_turb( - "coal_adv", - group, - units, - _Frac, - inpfilter={"commodity": ["coal"]}, - outfilter={"commodity": "electr"}, - ) - - vars["Coal|Electricity|w/o CCS|Hardcoal IGCC"] = _pe_elec_woCCSretro( - "igcc", - "igcc_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["coal"]}, - units=units, - _Frac=_Frac, - ) - - # --------------------------------------------- - # Primary Energy Electricity from coal with CCS - # --------------------------------------------- - - vars["Coal|Electricity|w/ CCS|Hardcoal Subcritical w/ FGD/DeNOx"] = ( + vars["Coal|Electricity|w/ CCS"] = ( _pe_elec_wCCSretro( "coal_ppl", "c_ppl_co2scr", @@ -5558,38 +5282,27 @@ def retr_pe(units, method=None): units=units, _Frac=_Frac, ) + + _pe_elec_wCCSretro( + "igcc", + "igcc_co2scr", + group, + inpfilter={"level": ["secondary"], "commodity": ["coal"]}, + units=units, + _Frac=_Frac, + ) + + _pe_elec_po_turb( + "coal_adv_ccs", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) + + _pe_elec_po_turb( + "igcc_ccs", group, units, _Frac, inpfilter={"commodity": ["coal"]} + ) ) - vars["Coal|Electricity|w/ CCS|Hardcoal Supercritical"] = _pe_elec_po_turb( - "coal_adv_ccs", - group, - units, - _Frac, - inpfilter={"commodity": ["coal"]}, - outfilter={"commodity": "electr"}, - ) - - vars["Coal|Electricity|w/ CCS|Hardcoal IGCC"] = _pe_elec_wCCSretro( - "igcc", - "igcc_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["coal"]}, - units=units, - _Frac=_Frac, - ) + _pe_elec_po_turb( - "igcc_ccs", - group, - units, - _Frac, - inpfilter={"commodity": ["coal"]}, - outfilter={"commodity": "electr"}, - ) - - # ----------------------------------------------- - # Primary Energy Electricity from gas without CCS - # ----------------------------------------------- + # ----------------------------------- + # Primary Energy Electricity from gas + # ----------------------------------- - vars["Gas|Electricity|w/o CCS|Natural Gas Steam Cycle"] = ( + _ElecGas_woCCS = ( _pe_elec_woCCSretro( "gas_ppl", "g_ppl_co2scr", @@ -5599,21 +5312,7 @@ def retr_pe(units, method=None): _Frac=_Frac, share=_gas_ppl_shr, ) - ) * (1 - _BGas_share) - - vars["Gas|Electricity|w/o CCS|Natural Gas Combustion Turbine"] = ( - _pe_elec_po_turb( - "gas_ct", - group, - units, - _Frac, - inpfilter={"commodity": ["gas"]}, - outfilter={"commodity": "electr"}, - ) - ) * (1 - _BGas_share) - - vars["Gas|Electricity|w/o CCS|Natural Gas Combined Cycle"] = ( - _pe_elec_woCCSretro( + + _pe_elec_woCCSretro( "gas_cc", "g_ppl_co2scr", group, @@ -5622,10 +5321,7 @@ def retr_pe(units, method=None): _Frac=_Frac, share=_gas_cc_shr, ) - ) * (1 - _BGas_share) - - vars["Gas|Electricity|w/o CCS|Natural Gas High Temperature Fuel Cell"] = ( - _pe_elec_woCCSretro( + + _pe_elec_woCCSretro( "gas_htfc", "gfc_co2scr", group, @@ -5633,15 +5329,16 @@ def retr_pe(units, method=None): units=units, _Frac=_Frac, ) - ) * (1 - _BGas_share) + + _pe_elec_po_turb( + "gas_ct", group, units, _Frac, inpfilter={"commodity": ["gas"]} + ) + ) - # ----------------------------------------------- - # Primary Energy Electricity from gas with CCS - # ----------------------------------------------- + vars["Gas|Electricity|w/o CCS"] = _ElecGas_woCCS * (1 - _BGas_share) - vars["Gas|Electricity|w/ CCS|Natural Gas Steam Cycle"] = ( + _ElecGas_wCCS = ( _pe_elec_wCCSretro( - "gas_ppl", + "gas_cc", "g_ppl_co2scr", group, inpfilter={"level": ["secondary"], "commodity": ["gas"]}, @@ -5649,30 +5346,16 @@ def retr_pe(units, method=None): _Frac=_Frac, share=_gas_cc_shr, ) - ) * (1 - _BGas_share) - - vars["Gas|Electricity|w/ CCS|Natural Gas Combined Cycle"] = ( - _pe_elec_wCCSretro( - "gas_cc", + + _pe_elec_wCCSretro( + "gas_ppl", "g_ppl_co2scr", group, inpfilter={"level": ["secondary"], "commodity": ["gas"]}, units=units, _Frac=_Frac, - share=_gas_cc_shr, - ) - + _pe_elec_po_turb( - "gas_cc_ccs", - group, - units, - _Frac, - inpfilter={"commodity": ["gas"]}, - outfilter={"commodity": "electr"}, + share=_gas_ppl_shr, ) - ) * (1 - _BGas_share) - - vars["Gas|Electricity|w/ CCS|Natural Gas High Temperature Fuel Cell"] = ( - _pe_elec_wCCSretro( + + _pe_elec_wCCSretro( "gas_htfc", "gfc_co2scr", group, @@ -5680,184 +5363,71 @@ def retr_pe(units, method=None): units=units, _Frac=_Frac, ) - ) * (1 - _BGas_share) - - # ----------------------------------------------- - # Primary Energy Electricity from oil without CCS - # ----------------------------------------------- - - vars["Oil|Electricity|w/o CCS|Crude Oil Steam Cycle"] = _pe_elec_po_turb( - "oil_ppl", - group, - units, - _Frac, - inpfilter={"commodity": ["crudeoil"]}, - outfilter={"commodity": "electr"}, - ) - - vars["Oil|Electricity|w/o CCS|Heavy Fuel Oil Steam Cycle"] = _pe_elec_po_turb( - "foil_ppl", - group, - units, - _Frac, - inpfilter={"commodity": ["fueloil"]}, - outfilter={"commodity": "electr"}, - ) - - vars["Oil|Electricity|w/o CCS|Light Fuel Oil Combined Cycle"] = ( - _pe_elec_po_turb( - "loil_cc", - group, - units, - _Frac, - inpfilter={"commodity": ["lightoil"]}, - outfilter={"commodity": "electr"}, + + _pe_elec_po_turb( + "gas_cc_ccs", group, units, _Frac, inpfilter={"commodity": ["gas"]} ) ) - vars["Oil|Electricity|w/o CCS|Light Fuel Oil Steam Cycle"] = _pe_elec_po_turb( - "loil_ppl", - group, - units, - _Frac, - inpfilter={"commodity": ["lightoil"]}, - outfilter={"commodity": "electr"}, - ) - - vars["Oil|Electricity|w/ CCS"] = pp_utils._make_zero() - - # --------------------------------------------------- - # Primary Energy Electricity from biomass without CCS - # --------------------------------------------------- - - vars["Biomass|Electricity|w/o CCS|Biomass Steam Cycle"] = _pe_elec_woCCSretro( - "bio_ppl", - "bio_ppl_co2scr", - group, - inpfilter={"level": ["primary"], "commodity": ["biomass"]}, - units=units, - _Frac=_Frac, - ) + vars["Gas|Electricity|w/ CCS"] = _ElecGas_wCCS * (1 - _BGas_share) - vars["Biomass|Electricity|w/o CCS|Biomass IGCC"] = _pe_elec_po_turb( - "bio_istig", - group, - units, - _Frac, - inpfilter={"commodity": ["biomass"]}, - outfilter={"commodity": "electr"}, - ) - - vars["Biomass|Electricity|w/o CCS|Biogas Steam Cycle"] = ( - _pe_elec_woCCSretro( - "gas_ppl", - "g_ppl_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, - units=units, - _Frac=_Frac, - share=_gas_ppl_shr, - ) - ) * _BGas_share + # ----------------------------------- + # Primary Energy Electricity from oil + # ----------------------------------- - vars["Biomass|Electricity|w/o CCS|Biogas Combustion Turbine"] = ( + vars["Oil|Electricity|w/o CCS"] = ( _pe_elec_po_turb( - "gas_ct", - group, - units, - _Frac, - inpfilter={"commodity": ["gas"]}, - outfilter={"commodity": "electr"}, + "foil_ppl", group, units, _Frac, inpfilter={"commodity": ["fueloil"]} ) - ) * _BGas_share - - vars["Biomass|Electricity|w/o CCS|Biogas Combined Cycle"] = ( - _pe_elec_woCCSretro( - "gas_cc", - "g_ppl_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, - units=units, - _Frac=_Frac, - share=_gas_cc_shr, + + _pe_elec_po_turb( + "loil_ppl", group, units, _Frac, inpfilter={"commodity": ["lightoil"]} ) - ) * _BGas_share - - vars["Biomass|Electricity|w/o CCS|Biogas High Temperature Fuel Cell"] = ( - _pe_elec_woCCSretro( - "gas_htfc", - "gfc_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, - units=units, - _Frac=_Frac, + + _pe_elec_po_turb( + "oil_ppl", group, units, _Frac, inpfilter={"commodity": ["crudeoil"]} + ) + + _pe_elec_po_turb( + "loil_cc", group, units, _Frac, inpfilter={"commodity": ["lightoil"]} ) - ) * _BGas_share - - # ------------------------------------------------ - # Primary Energy Electricity from biomass with CCS - # ------------------------------------------------ - - vars["Biomass|Electricity|w/ CCS|Biomass Steam Cycle"] = _pe_elec_wCCSretro( - "bio_ppl", - "bio_ppl_co2scr", - group, - inpfilter={"level": ["primary"], "commodity": ["biomass"]}, - units=units, - _Frac=_Frac, ) - vars["Biomass|Electricity|w/ CCS|Biomass IGCC"] = _pe_elec_po_turb( - "bio_istig_ccs", - group, - units, - _Frac, - inpfilter={"commodity": ["biomass"]}, - outfilter={"commodity": "electr"}, - ) + vars["Oil|Electricity|w/ CCS"] = pp_utils._make_zero() - vars["Biomass|Electricity|w/ CCS|Biogas Steam Cycle"] = ( - _pe_elec_wCCSretro( - "gas_ppl", - "g_ppl_co2scr", + # --------------------------------------- + # Primary Energy Electricity from biomass + # --------------------------------------- + + vars["Biomass|Electricity|w/o CCS"] = ( + _pe_elec_woCCSretro( + "bio_ppl", + "bio_ppl_co2scr", group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + inpfilter={"level": ["primary"], "commodity": ["biomass"]}, units=units, _Frac=_Frac, - share=_gas_cc_shr, ) - ) * _BGas_share + + _pe_elec_po_turb( + "bio_istig", group, units, _Frac, inpfilter={"commodity": ["biomass"]} + ) + + _ElecGas_woCCS * _BGas_share + ) - vars["Biomass|Electricity|w/ CCS|Biogas Combined Cycle"] = ( + vars["Biomass|Electricity|w/ CCS"] = ( _pe_elec_wCCSretro( - "gas_cc", - "g_ppl_co2scr", + "bio_ppl", + "bio_ppl_co2scr", group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, + inpfilter={"level": ["primary"], "commodity": ["biomass"]}, units=units, _Frac=_Frac, - share=_gas_cc_shr, ) + _pe_elec_po_turb( - "gas_cc_ccs", + "bio_istig_ccs", group, units, _Frac, - inpfilter={"commodity": ["gas"]}, - outfilter={"commodity": "electr"}, - ) - ) * _BGas_share - - vars["Biomass|Electricity|w/ CCS|Biogas High Temperature Fuel Cell"] = ( - _pe_elec_wCCSretro( - "gas_htfc", - "gfc_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, - units=units, - _Frac=_Frac, + inpfilter={"commodity": ["biomass"]}, ) - ) * _BGas_share + + _ElecGas_wCCS * _BGas_share + ) # ----------------------------------- # Primary Energy from biomass (other) @@ -5917,31 +5487,17 @@ def retr_pe(units, method=None): } ) - vars["Biomass|Residues|Crops"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Primary Energy|Biomass|Residues|Crops"], - } - ) - vars["Biomass|Residues|Forest industry"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Primary Energy|Biomass|Residues|Forest industry"], + "commodity": ["Primary Energy|Biomass|Residues|" "Forest industry"], } ) vars["Biomass|Residues|Logging"] = pp.land_out( lu_out_filter={ "level": ["land_use_reporting"], - "commodity": ["Primary Energy|Biomass|Residues|Logging"], - } - ) - - vars["Biomass|Roundwood harvest"] = pp.land_out( - lu_out_filter={ - "level": ["land_use_reporting"], - "commodity": ["Primary Energy|Biomass|Roundwood harvest"], + "commodity": ["Primary Energy|Biomass|Residues|" "Logging"], } ) @@ -5955,8 +5511,6 @@ def retr_pe(units, method=None): ["gas_bio"], units, inpfilter={"commodity": ["biomass"]} ) - vars["Biomass|Heat"] = pp.inp("bio_hpl", units) - vars["Biomass|Hydrogen"] = pp.inp( ["h2_bio", "h2_bio_ccs"], units, inpfilter={"commodity": ["biomass"]} ) @@ -5979,8 +5533,6 @@ def retr_pe(units, method=None): ["coal_gas"], units, inpfilter={"commodity": ["coal"]} ) - vars["Coal|Heat"] = pp.inp("coal_hpl", units) - vars["Coal|Hydrogen"] = pp.inp( ["h2_coal", "h2_coal_ccs"], units, inpfilter={"commodity": ["coal"]} ) @@ -6011,8 +5563,6 @@ def retr_pe(units, method=None): ["gas_t_d", "gas_t_d_ch4"], units, inpfilter={"commodity": ["gas"]} ) * (1 - _SynGas_share) - vars["Gas|Heat"] = pp.inp("gas_hpl", units) - vars["Gas|Hydrogen"] = pp.inp( ["h2_smr", "h2_smr_ccs"], units, inpfilter={"commodity": ["gas"]} ) @@ -6028,11 +5578,7 @@ def retr_pe(units, method=None): # ------------------------------------------------ vars["Oil|Gases"] = pp_utils._make_zero() - - vars["Oil|Heat"] = pp.inp("foil_hpl", units) - vars["Oil|Hydrogen"] = pp_utils._make_zero() - vars["Oil|Liquids"] = pp.inp( ["loil_t_d", "foil_t_d"], units, @@ -6062,160 +5608,44 @@ def retr_ppl_capparameters(prmfunc, units): Units to which variables should be converted. """ - cumulative = True if prmfunc == "pp.cumcap" else False - prmfunc = eval(prmfunc) vars = {} - # ------------------------------- - # Calculation of helper variables - # ------------------------------- - - # Calculate share of activity for later calculating distribution - # of gas-co2-scrubber capacity. - # Because a powerplant doesnt necessaarily need to have activity in the years - # when capacity installed for the scrubber, it is assumed that gas_cc "fills" - # the gap, hence NANs are replaced with 1 - if run_history != "True": - group = ["Region", "Vintage"] - else: - group = ["Region"] - _gas_cc_shr = pp.out("gas_cc", group=group) / pp.out( - ["gas_cc", "gas_ppl"], group=group - ) - _gas_ppl_shr = pp.out("gas_ppl", group=group) / pp.out( - ["gas_cc", "gas_ppl"], group=group - ) - - # Before filling NANs, if the values are being calculated for cumulative vraibles, - # then the forward fill needs to be done prior to filling NANs, or else the "0" - # cannot be preserved where there really is no activity. - if cumulative: - _gas_cc_shr = _gas_cc_shr.ffill(axis=1) - _gas_ppl_shr = _gas_ppl_shr.ffill(axis=1) - - _gas_cc_shr = _gas_cc_shr.fillna(1) - _gas_ppl_shr = _gas_ppl_shr.fillna(0) - - # ------------------------------ - # Start of calcualting variables - # ------------------------------ - # ["Note": "IGCC with CCS"] vars["Electricity|Biomass|w/ CCS|1"] = prmfunc("bio_istig_ccs") - # ["Note": "Steam cycle, scrubber retrofit only"] - # The scrubber retrofit is reported accounting for the efficiency. - # The MESSAGEV efficiency was .75; elec-input was .25 - # With the normlazing to output, the electricity input is now .333 (=.25/.75). - # The scrubber efficiency has been "lost" and hence using the current - # efficiency .66 (1.-.33), we underestimate the scrubber-capacity. - # The same applies to other scrubbers below. - vars["Electricity|Biomass|w/ CCS|2"] = _calc_scrubber_capacity( - prmfunc, "bio_ppl_co2scr", group, cumulative=cumulative - ) - # ["Note": "IGCC"] vars["Electricity|Biomass|w/o CCS|1"] = prmfunc("bio_istig") # ["Note": "Steam cycle"] - bio_wo_ccs2 = (prmfunc("bio_ppl") - prmfunc("bio_ppl_co2scr")).fillna(0) - # Ensure that all negative values are 0. This can occur when the - # scrubber instaallations dont match the yeaar in which powerplant - # capaacities are installed. - bio_wo_ccs2[bio_wo_ccs2 < 0] = 0 - vars["Electricity|Biomass|w/o CCS|2"] = bio_wo_ccs2 + vars["Electricity|Biomass|w/o CCS|2"] = prmfunc("bio_ppl") # ["Note": "IGCC with CCS"] - # The scrubber retrofit is reported accounting for the efficiency. - # The MESSAGEV efficiency was .85; elec-input was .15 - vars["Electricity|Coal|w/ CCS|1"] = ( - prmfunc("igcc_ccs") - + _calc_scrubber_capacity(prmfunc, "igcc_co2_scr", group, cumulative=cumulative) - ).fillna(0) + vars["Electricity|Coal|w/ CCS|1"] = prmfunc("igcc_ccs") # ["Note": "Steam cycle super critical with CCS"] vars["Electricity|Coal|w/ CCS|2"] = prmfunc("coal_adv_ccs") - # ["Note": "Steam cycle sub critical Filtered, scrubber retrofit only"] - # The scrubber retrofit is reported accounting for the efficiency. - # The MESSAGEV efficiency was .75; elec-input was .25 - vars["Electricity|Coal|w/ CCS|3"] = ( - prmfunc("c_ppl_co2scr") - + _calc_scrubber_capacity(prmfunc, "c_ppl_co2scr", group, cumulative=cumulative) - ).fillna(0) - # ["Note": "IGCC"] - coal_wo_ccs1 = (prmfunc("igcc") - prmfunc("igcc_co2scr")).fillna(0) - # Ensure that all negative values are 0. This can occur when the - # scrubber instaallations dont match the yeaar in which powerplant - # capaacities are installed. - coal_wo_ccs1[coal_wo_ccs1 < 0] = 0 - vars["Electricity|Coal|w/o CCS|1"] = coal_wo_ccs1 + vars["Electricity|Coal|w/o CCS|1"] = prmfunc("igcc") # ["Note": "Steam cycle super critical"] vars["Electricity|Coal|w/o CCS|2"] = prmfunc("coal_adv") # ["Note": "Steam cycle sub critical Filtered"] - coal_wo_ccs3 = (prmfunc("coal_ppl") - prmfunc("c_ppl_co2scr")).fillna(0) - # Ensure that all negative values are 0. This can occur when the - # scrubber instaallations dont match the yeaar in which powerplant - # capaacities are installed. - coal_wo_ccs3[coal_wo_ccs3 < 0] = 0 - vars["Electricity|Coal|w/o CCS|3"] = coal_wo_ccs3 + vars["Electricity|Coal|w/o CCS|3"] = prmfunc("coal_ppl") # ["Note": "Steam cycle sub critical unfiltered"] vars["Electricity|Coal|w/o CCS|4"] = prmfunc("coal_ppl_u") # ["Note": "Combined cycle with CCS"] - # The scrubber retrofit is reported accounting for the efficiency. - # The MESSAGEV efficiency was .87; elec-input was .13 - vars["Electricity|Gas|w/ CCS|1"] = ( - prmfunc("gas_cc_ccs") - + _calc_scrubber_capacity( - prmfunc, "g_ppl_co2scr", group, share=_gas_cc_shr, cumulative=cumulative - ) - ).fillna(0) - - vars["Electricity|Gas|w/ CCS|2"] = _calc_scrubber_capacity( - prmfunc, "g_ppl_co2scr", group, share=_gas_ppl_shr, cumulative=cumulative - ) + vars["Electricity|Gas|w/ CCS|1"] = prmfunc("gas_cc_ccs") # ["Note": "Combined cycle"] - gas_wo_ccs1 = ( - prmfunc("gas_cc") - - _calc_scrubber_capacity( - prmfunc, - "g_ppl_co2scr", - group, - share=_gas_cc_shr, - cumulative=cumulative, - efficiency=False, - ) - ).fillna(0) - # Ensure that all negative values are 0. This can occur when the - # scrubber instaallations dont match the yeaar in which powerplant - # capaacities are installed. - gas_wo_ccs1[gas_wo_ccs1 < 0] = 0 - vars["Electricity|Gas|w/o CCS|1"] = gas_wo_ccs1 + vars["Electricity|Gas|w/o CCS|1"] = prmfunc("gas_cc") # ["Note": "Steam cycle sub cricitcal"] - gas_wo_ccs2 = ( - prmfunc("gas_ppl") - - _calc_scrubber_capacity( - prmfunc, - "g_ppl_co2scr", - group, - share=_gas_ppl_shr, - cumulative=cumulative, - efficiency=False, - ) - ).fillna(0) - # Ensure that all negative values are 0. This can occur when the - # scrubber instaallations dont match the yeaar in which powerplant - # capaacities are installed. - gas_wo_ccs2[gas_wo_ccs2 < 0] = 0 - vars["Electricity|Gas|w/o CCS|2"] = gas_wo_ccs2 + vars["Electricity|Gas|w/o CCS|2"] = prmfunc("gas_ppl") # ["Note": "Combustion turbine"] vars["Electricity|Gas|w/o CCS|3"] = prmfunc("gas_ct") @@ -7030,95 +6460,6 @@ def retr_fe(units): vars = {} - # ---------------------------------------------------- - # Calculate share of hydrogen in FE - # based on method applied for allocating CO2 emissions - # ---------------------------------------------------- - - if run_history != "True": - group = ["Region", "Mode", "Vintage"] - else: - group = ["Region"] - - _inp_nonccs_gas_tecs = ( - pp.inp( - [ - "gas_rc", - "hp_gas_rc", - "gas_i", - "hp_gas_i", - "gas_trp", - # "gas_fs", - "gas_ppl", - "gas_ct", - "gas_cc", - "gas_htfc", - "gas_hpl", - ], - units, - inpfilter={"commodity": ["gas"]}, - ) - + pp.inp(["gas_t_d", "gas_t_d_ch4"], units, inpfilter={"commodity": ["gas"]}) - - pp.out(["gas_t_d", "gas_t_d_ch4"], units, outfilter={"commodity": ["gas"]}) - ) - - # Calculate shares for ppl feeding into g_ppl_co2scr (gas_cc and gas_ppl) - _gas_cc_shr = (pp.out("gas_cc") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) - - _gas_ppl_shr = (pp.out("gas_ppl") / pp.out(["gas_cc", "gas_ppl"])).fillna(0) - - _inp_nonccs_gas_tecs_wo_CCSRETRO = ( - _inp_nonccs_gas_tecs - - _pe_wCCSretro( - "gas_cc", - "g_ppl_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, - units=units, - share=_gas_cc_shr, - ) - - _pe_wCCSretro( - "gas_ppl", - "g_ppl_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, - units=units, - share=_gas_ppl_shr, - ) - - _pe_wCCSretro( - "gas_htfc", - "gfc_co2scr", - group, - inpfilter={"level": ["secondary"], "commodity": ["gas"]}, - units=units, - ) - ) - - _Hydrogen_tot = pp.out( - "h2_mix", - units, - ) - - _Hydrogen_ind = _Hydrogen_tot * ( - pp.inp(["gas_i", "hp_gas_i"], units, inpfilter={"commodity": ["gas"]}) - / _inp_nonccs_gas_tecs_wo_CCSRETRO - ).fillna(0) - - # _Hydrogen_fs = _Hydrogen_tot * ( - # pp.inp("gas_fs", units, inpfilter={"commodity": ["gas"]}) - # / _inp_nonccs_gas_tecs_wo_CCSRETRO - # ).fillna(0) - - _Hydrogen_res = _Hydrogen_tot * ( - pp.inp(["gas_rc", "hp_gas_rc"], units, inpfilter={"commodity": ["gas"]}) - / _inp_nonccs_gas_tecs_wo_CCSRETRO - ).fillna(0) - - _Hydrogen_trp = _Hydrogen_tot * ( - pp.inp("gas_trp", units, inpfilter={"commodity": ["gas"]}) - / _inp_nonccs_gas_tecs_wo_CCSRETRO - ).fillna(0) - # --------------- # Industry sector # --------------- @@ -7129,13 +6470,10 @@ def retr_fe(units): MethIND = pp.inp(["sp_meth_I", "meth_i", "methanol_fs"], units) EthIND = pp.inp(["eth_i", "ethanol_fs", "sp_eth_I"], units) - GasIND = ( - pp.inp( - ["gas_i", "hp_gas_i", "gas_fs"], - units, - inpfilter={"level": ["final"], "commodity": ["gas"]}, - ) - - _Hydrogen_ind + GasIND = pp.inp( + ["gas_i", "hp_gas_i", "gas_fs"], + units, + inpfilter={"level": ["final"], "commodity": ["gas"]}, ) CoalIND = pp.inp(["coal_i", "coal_fs"], units) @@ -7149,7 +6487,7 @@ def retr_fe(units): OnsitePVIND = pp.out("solar_pv_I", units) DheatIND = pp.inp("heat_i", units) SolThermIND = pp.out("solar_i", units) - H2IND = pp.inp(["h2_i", "h2_fc_I"], units) + _Hydrogen_ind + H2IND = pp.inp(["h2_i", "h2_fc_I"], units) vars["Industry|Electricity"] = ElecIND + OnsitePVIND vars["Industry|Electricity|Solar"] = OnsitePVIND @@ -7168,21 +6506,18 @@ def retr_fe(units): # Residential and commercial sector # --------------------------------- - BiomassRC = pp.inp("biomass_rc", units) - OilRC = pp.inp(["foil_rc", "loil_rc"], units) - MethRC = pp.inp("meth_rc", units) - EthRC = pp.inp("eth_rc", units) + BiomassRC = pp.inp(TECHS["rc biomass"], units) + OilRC = pp.inp(get_techs("rc", "foil loil"), units) + MethRC = pp.inp(TECHS["rc meth"], units) + EthRC = pp.inp(TECHS["rc eth"], units) - GasRC = ( - pp.inp( - ["gas_rc", "hp_gas_rc"], - units, - inpfilter={"level": ["final"], "commodity": ["gas"]}, - ) - - _Hydrogen_res + GasRC = pp.inp( + TECHS["rc gas"], + units, + inpfilter={"level": ["final"], "commodity": ["gas"]}, ) - CoalRC = pp.inp("coal_rc", units) + CoalRC = pp.inp(TECHS["rc coal"], units) ElecRC = pp.inp("sp_el_RC", units) # Comment OFR: In next variable - Added electricity requirements for gas @@ -7191,23 +6526,20 @@ def retr_fe(units): "hp_gas_rc", units, inpfilter={"level": ["final"], "commodity": ["electr"]} ) - ElecDirRC = pp.inp("elec_rc", units) + ElecDirRC = pp.inp(TECHS["rc elec"], units) ElecFcTRP = -1.0 * pp.out( - "h2_fc_trp", units, outfilter={"level": ["final"], "commodity": ["electr"]} + TECHS["trp h2"], units, outfilter={"level": ["final"], "commodity": ["electr"]} ) OnsitePVRC = pp.out("solar_pv_RC", units) - DheatRC = pp.inp("heat_rc", units) - SolThermRC = pp.out("solar_rc", units) + DheatRC = pp.inp(TECHS["rc heat"], units) + SolThermRC = pp.out(TECHS["rc solar"], units) - H2RC = ( - pp.inp( - ["h2_rc", "h2_fc_RC", "h2_fc_trp"], - units, - inpfilter={"level": ["final"], "commodity": ["hydrogen"]}, - ) - + _Hydrogen_res + H2RC = pp.inp( + TECHS["rc h2"] + TECHS["trp h2"], + units, + inpfilter={"level": ["final"], "commodity": ["hydrogen"]}, ) BiomassNC = pp.inp("biomass_nc", units) @@ -7235,21 +6567,20 @@ def retr_fe(units): # Transportation # -------------- - OilTRP = pp.inp(["foil_trp", "loil_trp"], units) + OilTRP = pp.inp(get_techs("trp", "foil loil"), units) OilTRP_shipping_foil = pp.inp(["foil_bunker"], units) OilTRP_shipping_loil = pp.inp(["loil_bunker"], units) - MethTRP = pp.inp(["meth_ic_trp", "meth_fc_trp"], units) + MethTRP = pp.inp(TECHS["trp meth"], units) MethTRP_shipping = pp.inp(["meth_bunker"], units) - EthTRP = pp.inp(["eth_ic_trp", "eth_fc_trp"], units) + EthTRP = pp.inp(TECHS["trp eth"], units) EthTRP_shipping = pp.inp(["eth_bunker"], units) - GasTRP = pp.inp(["gas_trp"], units) - _Hydrogen_trp + GasTRP = pp.inp(TECHS["trp gas"], units) GasTRP_shipping = pp.inp(["LNG_bunker"], units) - CoalTRP = pp.inp("coal_trp", units) - ElecTRP = pp.inp("elec_trp", units) + CoalTRP = pp.inp(TECHS["trp coal"], units) + ElecTRP = pp.inp(TECHS["trp elec"], units) - H2TRP = ( - pp.inp("h2_fc_trp", units, inpfilter={"level": ["final"], "commodity": ["lh2"]}) - + _Hydrogen_trp + H2TRP = pp.inp( + TECHS["trp h2"], units, inpfilter={"level": ["final"], "commodity": ["lh2"]} ) H2TRP_shipping = pp.inp("LH2_bunker", units) @@ -7878,7 +7209,7 @@ def retr_supply_inv(units_energy, units_emi, units_ene_mdl): # ------------------------------------------ _CCS_coal_elec = -1 * pp.emi( - ["c_ppl_co2scr", "coal_adv_ccs", "igcc_ccs", "cement_co2scr"], + ["c_ppl_co2scr", "coal_adv_ccs", "igcc_ccs"] + TECHS["cement with ccs"], units_ene_mdl, emifilter={"relation": ["CO2_Emission"]}, emission_units=units_emi, @@ -7933,13 +7264,13 @@ def retr_supply_inv(units_energy, units_emi, units_ene_mdl): "meth_ng_ccs", "h2_smr", "h2_smr_ccs", - "gas_rc", - "hp_gas_rc", "gas_i", "hp_gas_i", - "gas_trp", "gas_fs", ] + + TECHS["rc gas"] + + TECHS["trp gas"] + + TECHS["gas extra"] ) _Biogas_share = (_Biogas_use_tot / _Gas_use_tot).fillna(0) @@ -8045,8 +7376,8 @@ def retr_supply_inv(units_energy, units_emi, units_ene_mdl): ) vars["Liquids|Oil"] = pp.investment( - ["ref_lol", "ref_hil"], units=units_energy - ) + pp.tic_fom(["ref_hil", "ref_lol"], units=units_energy) + TECHS["refining"], units=units_energy + ) + pp.tic_fom(TECHS["refining"], units=units_energy) vars["Liquids"] = ( vars["Liquids|Coal and Gas"] @@ -8636,22 +7967,8 @@ def retr_water_use(units, method): + _pe_elec_wCCSretro( "igcc", "igcc_co2scr", group, inpfilter=inpfilter, units=units, _Frac=_Frac ) - + _pe_elec_po_turb( - "coal_adv_ccs", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) - + _pe_elec_po_turb( - "igcc_ccs", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + + _pe_elec_po_turb("coal_adv_ccs", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("igcc_ccs", group, units, _Frac, inpfilter=inpfilter) + _out_div_eff( ["meth_coal_ccs", "h2_coal_ccs", "syn_liq_ccs"], group, inpfilter, outfilter ) @@ -8678,22 +7995,8 @@ def retr_water_use(units, method): + _pe_elec_woCCSretro( "igcc", "igcc_co2scr", group, inpfilter=inpfilter, units=units, _Frac=_Frac ) - + _pe_elec_po_turb( - "coal_adv", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) - + _pe_elec_po_turb( - "coal_ppl_u", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + + _pe_elec_po_turb("coal_adv", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("coal_ppl_u", group, units, _Frac, inpfilter=inpfilter) + _out_div_eff(["meth_coal", "h2_coal", "syn_liq"], group, inpfilter, outfilter) ) @@ -8734,14 +8037,7 @@ def retr_water_use(units, method): units=units, _Frac=_Frac, ) - + _pe_elec_po_turb( - "gas_cc_ccs", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + + _pe_elec_po_turb("gas_cc_ccs", group, units, _Frac, inpfilter=inpfilter) + _out_div_eff(["h2_smr_ccs"], group, inpfilter, outfilter) ) @@ -8780,14 +8076,7 @@ def retr_water_use(units, method): units=units, _Frac=_Frac, ) - + _pe_elec_po_turb( - "gas_ct", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + + _pe_elec_po_turb("gas_ct", group, units, _Frac, inpfilter=inpfilter) + _out_div_eff(["h2_smr"], group, inpfilter, outfilter) ) @@ -8812,14 +8101,7 @@ def retr_water_use(units, method): units=units, _Frac=_Frac, ) - + _pe_elec_po_turb( - "bio_istig_ccs", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + + _pe_elec_po_turb("bio_istig_ccs", group, units, _Frac, inpfilter=inpfilter) + _out_div_eff( ["h2_bio_ccs", "eth_bio_ccs", "liq_bio_ccs"], group, inpfilter, outfilter ) @@ -8844,14 +8126,7 @@ def retr_water_use(units, method): units=units, _Frac=_Frac, ) - + _pe_elec_po_turb( - "bio_istig", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + + _pe_elec_po_turb("bio_istig", group, units, _Frac, inpfilter=inpfilter) + _out_div_eff(["h2_bio", "eth_bio", "liq_bio"], group, inpfilter, outfilter) ) @@ -8870,38 +8145,10 @@ def retr_water_use(units, method): ) _Direct_ele_oil = ( - _pe_elec_po_turb( - "foil_ppl", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) - + _pe_elec_po_turb( - "loil_ppl", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) - + _pe_elec_po_turb( - "oil_ppl", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) - + _pe_elec_po_turb( - "loil_cc", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + _pe_elec_po_turb("foil_ppl", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("loil_ppl", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("oil_ppl", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("loil_cc", group, units, _Frac, inpfilter=inpfilter) ) vars["Electricity|Oil"] = _Direct_ele_oil + _Cooling_ele_oil @@ -8915,12 +8162,7 @@ def retr_water_use(units, method): ) _Direct_ele_geo = _pe_elec_po_turb( - "geo_ppl", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, + "geo_ppl", group, units, _Frac, inpfilter=inpfilter ) vars["Electricity|Geothermal"] = _Direct_ele_geo + _Cooling_ele_geo @@ -8947,30 +8189,9 @@ def retr_water_use(units, method): ) _Direct_ele_nuc = ( - _pe_elec_po_turb( - "nuc_lc", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) - + _pe_elec_po_turb( - "nuc_hc", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) - + _pe_elec_po_turb( - "nuc_fbr", - group, - units, - _Frac, - inpfilter=inpfilter, - outfilter={"commodity": "electr"}, - ) + _pe_elec_po_turb("nuc_lc", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("nuc_hc", group, units, _Frac, inpfilter=inpfilter) + + _pe_elec_po_turb("nuc_fbr", group, units, _Frac, inpfilter=inpfilter) ) vars["Electricity|Nuclear"] = _Direct_ele_nuc + _Cooling_ele_nuc @@ -9081,7 +8302,7 @@ def retr_water_use(units, method): ) vars["Electricity|Other"] = pp.inp( - ["h2_fc_trp", "h2_fc_I", "h2_fc_RC"], units, inpfilter=inpfilter + ["h2_fc_I", "h2_fc_RC"] + TECHS["trp h2"], units, inpfilter=inpfilter ) vars["Electricity|Dry Cooling"] = ( @@ -9551,5 +8772,19 @@ def retr_water_use(units, method): ["ref_lol", "ref_hil"], units=units, emiffilter=emiffilter ) + _check_inf(vars) + df = pp_utils.make_outputdf(vars, units) return df + + +def _check_inf(vars: Mapping[str, pd.DataFrame]): + """Check for NaN or infinite values in `vars`.""" + fail = False + for name, data in vars.items(): + mask = (data.abs() == np.inf).any(axis=1) + if mask.any(): + fail = True + print(f"±inf values in {name!r}", data[mask].to_string(), sep="\n") + + assert not fail From a350708c725e18d80c215565768a96e1580dcead Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:10:05 +0100 Subject: [PATCH 07/10] Update .report.legacy.ENGAGE_SSP2_v417_tables Overwrite with version from iiasa/message_data branch `dev`. --- .../report/legacy/ENGAGE_SSP2_v417_tables.py | 359 +----------------- 1 file changed, 4 insertions(+), 355 deletions(-) diff --git a/message_ix_models/report/legacy/ENGAGE_SSP2_v417_tables.py b/message_ix_models/report/legacy/ENGAGE_SSP2_v417_tables.py index 8e76bbc857..1eab185fc8 100644 --- a/message_ix_models/report/legacy/ENGAGE_SSP2_v417_tables.py +++ b/message_ix_models/report/legacy/ENGAGE_SSP2_v417_tables.py @@ -1,6 +1,6 @@ import pandas as pd -from . import pp_utils +import message_data.tools.post_processing.pp_utils as pp_utils pp = None mu = None @@ -941,357 +941,6 @@ def retr_pop(units): return df -@_register -def retr_price( - units_CPrc_co2, - units_CPrc_co2_outp, - units_energy, - units_energy_outp, - units_CPrc_c, - conv_CPrc_co2_to_c, - units_agri, -): - """Price: Prices for Emissions, Energy and Land-Use. - - Parameters - ---------- - - units_CPrc_co2 : str - Units to which carbon price (in CO2 equivalent) should be converted. - units_CPrc_co2_outp : str - Units in which carbon price (in CO2 equivalent) should be reported in. - units_energy : str - Units to which energy variables should be converted. - units_energy_outp : str - Units to which variables should be converted. - units_CPrc_c : str - Units to which carbon price (in carbon equivalent) should be converted. - conv_CPrc_co2_to_c : number - Conversion factor for carbon price from CO2 to C. - units_agri : str - Units to which agricultural variables should be converted. - """ - - dfs = [] - - # ---------------------- - # Calculate Carbon Price - # ---------------------- - - vars = {} - vars["Carbon"] = pp.cprc(units_CPrc_co2) - dfs.append(pp_utils.make_outputdf(vars, units_CPrc_co2_outp, param="max")) - - # ------------------------------------------------- - # Calculate Final Energy Prices incl. carbon prices - # ------------------------------------------------- - - # Carbon price must also be multiplied by factor as retrieved - # prices are already converted to $/GJ - - _cbprc = pp.cprc(units_CPrc_c) * conv_CPrc_co2_to_c - vars = {} - # Final Energy prices are reported including the carbon price - vars["Final Energy|Residential|Electricity"] = pp.eneprc( - enefilter={"commodity": ["electr"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy|Residential|Gases|Natural Gas"] = pp.eneprc( - enefilter={"commodity": ["gas"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy|Residential|Liquids|Biomass"] = pp.eneprc( - enefilter={"commodity": ["ethanol"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy|Residential|Liquids|Oil"] = pp.eneprc( - enefilter={"commodity": ["lightoil"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy|Residential|Solids|Biomass"] = pp.eneprc( - enefilter={"commodity": ["biomass"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy|Residential|Solids|Coal"] = pp.eneprc( - enefilter={"commodity": ["coal"], "level": ["final"]}, units=units_energy - ) - - # ------------------------------------------------- - # Calculate Final Energy Prices excl. carbon prices - # ------------------------------------------------- - - vars["Final Energy wo carbon price|Residential|Electricity"] = pp.eneprc( - enefilter={"commodity": ["electr"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy wo carbon price|Residential|Gases|Natural Gas"] = ( - pp.eneprc( - enefilter={"commodity": ["gas"], "level": ["final"]}, units=units_energy - ) - - _cbprc * mu["crbcnt_gas"] - ) - - vars["Final Energy wo carbon price|Residential|Liquids|Biomass"] = pp.eneprc( - enefilter={"commodity": ["ethanol"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy wo carbon price|Residential|Liquids|Oil"] = ( - pp.eneprc( - enefilter={"commodity": ["lightoil"], "level": ["final"]}, - units=units_energy, - ) - - _cbprc * mu["crbcnt_oil"] - ) - - vars["Final Energy wo carbon price|Residential|Solids|Biomass"] = pp.eneprc( - enefilter={"commodity": ["biomass"], "level": ["final"]}, units=units_energy - ) - - vars["Final Energy wo carbon price|Residential|Solids|Coal"] = ( - pp.eneprc( - enefilter={"commodity": ["coal"], "level": ["final"]}, units=units_energy - ) - - _cbprc * mu["crbcnt_coal"] - ) - - # --------------------------------------------------- - # Calculate Primary Energy Prices excl. carbon prices - # --------------------------------------------------- - - # Primary energy prices are reported excluding the carbon price, hence the - # subtraction of the carbonprice * the carbon content of the fuel - - vars["Primary Energy w carbon price|Biomass"] = pp.eneprc( - enefilter={"commodity": ["biomass"], "level": ["primary"]}, units=units_energy - ) - - vars["Primary Energy|Biomass"] = vars["Primary Energy w carbon price|Biomass"] - - # ----------------------------------------------------- - # Calculate Secondary Energy Prices excl. carbon prices - # ----------------------------------------------------- - - # Secondary energy prices are reported excluding the carbon price, hence the - # subtraction of the carbonprice * the carbon content of the fuel - - # vars["Secondary Energy|Gases|Natural Gas"] =pp.eneprc( - # enefilter={"commodity": [""], - # "level": ["secondary"]}, - # units=units_energy) - - # vars["Secondary Energy|Liquids"] = pp.eneprc( - # enefilter={"commodity": [""], - # "level": ["secondary"]}, - # units=units_energy) - - vars["Secondary Energy|Liquids"] = pp_utils._make_zero() - - # vars["Secondary Energy|Solids|Biomass"] = pp.eneprc( - # enefilter={"commodity": [""], - # "level": ["secondary"]}, - # units=units_energy) - - # vars["Secondary Energy|Solids|Coal"] = pp.eneprc( - # enefilter={"commodity": [""], - # "level": ["secondary"]}, - # units=units_energy) - _cbprc * crbcnt_coal - - # Secondary energy prices are reported including the carbon price - vars["Secondary Energy w carbon price|Liquids"] = pp_utils._make_zero() - - dfs.append(pp_utils.make_outputdf(vars, units_energy_outp, param="max")) - - # ------------------------------------------------------ - # Calculate Energy Prices for variables where the GLOBAL - # value variable differs from regional value variable - # ------------------------------------------------------ - - vars = {} - - # Primary energy prices are reported excluding the carbon price, hence the - # subtraction of the carbonprice * the carbon content of the fuel - - # Coal uses primary-coal for regions and import-coal for GLB - vars["Primary Energy w carbon price|Coal"] = pd.concat( - [ - pp_utils.rem_glb( - pp.eneprc( - enefilter={"commodity": ["coal"], "level": ["primary"]}, - units=units_energy, - ) - ), - pp_utils.rem_reg( - pp.eneprc( - enefilter={"commodity": ["coal"], "level": ["import"]}, - units=units_energy, - ) - ), - ], - sort=True, - ) - - vars["Primary Energy|Coal"] = vars["Primary Energy w carbon price|Coal"].subtract( - pp_utils.rem_glb(_cbprc) * mu["crbcnt_coal"], fill_value=0 - ) - - # Gas uses primary-gas for regions and import-LNG for GLB - # Note: GLOBAL region uses LNG prices - vars["Primary Energy w carbon price|Gas"] = pd.concat( - [ - pp_utils.rem_glb( - pp.eneprc( - enefilter={"commodity": ["gas"], "level": ["primary"]}, - units=units_energy, - ) - ), - pp_utils.rem_reg( - pp.eneprc( - enefilter={"commodity": ["LNG"], "level": ["import"]}, - units=units_energy, - ) - ), - ], - sort=True, - ) - - vars["Primary Energy|Gas"] = vars["Primary Energy w carbon price|Gas"].subtract( - pp_utils.rem_glb(_cbprc) * mu["crbcnt_gas"], fill_value=0 - ) - - # Oil uses primary-crudeoil for regions and import-crudeoil for GLB - vars["Primary Energy w carbon price|Oil"] = pd.concat( - [ - pp_utils.rem_glb( - pp.eneprc( - enefilter={"commodity": ["crudeoil"], "level": ["secondary"]}, - units=units_energy, - ) - ), - pp_utils.rem_reg( - pp.eneprc( - enefilter={"commodity": ["crudeoil"], "level": ["import"]}, - units=units_energy, - ) - ), - ], - sort=True, - ) - - vars["Primary Energy|Oil"] = vars["Primary Energy w carbon price|Oil"].subtract( - pp_utils.rem_glb(_cbprc) * mu["crbcnt_oil"], fill_value=0 - ) - - # Secondary energy prices are reported including the carbon price - # Oil uses secondary-lightoil for regions and import-lightoil for GLB - vars["Secondary Energy w carbon price|Liquids|Oil"] = pd.concat( - [ - pp_utils.rem_glb( - pp.eneprc( - enefilter={"commodity": ["lightoil"], "level": ["secondary"]}, - units=units_energy, - ) - ), - pp_utils.rem_reg( - pp.eneprc( - enefilter={"commodity": ["lightoil"], "level": ["import"]}, - units=units_energy, - ) - ), - ], - sort=True, - ) - - vars["Secondary Energy|Liquids|Oil"] = vars[ - "Secondary Energy w carbon price|Liquids|Oil" - ].subtract(pp_utils.rem_glb(_cbprc) * mu["crbcnt_oil"], fill_value=0) - - # Biomass uses secondary-ethanol for regions and import-ethanol for GLB - vars["Secondary Energy w carbon price|Liquids|Biomass"] = pd.concat( - [ - pp_utils.rem_glb( - pp.eneprc( - enefilter={"commodity": ["ethanol"], "level": ["primary"]}, - units=units_energy, - ) - ), - pp_utils.rem_reg( - pp.eneprc( - enefilter={"commodity": ["ethanol"], "level": ["import"]}, - units=units_energy, - ) - ), - ], - sort=True, - ) - - # Carbon-price is NOT subtracted - vars["Secondary Energy|Liquids|Biomass"] = vars[ - "Secondary Energy w carbon price|Liquids|Biomass" - ] - - # Hydrogen uses secondary-hydrogen for regions and import-lh2 for GLB - vars["Secondary Energy w carbon price|Hydrogen"] = pd.concat( - [ - pp_utils.rem_glb( - pp.eneprc( - enefilter={"commodity": ["hydrogen"], "level": ["secondary"]}, - units=units_energy, - ) - ), - pp_utils.rem_reg( - pp.eneprc( - enefilter={"commodity": ["lh2"], "level": ["import"]}, - units=units_energy, - ) - ), - ], - sort=True, - ) - - # Carbon-price is NOT subtracted - vars["Secondary Energy|Hydrogen"] = vars["Secondary Energy w carbon price|Hydrogen"] - - dfs.append(pp_utils.make_outputdf(vars, units_energy_outp, glb=False)) - - # ------------------------------------------------------ - # Calculate Energy Prices for variables where the GLOBAL - # value is set to ZERO - # ------------------------------------------------------ - - vars = {} - - vars["Secondary Energy|Electricity"] = pp.eneprc( - enefilter={"commodity": ["electr"], "level": ["secondary"]}, units=units_energy - ) - - vars["Secondary Energy w carbon price|Electricity"] = pp.eneprc( - enefilter={"commodity": ["electr"], "level": ["secondary"]}, units=units_energy - ) - - dfs.append(pp_utils.make_outputdf(vars, units_energy_outp, glb=False)) - - # ---------------------------------------------------- - # Calculate Agricultural Prices where the global price - # is derived using a formula provided by GLOBIOM - # ---------------------------------------------------- - - vars = {} - - # Define the technology which represents the quantities used to derive - # sums accross regions values. - scale_tec = "Agricultural Production|Non-Energy Crops" - - vars["Agriculture|Non-Energy Crops and Livestock|Index"] = pp.retrieve_lu_price( - "Price|Agriculture|Non-Energy Crops and Livestock|Index", scale_tec - ) - vars["Agriculture|Non-Energy Crops|Index"] = pp.retrieve_lu_price( - "Price|Agriculture|Non-Energy Crops|Index", scale_tec - ) - dfs.append(pp_utils.make_outputdf(vars, units_agri, glb=False)) - - return pd.concat(dfs, sort=True) - - @_register def retr_kyoto(units): """Emissions: Kyoto Gases. @@ -3862,9 +3511,9 @@ def retr_pe(units, method=None): # Primary Energy Biomass # ---------------------- - vars["Biomass"] = pp.out( - ["biomass_imp", "land_use_biomass"], units, outfilter={"commodity": ["biomass"]} - ) - pp.inp(["biomass_exp"], units, inpfilter={"commodity": ["biomass"]}) + vars["Biomass"] = pp.land_out( + lu_out_filter={"level": ["land_use"], "commodity": ["bioenergy"]}, units=units + ) # Note OFR 20180412: Correction inserted. scrubber output per vintage cannot # be divided by powerplant eff per vintage. In most cases this does work except From 832882312a98372a9a58f6e57373dd657c68716d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:39:17 +0100 Subject: [PATCH 08/10] Exclude tests/report/legacy/ from ruff checking --- message_ix_models/tests/report/legacy/__init__.py | 0 pyproject.toml | 1 + 2 files changed, 1 insertion(+) create mode 100644 message_ix_models/tests/report/legacy/__init__.py diff --git a/message_ix_models/tests/report/legacy/__init__.py b/message_ix_models/tests/report/legacy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pyproject.toml b/pyproject.toml index 59f38e3361..be0c0dfe8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,6 +156,7 @@ exclude = [ "doc/_static/png_source_files/Land-use_emulator_figures.ipynb", "message_ix_models/model/material/report/Historical Power Sector Stock Reporting-.ipynb", "message_ix_models/report/legacy/", + "message_ix_models/tests/report/legacy/", "message_ix_models/util/compat/message_data/", ] From a6cd8a457f61af40290b4f73e78d504d2d149927 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:39:28 +0100 Subject: [PATCH 09/10] Add .tests.report.legacy.test_iamc_report_hackathon Copied from iiasa/message_data branch `dev`. --- .../legacy/test_iamc_report_hackathon.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 message_ix_models/tests/report/legacy/test_iamc_report_hackathon.py diff --git a/message_ix_models/tests/report/legacy/test_iamc_report_hackathon.py b/message_ix_models/tests/report/legacy/test_iamc_report_hackathon.py new file mode 100644 index 0000000000..ab568e8d35 --- /dev/null +++ b/message_ix_models/tests/report/legacy/test_iamc_report_hackathon.py @@ -0,0 +1,33 @@ +import pytest +from message_ix_models import testing +from message_ix_models.util import MESSAGE_DATA_PATH + +from message_data.tools.post_processing import iamc_report_hackathon + + +@pytest.mark.parametrize("set_out_dir", [False, True]) +def test_report(caplog, request, test_context, set_out_dir): + """Invoke :func:`.iamc_report_hackathon.report` directly.""" + # Mandatory arguments to report(): a scenario and its platform + scenario = testing.bare_res(request, test_context, solved=False) + + # Set dry_run = True to not actually perform any calculations or modifications + test_context.dry_run = True + + if set_out_dir: + expected = test_context.get_local_path("report", "foo") + args = dict(out_dir=expected) + else: + # Do not provide out_dir/no arguments + args = dict() + expected = MESSAGE_DATA_PATH.joinpath("reporting_output") + + # Call succeeds + iamc_report_hackathon.report( + scenario.platform, scenario, context=test_context, **args + ) + + # Dry-run message is logged + assert "DRY RUN" in caplog.messages[-1] + # Output directory is set + assert caplog.messages[-1].endswith(str(expected)) From d9bbbb8ef6eb48a7910a0596ce33f9b5e187dfb6 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 26 Nov 2024 13:40:05 +0100 Subject: [PATCH 10/10] Add .tests.report.legacy.test_pp_utils Copied from iiasa/message_data branch `dev`. --- .../tests/report/legacy/test_pp_utils.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 message_ix_models/tests/report/legacy/test_pp_utils.py diff --git a/message_ix_models/tests/report/legacy/test_pp_utils.py b/message_ix_models/tests/report/legacy/test_pp_utils.py new file mode 100644 index 0000000000..b5d7688453 --- /dev/null +++ b/message_ix_models/tests/report/legacy/test_pp_utils.py @@ -0,0 +1,109 @@ +from itertools import product + +import pandas as pd +import pytest +from message_ix_models.model.structure import get_codes +from message_ix_models.util import nodes_ex_world + +from message_data.tools.post_processing import pp_utils +from message_data.tools.post_processing.pp_utils import fil + +#: These format strings appear in .default_tables.retr_hfc() in calls to :func:`.fil`. +FACTOR_EXPR = "refAC foam Fire ANMDI mVac Solv".split() + +#: These values appear in default_run_config.yaml. The commented lines are those for +#: which :func:`.fil` is *not* called. +FACTOR_VAL_UNIT = ( + # (0, "kt HFC134a-equiv/yr"), # Total # Not called because hfc: "Total" + (125, "kt HFC125/yr"), # HFC125 + (134, "kt HFC134a/yr"), # HFC134a + (143, "kt HFC143a/yr"), # HFC143a + (227, "kt HFC227ea/yr"), # HFC227ea + # (0, "kt HFC23/yr"), # HFC23 # Not called because run: "empty" + (245, "kt HFC245fa/yr"), # HFC245fa + (32, "kt HFC32/yr"), # HFC32 + (431, "kt HFC43-10/yr"), # HFC430 + (365, "kt HFC365mfc/yr"), # HFC365mfc # Not called because run: False + (152, "kt HFC152a/yr"), # HFC152a + (236, "kt HFC236fa/yr"), # HFC236fa # Not called because run: False +) + + +#: Input data expected by :func:`.fil`. +#: +#: - Axis named "Region" with short labels, e.g. "ABC" rather than "R##_ABC" +#: - Some columns with period labels, as integers. +#: - Float or NaN values. +FIL_DF = pd.DataFrame( + [[1.0, None, 1.0]], columns=[2000, 2050, 2100], index=["AFR"] +).rename_axis("Region") + + +def pp_utils_globals(monkeypatch, regions): + """Set :mod:`.pp_utils` global module variables. + + This utility function uses pytest's `monkeypatch` fixture to set the variables; the + pre-existing values are restored after the end of each test. + + This only sets the globals that are used by :func:`.fil()`. + """ + # region_id: the ID of the node code list; i.e. Context.regions + monkeypatch.setattr(pp_utils, "region_id", regions) + + # regions: a mapping from model's native "R##_ABC" to "ABC", *plus* e.g. "R##_GLB" + # mapped to # "World". This normally set by .iamc_report_hackathon.report(). + nodes = get_codes(f"node/{regions}") + nodes = nodes[nodes.index("World")].child + regions_map = {n.id: n.id.split("_")[1] for n in nodes_ex_world(nodes)} + regions_map[f"{regions}_GLB"] = "World" + monkeypatch.setattr(pp_utils, "regions", regions_map) + + # unit_conversion: a mapping of mappings. fil() directly sets "???" on its data and + # then looks up mappings from "???" to `units_out` + monkeypatch.setattr( + pp_utils, + "unit_conversion", + {"???": {units: 1.0 for _, units in FACTOR_VAL_UNIT}}, + ) + + # Expected index of data frame returned by fil() + return set(regions_map.values()) + + +@pytest.mark.parametrize("regions", ["R11", "R12"]) +@pytest.mark.parametrize( + "fil_arg, min_filled_values", + [ + ("HFC_fac", 1), # The only supported value + ("foo", 0), # Other values are essentially a no-op + ], +) +def test_fil(monkeypatch, regions, fil_arg, min_filled_values): + """Test :func:`.pp_utils.fil`. + + The test is parametrized for `regions` and each argument to :func:`.fil`. + + `min_filled_values` checks that at least N values in the resulting data frame are + populated from the file(s). + """ + # Monkey-patch the global variables in the pp_utils module used by fil() and + # retrieve the expected index of the result + expected_regions = pp_utils_globals(monkeypatch, regions) + + # Iterate over all values for the `factor` and `units` arguments that fil() is + # expected to support + for factor_expr, (factor_val, units) in product(FACTOR_EXPR, FACTOR_VAL_UNIT): + # Mirror how values for the `factor` argument are constructed in + # .default_tables.retr_hfc(), e.g. "Fire134" + factor = f"{factor_expr}{factor_val}" + + # Function executes without error + result = fil(FIL_DF, fil_arg, factor, units) + + # Returned data are a pd.DataFrame with the expected index + assert expected_regions == set(result.index) + + # Number of NaN values in results is as expected + assert min_filled_values <= ( + result.notnull().sum().sum() - FIL_DF.notnull().sum().sum() + )