From c482dc87c89fa4aa605a54cd447c4be265c3212f Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 23 Aug 2024 14:13:49 +0200 Subject: [PATCH 1/3] add draft plotting all scenarios --- rules/postprocess.smk | 13 ++ scripts/plot_summary_all.py | 266 ++++++++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 scripts/plot_summary_all.py diff --git a/rules/postprocess.smk b/rules/postprocess.smk index edeff1ef4..fc7bbf45e 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -246,6 +246,19 @@ rule plot_summary: script: "../scripts/plot_summary.py" +rule plot_summary_all: + output: + costs="results/" + config["run"]["prefix"] + "/graphs/costs.svg", + balances="results/" + config["run"]["prefix"] + "/graphs/balances-energy.svg", + threads: 2 + resources: + mem_mb=10000, + log: + "results/" + config["run"]["prefix"] + "/logs/plot_summary_all.log", + conda: + "../envs/environment.yaml" + script: + "../scripts/plot_summary_all.py" STATISTICS_BARPLOTS = [ "capacity_factor", diff --git a/scripts/plot_summary_all.py b/scripts/plot_summary_all.py new file mode 100644 index 000000000..d86984f01 --- /dev/null +++ b/scripts/plot_summary_all.py @@ -0,0 +1,266 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Creates plots from summary CSV files. +""" + +import logging + +import matplotlib.pyplot as plt +import pandas as pd +from _helpers import configure_logging, set_scenario_config +from plot_summary import rename_techs, preferred_order + +logger = logging.getLogger(__name__) +plt.style.use("ggplot") + + +def plot_costs(cost_df, drop=None): + + + df = cost_df.groupby(cost_df.index.get_level_values(2)).sum() + + + # convert to billions + df = df / 1e9 + + df = df.groupby(df.index.map(rename_techs)).sum() + + to_drop = df.index[df.max(axis=1) < snakemake.config["plotting"]["costs_threshold"]] + + logger.info( + f"Dropping technology with costs below {snakemake.config['plotting']['costs_threshold']} EUR billion per year" + ) + logger.debug(df.loc[to_drop]) + + df = df.drop(to_drop) + + logger.info(f"Total system cost of {round(df.sum().iloc[0])} EUR billion per year") + + new_index = preferred_order.intersection(df.index).append( + df.index.difference(preferred_order) + ) + + # new_columns = df.sum().sort_values().index + if drop!=None: + df = df.droplevel([1,2,3], axis=1) + + planning_horizons = df.columns.get_level_values('planning_horizon').unique().sort_values() + scenarios = df.columns.get_level_values(0).unique() + + fig, axes = plt.subplots( + nrows=1, ncols=len(planning_horizons), + figsize=(12, 8), + sharey=True # This ensures that all subplots share the same y-axis + ) + + if len(planning_horizons) == 1: + axes = [axes] + + for ax, year in zip(axes, planning_horizons): + subset = df.xs(year, level='planning_horizon', axis=1) + + subset.T[new_index].plot( + kind="bar", + ax=ax, + stacked=True, + legend=False, + color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index] + ) + + # Set title and x-label + ax.set_title(year) + + ax.set_xlabel("") + + # Set x-ticks as scenario names (level=0) + ax.set_xticks(range(len(scenarios))) + + if ax == axes[0]: + ax.set_ylabel("System Cost [EUR billion per year]") + + + ax.grid(axis="x") + + ax.set_ylim([0, snakemake.config['plotting']["costs_max"]]) + + + handles, labels = ax.get_legend_handles_labels() + handles.reverse() + labels.reverse() + + axes[-1].legend( + handles, + labels, + ncol=1, + loc="upper left", + bbox_to_anchor=[1, 1], + frameon=False, + ) + + plt.tight_layout() + + fig.savefig(snakemake.output.costs, + bbox_inches="tight") + + + df.sum().unstack().T.plot() + plt.ylabel("System Cost [EUR billion per year]") + plt.xlabel("planning horizon") + plt.legend(bbox_to_anchor=(1,1)) + plt.savefig(snakemake.output.costs.split(".svg")[0]+"-total.svg", + bbox_inches="tight") + + + +def plot_balances(balances_df, drop=None): + + co2_carriers = ["co2", "co2 stored", "process emissions"] + balances = {i.replace(" ", "_"): [i] for i in balances_df.index.levels[0]} + balances["energy"] = [ + i for i in balances_df.index.levels[0] if i not in co2_carriers + ] + + for k, v in balances.items(): + df = balances_df.loc[v] + df = df.groupby(df.index.get_level_values(2)).sum() + + # convert MWh to TWh + df = df / 1e6 + + # remove trailing link ports + df.index = [ + ( + i[:-1] + if ( + (i not in ["co2", "NH3", "H2"]) + and (i[-1:] in ["0", "1", "2", "3", "4"]) + ) + else i + ) + for i in df.index + ] + + df = df.groupby(df.index.map(rename_techs)).sum() + + to_drop = df.index[ + df.abs().max(axis=1) < snakemake.config["plotting"]["energy_threshold"] / 10 + ] + + units = "MtCO2/a" if v[0] in co2_carriers else "TWh/a" + logger.debug( + f"Dropping technology energy balance smaller than {snakemake.config['plotting']['energy_threshold']/10} {units}" + ) + logger.debug(df.loc[to_drop]) + + df = df.drop(to_drop) + + logger.debug( + f"Total energy balance for {v} of {round(df.sum().iloc[0],2)} {units}" + ) + + if df.empty: + continue + + new_index = preferred_order.intersection(df.index).append( + df.index.difference(preferred_order) + ) + + if drop!=None: + df = df.droplevel([1,2,3], axis=1) + + + planning_horizons = df.columns.get_level_values('planning_horizon').unique().sort_values() + scenarios = df.columns.get_level_values(0).unique() + + fig, axes = plt.subplots( + nrows=1, ncols=len(planning_horizons), + figsize=(12, 8), + sharey=True # This ensures that all subplots share the same y-axis + ) + + if len(planning_horizons) == 1: + axes = [axes] + + for ax, year in zip(axes, planning_horizons): + subset = df.xs(year, level='planning_horizon', axis=1) + + subset.T[new_index].plot( + kind="bar", + ax=ax, + stacked=True, + legend=False, + color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index] + ) + + # Set title and x-label + ax.set_title(year) + + # Set x-ticks as scenario names (level=0) + ax.set_xticks(range(len(scenarios))) + + if ax == axes[0]: + if v[0] in co2_carriers: + ax.set_ylabel("CO2 [MtCO2/a]") + else: + ax.set_ylabel("Energy [TWh/a]") + + ax.grid(axis="x") + + + handles, labels = ax.get_legend_handles_labels() + handles.reverse() + labels.reverse() + + axes[-1].legend( + handles, + labels, + ncol=1, + loc="upper left", + bbox_to_anchor=[1, 1], + frameon=False, + ) + + plt.tight_layout() + + fig.savefig(snakemake.output.balances[:-10] + k + ".svg", bbox_inches="tight") + + +#%% +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + snakemake = mock_snakemake("plot_summary_all") + + configure_logging(snakemake) + set_scenario_config(snakemake) + + n_header = 4 + + prefix = snakemake.config["run"]["prefix"] + path = snakemake.output[0].split("graphs")[0] + scenarios = snakemake.config["run"]["name"] + + costs = {} + balances = {} + for scenario in scenarios: + try: + costs[scenario] = pd.read_csv(f"{path}/{scenario}/csvs/costs.csv", + index_col=list(range(3)), + header=list(range(n_header))) + balances[scenario] = pd.read_csv(f"{path}/{scenario}/csvs/supply_energy.csv", + index_col=list(range(3)), + header=list(range(n_header))) + except FileNotFoundError: + logger.info(f"{scenario} not solved yet.") + + costs = pd.concat(costs, axis=1) + balances = pd.concat(balances, axis=1) + + plot_costs(costs, drop=True) + + plot_balances(balances, drop=True) + + From 095f7c0650209f991b5d5a61d830c24e3b0a2f07 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 23 Aug 2024 14:16:08 +0200 Subject: [PATCH 2/3] add prefix function --- Snakefile | 2 +- scripts/_helpers.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index eb99437bf..c6cc73a47 100644 --- a/Snakefile +++ b/Snakefile @@ -10,7 +10,7 @@ from snakemake.utils import min_version min_version("8.11") -from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir +from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir, get_prefix copy_default_files(workflow) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a3b77c1c0..3679fc068 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -67,6 +67,11 @@ def get_rdir(run): return RDIR + +def get_prefix(run): + prefix = run.get("prefix", "") + return f"{prefix} + def get_run_path(fn, dir, rdir, shared_resources, exclude_from_shared): """ Dynamically provide paths based on shared resources and filename. From 527e4335556c0d26b6c1ea376f9a3493c7db1fac Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 12:20:18 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Snakefile | 8 +- rules/postprocess.smk | 2 + scripts/cluster_network.py | 4 +- scripts/plot_summary_all.py | 151 ++++++++++++++++++------------------ 4 files changed, 87 insertions(+), 78 deletions(-) diff --git a/Snakefile b/Snakefile index c6cc73a47..843f8ddd4 100644 --- a/Snakefile +++ b/Snakefile @@ -10,7 +10,13 @@ from snakemake.utils import min_version min_version("8.11") -from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir, get_prefix +from scripts._helpers import ( + path_provider, + copy_default_files, + get_scenarios, + get_rdir, + get_prefix, +) copy_default_files(workflow) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index fc7bbf45e..b4005ef56 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -246,6 +246,7 @@ rule plot_summary: script: "../scripts/plot_summary.py" + rule plot_summary_all: output: costs="results/" + config["run"]["prefix"] + "/graphs/costs.svg", @@ -260,6 +261,7 @@ rule plot_summary_all: script: "../scripts/plot_summary_all.py" + STATISTICS_BARPLOTS = [ "capacity_factor", "installed_capacity", diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 9e1db6a77..e2e1da915 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -511,9 +511,7 @@ def plot_busmap_for_n_clusters(n, n_clusters, solver_name="scip", fn=None): # Fast-path if no clustering is necessary busmap = n.buses.index.to_series() linemap = n.lines.index.to_series() - clustering = pypsa.clustering.spatial.Clustering( - n, busmap, linemap - ) + clustering = pypsa.clustering.spatial.Clustering(n, busmap, linemap) else: Nyears = n.snapshot_weightings.objective.sum() / 8760 diff --git a/scripts/plot_summary_all.py b/scripts/plot_summary_all.py index d86984f01..a742d6374 100644 --- a/scripts/plot_summary_all.py +++ b/scripts/plot_summary_all.py @@ -11,7 +11,7 @@ import matplotlib.pyplot as plt import pandas as pd from _helpers import configure_logging, set_scenario_config -from plot_summary import rename_techs, preferred_order +from plot_summary import preferred_order, rename_techs logger = logging.getLogger(__name__) plt.style.use("ggplot") @@ -19,9 +19,7 @@ def plot_costs(cost_df, drop=None): - df = cost_df.groupby(cost_df.index.get_level_values(2)).sum() - # convert to billions df = df / 1e9 @@ -44,53 +42,54 @@ def plot_costs(cost_df, drop=None): ) # new_columns = df.sum().sort_values().index - if drop!=None: - df = df.droplevel([1,2,3], axis=1) - - planning_horizons = df.columns.get_level_values('planning_horizon').unique().sort_values() + if drop != None: + df = df.droplevel([1, 2, 3], axis=1) + + planning_horizons = ( + df.columns.get_level_values("planning_horizon").unique().sort_values() + ) scenarios = df.columns.get_level_values(0).unique() - + fig, axes = plt.subplots( - nrows=1, ncols=len(planning_horizons), - figsize=(12, 8), - sharey=True # This ensures that all subplots share the same y-axis + nrows=1, + ncols=len(planning_horizons), + figsize=(12, 8), + sharey=True, # This ensures that all subplots share the same y-axis ) - + if len(planning_horizons) == 1: axes = [axes] - + for ax, year in zip(axes, planning_horizons): - subset = df.xs(year, level='planning_horizon', axis=1) - + subset = df.xs(year, level="planning_horizon", axis=1) + subset.T[new_index].plot( kind="bar", ax=ax, stacked=True, legend=False, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index] + color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index], ) - + # Set title and x-label ax.set_title(year) - + ax.set_xlabel("") - + # Set x-ticks as scenario names (level=0) ax.set_xticks(range(len(scenarios))) - + if ax == axes[0]: ax.set_ylabel("System Cost [EUR billion per year]") - - + ax.grid(axis="x") - - ax.set_ylim([0, snakemake.config['plotting']["costs_max"]]) - + + ax.set_ylim([0, snakemake.config["plotting"]["costs_max"]]) handles, labels = ax.get_legend_handles_labels() handles.reverse() labels.reverse() - + axes[-1].legend( handles, labels, @@ -99,24 +98,22 @@ def plot_costs(cost_df, drop=None): bbox_to_anchor=[1, 1], frameon=False, ) - + plt.tight_layout() - - fig.savefig(snakemake.output.costs, - bbox_inches="tight") - - + + fig.savefig(snakemake.output.costs, bbox_inches="tight") + df.sum().unstack().T.plot() plt.ylabel("System Cost [EUR billion per year]") plt.xlabel("planning horizon") - plt.legend(bbox_to_anchor=(1,1)) - plt.savefig(snakemake.output.costs.split(".svg")[0]+"-total.svg", - bbox_inches="tight") - + plt.legend(bbox_to_anchor=(1, 1)) + plt.savefig( + snakemake.output.costs.split(".svg")[0] + "-total.svg", bbox_inches="tight" + ) def plot_balances(balances_df, drop=None): - + co2_carriers = ["co2", "co2 stored", "process emissions"] balances = {i.replace(" ", "_"): [i] for i in balances_df.index.levels[0]} balances["energy"] = [ @@ -167,53 +164,56 @@ def plot_balances(balances_df, drop=None): new_index = preferred_order.intersection(df.index).append( df.index.difference(preferred_order) ) - - if drop!=None: - df = df.droplevel([1,2,3], axis=1) - - planning_horizons = df.columns.get_level_values('planning_horizon').unique().sort_values() + if drop != None: + df = df.droplevel([1, 2, 3], axis=1) + + planning_horizons = ( + df.columns.get_level_values("planning_horizon").unique().sort_values() + ) scenarios = df.columns.get_level_values(0).unique() - + fig, axes = plt.subplots( - nrows=1, ncols=len(planning_horizons), - figsize=(12, 8), - sharey=True # This ensures that all subplots share the same y-axis + nrows=1, + ncols=len(planning_horizons), + figsize=(12, 8), + sharey=True, # This ensures that all subplots share the same y-axis ) - + if len(planning_horizons) == 1: axes = [axes] - + for ax, year in zip(axes, planning_horizons): - subset = df.xs(year, level='planning_horizon', axis=1) - + subset = df.xs(year, level="planning_horizon", axis=1) + subset.T[new_index].plot( kind="bar", ax=ax, stacked=True, legend=False, - color=[snakemake.config["plotting"]["tech_colors"][i] for i in new_index] + color=[ + snakemake.config["plotting"]["tech_colors"][i] for i in new_index + ], ) - + # Set title and x-label ax.set_title(year) - + # Set x-ticks as scenario names (level=0) ax.set_xticks(range(len(scenarios))) - + if ax == axes[0]: if v[0] in co2_carriers: ax.set_ylabel("CO2 [MtCO2/a]") else: ax.set_ylabel("Energy [TWh/a]") - + ax.grid(axis="x") - handles, labels = ax.get_legend_handles_labels() handles.reverse() labels.reverse() - + axes[-1].legend( handles, labels, @@ -222,45 +222,48 @@ def plot_balances(balances_df, drop=None): bbox_to_anchor=[1, 1], frameon=False, ) - + plt.tight_layout() - + fig.savefig(snakemake.output.balances[:-10] + k + ".svg", bbox_inches="tight") - -#%% + +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake + snakemake = mock_snakemake("plot_summary_all") configure_logging(snakemake) set_scenario_config(snakemake) - + n_header = 4 - + prefix = snakemake.config["run"]["prefix"] path = snakemake.output[0].split("graphs")[0] scenarios = snakemake.config["run"]["name"] - + costs = {} balances = {} for scenario in scenarios: try: - costs[scenario] = pd.read_csv(f"{path}/{scenario}/csvs/costs.csv", - index_col=list(range(3)), - header=list(range(n_header))) - balances[scenario] = pd.read_csv(f"{path}/{scenario}/csvs/supply_energy.csv", - index_col=list(range(3)), - header=list(range(n_header))) + costs[scenario] = pd.read_csv( + f"{path}/{scenario}/csvs/costs.csv", + index_col=list(range(3)), + header=list(range(n_header)), + ) + balances[scenario] = pd.read_csv( + f"{path}/{scenario}/csvs/supply_energy.csv", + index_col=list(range(3)), + header=list(range(n_header)), + ) except FileNotFoundError: logger.info(f"{scenario} not solved yet.") - + costs = pd.concat(costs, axis=1) balances = pd.concat(balances, axis=1) - + plot_costs(costs, drop=True) plot_balances(balances, drop=True) - -