diff --git a/rime/generate_aggregated_inputs.py b/rime/generate_aggregated_inputs.py index 3035499..4e3c2a4 100644 --- a/rime/generate_aggregated_inputs.py +++ b/rime/generate_aggregated_inputs.py @@ -4,35 +4,38 @@ @author: byers """ - +# from alive_progress import alive_bar import glob # import numpy as np import os -import pandas as pd import pyam import re import xarray as xr import time -from alive_progress import alive_bar - -start = time.time() +# try: -from process_config import * +# ab_present = True +# except: +# print("alive_progress not installed") +# ab_present = False -# Load impact, interpolate +# start = time.time() +from rime.rime_functions import * +from rime.process_config import * -# few_vars = True +few_vars = False # %% Settings config - years = list(range(yr_start, yr_end + 1)) # %% List of indicators and files + +# Define a list of table data files to read in os.chdir(wdtable_input) files = glob.glob(table_output_format.replace("|", "*")) @@ -49,10 +52,8 @@ # files = files[4:5] # heatwav & iavar # files = files[5:10] # files = files[10:12] -files = files[12:15] -# files = files[15:] - -# batch = 1 +# files = files[12:15] +files = files[15:] indicators = [] @@ -111,72 +112,67 @@ # df.drop(columns=['model', 'scenario'], inplace=True) # if few_vars: - # df = df.loc[df.variable.str.contains('High')] small_vars = list(set([x.split("|")[0] for x in dfin.variable])) - - with alive_bar( - total=len(small_vars), - title="Processing", - length=10, - force_tty=True, - bar="circles", - spinner="elements", - ) as bar: - for vari in small_vars: - df_ind = loop_interpolate_gmt(df.loc[df.variable.str.startswith(vari)]) - # dfbig = pd.concat([dfbig, df_ind]) - print(f"dfbig: indicator {ind[0]}: {time.time()-istart}") - - # % Convert and save out to xarray - # dfbig.dropna(how='all') - - dfp = df_ind.melt( - id_vars=[ - "model", - "scenario", - "variable", - "region", - "unit", - "SSP", - "GMT", - ], - value_vars=years, - var_name="year", - ) # change to df_big if concatenating multiple - - dfp.columns = [x.lower() for x in dfp.columns] - - dsout = xr.Dataset() - - for indicator in dfp.variable.unique(): - print(indicator) - dx = ( - dfp.loc[dfp.variable == indicator] - .set_index(["gmt", "year", "ssp", "region"]) - .to_xarray() - ) - # dx.attrs['unit'] = dx.assign_coords({'unit':dx.unit.values[0,0,0,0]}) - - dsout[indicator] = dx["value"].to_dataset(name=indicator)[indicator] - dsout[indicator].attrs["unit"] = dx.unit.values[0, 0, 0, 0] - # dsout = dsout[indicator].assign_coords({'unit':dx.unit.values[0,0,0,0]}) - - # bar() - - dsout["ssp"] = [x.upper() for x in dsout["ssp"].values] - # dsout = dsout.drop_vars('unit') - - # % Write out - print("Writing out... ") - comp = dict(zlib=True, complevel=5) - encoding = {var: comp for var in dsout.data_vars} - - # ind = indicator.replace('|', '_') - filename = f"{output_dir}{vari}_{region}.nc" - - dsout.to_netcdf(filename, encoding=encoding) - bar() + # if ab_present: + # with alive_bar(total=len(small_vars), + # title='Processing', length=10, force_tty=True, + # bar='circles', + # spinner='elements') as bar: + + # print('alive bar present') + # Apply function here + for vari in small_vars: + df_ind = loop_interpolate_gmt( + df.loc[df.variable.str.startswith(vari)], yr_start, yr_end + ) + # dfbig = pd.concat([dfbig, df_ind]) + print(f"dfbig: indicator {ind[0]}: {time.time()-istart}") + + # % Convert and save out to xarray + # dfbig.dropna(how='all') + + dfp = df_ind.melt( + id_vars=[ + "model", + "scenario", + "variable", + "region", + "unit", + "SSP", + "GMT", + ], + value_vars=years, + var_name="year", + ) # change to df_big if concatenating multiple + + dfp.columns = [x.lower() for x in dfp.columns] + + dsout = xr.Dataset() + + for indicator in dfp.variable.unique(): + print(indicator) + dx = ( + dfp.loc[dfp.variable == indicator] + .set_index(["gmt", "year", "ssp", "region"]) + .to_xarray() + ) + # dx.attrs['unit'] = dx.assign_coords({'unit':dx.unit.values[0,0,0,0]}) + dsout[indicator] = dx["value"].to_dataset(name=indicator)[indicator] + dsout[indicator].attrs["unit"] = dx.unit.values[0, 0, 0, 0] + # dsout = dsout[indicator].assign_coords({'unit':dx.unit.values[0,0,0,0]}) + + dsout["ssp"] = [x.upper() for x in dsout["ssp"].values] + # dsout = dsout.drop_vars('unit') + + # % Write out + print("Writing out... ") + comp = dict(zlib=True, complevel=5) + encoding = {var: comp for var in dsout.data_vars} + filename = f"{output_dir}{vari}_{region}.nc" + dsout.to_netcdf(filename, encoding=encoding) + # if ab_present: + # bar() # ============================================================================= diff --git a/rime/plot_dashboards.py b/rime/plot_dashboards.py new file mode 100644 index 0000000..8f4377a --- /dev/null +++ b/rime/plot_dashboards.py @@ -0,0 +1,544 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Aug 29 09:20:24 2023 + +@author: werning +""" + +import sys + +sys.path.append("H:\\git\\climate_impacts_processing") + +import seaborn as sns +import xarray as xr +import numpy as np +import hotspots_functions_tables as hft +import hotspots_functions_pp as hfp +import os +import matplotlib.pyplot as plt +import holoviews as hv +import hvplot.pandas +import hvplot.xarray +import itertools as it +from scipy.interpolate import interp1d + +from rime.process_config import * +from rime.rime_functions import * + + +input_dir = "C:\\Users\\{user}\\IIASA\\ECE.prog - Research Theme - NEXUS\\Hotspots_Explorer_2p0\\Data\\1_mmm_stats" +score_dir = "C:\\Users\\{user}\\IIASA\ECE.prog - Research Theme - NEXUS\\Hotspots_Explorer_2p0\\Data\\3_scores" +diff_dir = "C:\\Users\\{user}\\IIASA\ECE.prog - Research Theme - NEXUS\\Hotspots_Explorer_2p0\\Data\\2_differences" +std_dir = "C:\\Users\\{user}\\IIASA\\ECE.prog - Research Theme - NEXUS\\Hotspots_Explorer_2p0\\Data\\score_revision\\hist_standard_deviations_V2\\" +# plot_dir = "C:\\Users\\{user}\\IIASA\ECE.prog - Research Theme - NEXUS\\Hotspots_Explorer_2p0\\Data\\score_revision\\bivariate" +# yaml_path = "H:\\git\\climate_impacts_processing\\hotspots.yml" +# landmask_path = "H:\\git\\climate_impacts_processing\\landareamaskmap0.nc" +# kg_class_path = "H:\\git\\climate_impacts_processing\\kg_class.nc" + +# Set interpolation & load landarea mask and parameters +land_mask = hfp.load_landmask(landmask_path) +params = hfp.load_parameters(yaml_path) +with xr.open_dataset(kg_class_path, engine="netcdf4") as kg_class: + kg_class.load() +kg_class = kg_class.sel(band=0) + +indicators = [ + "cdd", + "dri", + "dri_qtot", + "ia_var", + "ia_var_qtot", + "precip", + "sdd", + "sdd_24p0", + "sdd_20p0", + "sdd_18p3", + "seas", + "seas_qtot", + "tr20", + "wsi", +] +# indicators = ['seas'] +gmt_threshold = 3.0 + +# %% Functions +# ----------------------------------------------------------------------------- + + + +def bin_data(data, bins): + data = xr.where(data > bins[3], bins[3], data) + data = xr.where(data < bins[0], bins[0], data) + + m = interp1d(bins, [0, 1, 2, 3]) + pt = m(data.values) + data.values = pt + + return data + + +# combination = [f'{ind}_{i}' for i in (variables)] + +# indic_bins = {i: params[i]['score_range'] for i in variables} +# # indic_bins = {i: score_ranges[i] for i in combination} + +# for var in variables: + +# # data[var] = xr.where(data[var] > max(indic_bins[f'{ind}_{var}']), +# # max(indic_bins[f'{ind}_{var}']), data[var]) +# # data[var] = xr.where(data[var] < min(indic_bins[f'{ind}_{var}']), +# # min(indic_bins[f'{ind}_{var}']), data[var]) + +# # m = interp1d(indic_bins[f'{ind}_{var}'], score_bins) + +# data[var] = xr.where(data[var] > max(indic_bins[var]), +# max(indic_bins[var]), data[var]) +# data[var] = xr.where(data[var] < min(indic_bins[var]), +# min(indic_bins[var]), data[var]) + +# m = interp1d(indic_bins[var], score_bins) + + +# pt = m(data[var].values) +# data[var].values = pt + +# return data + +# %% Calculation +# ----------------------------------------------------------------------------- + +plot_list = [] + +for ind in indicators: + # Select mean, apply land mask and remove inf + historical = xr.open_dataset( + os.path.join( + input_dir, + "Historical", + f'ISIMIP{params["protocol"][ind]}_MM_historical_{ind}.nc4', + ) + ) + historical = hfp.apply_land_mask(historical, land_mask) + historical = historical.where(historical.pipe(np.isfinite)).fillna(np.nan) + + # Select mean, apply land mask and remove inf + future = xr.open_dataset( + os.path.join(input_dir, f'ISIMIP{params["protocol"][ind]}_MM_{ind}_0p1.nc4') + ) + future = hfp.apply_land_mask(future, land_mask) + future = future.where(future.pipe(np.isfinite)).fillna(np.nan) + + if ind == "seas" or ind == "seas_qtot": + historical = historical.where((historical < 5) & (historical > -5)) + future = future.where((future < 5) & (future > -5)) + + if ind == "ia_var" or ind == "ia_var_qtot": + historical = historical.where((historical < 2) & (historical > -2)) + future = future.where((future < 2) & (future > -2)) + + # Load score data for plotting + scores = xr.open_dataset( + os.path.join(score_dir, f'ISIMIP{params["protocol"][ind]}_Scores_{ind}_0p1.nc4') + ) + + # Load diff data for plotting + diff = xr.open_dataset( + os.path.join(diff_dir, f'ISIMIP{params["protocol"][ind]}_Diff_{ind}_0p1.nc4') + ) + diff = diff.where(diff.pipe(np.isfinite)).fillna(np.nan) + + for var in params["indicators"][ind]: + # Select variable and rename it + hist_data = historical[var][1].rename("hist") + plot_hist = hist_data.hvplot( + x="lon", + y="lat", + shared_axes=False, + cmap=params["indicators"][ind][var]["ind_cmap"], + clim=( + params["indicators"][ind][var]["ind_min"], + params["indicators"][ind][var]["ind_max"], + ), + title="Absolute - historical", + ) + + # Select variable and rename it + future_data = future.sel({"threshold": gmt_threshold}) + future_data = future_data[var][1, :, :].rename( + str(gmt_threshold).replace(".", "p") + ) + # plot_future = future_data.hvplot(x='lon', y='lat', shared_axes=False, cmap = params["indicators"][ind][var]['ind_cmap'], clim=(params["indicators"][ind][var]['ind_min'], params["indicators"][ind][var]['ind_max']), title=f'Absolute - {str(gmt_threshold).replace(".", "p")}') + + # Select variable and rename it + diff_data = diff.sel({"threshold": gmt_threshold}) + diff_data = diff_data[var][0].rename("diff") + # plot_diff = diff_data.hvplot(x='lon', y='lat', shared_axes=False, cmap=params["indicators"][ind][var]['diff_cmap'], clim=(params["indicators"][ind][var]['diff_min'], params["indicators"][ind][var]['diff_max']), title=f'Difference - {str(gmt_threshold).replace(".", "p")}') + + # Select variable and rename it + score_data = scores.sel({"threshold": gmt_threshold}) + score_data = score_data[var][0].rename("score") + # plot_score = score_data.hvplot(x='lon', y='lat', cmap='magma_r', shared_axes=False, title=f'Score - {str(gmt_threshold).replace(".", "p")}') + + # Load standard deviation data, apply land mask, remove inf and rename + std_dev = hft.load_netcdf( + os.path.join( + std_dir, + f'{params["indicators"][ind][var]["short_name"]}_hist_std_GCM_avg.nc', + ) + ) + std_dev = hfp.apply_land_mask(std_dev, land_mask) + std_dev = std_dev.where(std_dev.pipe(np.isfinite)).fillna(np.nan) + std_dev = std_dev.rename("std") + + # Calculate change in standard deviation + std_change = (future_data - hist_data) / std_dev + std_change = std_change.where(std_change.pipe(np.isfinite)).fillna(np.nan) + std_change = std_change.rename("std_change") + # plot_z = std_change.hvplot(x='lon', y='lat', cmap='magma_r', clim=(0,3), shared_axes=False, title='Z score') + + # Joint plots + # hist_plot = sns.jointplot(x=hist_data.to_dataframe()['hist'], y=std_change.to_dataframe()['std_change']) + # hist_plot.ax_marg_x.set_xlim(0, 1800) + # hist_plot.ax_marg_y.set_ylim(0, 50) + # hist_plot.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_joint_hist_std_change.png')) + # future_plot = sns.jointplot(x=future_data.to_dataframe()[str(gmt_threshold).replace(".", "p")], y=std_change.to_dataframe()['std_change']) + # future_plot.ax_marg_x.set_xlim(0, 1800) + # future_plot.ax_marg_y.set_ylim(0, 50) + # future_plot.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_joint_{str(gmt_threshold).replace(".", "p")}_std_change.png')) + + joint_merged = xr.merge([hist_data, std_change]).to_dataframe() + # plot_joint_hist = joint_merged.hvplot.scatter(x='hist', y='std_change', shared_axes=False, cmap=['blue'], title='Joint plot hist - z score') + + # Bin data + std_change_binned = bin_data(std_change, [0, 1, 2, 3]) + + # Calculate quartiles + bins = hist_data.quantile([0, 0.25, 0.5, 0.75]) + + # Quartile bins + hist_data_binned = bin_data(hist_data, bins) + future_data_binned = bin_data(future_data, bins) + + # Create bivariate score + hist_bivariate = std_change_binned + hist_data_binned + hist_bivariate = hist_bivariate.rename("std_score") + future_bivariate = std_change_binned + future_data_binned + future_bivariate = future_bivariate.rename("std_score") + # plot_score_quartile = future_bivariate.hvplot(x='lon', y='lat', cmap='magma_r', shared_axes=False, clim=(0,6), title='Score - Quartiles') + + color_cycle = hv.Cycle( + [ + "#fcfdbf", + "#feb078", + "#f1605d", + "#b5367a", + "#721f81", + "#2c115f", + "#000004", + ] + ) + explicit_mapping = { + "0": "#fcfdbf", + "1": "#feb078", + "2": "#f1605d", + "3": "#b5367a", + "4": "#721f81", + "5": "#2c115f", + "6": "#000004", + } + + # + joint_hist_quartiles_merged = xr.merge( + [hist_data, std_change, hist_bivariate] + ).to_dataframe() + # plot_joint_hist_quartiles = joint_hist_quartiles_merged.hvplot.scatter(x='hist', y='std_change', by='std_score', color=explicit_mapping, title='Joint plot hist - z score quartiles') + joint_future_quartiles_merged = xr.merge( + [future_data, std_change, future_bivariate] + ).to_dataframe() + # plot_joint_hist_quartiles = joint_hist_quartiles_merged.hvplot.scatter(x='hist', y='std_change', color='std_score', cmap=explicit_mapping, title='Joint plot hist - z score quartiles') + # plot_joint_future_quartiles = joint_future_quartiles_merged.hvplot.scatter(x=f'{str(gmt_threshold).replace(".", "p")}', y='std_change', color='std_score', cmap=explicit_mapping, title=f'Joint plot {str(gmt_threshold).replace(".", "p")} - z score quartiles') + + # plot_joint_hist_quartiles = joint_hist_quartiles_merged.hvplot.scatter(x='hist', y='std_change', color='std_score', cmap='magma_r', title='Joint plot hist - z score quartiles').redim.range(std_score=(0, 6)) + # plot_joint_future_quartiles = joint_future_quartiles_merged.hvplot.points(x=f'{str(gmt_threshold).replace(".", "p")}', y='std_change', color='std_score', cmap='magma_r', title=f'Joint plot {str(gmt_threshold).replace(".", "p")} - z score quartiles').redim.range(std_score=(0, 6)) + + # # Joint plot - coloured with score + # joint_hist = xr.merge([hist_data, std_change, hist_bivariate]).to_dataframe() + # sns.jointplot(data=joint_hist, x='hist', y='std_change', c=joint_hist.std_score.dropna(), joint_kws={"color":None, 'cmap':'magma_r'}) + # joint_future = xr.merge([future_data, std_change, future_bivariate]).to_dataframe() + # sns.jointplot(data=joint_future, x='2p0', y='std_change', c=joint_future.std_score.dropna(), joint_kws={"color":None, 'cmap':'magma_r'}) + + kg_hist_all = xr.DataArray( + data=np.full([len(hist_data.lat), len(hist_data.lon)], np.nan), + coords={"lat": hist_data.lat, "lon": hist_data.lon}, + ) + kg_future_all = xr.DataArray( + data=np.full([len(hist_data.lat), len(hist_data.lon)], np.nan), + coords={"lat": hist_data.lat, "lon": hist_data.lon}, + ) + + for k in range(1, 6): + kg_hist_data = hist_data.where(kg_class == k).kg_class + kg_future_data = future_data.where(kg_class == k).kg_class + kg_std_change_binned = std_change_binned.where(kg_class == k).kg_class + + if var == "wsi": + kg_bins = [0.1, 0.2, 0.3, 0.4] + else: + kg_bins = kg_hist_data.quantile([0, 0.25, 0.5, 0.75]).values + print(f"{k} - {kg_bins}") + kg_hist_data_binned = bin_data(kg_hist_data, kg_bins) + kg_future_data_binned = bin_data(kg_future_data, kg_bins) + + kg_hist_bivariate = kg_std_change_binned + kg_hist_data_binned + kg_future_bivariate = kg_std_change_binned + kg_future_data_binned + + kg_hist_all = xr.where(kg_class == k, kg_hist_bivariate, kg_hist_all) + kg_future_all = xr.where(kg_class == k, kg_future_bivariate, kg_future_all) + + # fig = plt.figure() + # kg_hist_bivariate.plot(cmap='magma_r', vmin=0, vmax=6) + # plt.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_kg_class_{k}_score_hist.png')) + # plt.close() + + # fig = plt.figure() + # kg_future_bivariate.plot(cmap='magma_r', vmin=0, vmax=6) + # plt.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_kg_class_{k}_score_{str(gmt_threshold).replace(".", "p")}.png')) + # plt.close() + + # plot_kg_score_quartile = kg_future_all.hvplot(x='lon', y='lat', cmap='magma_r', shared_axes=False, clim=(0,6), title='Score - Quartiles - KG') + plot_kg_score_quartile = kg_future_all.hvplot( + x="lon", + y="lat", + cmap="magma_r", + shared_axes=False, + clim=(0, 6), + title=f'{params["indicators"][ind][var]["long_name"]}', + ) + + # Create kg bivariate score + kg_hist_all = kg_hist_all.kg_class.rename("std_score") + kg_future_all = kg_future_all.kg_class.rename("std_score") + + # + joint_hist_kg_quartiles_merged = xr.merge( + [hist_data, std_change, kg_hist_all] + ).to_dataframe() + joint_future_kg_quartiles_merged = xr.merge( + [future_data, std_change, kg_future_all] + ).to_dataframe() + # plot_joint_hist_kg_quartiles = joint_hist_kg_quartiles_merged.hvplot.scatter(x='hist', y='std_change', by='std_score', color=color_cycle, title='Joint plot hist - z score quartiles KG') + # plot_joint_future_kg_quartiles = joint_future_kg_quartiles_merged.hvplot.scatter(x=f'{str(gmt_threshold).replace(".", "p")}', y='std_change', by='std_score', color=color_cycle, title=f'Joint plot {str(gmt_threshold).replace(".", "p")} - z score quartiles KG') + # plot_joint_hist_kg_quartiles = joint_hist_kg_quartiles_merged.hvplot.scatter(x='hist', y='std_change', color='std_score', cmap='magma_r', title='Joint plot hist - z score quartiles KG').redim.range(std_score=(0, 6)) + # plot_joint_future_kg_quartiles = joint_future_kg_quartiles_merged.hvplot.scatter(x=f'{str(gmt_threshold).replace(".", "p")}', y='std_change', color='std_score', cmap='magma_r', title=f'Joint plot {str(gmt_threshold).replace(".", "p")} - z score quartiles KG').redim.range(std_score=(0, 6)) + + # print('saving plot') + # plot_list = [plot_future, plot_diff, plot_score, + # plot_joint_hist, plot_hist, plot_z, + # plot_joint_hist_quartiles, plot_joint_future_quartiles, plot_score_quartile, + # plot_joint_hist_kg_quartiles, plot_joint_future_kg_quartiles, plot_kg_score_quartile] + + # plot_list = [plot_future, plot_diff, plot_score, + # plot_joint_hist, plot_hist, plot_z, + # plot_joint_hist_quartiles, plot_joint_future_quartiles, plot_score_quartile] + + plot_list = plot_list + [plot_kg_score_quartile] + + # plot = hv.Layout(plot_list).cols(3) + # hvplot.save(plot, f'{params["indicators"][ind][var]["short_name"]}_bivariate_dashboard_{str(gmt_threshold).replace(".", "p")}_interp.html') + + plot = ( + hv.Layout(plot_list) + .cols(3) + .opts(title=f'{str(gmt_threshold).replace(".", "p")}') + ) + hvplot.save( + plot, + f'All_indicators_bivariate_dashboard_{str(gmt_threshold).replace(".", "p")}_interp.html', + ) + + # fig = plt.figure() + # kg_hist_all.kg_class.plot(cmap='magma_r', vmin=0, vmax=6) + # plt.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_kg_quartiles_score_hist.png')) + # plt.close() + + # fig = plt.figure() + # kg_future_all.kg_class.plot(cmap='magma_r', vmin=0, vmax=6) + # plt.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_kg_quartiles_score_{str(gmt_threshold).replace(".", "p")}.png')) + # plt.close() + + # Manual bins + # hist_data_binned = bin_data(hist_data, [10, 50, 100]) + # future_data_binned = bin_data(future_data, [10, 50, 100]) + + # fig = plt.figure() + # hist_bivariate.plot(cmap='magma_r', vmin=0, vmax=6) + # plt.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_quartiles_score_hist.png')) + # plt.close() + + # fig = plt.figure() + # future_bivariate.plot(cmap='magma_r', vmin=0, vmax=6) + # plt.savefig(os.path.join(plot_dir, f'{params["indicators"][ind][var]["short_name"]}_quartiles_score_{str(gmt_threshold).replace(".", "p")}.png')) + # plt.close() + +# temp4 = bin_data(t20_std_change, [1, 2, 3]) +# temp4_abs = bin_data(t20_hist, [10, 50, 100]) +# temp4_2p0 = bin_data(t20_2p0, [10, 50, 100]) + +# temp4 = bin_data(sdd_std_change, [1, 2, 3]) +# temp4_abs = bin_data(sdd_hist, [250, 750, 1250]) +# temp4_2p0 = bin_data(sdd_2p0, [250, 750, 1250]) + +# test = bin_data(pr20_std_change, [1, 2, 3]) +# temp_abs = bin_data(pr20_hist, [5, 10, 20]) +# temp4_2p0 = bin_data(pr20_2p0, [5, 10, 20]) + + + +#%% + + +dot = [3, 5, 7, 10] +quantiles = [0.95, 0.97, 0.99] +thresholds = [1.2, 1.5, 2.0, 2.5, 3.0, 3.5] + + + +land_mask = xr.open_dataset('H:\\git\\climate_impacts_processing\\landareamaskmap0.nc') + + + +output_dir = 'H:\\hotspots_explorer\\outputs\\test_zeros\\multi-model' +os.chdir(output_dir) + + + +hw_mm = xr.open_dataset('H:/hotspots_explorer/outputs/test_zeros/multi-model/ISIMIP3b_MM_heatwave.nc4') +hw_diff = xr.open_dataset('H:/hotspots_explorer/outputs/test_zeros/multi-model/ISIMIP3b_Diff_heatwave.nc4') +hw_scores = xr.open_dataset('H:/hotspots_explorer/outputs/test_zeros/multi-model/ISIMIP3b_Scores_heatwave.nc4') + + + +hw_mm = hw_mm.where(land_mask['land area'] > 0) + + +#%% STARTS here new + +fn_ds = 'C:\\Users\\byers\\IIASA\\ECE.prog - Documents\\Research Theme - NEXUS\\Hotspots_Explorer_2p0\\rcre_testing\\testing_2\\output\\maps\\' + +ds = xr.open_dataset(fn_ds+'scenario_maps_multiindicator_score.nc') + + +#%% +plot_list = [] +year = 2055 +for v in ds.data_vars: + + new_plot = ds[v].sel(year=year).hvplot(x='lon', y='lat', cmap='magma_r', shared_axes=True) + plot_list = plot_list + [new_plot] + +plot = hv.Layout(plot_list).cols(3) +hvplot.save(plot, f'{fn_ds}_test_dashboard_score.html') + + +def plot_maps_dashboard(ds, filename=None, indicators=None, year=2050, cmap='magma_r', shared_axes=True, clim=None): + + + # if indicators==None: + # indicators = list(ds.data_vars) + # elif isinstance(indicators, list): + # if not all(x in ds.data_vars for x in indicators): + # print('') + # else: + # try: + # Your code here + # except Exception as e: + # print(f"Error: not all items in indicators were found in ds.") + # elif not isinstance(indicators, list): + # print('') + # try: + # nothing + # except Exception as e: + # print(f"Error: indicators must be of type list.") + + + + # Subset the dataset. Check dims and length + + ds = check_ds_dims(ds) + + + # if 'year' in ds.dims: + # ds = ds.sel(year=year) + # elif len(ds.dims) != 2: + # except Exception as e: + # print(f"Error: Year not a dimension and more than 2 dimensions in dataset") + + + + ds = ds.sel(year=year) + + + for i in indicators: + + new_plot = ds[i].sel(year=year).hvplot(x='lon', y='lat', cmap='magma_r', shared_axes=True) + plot_list = plot_list + [new_plot] + + plot = hv.Layout(plot_list).cols(3) + + + + # Plot - check filename + # if type(filename) is None: + # filename = 'maps_dashboard_{model}_{scenario}.html' + + # elif (type(filename) is str): + # if (filename[:-5]) != '.html': + # except Exception as e: + # print(f"filename {filename} must end with '.html'") + # else: + # except Exception as e: + # print(f"filename must be string and end with '.html'") + + + + hvplot.save(plot, filename) + + +#%% +for q, dt in it.product(range(0, len(quantiles)), range(0,len(dot))): + + print(q,dt) + + plot_list = [] + + for t in range(0, len(thresholds)): + + print(t) + + plot_mm = hw_mm.mean_dur[t,1,:,:,q,dt].hvplot(x='lon', y='lat', clim=(0,50), cmap='YlOrRd', shared_axes=False) + plot_diff = hw_diff.mean_dur[t,0,:,:,q,dt].hvplot(x='lon', y='lat', clim=(0,500), cmap='Reds', shared_axes=False) + plot_score = hw_scores.mean_dur[t,0,:,:,q,dt].hvplot(x='lon', y='lat', clim=(0,3), cmap='magma_r', shared_axes=False) + plot_list = plot_list + [plot_mm, plot_diff, plot_score] + + plot = hv.Layout(plot_list).cols(3) + hvplot.save(plot, f'hw_{str(quantiles[q])[2:]}_{dot[dt]}.html') + del plot, plot_mm, plot_diff, plot_score, plot_list + + + + + + + + + + + + + + + + + + + diff --git a/rime/pp_process_maps.ipynb b/rime/pp_process_maps.ipynb new file mode 100644 index 0000000..4133af9 --- /dev/null +++ b/rime/pp_process_maps.ipynb @@ -0,0 +1,3980 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9c1ac612", + "metadata": {}, + "source": [ + "# Overview\n", + "\n", + "**Inputs:**\n", + " - Emissions or global mean temperature (GMT) scenarios from IAM. \n", + " - Climate impacts maps by Global Warming Level. \n", + "\n", + "**Outputs:**\n", + " - Maps of climate impacts through time, per scenario.\n", + "\n", + "Steps \n", + "1. Import climate impacts database. NetCDF 4-d file with different impacts as variables.\n", + "2. Import the scenarios to be run. Can be global mean temp scenarios or Emissions|CO2 scenarios.\n", + "3. Preprocessing steps.\n", + "4. Run.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b9011583", + "metadata": {}, + "outputs": [], + "source": [ + "## Add figure here" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1584932a", + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": [ + "if (typeof IPython !== 'undefined') { IPython.OutputArea.prototype._should_scroll = function(lines){ return false; }}" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, js_modules, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + " if (js_modules == null) js_modules = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls.length === 0 && js_modules.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error() {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (var i = 0; i < css_urls.length; i++) {\n", + " var url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " var skip = [];\n", + " if (window.requirejs) {\n", + " window.requirejs.config({'packages': {}, 'paths': {'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@4.2.5/dist/gridstack-h5', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'gridstack': {'exports': 'GridStack'}}});\n", + " require([\"gridstack\"], function(GridStack) {\n", + "\twindow.GridStack = GridStack\n", + "\ton_load()\n", + " })\n", + " require([\"notyf\"], function() {\n", + "\ton_load()\n", + " })\n", + " root._bokeh_is_loading = css_urls.length + 2;\n", + " } else {\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length;\n", + " } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n", + " var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/gridstack/gridstack@4.2.5/dist/gridstack-h5.js'];\n", + " for (var i = 0; i < urls.length; i++) {\n", + " skip.push(urls[i])\n", + " }\n", + " } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n", + " var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n", + " for (var i = 0; i < urls.length; i++) {\n", + " skip.push(urls[i])\n", + " }\n", + " } for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " if (skip.indexOf(url) >= 0) {\n", + "\tif (!window.requirejs) {\n", + "\t on_load();\n", + "\t}\n", + "\tcontinue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (var i = 0; i < js_modules.length; i++) {\n", + " var url = js_modules[i];\n", + " if (skip.indexOf(url) >= 0) {\n", + "\tif (!window.requirejs) {\n", + "\t on_load();\n", + "\t}\n", + "\tcontinue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " if (!js_urls.length && !js_modules.length) {\n", + " on_load()\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\", \"https://unpkg.com/@holoviz/panel@0.14.4/dist/panel.min.js\"];\n", + " var js_modules = [];\n", + " var css_urls = [\"https://cdn.holoviz.org/panel/0.14.4/dist/css/alerts.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/card.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/dataframe.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/debugger.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/json.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/loading.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/markdown.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/widgets.css\"];\n", + " var inline_js = [ function(Bokeh) {\n", + " inject_raw_css(\"\\n .bk.pn-loading.arc:before {\\n background-image: url(\\\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHN0eWxlPSJtYXJnaW46IGF1dG87IGJhY2tncm91bmQ6IG5vbmU7IGRpc3BsYXk6IGJsb2NrOyBzaGFwZS1yZW5kZXJpbmc6IGF1dG87IiB2aWV3Qm94PSIwIDAgMTAwIDEwMCIgcHJlc2VydmVBc3BlY3RSYXRpbz0ieE1pZFlNaWQiPiAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjYzNjM2MzIiBzdHJva2Utd2lkdGg9IjEwIiByPSIzNSIgc3Ryb2tlLWRhc2hhcnJheT0iMTY0LjkzMzYxNDMxMzQ2NDE1IDU2Ljk3Nzg3MTQzNzgyMTM4Ij4gICAgPGFuaW1hdGVUcmFuc2Zvcm0gYXR0cmlidXRlTmFtZT0idHJhbnNmb3JtIiB0eXBlPSJyb3RhdGUiIHJlcGVhdENvdW50PSJpbmRlZmluaXRlIiBkdXI9IjFzIiB2YWx1ZXM9IjAgNTAgNTA7MzYwIDUwIDUwIiBrZXlUaW1lcz0iMDsxIj48L2FuaW1hdGVUcmFuc2Zvcm0+ICA8L2NpcmNsZT48L3N2Zz4=\\\");\\n background-size: auto calc(min(50%, 400px));\\n }\\n \");\n", + " }, function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {} // ensure no trailing comma for IE\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if ((root.Bokeh !== undefined) || (force === true)) {\n", + " for (var i = 0; i < inline_js.length; i++) {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " }} else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " }\n", + " }\n", + "\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(css_urls, js_urls, js_modules, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(window));" + ], + "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@4.2.5/dist/gridstack-h5', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'gridstack': {'exports': 'GridStack'}}});\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 2;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length;\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/gridstack/gridstack@4.2.5/dist/gridstack-h5.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) >= 0) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) >= 0) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\", \"https://unpkg.com/@holoviz/panel@0.14.4/dist/panel.min.js\"];\n var js_modules = [];\n var css_urls = [\"https://cdn.holoviz.org/panel/0.14.4/dist/css/alerts.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/card.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/dataframe.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/debugger.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/json.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/loading.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/markdown.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/widgets.css\"];\n var inline_js = [ function(Bokeh) {\n inject_raw_css(\"\\n .bk.pn-loading.arc:before {\\n background-image: url(\\\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHN0eWxlPSJtYXJnaW46IGF1dG87IGJhY2tncm91bmQ6IG5vbmU7IGRpc3BsYXk6IGJsb2NrOyBzaGFwZS1yZW5kZXJpbmc6IGF1dG87IiB2aWV3Qm94PSIwIDAgMTAwIDEwMCIgcHJlc2VydmVBc3BlY3RSYXRpbz0ieE1pZFlNaWQiPiAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjYzNjM2MzIiBzdHJva2Utd2lkdGg9IjEwIiByPSIzNSIgc3Ryb2tlLWRhc2hhcnJheT0iMTY0LjkzMzYxNDMxMzQ2NDE1IDU2Ljk3Nzg3MTQzNzgyMTM4Ij4gICAgPGFuaW1hdGVUcmFuc2Zvcm0gYXR0cmlidXRlTmFtZT0idHJhbnNmb3JtIiB0eXBlPSJyb3RhdGUiIHJlcGVhdENvdW50PSJpbmRlZmluaXRlIiBkdXI9IjFzIiB2YWx1ZXM9IjAgNTAgNTA7MzYwIDUwIDUwIiBrZXlUaW1lcz0iMDsxIj48L2FuaW1hdGVUcmFuc2Zvcm0+ICA8L2NpcmNsZT48L3N2Zz4=\\\");\\n background-size: auto calc(min(50%, 400px));\\n }\\n \");\n }, function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, js_modules, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", + " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", + "}\n", + "\n", + "\n", + " function JupyterCommManager() {\n", + " }\n", + "\n", + " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", + " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " comm_manager.register_target(comm_id, function(comm) {\n", + " comm.on_msg(msg_handler);\n", + " });\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", + " comm.onMsg = msg_handler;\n", + " });\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " console.log(message)\n", + " var content = {data: message.data, comm_id};\n", + " var buffers = []\n", + " for (var buffer of message.buffers || []) {\n", + " buffers.push(new DataView(buffer))\n", + " }\n", + " var metadata = message.metadata || {};\n", + " var msg = {content, buffers, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " })\n", + " }\n", + " }\n", + "\n", + " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", + " if (comm_id in window.PyViz.comms) {\n", + " return window.PyViz.comms[comm_id];\n", + " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", + " if (msg_handler) {\n", + " comm.on_msg(msg_handler);\n", + " }\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", + " comm.open();\n", + " if (msg_handler) {\n", + " comm.onMsg = msg_handler;\n", + " }\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", + " comm_promise.then((comm) => {\n", + " window.PyViz.comms[comm_id] = comm;\n", + " if (msg_handler) {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data};\n", + " var metadata = message.metadata || {comm_id};\n", + " var msg = {content, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " }) \n", + " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", + " return comm_promise.then((comm) => {\n", + " comm.send(data, metadata, buffers, disposeOnDone);\n", + " });\n", + " };\n", + " var comm = {\n", + " send: sendClosure\n", + " };\n", + " }\n", + " window.PyViz.comms[comm_id] = comm;\n", + " return comm;\n", + " }\n", + " window.PyViz.comm_manager = new JupyterCommManager();\n", + " \n", + "\n", + "\n", + "var JS_MIME_TYPE = 'application/javascript';\n", + "var HTML_MIME_TYPE = 'text/html';\n", + "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", + "var CLASS_NAME = 'output';\n", + "\n", + "/**\n", + " * Render data to the DOM node\n", + " */\n", + "function render(props, node) {\n", + " var div = document.createElement(\"div\");\n", + " var script = document.createElement(\"script\");\n", + " node.appendChild(div);\n", + " node.appendChild(script);\n", + "}\n", + "\n", + "/**\n", + " * Handle when a new output is added\n", + " */\n", + "function handle_add_output(event, handle) {\n", + " var output_area = handle.output_area;\n", + " var output = handle.output;\n", + " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + " if (id !== undefined) {\n", + " var nchildren = toinsert.length;\n", + " var html_node = toinsert[nchildren-1].children[0];\n", + " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var scripts = [];\n", + " var nodelist = html_node.querySelectorAll(\"script\");\n", + " for (var i in nodelist) {\n", + " if (nodelist.hasOwnProperty(i)) {\n", + " scripts.push(nodelist[i])\n", + " }\n", + " }\n", + "\n", + " scripts.forEach( function (oldScript) {\n", + " var newScript = document.createElement(\"script\");\n", + " var attrs = [];\n", + " var nodemap = oldScript.attributes;\n", + " for (var j in nodemap) {\n", + " if (nodemap.hasOwnProperty(j)) {\n", + " attrs.push(nodemap[j])\n", + " }\n", + " }\n", + " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", + " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", + " oldScript.parentNode.replaceChild(newScript, oldScript);\n", + " });\n", + " if (JS_MIME_TYPE in output.data) {\n", + " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", + " }\n", + " output_area._hv_plot_id = id;\n", + " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", + " window.PyViz.plot_index[id] = Bokeh.index[id];\n", + " } else {\n", + " window.PyViz.plot_index[id] = null;\n", + " }\n", + " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " var bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var script_attrs = bk_div.children[0].attributes;\n", + " for (var i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + "function handle_clear_output(event, handle) {\n", + " var id = handle.cell.output_area._hv_plot_id;\n", + " var server_id = handle.cell.output_area._bokeh_server_id;\n", + " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", + " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", + " if (server_id !== null) {\n", + " comm.send({event_type: 'server_delete', 'id': server_id});\n", + " return;\n", + " } else if (comm !== null) {\n", + " comm.send({event_type: 'delete', 'id': id});\n", + " }\n", + " delete PyViz.plot_index[id];\n", + " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", + " var doc = window.Bokeh.index[id].model.document\n", + " doc.clear();\n", + " const i = window.Bokeh.documents.indexOf(doc);\n", + " if (i > -1) {\n", + " window.Bokeh.documents.splice(i, 1);\n", + " }\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle kernel restart event\n", + " */\n", + "function handle_kernel_cleanup(event, handle) {\n", + " delete PyViz.comms[\"hv-extension-comm\"];\n", + " window.PyViz.plot_index = {}\n", + "}\n", + "\n", + "/**\n", + " * Handle update_display_data messages\n", + " */\n", + "function handle_update_output(event, handle) {\n", + " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", + " handle_add_output(event, handle)\n", + "}\n", + "\n", + "function register_renderer(events, OutputArea) {\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " var toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[0]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " events.on('output_added.OutputArea', handle_add_output);\n", + " events.on('output_updated.OutputArea', handle_update_output);\n", + " events.on('clear_output.CodeCell', handle_clear_output);\n", + " events.on('delete.Cell', handle_clear_output);\n", + " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", + "\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " safe: true,\n", + " index: 0\n", + " });\n", + "}\n", + "\n", + "if (window.Jupyter !== undefined) {\n", + " try {\n", + " var events = require('base/js/events');\n", + " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " } catch(err) {\n", + " }\n", + "}\n" + ], + "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import dask\n", + "import glob\n", + "import numpy as np\n", + "import os\n", + "import pyam\n", + "import time\n", + "import xarray as xr\n", + "import yaml\n", + "from dask import delayed\n", + "from dask.distributed import Client\n", + "from process_config import *\n", + "from rime_functions import *\n", + "\n", + "with open(yaml_path, \"r\") as f:\n", + " params = yaml.full_load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e10671eb", + "metadata": {}, + "outputs": [], + "source": [ + "dask.config.set(scheduler=\"processes\")\n", + "dask.config.set(num_workers=num_workers)\n", + "client = Client()\n" + ] + }, + { + "cell_type": "markdown", + "id": "356f0d3a", + "metadata": {}, + "source": [ + "Open http://localhost:8787/status" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "43c3cede", + "metadata": {}, + "outputs": [], + "source": [ + "### Import the climate impacts database files\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1b29ceb", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "55bc4121", + "metadata": {}, + "source": [ + "### Import scenarios data\n", + "\n", + "Load an IAMC scenarios dataset. \n", + "Decide whether using global mean temperature or CO2 mode. \n", + "Assign SSPs if missing and fix duplicate temperatures. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "0d4c1405", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "pyam - INFO: Running in a notebook, setting up a basic logging at level INFO\n", + "pyam.core - INFO: Reading file C:\\Users\\byers\\IIASA\\ECE.prog - Documents\\Research Theme - NEXUS\\Hotspots_Explorer_2p0\\rcre_testing\\testing_2\\emissions_temp_AR6_small.xlsx\n", + "pyam.core - INFO: Reading meta indicators\n" + ] + } + ], + "source": [ + "df_scens_in = pyam.IamDataFrame(fname_input_scenarios)\n", + "dft = df_scens_in.filter(variable=temp_variable)\n", + "# dft = np.round(dft.as_pandas()[pyam.IAMC_IDX + [\"year\", \"value\", \"Ssp_family\"]], 2)\n", + "# dft = pyam.IamDataFrame(dft)\n", + "# Replace & fill missing SSP scenario allocation\n", + "dft = ssp_helper(dft, ssp_meta_col=\"Ssp_family\", default_ssp=\"SSP2\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6f3c225d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\n", + "Index:\n", + " * model : AIM/CGE 2.0, AIM/CGE 2.1, AIM/CGE 2.2, ... WITCH-GLOBIOM 4.4 (44)\n", + " * scenario : 1.5C, 2.5C, 2C, 2CNow_Gradual, ... WB2C (559)\n", + "Timeseries data coordinates:\n", + " region : World (1)\n", + " variable : ... (1)\n", + " unit : K (1)\n", + " year : 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, ... 2100 (86)\n", + " ssp_family : SSP1, SSP2, SSP3, SSP4, SSP5 (5)\n", + "Meta indicators:\n", + " exclude (bool) False (1)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dft" + ] + }, + { + "cell_type": "markdown", + "id": "4b3ecfc0", + "metadata": {}, + "source": [ + "## Example multiple scenarios, 1 climate indicator" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b93f6463", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test multiple scenarios, 1 indicator\n", + "Single indicator mode\n", + "POLES_ADVANCE_ADVANCE_2020_1.5C-2100\n", + "POLES_ADVANCE_ADVANCE_2020_Med2C\n", + "POLES_ADVANCE_ADVANCE_2020_WB2C\n", + "POLES_ADVANCE_ADVANCE_2030_1.5C-2100\n", + "POLES_ADVANCE_ADVANCE_2030_Med2C\n", + "POLES_ADVANCE_ADVANCE_2030_Price1.5C\n", + "POLES_ADVANCE_ADVANCE_2030_WB2C\n", + "POLES_ADVANCE_ADVANCE_INDC\n", + "POLES_ADVANCE_ADVANCE_NoPolicy\n", + "POLES_ADVANCE_ADVANCE_Reference\n", + "FINISHED Test multiple scenarios, 1 indicator\n", + "10.552809476852417\n" + ] + } + ], + "source": [ + "print(\"Test multiple scenarios, 1 indicator\")\n", + "start = time.time()\n", + "\n", + "ind = \"precip\"\n", + "var = \"sdii\"\n", + "short = params[\"indicators\"][ind][var][\"short_name\"]\n", + "\n", + "ssp = \"ssp2\"\n", + "\n", + "\n", + "\n", + "files = glob.glob(os.path.join(impact_data_dir, ind, f\"*{short}_{ssp}*{ftype}.nc4\"))\n", + "mapdata = xr.open_mfdataset(\n", + " files, preprocess=remove_ssp_from_ds, combine=\"nested\", concat_dim=\"gmt\"\n", + ")\n", + "\n", + "\n", + "map_out_MS = map_transform_gmt_multi_dask(\n", + " dft.filter(model=\"POLES ADVANCE\"),\n", + " mapdata,\n", + " years,\n", + " use_dask=True,\n", + " gmt_name=\"threshold\",\n", + " interpolation=interpolation,\n", + ")\n", + "\n", + "comp = dict(zlib=True, complevel=5)\n", + "encoding = {var: comp for var in map_out_MS.data_vars}\n", + "filename = f\"{output_folder_maps}scenario_maps_multiscenario_{ftype}_test_notebook.nc\"\n", + "map_out_MS.to_netcdf(filename, encoding=encoding)\n", + "\n", + "print(\"FINISHED Test multiple scenarios, 1 indicator\")\n", + "print(f\"{time.time()-start}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "c16fbae3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:                           (lat: 360, lon: 720, year: 9)\n",
+       "Coordinates:\n",
+       "  * lon                               (lon) int64 0 1 2 3 4 ... 716 717 718 719\n",
+       "  * lat                               (lat) int64 0 1 2 3 4 ... 356 357 358 359\n",
+       "  * year                              (year) int32 2015 2025 2035 ... 2085 2095\n",
+       "Data variables:\n",
+       "    POLES_ADVANCE_ADVANCE_2020_Med2C  (lat, lon, year) float64 nan nan ... nan\n",
+       "    POLES_ADVANCE_ADVANCE_2020_WB2C   (lat, lon, year) float64 nan nan ... nan\n",
+       "    POLES_ADVANCE_ADVANCE_2030_Med2C  (lat, lon, year) float64 nan nan ... nan\n",
+       "    POLES_ADVANCE_ADVANCE_INDC        (lat, lon, year) float64 nan nan ... nan\n",
+       "    POLES_ADVANCE_ADVANCE_Reference   (lat, lon, year) float64 nan nan ... nan
" + ], + "text/plain": [ + "\n", + "Dimensions: (lat: 360, lon: 720, year: 9)\n", + "Coordinates:\n", + " * lon (lon) int64 0 1 2 3 4 ... 716 717 718 719\n", + " * lat (lat) int64 0 1 2 3 4 ... 356 357 358 359\n", + " * year (year) int32 2015 2025 2035 ... 2085 2095\n", + "Data variables:\n", + " POLES_ADVANCE_ADVANCE_2020_Med2C (lat, lon, year) float64 nan nan ... nan\n", + " POLES_ADVANCE_ADVANCE_2020_WB2C (lat, lon, year) float64 nan nan ... nan\n", + " POLES_ADVANCE_ADVANCE_2030_Med2C (lat, lon, year) float64 nan nan ... nan\n", + " POLES_ADVANCE_ADVANCE_INDC (lat, lon, year) float64 nan nan ... nan\n", + " POLES_ADVANCE_ADVANCE_Reference (lat, lon, year) float64 nan nan ... nan" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "map_out_MS" + ] + }, + { + "cell_type": "markdown", + "id": "3ce5dffd", + "metadata": {}, + "source": [ + "## Example: 1 scenario, multiple indicators Faster with non dask\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "d6aaab79", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test 1 scenario, multiple indicators\n", + "cdd\n", + "precip\n", + "dri\n", + "dri_qtot\n", + "ia_var\n", + "ia_var_qtot\n", + "sdd_18p3\n", + "sdd_24p0\n" + ] + } + ], + "source": [ + "\n", + "print(\"Test 1 scenario, multiple indicators\")\n", + "start = time.time()\n", + "ssp = \"ssp2\"\n", + "mapdata = xr.Dataset()\n", + "indicators = [\n", + " \"cdd\",\n", + " \"precip\",\n", + " \"dri\",\n", + " \"dri_qtot\",\n", + " \"ia_var\",\n", + " \"ia_var_qtot\",\n", + " \"sdd_18p3\",\n", + " \"sdd_24p0\",\n", + "] #'heatwave']\n", + "\n", + "for ind in indicators:\n", + " print(ind)\n", + " for var in params[\"indicators\"][ind]:\n", + " short = params[\"indicators\"][ind][var][\"short_name\"]\n", + " files = glob.glob(\n", + " os.path.join(impact_data_dir, ind, f\"*{short}_{ssp}*{ftype}.nc4\")\n", + " )\n", + " mapdata[short] = xr.open_mfdataset(\n", + " files, preprocess=remove_ssp_from_ds, combine=\"nested\", concat_dim=\"gmt\"\n", + " )[short]\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "3d38fbd5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:     (lat: 360, lon: 720, gmt: 6)\n",
+       "Coordinates:\n",
+       "    stats       object 'mean'\n",
+       "  * lat         (lat) float64 89.75 89.25 88.75 88.25 ... -88.75 -89.25 -89.75\n",
+       "  * lon         (lon) float64 -179.8 -179.2 -178.8 -178.2 ... 178.8 179.2 179.8\n",
+       "    threshold   (gmt) float64 1.2 1.5 2.0 2.5 3.0 3.5\n",
+       "    quantile    float64 0.95\n",
+       "Dimensions without coordinates: gmt\n",
+       "Data variables:\n",
+       "    cdd         (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    pr_r10      (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    pr_r20      (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    pr_r95p     (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    pr_r99p     (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    sdii        (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    dri         (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    dri_qtot    (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    iavar       (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    iavar_qtot  (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    sdd_c_18p3  (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>\n",
+       "    sdd_c_24p0  (gmt, lat, lon) float64 dask.array<chunksize=(1, 360, 720), meta=np.ndarray>
" + ], + "text/plain": [ + "\n", + "Dimensions: (lat: 360, lon: 720, gmt: 6)\n", + "Coordinates:\n", + " stats object 'mean'\n", + " * lat (lat) float64 89.75 89.25 88.75 88.25 ... -88.75 -89.25 -89.75\n", + " * lon (lon) float64 -179.8 -179.2 -178.8 -178.2 ... 178.8 179.2 179.8\n", + " threshold (gmt) float64 1.2 1.5 2.0 2.5 3.0 3.5\n", + " quantile float64 0.95\n", + "Dimensions without coordinates: gmt\n", + "Data variables:\n", + " cdd (gmt, lat, lon) float64 dask.array\n", + " pr_r10 (gmt, lat, lon) float64 dask.array\n", + " pr_r20 (gmt, lat, lon) float64 dask.array\n", + " pr_r95p (gmt, lat, lon) float64 dask.array\n", + " pr_r99p (gmt, lat, lon) float64 dask.array\n", + " sdii (gmt, lat, lon) float64 dask.array\n", + " dri (gmt, lat, lon) float64 dask.array\n", + " dri_qtot (gmt, lat, lon) float64 dask.array\n", + " iavar (gmt, lat, lon) float64 dask.array\n", + " iavar_qtot (gmt, lat, lon) float64 dask.array\n", + " sdd_c_18p3 (gmt, lat, lon) float64 dask.array\n", + " sdd_c_24p0 (gmt, lat, lon) float64 dask.array" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mapdata" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e8b78c05", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Single scenario mode\n", + "cdd\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "pr_r10\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "pr_r20\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "pr_r95p\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "pr_r99p\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "sdii\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "dri\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "dri_qtot\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "iavar\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "iavar_qtot\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "sdd_c_18p3\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "sdd_c_24p0\n", + "Warning! Min temperature below 1.2°C 1.12, data before not possible\n", + "FINISHED 1 scenario, multiple indicators\n", + "15.842880964279175\n" + ] + } + ], + "source": [ + "map_out_MI = map_transform_gmt_multi_dask(\n", + " dft.filter(model=\"AIM*\", scenario=\"SSP1-34\"),\n", + " mapdata,\n", + " years,\n", + " use_dask=False,\n", + " gmt_name=\"threshold\",\n", + ")\n", + "\n", + "comp = dict(zlib=True, complevel=5)\n", + "encoding = {var: comp for var in map_out_MI.data_vars}\n", + "filename = f\"{output_folder_maps}scenario_maps_multiindicator_{ftype}_test_notebook.nc\"\n", + "# map_out_MI.to_netcdf(filename, encoding=encoding)\n", + "\n", + "print(\"FINISHED 1 scenario, multiple indicators\")\n", + "print(f\"{time.time()-start}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c99b2928", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:     (lat: 360, lon: 720, year: 9)\n",
+       "Coordinates:\n",
+       "  * lon         (lon) int64 0 1 2 3 4 5 6 7 ... 712 713 714 715 716 717 718 719\n",
+       "  * lat         (lat) int64 0 1 2 3 4 5 6 7 ... 352 353 354 355 356 357 358 359\n",
+       "  * year        (year) int32 2015 2025 2035 2045 2055 2065 2075 2085 2095\n",
+       "Data variables:\n",
+       "    cdd         (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    pr_r10      (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    pr_r20      (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    pr_r95p     (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    pr_r99p     (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    sdii        (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    dri         (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    dri_qtot    (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    iavar       (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    iavar_qtot  (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    sdd_c_18p3  (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n",
+       "    sdd_c_24p0  (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan
" + ], + "text/plain": [ + "\n", + "Dimensions: (lat: 360, lon: 720, year: 9)\n", + "Coordinates:\n", + " * lon (lon) int64 0 1 2 3 4 5 6 7 ... 712 713 714 715 716 717 718 719\n", + " * lat (lat) int64 0 1 2 3 4 5 6 7 ... 352 353 354 355 356 357 358 359\n", + " * year (year) int32 2015 2025 2035 2045 2055 2065 2075 2085 2095\n", + "Data variables:\n", + " cdd (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " pr_r10 (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " pr_r20 (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " pr_r95p (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " pr_r99p (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " sdii (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " dri (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " dri_qtot (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " iavar (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " iavar_qtot (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " sdd_c_18p3 (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan\n", + " sdd_c_24p0 (lat, lon, year) float64 nan nan nan nan nan ... nan nan nan nan" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "map_out_MI" + ] + }, + { + "cell_type": "markdown", + "id": "d2c328f0", + "metadata": {}, + "source": [ + "## Plot dashboard" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6e65d9ab", + "metadata": {}, + "outputs": [], + "source": [ + "from rime_functions import plot_maps_dashboard" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "830e067b", + "metadata": {}, + "outputs": [ + { + "ename": "Exception", + "evalue": "filename must be string and end with '.html'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mException\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_7828\\1221313485.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mplot_maps_dashboard\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmap_out_MI\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindicators\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'dri'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'dri_qtot'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myear\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2055\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'magma_r'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshared_axes\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mclim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;32mC:\\Github\\rime\\rime\\rime_functions.py\u001b[0m in \u001b[0;36mplot_maps_dashboard\u001b[1;34m(ds, filename, indicators, year, cmap, shared_axes, clim)\u001b[0m\n\u001b[0;32m 636\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 637\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 638\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf\"filename must be string and end with '.html'\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 639\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 640\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mException\u001b[0m: filename must be string and end with '.html'" + ] + } + ], + "source": [ + "plot_maps_dashboard(map_out_MI, filename=None, indicators=['dri','dri_qtot',], year=2055, cmap='magma_r', shared_axes=True, clim=None)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3197ec9d", + "metadata": {}, + "outputs": [], + "source": [ + "ds = map_out_MI\n", + "year=2055\n", + "model = 'AIM'\n", + "scenario = \"SSP1-34\"\n", + "# if indicators==None:\n", + "# indicators = list(ds.data_vars)\n", + "# elif isinstance(indicators, list):\n", + "# if all(x in ds.data_vars for x in indicators)==False:\n", + "# raise Exception(f\"Error: not all items in indicators were found in ds.\")\n", + "# elif isinstance(indicators, list)==False:\n", + "# raise Exception(f\"Error: indicators must be of type list.\")\n", + "\n", + "\n", + "# Subset the dataset. Check dims and length\n", + "\n", + "ds = check_ds_dims(ds)\n", + "\n", + "\n", + "if 'year' in ds.dims:\n", + " ds = ds.sel(year=year).squeeze()\n", + "elif len(ds.dims) != 2:\n", + " raise Exception(f\"Error: Year not a dimension and more than 2 dimensions in dataset\")\n", + "\n", + "plot_list = []\n", + "# Run loop through indicators (variables)\n", + "indicators = ['dri','dri_qtot']\n", + "for i in indicators:\n", + "\n", + " new_plot = ds[i].hvplot(x='lon', y='lat', cmap='magma_r', shared_axes=True)\n", + " plot_list = plot_list + [new_plot]\n", + "\n", + "plot = hv.Layout(plot_list).cols(3)\n", + "\n", + "filename = f'maps_dashboard_{model}_{scenario}.html'\n", + "\n", + "\n", + "# Plot - check filename\n", + "# if type(filename) is None:\n", + "# filename = 'maps_dashboard_{model}_{scenario}.html'\n", + "\n", + "# elif (type(filename) is str):\n", + "# if (filename[:-5]) != '.html':\n", + "# raise Exception(f\"filename {filename} must end with '.html'\")\n", + "\n", + "# else:\n", + "# raise Exception(f\"filename must be string and end with '.html'\")\n", + "\n", + "\n", + "\n", + "hvplot.save(plot, filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cf038af", + "metadata": {}, + "outputs": [], + "source": [ + "import holoviews as hv\n", + "import hvplot.pandas\n", + "import hvplot.xarray" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d858e2a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/rime/pp_process_tabledata.ipynb b/rime/pp_process_tabledata.ipynb new file mode 100644 index 0000000..a22bbe6 --- /dev/null +++ b/rime/pp_process_tabledata.ipynb @@ -0,0 +1,2411 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "a2a0be2b", + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " var force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, js_modules, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + " if (js_modules == null) js_modules = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls.length === 0 && js_modules.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error() {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (var i = 0; i < css_urls.length; i++) {\n", + " var url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " var skip = [];\n", + " if (window.requirejs) {\n", + " window.requirejs.config({'packages': {}, 'paths': {'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@4.2.5/dist/gridstack-h5', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'gridstack': {'exports': 'GridStack'}}});\n", + " require([\"gridstack\"], function(GridStack) {\n", + "\twindow.GridStack = GridStack\n", + "\ton_load()\n", + " })\n", + " require([\"notyf\"], function() {\n", + "\ton_load()\n", + " })\n", + " root._bokeh_is_loading = css_urls.length + 2;\n", + " } else {\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length;\n", + " } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n", + " var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/gridstack/gridstack@4.2.5/dist/gridstack-h5.js'];\n", + " for (var i = 0; i < urls.length; i++) {\n", + " skip.push(urls[i])\n", + " }\n", + " } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n", + " var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n", + " for (var i = 0; i < urls.length; i++) {\n", + " skip.push(urls[i])\n", + " }\n", + " } for (var i = 0; i < js_urls.length; i++) {\n", + " var url = js_urls[i];\n", + " if (skip.indexOf(url) >= 0) {\n", + "\tif (!window.requirejs) {\n", + "\t on_load();\n", + "\t}\n", + "\tcontinue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " for (var i = 0; i < js_modules.length; i++) {\n", + " var url = js_modules[i];\n", + " if (skip.indexOf(url) >= 0) {\n", + "\tif (!window.requirejs) {\n", + "\t on_load();\n", + "\t}\n", + "\tcontinue;\n", + " }\n", + " var element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error;\n", + " element.async = false;\n", + " element.src = url;\n", + " element.type = \"module\";\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " if (!js_urls.length && !js_modules.length) {\n", + " on_load()\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\", \"https://unpkg.com/@holoviz/panel@0.14.4/dist/panel.min.js\"];\n", + " var js_modules = [];\n", + " var css_urls = [\"https://cdn.holoviz.org/panel/0.14.4/dist/css/alerts.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/card.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/dataframe.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/debugger.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/json.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/loading.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/markdown.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/widgets.css\"];\n", + " var inline_js = [ function(Bokeh) {\n", + " inject_raw_css(\"\\n .bk.pn-loading.arc:before {\\n background-image: url(\\\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHN0eWxlPSJtYXJnaW46IGF1dG87IGJhY2tncm91bmQ6IG5vbmU7IGRpc3BsYXk6IGJsb2NrOyBzaGFwZS1yZW5kZXJpbmc6IGF1dG87IiB2aWV3Qm94PSIwIDAgMTAwIDEwMCIgcHJlc2VydmVBc3BlY3RSYXRpbz0ieE1pZFlNaWQiPiAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjYzNjM2MzIiBzdHJva2Utd2lkdGg9IjEwIiByPSIzNSIgc3Ryb2tlLWRhc2hhcnJheT0iMTY0LjkzMzYxNDMxMzQ2NDE1IDU2Ljk3Nzg3MTQzNzgyMTM4Ij4gICAgPGFuaW1hdGVUcmFuc2Zvcm0gYXR0cmlidXRlTmFtZT0idHJhbnNmb3JtIiB0eXBlPSJyb3RhdGUiIHJlcGVhdENvdW50PSJpbmRlZmluaXRlIiBkdXI9IjFzIiB2YWx1ZXM9IjAgNTAgNTA7MzYwIDUwIDUwIiBrZXlUaW1lcz0iMDsxIj48L2FuaW1hdGVUcmFuc2Zvcm0+ICA8L2NpcmNsZT48L3N2Zz4=\\\");\\n background-size: auto calc(min(50%, 400px));\\n }\\n \");\n", + " }, function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {} // ensure no trailing comma for IE\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if ((root.Bokeh !== undefined) || (force === true)) {\n", + " for (var i = 0; i < inline_js.length; i++) {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " }} else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " }\n", + " }\n", + "\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(css_urls, js_urls, js_modules, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(window));" + ], + "application/vnd.holoviews_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {'gridstack': 'https://cdn.jsdelivr.net/npm/gridstack@4.2.5/dist/gridstack-h5', 'notyf': 'https://cdn.jsdelivr.net/npm/notyf@3/notyf.min'}, 'shim': {'gridstack': {'exports': 'GridStack'}}});\n require([\"gridstack\"], function(GridStack) {\n\twindow.GridStack = GridStack\n\ton_load()\n })\n require([\"notyf\"], function() {\n\ton_load()\n })\n root._bokeh_is_loading = css_urls.length + 2;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length;\n } if (((window['GridStack'] !== undefined) && (!(window['GridStack'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/gridstack/gridstack@4.2.5/dist/gridstack-h5.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } if (((window['Notyf'] !== undefined) && (!(window['Notyf'] instanceof HTMLElement))) || window.requirejs) {\n var urls = ['https://cdn.holoviz.org/panel/0.14.4/dist/bundled/notificationarea/notyf@3/notyf.min.js'];\n for (var i = 0; i < urls.length; i++) {\n skip.push(urls[i])\n }\n } for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) >= 0) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) >= 0) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\", \"https://unpkg.com/@holoviz/panel@0.14.4/dist/panel.min.js\"];\n var js_modules = [];\n var css_urls = [\"https://cdn.holoviz.org/panel/0.14.4/dist/css/alerts.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/card.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/dataframe.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/debugger.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/json.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/loading.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/markdown.css\", \"https://cdn.holoviz.org/panel/0.14.4/dist/css/widgets.css\"];\n var inline_js = [ function(Bokeh) {\n inject_raw_css(\"\\n .bk.pn-loading.arc:before {\\n background-image: url(\\\"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHN0eWxlPSJtYXJnaW46IGF1dG87IGJhY2tncm91bmQ6IG5vbmU7IGRpc3BsYXk6IGJsb2NrOyBzaGFwZS1yZW5kZXJpbmc6IGF1dG87IiB2aWV3Qm94PSIwIDAgMTAwIDEwMCIgcHJlc2VydmVBc3BlY3RSYXRpbz0ieE1pZFlNaWQiPiAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSIjYzNjM2MzIiBzdHJva2Utd2lkdGg9IjEwIiByPSIzNSIgc3Ryb2tlLWRhc2hhcnJheT0iMTY0LjkzMzYxNDMxMzQ2NDE1IDU2Ljk3Nzg3MTQzNzgyMTM4Ij4gICAgPGFuaW1hdGVUcmFuc2Zvcm0gYXR0cmlidXRlTmFtZT0idHJhbnNmb3JtIiB0eXBlPSJyb3RhdGUiIHJlcGVhdENvdW50PSJpbmRlZmluaXRlIiBkdXI9IjFzIiB2YWx1ZXM9IjAgNTAgNTA7MzYwIDUwIDUwIiBrZXlUaW1lcz0iMDsxIj48L2FuaW1hdGVUcmFuc2Zvcm0+ICA8L2NpcmNsZT48L3N2Zz4=\\\");\\n background-size: auto calc(min(50%, 400px));\\n }\\n \");\n }, function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, js_modules, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "\n", + "if ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n", + " window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n", + "}\n", + "\n", + "\n", + " function JupyterCommManager() {\n", + " }\n", + "\n", + " JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n", + " if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " comm_manager.register_target(comm_id, function(comm) {\n", + " comm.on_msg(msg_handler);\n", + " });\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n", + " comm.onMsg = msg_handler;\n", + " });\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " console.log(message)\n", + " var content = {data: message.data, comm_id};\n", + " var buffers = []\n", + " for (var buffer of message.buffers || []) {\n", + " buffers.push(new DataView(buffer))\n", + " }\n", + " var metadata = message.metadata || {};\n", + " var msg = {content, buffers, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " })\n", + " }\n", + " }\n", + "\n", + " JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n", + " if (comm_id in window.PyViz.comms) {\n", + " return window.PyViz.comms[comm_id];\n", + " } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n", + " var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n", + " var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n", + " if (msg_handler) {\n", + " comm.on_msg(msg_handler);\n", + " }\n", + " } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n", + " var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n", + " comm.open();\n", + " if (msg_handler) {\n", + " comm.onMsg = msg_handler;\n", + " }\n", + " } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n", + " var comm_promise = google.colab.kernel.comms.open(comm_id)\n", + " comm_promise.then((comm) => {\n", + " window.PyViz.comms[comm_id] = comm;\n", + " if (msg_handler) {\n", + " var messages = comm.messages[Symbol.asyncIterator]();\n", + " function processIteratorResult(result) {\n", + " var message = result.value;\n", + " var content = {data: message.data};\n", + " var metadata = message.metadata || {comm_id};\n", + " var msg = {content, metadata}\n", + " msg_handler(msg);\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " return messages.next().then(processIteratorResult);\n", + " }\n", + " }) \n", + " var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n", + " return comm_promise.then((comm) => {\n", + " comm.send(data, metadata, buffers, disposeOnDone);\n", + " });\n", + " };\n", + " var comm = {\n", + " send: sendClosure\n", + " };\n", + " }\n", + " window.PyViz.comms[comm_id] = comm;\n", + " return comm;\n", + " }\n", + " window.PyViz.comm_manager = new JupyterCommManager();\n", + " \n", + "\n", + "\n", + "var JS_MIME_TYPE = 'application/javascript';\n", + "var HTML_MIME_TYPE = 'text/html';\n", + "var EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\n", + "var CLASS_NAME = 'output';\n", + "\n", + "/**\n", + " * Render data to the DOM node\n", + " */\n", + "function render(props, node) {\n", + " var div = document.createElement(\"div\");\n", + " var script = document.createElement(\"script\");\n", + " node.appendChild(div);\n", + " node.appendChild(script);\n", + "}\n", + "\n", + "/**\n", + " * Handle when a new output is added\n", + " */\n", + "function handle_add_output(event, handle) {\n", + " var output_area = handle.output_area;\n", + " var output = handle.output;\n", + " if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + " var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + " if (id !== undefined) {\n", + " var nchildren = toinsert.length;\n", + " var html_node = toinsert[nchildren-1].children[0];\n", + " html_node.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var scripts = [];\n", + " var nodelist = html_node.querySelectorAll(\"script\");\n", + " for (var i in nodelist) {\n", + " if (nodelist.hasOwnProperty(i)) {\n", + " scripts.push(nodelist[i])\n", + " }\n", + " }\n", + "\n", + " scripts.forEach( function (oldScript) {\n", + " var newScript = document.createElement(\"script\");\n", + " var attrs = [];\n", + " var nodemap = oldScript.attributes;\n", + " for (var j in nodemap) {\n", + " if (nodemap.hasOwnProperty(j)) {\n", + " attrs.push(nodemap[j])\n", + " }\n", + " }\n", + " attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n", + " newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n", + " oldScript.parentNode.replaceChild(newScript, oldScript);\n", + " });\n", + " if (JS_MIME_TYPE in output.data) {\n", + " toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n", + " }\n", + " output_area._hv_plot_id = id;\n", + " if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n", + " window.PyViz.plot_index[id] = Bokeh.index[id];\n", + " } else {\n", + " window.PyViz.plot_index[id] = null;\n", + " }\n", + " } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " var bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " var script_attrs = bk_div.children[0].attributes;\n", + " for (var i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + "function handle_clear_output(event, handle) {\n", + " var id = handle.cell.output_area._hv_plot_id;\n", + " var server_id = handle.cell.output_area._bokeh_server_id;\n", + " if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n", + " var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n", + " if (server_id !== null) {\n", + " comm.send({event_type: 'server_delete', 'id': server_id});\n", + " return;\n", + " } else if (comm !== null) {\n", + " comm.send({event_type: 'delete', 'id': id});\n", + " }\n", + " delete PyViz.plot_index[id];\n", + " if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n", + " var doc = window.Bokeh.index[id].model.document\n", + " doc.clear();\n", + " const i = window.Bokeh.documents.indexOf(doc);\n", + " if (i > -1) {\n", + " window.Bokeh.documents.splice(i, 1);\n", + " }\n", + " }\n", + "}\n", + "\n", + "/**\n", + " * Handle kernel restart event\n", + " */\n", + "function handle_kernel_cleanup(event, handle) {\n", + " delete PyViz.comms[\"hv-extension-comm\"];\n", + " window.PyViz.plot_index = {}\n", + "}\n", + "\n", + "/**\n", + " * Handle update_display_data messages\n", + " */\n", + "function handle_update_output(event, handle) {\n", + " handle_clear_output(event, {cell: {output_area: handle.output_area}})\n", + " handle_add_output(event, handle)\n", + "}\n", + "\n", + "function register_renderer(events, OutputArea) {\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " var toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[0]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " events.on('output_added.OutputArea', handle_add_output);\n", + " events.on('output_updated.OutputArea', handle_update_output);\n", + " events.on('clear_output.CodeCell', handle_clear_output);\n", + " events.on('delete.Cell', handle_clear_output);\n", + " events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n", + "\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " safe: true,\n", + " index: 0\n", + " });\n", + "}\n", + "\n", + "if (window.Jupyter !== undefined) {\n", + " try {\n", + " var events = require('base/js/events');\n", + " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " } catch(err) {\n", + " }\n", + "}\n" + ], + "application/vnd.holoviews_load.v0+json": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "if (typeof IPython !== 'undefined') { IPython.OutputArea.prototype._should_scroll = function(lines){ return false; }}" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from process_config import *\n", + "from rime_functions import *\n", + "import dask.dataframe as dd\n", + "from dask.diagnostics import ProgressBar\n", + "from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler\n", + "from dask.distributed import Client\n", + "import glob\n", + "import numpy as np\n", + "import pandas as pd\n", + "import pyam\n", + "import re\n", + "import time\n", + "import xarray as xr\n", + "import dask" + ] + }, + { + "cell_type": "markdown", + "id": "b759c865", + "metadata": {}, + "source": [ + "# Overview\n", + "\n", + "**Inputs:**\n", + " - Emissions or GMT scenarios from IAM. \n", + " - Climate impacts database aggregated to region (e.g. country or R10) by Global Warming Level. \n", + "\n", + "**Outputs:**\n", + " - Table data of climate impacts through time in IAMC format, per scenario.\n", + "\n", + "Steps \n", + "1. Import climate impacts database. NetCDF 4D file with different impacts as variables.\n", + "2. Import the scenarios to be run. Can be global mean temp scenarios or Emissions|CO2 scenarios.\n", + "3. Preprocessing steps.\n", + "4. Run.\n" + ] + }, + { + "cell_type": "markdown", + "id": "7d83ac6c", + "metadata": {}, + "source": [ + "### Import the climate impacts database files\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bfee2567", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:                                     (gmt: 251, year: 9, ssp: 3,\n",
+       "                                                 region: 226)\n",
+       "Coordinates:\n",
+       "  * gmt                                         (gmt) float64 1.2 1.206 ... 3.5\n",
+       "  * year                                        (year) int64 2015 2025 ... 2095\n",
+       "  * ssp                                         (ssp) object 'SSP1' ... 'SSP3'\n",
+       "  * region                                      (region) object 'AFG' ... 'wo...\n",
+       "Data variables:\n",
+       "    sdii|Exposure|Land area                     (gmt, year, ssp, region) float64 dask.array<chunksize=(251, 9, 3, 226), meta=np.ndarray>\n",
+       "    sdii|Exposure|Population                    (gmt, year, ssp, region) float64 dask.array<chunksize=(251, 9, 3, 226), meta=np.ndarray>\n",
+       "    sdii|Exposure|Population|%                  (gmt, year, ssp, region) float64 dask.array<chunksize=(251, 9, 3, 226), meta=np.ndarray>\n",
+       "    sdii|Hazard|Absolute|Land area weighted     (gmt, year, ssp, region) float64 dask.array<chunksize=(251, 9, 3, 226), meta=np.ndarray>\n",
+       "    sdii|Hazard|Absolute|Population weighted    (gmt, year, ssp, region) float64 dask.array<chunksize=(251, 9, 3, 226), meta=np.ndarray>\n",
+       "    sdii|Hazard|Risk score|Population weighted  (gmt, year, ssp, region) float64 dask.array<chunksize=(251, 9, 3, 226), meta=np.ndarray>
" + ], + "text/plain": [ + "\n", + "Dimensions: (gmt: 251, year: 9, ssp: 3,\n", + " region: 226)\n", + "Coordinates:\n", + " * gmt (gmt) float64 1.2 1.206 ... 3.5\n", + " * year (year) int64 2015 2025 ... 2095\n", + " * ssp (ssp) object 'SSP1' ... 'SSP3'\n", + " * region (region) object 'AFG' ... 'wo...\n", + "Data variables:\n", + " sdii|Exposure|Land area (gmt, year, ssp, region) float64 dask.array\n", + " sdii|Exposure|Population (gmt, year, ssp, region) float64 dask.array\n", + " sdii|Exposure|Population|% (gmt, year, ssp, region) float64 dask.array\n", + " sdii|Hazard|Absolute|Land area weighted (gmt, year, ssp, region) float64 dask.array\n", + " sdii|Hazard|Absolute|Population weighted (gmt, year, ssp, region) float64 dask.array\n", + " sdii|Hazard|Risk score|Population weighted (gmt, year, ssp, region) float64 dask.array" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "filesall = glob.glob(fname_input_climate)\n", + "files = filesall[12:14]\n", + "f = files[0]\n", + "# load input climate impacts data file\n", + "ds = xr.open_mfdataset(f)\n", + "ds = ds.sel(year=years)\n", + "ds" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "576d3d29", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# of variables = 6\n" + ] + } + ], + "source": [ + "varis = list(ds.data_vars.keys())[:lvaris]\n", + "dsi = ds[varis]\n", + "print(f\"# of variables = {len(varis)}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "90a923db", + "metadata": {}, + "source": [ + "### Import scenarios data\n", + "\n", + "Load an IAMC scenarios dataset. \n", + "Decide whether using global mean temperature or CO2 mode. \n", + "Assign SSPs if missing and fix duplicate temperatures. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9eb98848", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "pyam - INFO: Running in a notebook, setting up a basic logging at level INFO\n", + "pyam.core - INFO: Reading file C:\\Users\\byers\\IIASA\\ECE.prog - Documents\\Research Theme - NEXUS\\Hotspots_Explorer_2p0\\rcre_testing\\testing_2\\emissions_temp_AR6_small.xlsx\n", + "pyam.core - INFO: Reading meta indicators\n" + ] + } + ], + "source": [ + "df_scens_in = pyam.IamDataFrame(fname_input_scenarios)\n", + "# SCENARIO DATA PRE-PROCESSING\n", + "# Filter for temperature variable" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "0caacb05", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\n", + "Index:\n", + " * model : AIM/CGE 2.0, AIM/CGE 2.1, AIM/CGE 2.2, ... WITCH-GLOBIOM 4.4 (44)\n", + " * scenario : 1.5C, 2.5C, 2C, 2CNow_Gradual, ... WB2C (559)\n", + "Timeseries data coordinates:\n", + " region : World (1)\n", + " variable : AR6 climate diagnostics|Infilled|Emissions|CO2, ... (3)\n", + " unit : K, Mt CO2/yr (2)\n", + " year : 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, ... 2100 (86)\n", + "Meta indicators:\n", + " Category (object) C3, C5, C6, C7, C4, C1, C2, C8 (8)\n", + " Category_name (object) C3: limit warming to 2°C (>67%), ... (8)\n", + " Category_subset (object) C3y_+veGHGs, C5, C6, C7, C4, C1a_NZGHGs, ... C8 (10)\n", + " Subset_Ch4 (object) Limit to 2C (>67%) immediate 2020 action, ... (6)\n", + " Category_Vetting_historical (object) C3, C5, C6, C7, C4, C1, C2, C8 (8)\n", + " ..." + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_scens_in" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "3c03efcb", + "metadata": {}, + "outputs": [], + "source": [ + "mode = 'CO2'\n", + "if mode == \"GMT\":\n", + " dfp = df_scens_in.filter(variable=temp_variable)\n", + "elif mode == \"CO2\":\n", + " dfp = prepare_cumCO2(df_scens_in, years=years, use_dask=True)\n", + " ts = dfp.timeseries().apply(co2togmt_simple)\n", + " ts = pyam.IamDataFrame(ts)\n", + " ts.rename(\n", + " {\n", + " \"variable\": {ts.variable[0]: \"RIME|GSAT_tcre\"},\n", + " \"unit\": {ts.unit[0]: \"°C\"},\n", + " },\n", + " inplace=True,\n", + " )\n", + " # Export data to check error and relationships\n", + " # ts.append(dfp).to_csv('c://users//byers//downloads//tcre_gmt_output.csv')\n", + " dfp = ts\n", + " dfp.meta = df_scens_in.meta.copy()\n", + "dfp = dfp.filter(year=years)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "010f68db", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\n", + "Index:\n", + " * model : AIM/CGE 2.1, AIM/CGE 2.2, AIM/Hub-Global 2.0, ... WITCH-GLOBIOM 4.4 (34)\n", + " * scenario : 1.5C, 2.5C, 2C, 2CNow_Gradual, ... WB2C (511)\n", + "Timeseries data coordinates:\n", + " region : World (1)\n", + " variable : RIME|GSAT_tcre (1)\n", + " unit : °C (1)\n", + " year : 2015, 2025, 2035, 2045, 2055, 2065, 2075, 2085, 2095 (9)\n", + " ssp_family : SSP1, SSP2, SSP3, SSP4, SSP5 (5)\n", + "Meta indicators:\n", + " exclude (bool) False (1)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dfp" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "836d6445", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelscenarioregionvariableunitssp_family201520252035204520552065207520852095
0AIM/CGE 2.1CD-LINKS_INDC2030i_1600WorldRIME|GSAT_tcre°CSSP21.3211.5161.6831.7781.8421.8811.9150001.9551.996
1AIM/CGE 2.1CD-LINKS_INDCiWorldRIME|GSAT_tcre°CSSP21.3211.5171.7061.9022.1042.3172.5420002.7803.033
2AIM/CGE 2.1CD-LINKS_NDC2030i_1000WorldRIME|GSAT_tcre°CSSP21.3211.5161.6831.7641.7841.7731.7550001.7411.728
3AIM/CGE 2.1CD-LINKS_NPi2020_1000WorldRIME|GSAT_tcre°CSSP21.3211.5051.6191.6781.7121.7181.7141481.7221.727
4AIM/CGE 2.1CD-LINKS_NPi2020_1600WorldRIME|GSAT_tcre°CSSP21.3211.5091.6361.7341.8131.8681.9190001.9762.034
................................................
988WITCH-GLOBIOM 4.4CD-LINKS_NPiWorldRIME|GSAT_tcre°CSSP21.3221.5351.7962.0962.4262.7843.1690003.5793.998
989WITCH-GLOBIOM 4.4CD-LINKS_NPi2020_1000WorldRIME|GSAT_tcre°CSSP21.3221.5071.6161.6931.7441.7691.7770001.7661.737
990WITCH-GLOBIOM 4.4CD-LINKS_NPi2020_1600WorldRIME|GSAT_tcre°CSSP21.3221.5161.6621.7731.8641.9351.9890002.0182.025
991WITCH-GLOBIOM 4.4CD-LINKS_NPi2020_400WorldRIME|GSAT_tcre°CSSP21.3221.5031.5841.6211.6291.6141.5810001.5241.451
992WITCH-GLOBIOM 4.4CD-LINKS_NoPolicyWorldRIME|GSAT_tcre°CSSP21.3221.5421.8112.1192.4572.8223.2140003.6304.056
\n", + "

993 rows × 15 columns

\n", + "
" + ], + "text/plain": [ + " model scenario region variable unit \\\n", + "0 AIM/CGE 2.1 CD-LINKS_INDC2030i_1600 World RIME|GSAT_tcre °C \n", + "1 AIM/CGE 2.1 CD-LINKS_INDCi World RIME|GSAT_tcre °C \n", + "2 AIM/CGE 2.1 CD-LINKS_NDC2030i_1000 World RIME|GSAT_tcre °C \n", + "3 AIM/CGE 2.1 CD-LINKS_NPi2020_1000 World RIME|GSAT_tcre °C \n", + "4 AIM/CGE 2.1 CD-LINKS_NPi2020_1600 World RIME|GSAT_tcre °C \n", + ".. ... ... ... ... ... \n", + "988 WITCH-GLOBIOM 4.4 CD-LINKS_NPi World RIME|GSAT_tcre °C \n", + "989 WITCH-GLOBIOM 4.4 CD-LINKS_NPi2020_1000 World RIME|GSAT_tcre °C \n", + "990 WITCH-GLOBIOM 4.4 CD-LINKS_NPi2020_1600 World RIME|GSAT_tcre °C \n", + "991 WITCH-GLOBIOM 4.4 CD-LINKS_NPi2020_400 World RIME|GSAT_tcre °C \n", + "992 WITCH-GLOBIOM 4.4 CD-LINKS_NoPolicy World RIME|GSAT_tcre °C \n", + "\n", + " ssp_family 2015 2025 2035 2045 2055 2065 2075 2085 \\\n", + "0 SSP2 1.321 1.516 1.683 1.778 1.842 1.881 1.915000 1.955 \n", + "1 SSP2 1.321 1.517 1.706 1.902 2.104 2.317 2.542000 2.780 \n", + "2 SSP2 1.321 1.516 1.683 1.764 1.784 1.773 1.755000 1.741 \n", + "3 SSP2 1.321 1.505 1.619 1.678 1.712 1.718 1.714148 1.722 \n", + "4 SSP2 1.321 1.509 1.636 1.734 1.813 1.868 1.919000 1.976 \n", + ".. ... ... ... ... ... ... ... ... ... \n", + "988 SSP2 1.322 1.535 1.796 2.096 2.426 2.784 3.169000 3.579 \n", + "989 SSP2 1.322 1.507 1.616 1.693 1.744 1.769 1.777000 1.766 \n", + "990 SSP2 1.322 1.516 1.662 1.773 1.864 1.935 1.989000 2.018 \n", + "991 SSP2 1.322 1.503 1.584 1.621 1.629 1.614 1.581000 1.524 \n", + "992 SSP2 1.322 1.542 1.811 2.119 2.457 2.822 3.214000 3.630 \n", + "\n", + " 2095 \n", + "0 1.996 \n", + "1 3.033 \n", + "2 1.728 \n", + "3 1.727 \n", + "4 2.034 \n", + ".. ... \n", + "988 3.998 \n", + "989 1.737 \n", + "990 2.025 \n", + "991 1.451 \n", + "992 4.056 \n", + "\n", + "[993 rows x 15 columns]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "few_scenarios = False\n", + "very_few_scenarios = False\n", + "\n", + "if few_scenarios:\n", + " dfp = dfp.filter(Category=[\"C1*\"])\n", + " if very_few_scenarios:\n", + " dfp = dfp.filter(model=\"REMIND 2.1*\", scenario=\"*\") \n", + " \n", + " \n", + "# Assign SSP to meta and select SSP2 in case SSP not present in name\n", + "dfp = ssp_helper(dfp, ssp_meta_col=\"Ssp_family\", default_ssp=\"SSP2\")\n", + "\n", + "# Fix duplicate temperatures\n", + "dft = dfp.timeseries().reset_index()\n", + "dft = dft.apply(fix_duplicate_temps, years=years, axis=1)\n", + "dft.reset_index(inplace=True, drop=True)\n", + "dft" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78e605c3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "4d915d62", + "metadata": {}, + "source": [ + "### START PROCESSING" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4348fd06", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ ] | 0% Completed | 124.97 ms" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\byers\\Anaconda3\\envs\\py310\\lib\\site-packages\\dask\\dataframe\\utils.py:291: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n", + " return pd.Series([], dtype=dtype, name=name, index=index)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[########################################] | 100% Completed | 105.32 s\n", + " Done: 105.48372197151184\n", + " Saved: COUNTRIES yrs=10\n", + " 105.48372197151184\n", + "6 variables, 1202 scenarios\n" + ] + } + ], + "source": [ + "start = time.time()\n", + "year_res = 10\n", + "parallel = True\n", + "if parallel:\n", + " \"\"\"\n", + " For parallel processing, convert dft as a wide IAMC pd.Dataframe\n", + " into a dask.DataFrame.\n", + " \"\"\"\n", + " ddf = dd.from_pandas(dft, npartitions=1000)\n", + "\n", + " # dfx = dft.iloc[0].squeeze() # FOR DEBUIGGING THE FUNCTION\n", + " outd = ddf.apply(\n", + " calculate_impacts_gmt, dsi=dsi, axis=1, meta=(\"result\", None)\n", + " )\n", + "\n", + " with ProgressBar():\n", + " # try:\n", + " df_new = outd.compute(num_workers=num_workers)\n", + "else:\n", + " df_new = dft.apply(calculate_impacts_gmt, dsi=dsi, axis=1)\n", + "\n", + "expandeddGMT = pd.concat([df_new[x] for x in df_new.index])\n", + "print(f\" Done: {time.time()-start}\")\n", + "\n", + "filename = f\"RIME_output_{region}_{year_res}yr.csv\"\n", + "\n", + "# expandedd.to_csv(filename, encoding=\"utf-8\", index=False)\n", + "print(f\" Saved: {region} yrs={year_res}\\n {time.time()-start}\")\n", + "print(f\"{len(dsi.data_vars)} variables, {len(dfp.meta)} scenarios\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "2421034d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelscenarioregionvariableunit201520252035204520552065207520852095
0AIM/CGE 2.0SSP1-26AFGRIME|sdii|Exposure|Land areakm20.0000000.000000445.506108954.6559461145.5871351145.5871351018.299676954.655946827.368486
1AIM/CGE 2.0SSP1-26AGORIME|sdii|Exposure|Land areakm20.0000000.000000732.1867341568.9715721882.7658861882.7658861673.5696771568.9715721359.775362
2AIM/CGE 2.0SSP1-26ALBRIME|sdii|Exposure|Land areakm20.0000000.0000000.0000000.0000000.0000000.0000000.0000000.0000000.000000
3AIM/CGE 2.0SSP1-26ANDRIME|sdii|Exposure|Land areakm20.0000000.0000000.0000000.0000000.0000000.0000000.0000000.0000000.000000
4AIM/CGE 2.0SSP1-26ARERIME|sdii|Exposure|Land areakm218898.0824458425.0010251839.3983952292.4859962462.3938462462.3938462349.1219462292.4859962179.214096
.............................................
1351WITCH-GLOBIOM 4.4CD-LINKS_NoPolicyYEMRIME|sdii|Hazard|Risk score|Population weightedrisk score0.1278710.1907850.2327190.1924190.2880640.3387430.3262230.2107270.191489
1352WITCH-GLOBIOM 4.4CD-LINKS_NoPolicyZAFRIME|sdii|Hazard|Risk score|Population weightedrisk score0.2818090.2583690.2297940.2116870.2787530.2785460.2281410.2002230.195988
1353WITCH-GLOBIOM 4.4CD-LINKS_NoPolicyZMBRIME|sdii|Hazard|Risk score|Population weightedrisk score0.0127700.0120990.0114370.0115760.0265420.0320430.0635570.2116310.236261
1354WITCH-GLOBIOM 4.4CD-LINKS_NoPolicyZWERIME|sdii|Hazard|Risk score|Population weightedrisk score0.0125580.0087380.0062240.0085280.0428630.0650080.0781110.0858550.087229
1355WITCH-GLOBIOM 4.4CD-LINKS_NoPolicyworldRIME|sdii|Hazard|Risk score|Population weightedrisk score0.1980080.2206270.2603670.3095290.4167210.5049520.5798030.6207980.631845
\n", + "

1629912 rows × 14 columns

\n", + "
" + ], + "text/plain": [ + " model scenario region \\\n", + "0 AIM/CGE 2.0 SSP1-26 AFG \n", + "1 AIM/CGE 2.0 SSP1-26 AGO \n", + "2 AIM/CGE 2.0 SSP1-26 ALB \n", + "3 AIM/CGE 2.0 SSP1-26 AND \n", + "4 AIM/CGE 2.0 SSP1-26 ARE \n", + "... ... ... ... \n", + "1351 WITCH-GLOBIOM 4.4 CD-LINKS_NoPolicy YEM \n", + "1352 WITCH-GLOBIOM 4.4 CD-LINKS_NoPolicy ZAF \n", + "1353 WITCH-GLOBIOM 4.4 CD-LINKS_NoPolicy ZMB \n", + "1354 WITCH-GLOBIOM 4.4 CD-LINKS_NoPolicy ZWE \n", + "1355 WITCH-GLOBIOM 4.4 CD-LINKS_NoPolicy world \n", + "\n", + " variable unit \\\n", + "0 RIME|sdii|Exposure|Land area km2 \n", + "1 RIME|sdii|Exposure|Land area km2 \n", + "2 RIME|sdii|Exposure|Land area km2 \n", + "3 RIME|sdii|Exposure|Land area km2 \n", + "4 RIME|sdii|Exposure|Land area km2 \n", + "... ... ... \n", + "1351 RIME|sdii|Hazard|Risk score|Population weighted risk score \n", + "1352 RIME|sdii|Hazard|Risk score|Population weighted risk score \n", + "1353 RIME|sdii|Hazard|Risk score|Population weighted risk score \n", + "1354 RIME|sdii|Hazard|Risk score|Population weighted risk score \n", + "1355 RIME|sdii|Hazard|Risk score|Population weighted risk score \n", + "\n", + " 2015 2025 2035 2045 2055 \\\n", + "0 0.000000 0.000000 445.506108 954.655946 1145.587135 \n", + "1 0.000000 0.000000 732.186734 1568.971572 1882.765886 \n", + "2 0.000000 0.000000 0.000000 0.000000 0.000000 \n", + "3 0.000000 0.000000 0.000000 0.000000 0.000000 \n", + "4 18898.082445 8425.001025 1839.398395 2292.485996 2462.393846 \n", + "... ... ... ... ... ... \n", + "1351 0.127871 0.190785 0.232719 0.192419 0.288064 \n", + "1352 0.281809 0.258369 0.229794 0.211687 0.278753 \n", + "1353 0.012770 0.012099 0.011437 0.011576 0.026542 \n", + "1354 0.012558 0.008738 0.006224 0.008528 0.042863 \n", + "1355 0.198008 0.220627 0.260367 0.309529 0.416721 \n", + "\n", + " 2065 2075 2085 2095 \n", + "0 1145.587135 1018.299676 954.655946 827.368486 \n", + "1 1882.765886 1673.569677 1568.971572 1359.775362 \n", + "2 0.000000 0.000000 0.000000 0.000000 \n", + "3 0.000000 0.000000 0.000000 0.000000 \n", + "4 2462.393846 2349.121946 2292.485996 2179.214096 \n", + "... ... ... ... ... \n", + "1351 0.338743 0.326223 0.210727 0.191489 \n", + "1352 0.278546 0.228141 0.200223 0.195988 \n", + "1353 0.032043 0.063557 0.211631 0.236261 \n", + "1354 0.065008 0.078111 0.085855 0.087229 \n", + "1355 0.504952 0.579803 0.620798 0.631845 \n", + "\n", + "[1629912 rows x 14 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "expandeddGMT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a77ed04", + "metadata": {}, + "outputs": [], + "source": [ + "# expandeddGMT" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5bbb046", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57d63356", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/rime/process_config.py b/rime/process_config.py index c7c96ed..aaa2825 100644 --- a/rime/process_config.py +++ b/rime/process_config.py @@ -13,22 +13,27 @@ # ============================================================================= import os +# Run and environment settings +user = "byers" +env = "pc" +# env = 'server' +# env = 'ebro3' + +# git_path = f"C:\\users\\{user}\\Github\\" +git_path = f"C:\\Github\\" + + # From generate_aggregated_inputs.py -region = "COUNTRIES" -# region = 'R10' +region = "COUNTRIES" # 'R10' or 'COUNTRIES' table_output_format = f"table_output_|_{region}.csv" yr_start = 2010 yr_end = 2100 -# Run and environment settings -user = "byers" -env = "pc" -# env = 'server' -# env = 'ebro3' + num_workers = 24 parallel = True # Uses Dask in processing the IAMC scenarios @@ -41,7 +46,10 @@ # %% Working directories # ============================================================================= -yaml_path = "C:\\users\\byers\\Github\\climate_impacts_processing\\hotspots.yml" + +yaml_path = git_path+"climate_impacts_processing\\hotspots.yml" +landmask_path = git_path+"climate_impacts_processing\\landareamaskmap0.nc" +kg_class_path = git_path+"climate_impacts_processing\\kg_class.nc" if env == "pc": # Main working directory @@ -64,7 +72,7 @@ elif env == "ebro3": - # wd = 'H:\\' + wd = "H:\\" wd2 = "H:\\rcre_testing\\" # Input source of processed climate data by ssp/year/variable/region @@ -86,7 +94,6 @@ year_resols = [5] input_scenarios_name = "AR6full" - temp_variable = ( "AR6 climate diagnostics|Surface Temperature (GSAT)|MAGICCv7.5.3|50.0th Percentile" ) @@ -114,7 +121,7 @@ # impact data settings indicators = ["cdd", "precip"] -ftype = "score" +ftype = "score" #score interpolation = 0.01 diff --git a/rime/process_maps.py b/rime/process_maps.py index ea7c955..30ca983 100644 --- a/rime/process_maps.py +++ b/rime/process_maps.py @@ -4,6 +4,7 @@ Created on Thu Apr 6 11:36:24 2023 @author: werning, byers +Execute this script ideally in the top level of the RIME directory To Do: @@ -11,28 +12,28 @@ """ - +import dask import glob import numpy as np import os -import xarray as xr -import yaml import pyam import time -import dask -import dask.array as da +import xarray as xr +import yaml from dask import delayed from dask.distributed import Client -from process_config import * -from rime_functions import * +from rime.process_config import * +from rime.rime_functions import * # from dask.diagnostics import Profiler, ResourceProfiles, CacheProfiler dask.config.set(scheduler="processes") -dask.config.set(num_workers=72) - +dask.config.set(num_workers=num_workers) client = Client() +# dask.config.set(scheduler='synchronous') + + with open(yaml_path, "r") as f: params = yaml.full_load(f) @@ -44,7 +45,7 @@ dft = np.round(dft.as_pandas()[pyam.IAMC_IDX + ["year", "value", "Ssp_family"]], 2) # Replace & fill missing SSP scenario allocation # dft.Ssp_family.replace(sspdic, inplace=True) # metadata must have Ssp_faily column. If not SSP2 automatically chosen -# dft.loc[dft.Ssp_family.isnull(), ssp_meta_col] = 'ssp2' +dft.loc[dft.Ssp_family.isnull(), ssp_meta_col] = "ssp2" dft = pyam.IamDataFrame(dft) @@ -64,13 +65,18 @@ # Test multiple scenarios, 1 indicator files = glob.glob(os.path.join(impact_data_dir, ind, f"*{short}_{ssp}*{ftype}.nc4")) mapdata = xr.open_mfdataset( - files, preprocess=preprocess, combine="nested", concat_dim="gmt" + files, preprocess=remove_ssp_from_ds, combine="nested", concat_dim="gmt" ) # df = pyam.IamDataFrame(dft) map_out_MS = map_transform_gmt_multi_dask( - dft.filter(model="POLES ADVANCE"), mapdata, use_dask=True, gmt_name="threshold" + dft.filter(model="POLES ADVANCE"), + mapdata, + years, + use_dask=True, + gmt_name="threshold", + interpolation=interpolation, ) comp = dict(zlib=True, complevel=5) @@ -117,12 +123,13 @@ os.path.join(impact_data_dir, ind, f"*{short}_{ssp}*{ftype}.nc4") ) mapdata[short] = xr.open_mfdataset( - files, preprocess=preprocess, combine="nested", concat_dim="gmt" + files, preprocess=remove_ssp_from_ds, combine="nested", concat_dim="gmt" )[short] map_out_MI = map_transform_gmt_multi_dask( dft.filter(model="AIM*", scenario="SSP1-34"), mapdata, + years, use_dask=False, gmt_name="threshold", ) @@ -137,13 +144,5 @@ # 1 scenarios, 5 indis = 14 seconds # 1 scenario, 8 indis = 22s, = 1.8s/s dask=False +# 1 scenario, 12 indis = 62s, = 5s/s dask=False, 24 workers. # 1... 32s dask=True - -# %% Define functions - - -# def test_map_transform_gmt(): -# return x - - -# def test_map_transform_gmt_multi(): diff --git a/rime/process_tabledata.py b/rime/process_tabledata.py index de22b12..78ab15c 100644 --- a/rime/process_tabledata.py +++ b/rime/process_tabledata.py @@ -6,6 +6,7 @@ """ if __name__ == "__main__": from process_config import * + from rime_functions import * # from alive_progress import alive_bar import dask.dataframe as dd @@ -20,7 +21,6 @@ import time import xarray as xr import dask - from rime_functions import * # from pandas import InvalidIndexError # dask.config.set(scheduler='threads') # overwrite default with multiprocessing scheduler @@ -32,16 +32,24 @@ # files = filesall[2:6] # files = filesall[7:9] # problem in 6? # files = filesall[9:12] - # files = filesall[12:15] - files = filesall[15:] + files = filesall[12:15] + # files = filesall[15:] if len(files) == 0: - print("no files!") - dsfsdfsdfsdf + raise Exception("No files!") # load input IAMC scenarios file df_scens_in = pyam.IamDataFrame(fname_input_scenarios) + mode = "CO2" + if mode == "CO2": + print( + "CO2 mode: Global mean temperatures will be derived from response \ + to cumulative CO2 emissions." + ) + elif mode == "GMT": + print("GMT mode: Global mean temperatures provided as input.") + if parallel: dask.config.set( scheduler="processes" @@ -59,76 +67,81 @@ # % # Load aggregated file, and test with temperature pathway # ============================================================================= - # %% + # %% Normal testing with temperature variable for year_res in year_resols: for f in files: start = time.time() - # Get variable name + # Get variable name for the filename output v1 = f.split(f"_{region}")[0] v2 = v1.split("\\")[-1] years = range(2015, 2101, year_res) + ############################# + # CLIMATE DATA PRE-PROCESSING # load input climate impacts data file ds = xr.open_mfdataset(f) ds = ds.sel(year=years) - # Filter for temperature variable - dft = df_scens_in.filter(variable=temp_variable) - - if few_scenarios: - dft = dft.filter(scenario="*SSP*") - dft = dft.filter(Category=["C1", "C2", "C3", "C8"]) - dft = dft.filter(Category=["C1*"]) - # dft = dft.filter(scenario='R2p1_SSP2-PkBudg900', keep=False) - if very_few_scenarios: - dft = dft.filter(model="WIT*", scenario="*") - - # assign SSPs - dft = dft.filter(year=years) - dft = np.round( - dft.as_pandas()[pyam.IAMC_IDX + ["year", "value", "Ssp_family"]], 3 - ) - sspdic = {1.0: "SSP1", 2.0: "SSP2", 3.0: "SSP3", 4.0: "SSP4", 5.0: "SSP5"} - dft.Ssp_family.replace( - sspdic, inplace=True - ) # metadata must have Ssp_faily column. If not SSP2 automatically chosen - dft.loc[dft.Ssp_family.isnull(), ssp_meta_col] = "SSP2" - dfX = pyam.IamDataFrame(dft) - # % Full thing - varis = list(ds.data_vars.keys())[:lvaris] - # Subset to fewer indicators # varis = list(ds.data_vars.keys()) # varis = [str for str in list(ds.data_vars.keys()) if any(sub in str for sub in ['High','Low'])==False] # lvaris = len(varis) # dsi = ds[list(ds.data_vars.keys())[:x]] - dsi = ds[varis] - print(f"real len variables = {len(varis)}") + print(f"# of variables = {len(varis)}") - dft = dfX.timeseries().reset_index() + ############################## + # SCENARIO DATA PRE-PROCESSING + # Filter for temperature variable + + if mode == "GMT": + dfp = df_scens_in.filter(variable=temp_variable) + elif mode == "CO2": + dfp = prepare_cumCO2(df_scens_in, years=years, use_dask=True) + ts = dfp.timeseries().apply(co2togmt_simple) + ts = pyam.IamDataFrame(ts) + ts.rename( + { + "variable": {ts.variable[0]: "RIME|GSAT_tcre"}, + "unit": {ts.unit[0]: "°C"}, + }, + inplace=True, + ) + # Export data to check error and relationships + # ts.append(dfp).to_csv('c://users//byers//downloads//tcre_gmt_output.csv') + dfp = ts + dfp.meta = df_scens_in.meta.copy() + dfp = dfp.filter(year=years) + + if few_scenarios: + dfp = dfp.filter(scenario="*SSP*") + dfp = dfp.filter(Category=["C1", "C2", "C3", "C8"]) + dfp = dfp.filter(Category=["C1*"]) + # dfp = dfp.filter(scenario='R2p1_SSP2-PkBudg900', keep=False) + if very_few_scenarios: + dfp = dfp.filter(model="REMIND 2.1*", scenario="*") # (4) + + # assign SSPs + dfp = ssp_helper(dfp, ssp_meta_col="Ssp_family", default_ssp="SSP2") # Fix duplicate temperatures - dft = dft.apply(fix_dupes, axis=1) + dft = dfp.timeseries().reset_index() + dft = dft.apply(fix_duplicate_temps, years=years, axis=1) dft.reset_index(inplace=True, drop=True) - # Convert to dask array - # meta_df = pd.DataFrame(columns=dft.columns, dtype=float)#, name='meta') - # dic = {k:str for k in ['model', - # 'scenario', - # 'region', - # 'variable', - # 'unit', - # 'ssp_family',]} - - # meta_df = meta_df.astype(dic) + ########################### + # START PROCESSING if parallel: + """ + For parallel processing, convert dft as a wide IAMC pd.Dataframe + into a dask.DataFrame. + """ ddf = dd.from_pandas(dft, npartitions=1000) # dfx = dft.iloc[0].squeeze() # FOR DEBUIGGING THE FUNCTION @@ -139,32 +152,18 @@ with ProgressBar(): # try: - df_new = outd.compute( - num_workers=num_workers - ) # scheduler='processes') + df_new = outd.compute(num_workers=num_workers) print(f" Applied: {time.time()-start}") # except(InvalidIndexError): # print(f'PROBLEM {f}') else: - df_new = dft.apply( - calculate_impacts_gmt, dsi=dsi, axis=1 - ) # .compute(scheduler='processes') + df_new = dft.apply(calculate_impacts_gmt, dsi=dsi, axis=1) expandedd = pd.concat([df_new[x] for x in df_new.index]) print(f" Done: {time.time()-start}") - filename = f"{wd}{wd2}{output_folder_tables}{input_scenarios_name}_rcre_output_{region}_{v2}_{year_res}yr{test}.csv" + filename = f"{wd}{wd2}{output_folder_tables}{input_scenarios_name}_RIME_output_{region}_{v2}_{year_res}yr{test}.csv" - expandedd.to_csv(filename, encoding="utf-8", index=False) + # expandedd.to_csv(filename, encoding="utf-8", index=False) print(f" Saved: {region} yrs={year_res}\n {time.time()-start}") - print(f"{len(dsi.data_vars)} variables") - - # keep_cols = [x for x in expandedd.columns if type(x)==str] - # keep_cols = keep_cols + list(range(2015,2101,5)) - # out_small = expandedd[keep_cols] - # filename_small = f'{wd}{wd2}{output_folder}{input_scenarios_name}_rcre_output_10yr_{region}.csv' - # out_small.to_csv(filename_small, encoding='utf-8') - # print(f' Saved: {time.time()-start}') - - -# %% + print(f"{len(dsi.data_vars)} variables, {len(dfp.meta)} scenarios") diff --git a/rime/rime_functions.py b/rime/rime_functions.py index b3b7bbb..3d13a73 100644 --- a/rime/rime_functions.py +++ b/rime/rime_functions.py @@ -1,22 +1,38 @@ # -*- coding: utf-8 -*- """ rime_functions.py +Set of core functions for RIME """ +import dask +import dask.dataframe as dd +from dask import delayed +import holoviews as hv +import hvplot.pandas +import hvplot.xarray +import numpy as np +import pandas as pd +import pyam +from scipy.stats import linregress +import xarray as xr -def loop_interpolate_gmt(df): + +def loop_interpolate_gmt(df, yr_start, yr_end): """ - Loop through the variables and regions in df and interpolate the impacts data + Loop through the variables and regions in df and interpolate the impacts data between the global warming levels Parameters ---------- df : pandas.DataFrame() - Df is in IAMC wide format, with columns model/scenario/region/unit/variable/[years] + df is in IAMC wide format, with columns model/scenario/region/unit/variable/[years] + yr_start : int # 2010 # 2015 + start year of the interpolated data + yr_end : int # 2100 # 2099 + end year of the interpolated data Returns ------- - df_ind : TYPE - DESCRIPTION. + df_ind : pandas.DataFrame() """ # if type(df_ind)==None: @@ -26,9 +42,7 @@ def loop_interpolate_gmt(df): for variable in df.variable.unique(): print(variable) - # vstart = time.time() for ssp in SSPs: - # print(ssp) for region in regions: dfs = df.loc[ (df.region == region) & (df.SSP == ssp) & (df.variable == variable) @@ -46,7 +60,6 @@ def loop_interpolate_gmt(df): dfs[cols] = dfs[cols].interpolate() dfs.drop_duplicates(inplace=True) df_ind = pd.concat([df_ind, dfs]) - # print(f'{time.time()-vstart}') return df_ind @@ -57,26 +70,27 @@ def loop_interpolate_gmt(df): def calculate_impacts_gmt( - dfx, dsi, prefix_indicator="RIME|", ssp_meta_col="ssp_family", gmt_below=1.5 + dfx, + dsi, + prefix_indicator="RIME|", + ssp_meta_col="ssp_family", ): """ - Takes in a set of scenarios of GMT and dataset of climate impacts by GWL, + Takes in a set of scenarios of GMT and a dataset of climate impacts by GWL, and returns a table of climate impacts per scenario through time. - + Parameters ---------- dfx : dask.DataFrame dask.DataFrame in the format of an IAMC table (see pyam.IamDataFrame) with scenario(s) as rows and only 1 variable for the global mean temperature timeseries. Needs to have a column specifying the SSP to be assumed. dsi : xarray.Dataset dimensions ['gmt','year','ssp','region'] and each variable is a climate indicator - filename : str, optional - DESCRIPTION. The default is NONE. output filename as .csv (utf-8), otherwise returns idf prefix_indicator : str, optional - DESCRIPTION. The default is 'RIME|'. Use this to change the output indicator prefix. + DESCRIPTION. The default is 'RIME|'. Use this to change the output indicator prefix in the table data variable column. ssp_meta_col : str, optional DESCRIPTION. The default is 'Ssp_family'. Use this to change the name of the meta column used to assin the SSP per scenario - gmt_below : float, optional - DESCRIPTION. The default is 1.5. Assign all gmt values below gmt_below to the value of gmt_below (in case not enough data) + # gmt_below : float, optional + # DESCRIPTION. The default is 1.5. Assign all gmt values below gmt_below to the value of gmt_below (in case not enough data) Returns ------- @@ -136,22 +150,22 @@ def calculate_impacts_gmt( return idf -def fix_dupes(dfti): +def fix_duplicate_temps(df, years): """ Function that modifies GMT temperatures minutely, in case there are duplicates in the series. Parameters ---------- - dfti : pandas.DataFrame, + df : pandas.DataFrame, a file from which to take transform and latitude objects Returns ------- """ - vs = dfti[years] - ld = len(dfti[years]) - lu = len(set(dfti[years])) + vs = df[years] + ld = len(df[years]) + lu = len(set(df[years])) if ld != lu: # print('') @@ -162,8 +176,8 @@ def fix_dupes(dfti): vsn.append(x) else: vsn.append(x + np.random.uniform(-0.005, 0.005)) - dfti[years] = vsn - return dfti + df[years] = vsn + return df # ============================================================================= @@ -175,16 +189,42 @@ def map_transform_gmt( df1, mapdata, var_name, + years, map_array=xr.Dataset(), + caution_checks=True, + drawdown_max=0.15, gmt_name="threshold", + interpolation=0.01, temp_min=1.2, temp_max=3.5, - drawdown_max=0.15, ): + """ + Takes in a set of scenarios of GMT and dataset of climate impacts by GWL, + and returns a table of climate impacts per scenario through time. + + Parameters + ---------- + df1 : dask.DataFrame + dask.DataFrame in the format of an IAMC table (see pyam.IamDataFrame) with scenario(s) as rows and only 1 variable for the global mean temperature timeseries. Needs to have a column specifying the SSP to be assumed. + mapdata : xarray.Dataset + dimensions ['gmt','year','ssp','region'] and each variable is a climate indicator + ..... + .... + ... + + Returns + ------- + idf : pyam.IamDataFrame + pyam.IamDataFrame with variables for the impact indicators for each scenario(s). + + """ + if len(df1.variable) > 1: - print("Error: more than 1 variable in DataFrame") - breakss + raise Exception("Error: more than 1 variable in DataFrame") + if len(df1.meta) > 1: + raise Exception("Error: more than 1 model-scenario in DataFrame") + # Should only be 1 scenario for model, scenario in df1.index: # Get the GMT values from df gmt_values = df1.filter(year=years).data["value"].values @@ -216,6 +256,8 @@ def map_transform_gmt( except ValueError: print("") + # Get indicator name + short = list(mapdata.data_vars)[0] # provide new interpolated vector (e.g. 0.01 gmt resolution) new_thresholds = np.arange( mapdata.gmt.min(), mapdata.gmt.max() + interpolation, interpolation @@ -267,12 +309,40 @@ def map_transform_gmt( def map_transform_gmt_multi_dask( df, mapdata, + years, gmt_name="threshold", use_dask=True, temp_min=1.2, temp_max=3.5, drawdown_max=0.15, + interpolation=0.01, ): + """ + Takes in a set of scenarios of GMT and dataset of climate impacts by GWL, + and returns a table of climate impacts per scenario through time. + + Parameters + ---------- + df : dask.DataFrame + dask.DataFrame in the format of an IAMC table (see pyam.IamDataFrame) with scenario(s) as rows and only 1 variable for the global mean temperature timeseries. Needs to have a column specifying the SSP to be assumed. + mapdata : xarray.Dataset + dimensions ['gmt','year','ssp','region'] and each variable is a climate indicator + filename : str, optional + DESCRIPTION. The default is NONE. output filename as .csv (utf-8), otherwise returns idf + # prefix_indicator : str, optional + # DESCRIPTION. The default is 'RIME|'. Use this to change the output indicator prefix. + # ssp_meta_col : str, optional + # DESCRIPTION. The default is 'Ssp_family'. Use this to change the name of the meta column used to assin the SSP per scenario + # gmt_below : float, optional + # DESCRIPTION. The default is 1.5. Assign all gmt values below gmt_below to the value of gmt_below (in case not enough data) + + Returns + ------- + idf : pyam.IamDataFrame + pyam.IamDataFrame with variables for the impact indicators for each scenario(s). + + """ + map_array = xr.Dataset() if len(df.index) > 1: @@ -296,15 +366,12 @@ def map_transform_gmt_multi_dask( if use_dask: # Create delayed task for map_transform_gmt delayed_map_transform = delayed(map_transform_gmt)( - df1, mapdata, var_name, map_array + df1, mapdata, var_name, years, map_array ) delayed_tasks.append(delayed_map_transform) else: map_array = map_transform_gmt( - df1, - mapdata, - var_name, - map_array, + df1, mapdata, var_name, years, map_array ) if use_dask: # Compute delayed tasks concurrently @@ -334,16 +401,11 @@ def map_transform_gmt_multi_dask( if use_dask: # Create delayed task for map_transform_gmt_dask delayed_map_transform = delayed(map_transform_gmt)( - df1, mapdata, var_name, map_array + df1, mapdata, var_name, years, map_array ) delayed_tasks.append(delayed_map_transform) else: - map_array = map_transform_gmt( - df1, - mapdata, - var_name, - map_array, - ) + map_array = map_transform_gmt(df1, mapdata, var_name, years, map_array) if use_dask: # Compute delayed tasks concurrently computed_results = dask.compute(*delayed_tasks) @@ -359,53 +421,167 @@ def map_transform_gmt_multi_dask( return map_array +def calculate_cumulative_CO2(ts, first_year, year, variable): + """ + + + Parameters + ---------- + ts : TYPE + DESCRIPTION. + first_year : TYPE + DESCRIPTION. + year : TYPE + DESCRIPTION. + variable : TYPE + DESCRIPTION. + + Returns + ------- + dfo : TYPE + DESCRIPTION. + + """ + dfo = ( + ts.apply( + pyam.cumulative, raw=False, axis=1, first_year=first_year, last_year=year + ) + .to_frame() + .reset_index() + ) + dfo["year"] = year + dfo["variable"] = f"{variable}|Cumulative|from {first_year}" + dfo.rename(columns={0: "value"}, inplace=True) + return dfo + + +def prepare_cumCO2( + df, + variable="Emissions|CO2", + unit_in="Mt CO2/yr", + unit_out="Gt CO2/yr", + years=None, + first_year=2020, + last_year=2100, + use_dask=False, +): + """ + Prepares a dataframe, by taking in CO2 emissions, filtering, converting + units to Gt CO2, and then calculating cumulative CO2. + + Parameters + ---------- + df : pyam.IamDataFrame + DESCRIPTION. + variable : str, optional + DESCRIPTION. The default is 'Emissions|CO2'. + unit_in : str, optional + DESCRIPTION. The default is 'Mt CO2/yr'. + years : list or np.array, optional + If list or range is provided, cumulative emissions are calculated + iteratively between the first value to each subsequent interval. If NONE, + first_year and last)year are used. + first_year : int, optional + DESCRIPTION. The default is 2020. + last_year : int, optional + DESCRIPTION. The default is 2100. + use_dask : boolean, optional + If the datasets are very large, with many scenarios or high resolution + of years, dask may perform faster. For smaller sets, keep as False. + + Returns + ------- + df : pyam.IamDataFrame + Returns df with cumulative CO2 (Gt CO2/yr) through time. + + """ + + if years is not None: + years = list(years) + if len(years) < 2: + raise Exception("Error: years must be at least 2 items long") + first_year = years[0] + last_year = years[-1] + else: + years = [first_year, last_year] + + df_cumCO2 = pd.DataFrame() + + # filter and convert + ts = ( + df.filter( + variable=variable, + ) + .convert_unit(unit_in, "Gt CO2/yr") + .timeseries() + ) + + if use_dask: + # Create a list of delayed computations + dask_dfs = [ + delayed(calculate_cumulative_CO2)(ts, first_year, year, variable) + for year in years + ] + + # Compute the delayed computations and concatenate the results + df_cumCO2 = dd.from_delayed(dask_dfs) + df_cumCO2 = df_cumCO2.compute() + else: + df_cumCO2 = pd.concat( + [calculate_cumulative_CO2(ts, first_year, year, variable) for year in years] + ) + + df_cumCO2["unit"] = unit_out + + return pyam.IamDataFrame(df_cumCO2) + + def co2togmt_simple(cum_CO2, regr=None): """ - Takes in vector of cumulative CO2 values and calculates Global mean surface + Takes in vector of CO2 values and calculates Global mean surface air temperature (p50). Parameters ---------- cum_CO2 : int, float, np.array, pandas.Series - Value of cumulative CO2, from 2020 to net-zero CO2 emissions or end of century, in Gt CO2. + Value of cumulative CO2, from 2020 onwards, in Gt CO2. regr : dict, optional 'slope' and 'intercept' values for line. The default is None, in which case parameters from AR6 assessment are used. Provide {'slope': m, - 'intercept': x} to define own linear relationship, otherwise 'AR5'. + 'intercept': x} to define own linear relationship. Returns ------- Global mean surface air temperature (p50). """ - import pandas as pd - import numpy as np + + # if isinstance(cum_CO2, pyam.IamDataFrame): + + cum_CO2 = np.array(cum_CO2) if regr == None: # use default AR6 paramters slope = 0.0005099869587542405 intercept = 1.3024249460191835 - elif type(regr) == dict: + elif isinstance(regr, dict): # User provided parameters slope = regr["slope"] intercept = regr["intercept"] - elif type(regr) == str: + elif isinstance(regr, str): if regr == "AR6": # use default AR6 paramters slope = 0.0005099869587542405 intercept = 1.3024249460191835 - elif regr == "AR5": - # use default AR6 paramters - slope = 0.0043534 - intercept = 1.36555 + # elif regr == "AR5": + # # use default AR6 paramters + # slope = 0.0043534 + # intercept = 1.36555 else: print("Warning: specification not recognized") - raise ("Error: specification not recognized") - elif type(regr) == pd.DataFrame: - from scipy.stats import linregress - - # use the first two columns - x = regr.columns[0] - y = regr.columns[1] + raise Exception("Error: specification not recognized") + elif isinstance(regr, pd.DataFrame): + # Use linear regression based on DataFrame + x, y = regr.columns[0], regr.columns[1] slope, intercept, r, p, se = linregress(regr[x], regr[y]) print(f"Slope: {slope}") print(f"Intercept: {intercept}") @@ -416,7 +592,131 @@ def co2togmt_simple(cum_CO2, regr=None): return gmt -def preprocess(ds): +def plot_maps_dashboard(ds, filename=None, indicators=None, year=2050, cmap='magma_r', shared_axes=True, clim=None): + + + if indicators==None: + indicators = list(ds.data_vars) + elif isinstance(indicators, list): + if all(x in ds.data_vars for x in indicators)==False: + raise Exception(f"Error: not all items in indicators were found in ds.") + elif isinstance(indicators, list)==False: + raise Exception(f"Error: indicators must be of type list.") + + + # Subset the dataset. Check dims and length + + ds = check_ds_dims(ds) + + + if 'year' in ds.dims: + ds = ds.sel(year=year).squeeze() + elif len(ds.dims) != 2: + raise Exception(f"Error: Year not a dimension and more than 2 dimensions in dataset") + + plot_list = [] + + # Run loop through indicators (variables) + for i in indicators: + + new_plot = ds[i].hvplot(x='lon', y='lat', cmap='magma_r', shared_axes=True) + plot_list = plot_list + [new_plot] + + plot = hv.Layout(plot_list).cols(3) + + + + # Plot - check filename + if type(filename) is None: + filename = 'maps_dashboard_{model}_{scenario}.html' + + elif (type(filename) is str): + if (filename[:-5]) != '.html': + raise Exception(f"filename {filename} must end with '.html'") + + else: + raise Exception(f"filename must be string and end with '.html'") + + + + hvplot.save(plot, filename) + + + +def remove_ssp_from_ds(ds): + """ + Preprocess input netCDF datasets to remove ssp from the variable names. + Passed to the `preprocess` argument of xr.open_mfdataset() + + Parameters + ---------- + ds : xarray.Dataset + DESCRIPTION. + + Returns + ------- + xarray.Dataset + DESCRIPTION. + + """ var = list(ds.keys())[0] short = f'{var.rsplit("_ssp")[0]}' + return ds.rename({list(ds.keys())[0]: short}) + + +def ssp_helper(dft, ssp_meta_col="Ssp_family", default_ssp="SSP2"): + """ + Function to fill out and assign SSP to a meta column called Ssp_family. If + there is no meta column with SSP information, automatically filled with + default_ssp. + + Parameters + ---------- + dft : pyam.IamDataFram + input + ssp_meta_col : Str, optional + DESCRIPTION. The default is "Ssp_family". + + Returns + ------- + None. + + """ + dft = np.round(dft.as_pandas()[pyam.IAMC_IDX + ["year", "value", ssp_meta_col]], 3) + sspdic = {1.0: "SSP1", 2.0: "SSP2", 3.0: "SSP3", 4.0: "SSP4", 5.0: "SSP5"} + dft[ssp_meta_col].replace( + sspdic, inplace=True + ) # metadata must have Ssp_family column. If not SSP2 automatically chosen + dft.loc[dft[ssp_meta_col].isnull(), ssp_meta_col] = default_ssp + + return pyam.IamDataFrame(dft) + + +def check_ds_dims(ds): + """ + Function to check the dimensions present in dataset before passing to plot maps + + Parameters + ---------- + ds : xarray.Dataset + If 2 dimensions, must be either x/y or lon/lat (former is renames to lon/lat). Third dimension can be 'year'. Otherwise errors are raised. + + Returns + ------- + ds : xarray.Dataset with renamed dimensions, if necessary + + + """ + if len(ds.dims) >= 3: + if 'year' not in ds.dims: + raise ValueError("The dataset contains 3 or more dimensions, but 'year' dimension is missing.") + + if 'lat' in ds.dims and 'lon' in ds.dims: + return ds + elif 'x' in ds.dims and 'y' in ds.dims: + ds = ds.rename({'x': 'lat', 'y': 'lon'}) + return ds + else: + raise ValueError("The dataset does not contain 'lat' and 'lon' or 'x' and 'y' dimensions.") +