Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor scripts for functions and add TCRE #1

Merged
merged 7 commits into from
Oct 30, 2023
Merged

Refactor scripts for functions and add TCRE #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
217e0ec
testing functions import, black format, and add mapo plot script
byersiiasa Oct 9, 2023
6037fa7
imrpove functions, add tcre option.
byersiiasa Oct 9, 2023
5e12fe8
start plot dashboards
byersiiasa Oct 17, 2023
a5e4ada
do dashboards and update function
byersiiasa Oct 23, 2023
55c567a
start notebook examples
byersiiasa Oct 23, 2023
7b1fe91
start notebooks
byersiiasa Oct 30, 2023
a9f0bdb
fix syntax errors
byersiiasa Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 73 additions & 77 deletions rime/generate_aggregated_inputs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,38 @@

@author: byers
"""

# from alive_progress import alive_bar
import glob

# import numpy as np
import os
import pandas as pd
import pyam
import re
import xarray as xr
import time
from alive_progress import alive_bar

start = time.time()

# try:

from process_config import *
# ab_present = True
# except:
# print("alive_progress not installed")
# ab_present = False

# Load impact, interpolate
# start = time.time()

from rime.rime_functions import *
from rime.process_config import *

# few_vars = True
few_vars = False

# %% Settings config


years = list(range(yr_start, yr_end + 1))


# %% List of indicators and files

# Define a list of table data files to read in
os.chdir(wdtable_input)
files = glob.glob(table_output_format.replace("|", "*"))

Expand All @@ -49,10 +52,8 @@
# files = files[4:5] # heatwav & iavar
# files = files[5:10]
# files = files[10:12]
files = files[12:15]
# files = files[15:]

# batch = 1
# files = files[12:15]
files = files[15:]


indicators = []
Expand Down Expand Up @@ -111,72 +112,67 @@
# df.drop(columns=['model', 'scenario'], inplace=True)

# if few_vars:

# df = df.loc[df.variable.str.contains('High')]

small_vars = list(set([x.split("|")[0] for x in dfin.variable]))

with alive_bar(
total=len(small_vars),
title="Processing",
length=10,
force_tty=True,
bar="circles",
spinner="elements",
) as bar:
for vari in small_vars:
df_ind = loop_interpolate_gmt(df.loc[df.variable.str.startswith(vari)])
# dfbig = pd.concat([dfbig, df_ind])
print(f"dfbig: indicator {ind[0]}: {time.time()-istart}")

# % Convert and save out to xarray
# dfbig.dropna(how='all')

dfp = df_ind.melt(
id_vars=[
"model",
"scenario",
"variable",
"region",
"unit",
"SSP",
"GMT",
],
value_vars=years,
var_name="year",
) # change to df_big if concatenating multiple

dfp.columns = [x.lower() for x in dfp.columns]

dsout = xr.Dataset()

for indicator in dfp.variable.unique():
print(indicator)
dx = (
dfp.loc[dfp.variable == indicator]
.set_index(["gmt", "year", "ssp", "region"])
.to_xarray()
)
# dx.attrs['unit'] = dx.assign_coords({'unit':dx.unit.values[0,0,0,0]})

dsout[indicator] = dx["value"].to_dataset(name=indicator)[indicator]
dsout[indicator].attrs["unit"] = dx.unit.values[0, 0, 0, 0]
# dsout = dsout[indicator].assign_coords({'unit':dx.unit.values[0,0,0,0]})

# bar()

dsout["ssp"] = [x.upper() for x in dsout["ssp"].values]
# dsout = dsout.drop_vars('unit')

# % Write out
print("Writing out... ")
comp = dict(zlib=True, complevel=5)
encoding = {var: comp for var in dsout.data_vars}

# ind = indicator.replace('|', '_')
filename = f"{output_dir}{vari}_{region}.nc"

dsout.to_netcdf(filename, encoding=encoding)
bar()
# if ab_present:
# with alive_bar(total=len(small_vars),
# title='Processing', length=10, force_tty=True,
# bar='circles',
# spinner='elements') as bar:

# print('alive bar present')
# Apply function here
for vari in small_vars:
df_ind = loop_interpolate_gmt(
df.loc[df.variable.str.startswith(vari)], yr_start, yr_end
)
# dfbig = pd.concat([dfbig, df_ind])
print(f"dfbig: indicator {ind[0]}: {time.time()-istart}")

# % Convert and save out to xarray
# dfbig.dropna(how='all')

dfp = df_ind.melt(
id_vars=[
"model",
"scenario",
"variable",
"region",
"unit",
"SSP",
"GMT",
],
value_vars=years,
var_name="year",
) # change to df_big if concatenating multiple

dfp.columns = [x.lower() for x in dfp.columns]

dsout = xr.Dataset()

for indicator in dfp.variable.unique():
print(indicator)
dx = (
dfp.loc[dfp.variable == indicator]
.set_index(["gmt", "year", "ssp", "region"])
.to_xarray()
)
# dx.attrs['unit'] = dx.assign_coords({'unit':dx.unit.values[0,0,0,0]})
dsout[indicator] = dx["value"].to_dataset(name=indicator)[indicator]
dsout[indicator].attrs["unit"] = dx.unit.values[0, 0, 0, 0]
# dsout = dsout[indicator].assign_coords({'unit':dx.unit.values[0,0,0,0]})

dsout["ssp"] = [x.upper() for x in dsout["ssp"].values]
# dsout = dsout.drop_vars('unit')

# % Write out
print("Writing out... ")
comp = dict(zlib=True, complevel=5)
encoding = {var: comp for var in dsout.data_vars}
filename = f"{output_dir}{vari}_{region}.nc"
dsout.to_netcdf(filename, encoding=encoding)
# if ab_present:
# bar()

# =============================================================================
Loading
Loading