Merge pull request #2 from iiasa/python_setup

Python setup

requirements.txt

@@ -0,0 +1,13 @@
+dask~=2022.7.0
+holoviews~=1.17.1
+hvplot~=0.9.0
+matplotlib~=3.5.2
+numpy~=1.23.5
+pandas~=1.5.2
+pyam_iamc~=1.7.1.dev3+gbec0295
+PyYAML~=6.0
+PyYAML~=6.0.1
+rich~=13.6.0
+scipy~=1.9.3
+seaborn~=0.13.0
+xarray~=2022.11.0

rime/generate_aggregated_inputs.py

@@ -4,35 +4,38 @@
 
 @author: byers
 """
-
+# from alive_progress import alive_bar
 import glob
 
 # import numpy as np
 import os
-import pandas as pd
 import pyam
 import re
 import xarray as xr
 import time
-from alive_progress import alive_bar
-
-start = time.time()
 
+# try:
 
-from process_config import *
+# ab_present = True
+# except:
+#     print("alive_progress not installed")
+#     ab_present = False
 
-# Load impact, interpolate
+# start = time.time()
 
+from rime.rime_functions import *
+from rime.process_config import *
 
-# few_vars = True
+few_vars = False
 
 # %% Settings config
 
-
 years = list(range(yr_start, yr_end + 1))
 
 
 # %% List of indicators and files
+
+# Define a list of table data files to read in
 os.chdir(wdtable_input)
 files = glob.glob(table_output_format.replace("|", "*"))
 
@@ -49,10 +52,8 @@
 # files = files[4:5] # heatwav & iavar
 # files = files[5:10]
 # files = files[10:12]
-files = files[12:15]
-# files = files[15:]
-
-# batch = 1
+# files = files[12:15]
+files = files[15:]
 
 
 indicators = []
@@ -111,72 +112,67 @@
     # df.drop(columns=['model', 'scenario'], inplace=True)
 
     # if few_vars:
-
     #     df = df.loc[df.variable.str.contains('High')]
 
     small_vars = list(set([x.split("|")[0] for x in dfin.variable]))
-
-    with alive_bar(
-        total=len(small_vars),
-        title="Processing",
-        length=10,
-        force_tty=True,
-        bar="circles",
-        spinner="elements",
-    ) as bar:
-        for vari in small_vars:
-            df_ind = loop_interpolate_gmt(df.loc[df.variable.str.startswith(vari)])
-            # dfbig = pd.concat([dfbig, df_ind])
-            print(f"dfbig: indicator {ind[0]}: {time.time()-istart}")
-
-            # % Convert and save out to xarray
-            # dfbig.dropna(how='all')
-
-            dfp = df_ind.melt(
-                id_vars=[
-                    "model",
-                    "scenario",
-                    "variable",
-                    "region",
-                    "unit",
-                    "SSP",
-                    "GMT",
-                ],
-                value_vars=years,
-                var_name="year",
-            )  # change to df_big if concatenating multiple
-
-            dfp.columns = [x.lower() for x in dfp.columns]
-
-            dsout = xr.Dataset()
-
-            for indicator in dfp.variable.unique():
-                print(indicator)
-                dx = (
-                    dfp.loc[dfp.variable == indicator]
-                    .set_index(["gmt", "year", "ssp", "region"])
-                    .to_xarray()
-                )
-                # dx.attrs['unit'] = dx.assign_coords({'unit':dx.unit.values[0,0,0,0]})
-
-                dsout[indicator] = dx["value"].to_dataset(name=indicator)[indicator]
-                dsout[indicator].attrs["unit"] = dx.unit.values[0, 0, 0, 0]
-                # dsout = dsout[indicator].assign_coords({'unit':dx.unit.values[0,0,0,0]})
-
-                # bar()
-
-            dsout["ssp"] = [x.upper() for x in dsout["ssp"].values]
-            # dsout = dsout.drop_vars('unit')
-
-            # % Write out
-            print("Writing out... ")
-            comp = dict(zlib=True, complevel=5)
-            encoding = {var: comp for var in dsout.data_vars}
-
-            # ind = indicator.replace('|', '_')
-            filename = f"{output_dir}{vari}_{region}.nc"
-
-            dsout.to_netcdf(filename, encoding=encoding)
-            bar()
+    # if ab_present:
+    # with alive_bar(total=len(small_vars),
+    #                title='Processing', length=10, force_tty=True,
+    #                bar='circles',
+    #                spinner='elements') as bar:
+
+    #         print('alive bar present')
+    # Apply function here
+    for vari in small_vars:
+        df_ind = loop_interpolate_gmt(
+            df.loc[df.variable.str.startswith(vari)], yr_start, yr_end
+        )
+        # dfbig = pd.concat([dfbig, df_ind])
+        print(f"dfbig: indicator {ind[0]}: {time.time()-istart}")
+
+        # % Convert and save out to xarray
+        # dfbig.dropna(how='all')
+
+        dfp = df_ind.melt(
+            id_vars=[
+                "model",
+                "scenario",
+                "variable",
+                "region",
+                "unit",
+                "SSP",
+                "GMT",
+            ],
+            value_vars=years,
+            var_name="year",
+        )  # change to df_big if concatenating multiple
+
+        dfp.columns = [x.lower() for x in dfp.columns]
+
+        dsout = xr.Dataset()
+
+        for indicator in dfp.variable.unique():
+            print(indicator)
+            dx = (
+                dfp.loc[dfp.variable == indicator]
+                .set_index(["gmt", "year", "ssp", "region"])
+                .to_xarray()
+            )
+            # dx.attrs['unit'] = dx.assign_coords({'unit':dx.unit.values[0,0,0,0]})
+            dsout[indicator] = dx["value"].to_dataset(name=indicator)[indicator]
+            dsout[indicator].attrs["unit"] = dx.unit.values[0, 0, 0, 0]
+            # dsout = dsout[indicator].assign_coords({'unit':dx.unit.values[0,0,0,0]})
+
+        dsout["ssp"] = [x.upper() for x in dsout["ssp"].values]
+        # dsout = dsout.drop_vars('unit')
+
+        # % Write out
+        print("Writing out... ")
+        comp = dict(zlib=True, complevel=5)
+        encoding = {var: comp for var in dsout.data_vars}
+        filename = f"{output_dir}{vari}_{region}.nc"
+        dsout.to_netcdf(filename, encoding=encoding)
+        # if ab_present:
+        # bar()
 
     # =============================================================================