From 1051987043583ed1de4225891e9f785c9f4cb990 Mon Sep 17 00:00:00 2001 From: alexoort <63342347+alexoort@users.noreply.github.com> Date: Thu, 20 Jul 2023 14:14:16 -0400 Subject: [PATCH 01/12] Point source visualization and implementation into clustering algorithm (#150) Enables integration of SRON-detected methane plumes into clustering algorithm. Allows users to visualize point sources with IMI preview. --- config.yml | 2 +- envs/Harvard-Cannon/imi_env.yml | 3 +- run_imi.sh | 1 + .../statevector_component/aggregation.py | 147 +++++++++++++++++- src/geoschem_run_scripts/ch4_run.template | 1 - .../run_jacobian_simulations.sh | 1 - src/inversion_scripts/imi_preview.py | 2 + src/inversion_scripts/utils.py | 90 +++++++++++ src/notebooks/visualization_notebook.ipynb | 46 ++++++ 9 files changed, 287 insertions(+), 6 deletions(-) diff --git a/config.yml b/config.yml index 15059653..a460f1e8 100644 --- a/config.yml +++ b/config.yml @@ -179,4 +179,4 @@ PreviewDryRun: true SpinupDryrun: true ProductionDryRun: true PosteriorDryRun: true -BCdryrun: true +BCdryrun: true \ No newline at end of file diff --git a/envs/Harvard-Cannon/imi_env.yml b/envs/Harvard-Cannon/imi_env.yml index 089eaf4b..69cec6c9 100644 --- a/envs/Harvard-Cannon/imi_env.yml +++ b/envs/Harvard-Cannon/imi_env.yml @@ -22,4 +22,5 @@ dependencies: - ipykernel=6.15.0 - jupyter=1.0.0 - bottleneck=1.3.5 - - boto3=1.26.161 \ No newline at end of file + - bs4=4.12.2 + - boto3=1.26.161 diff --git a/run_imi.sh b/run_imi.sh index 7e4baa84..6ee7f003 100755 --- a/run_imi.sh +++ b/run_imi.sh @@ -4,6 +4,7 @@ #SBATCH -n 1 #SBATCH -o "imi_output.log" + # This script will run the Integrated Methane Inversion (IMI) with GEOS-Chem. # For documentation, see https://imi.readthedocs.io. # diff --git a/src/components/statevector_component/aggregation.py b/src/components/statevector_component/aggregation.py index 14e945d1..e4a6e70d 100755 --- a/src/components/statevector_component/aggregation.py +++ b/src/components/statevector_component/aggregation.py @@ -3,11 +3,17 @@ import sys import yaml +import datetime +import requests import xarray as xr import numpy as np import pandas as pd -import yaml -import sys +import geopandas as gpd +from bs4 import BeautifulSoup +import matplotlib.pyplot as plt +from shapely.geometry import Point +from dateutil.relativedelta import relativedelta + from src.inversion_scripts.imi_preview import ( estimate_averaging_kernel, map_sensitivities_to_sv, @@ -296,6 +302,115 @@ def generate_cluster_pairs(config, sensitivities): return sorted(cluster_pairs, key=lambda x: x[0]) +def get_plumes(month, year): + """ + Description: + Scrapes the SRON database for weekly methane plumes, saving each week's data as csv files + arguments: + month String : the month (number) for which to select the plumes + year String : the year (number) for which to select the plumes + Returns: pd.Dataframe() : pandas dataframe with all of the plumes detected for that month + """ + sron_url = "https://earth.sron.nl/wp-content/uploads/" #URL of the SRON database for weekly methane plumes + url = sron_url + year + "/" + month.zfill(2) + response = requests.get(url) + parser = BeautifulSoup(response.content, 'html.parser') + plume = pd.DataFrame() + for link in parser.find_all('a'): + if ('.csv' in link.get('href') and 'SRON_Weekly_Methane_Plumes' in link.get('href')): #filters through all .csv files containing "SRON_Weekly_Methane_Plumes" + csvUrl = url + "/" + link.get('href') + dates = csvUrl.split("_v")[1] + try: + rcsv = requests.get(csvUrl, allow_redirects=True) + file = f"SRON_{dates}" + open(file, 'wb').write(rcsv.content) # downloads all of the plumes from that week in a CSV file in the current directory + df = pd.read_csv(file) # reads from the csv file into a pandas dataframe + plume = plume.append(df, ignore_index=True) + except: + print(f"Warning: Unable to access data for csv file at {csvUrl}. The file may not exist or there may be a connection problem.") + return plume + + +def shapefile_filter(plumes, shapefile_path): + """ + Description: + Removes any plumes (coordinates) that are not within a given shapefile + arguments: + plumes pd.Dataframe() : a pandas dataframe with columns 'lon' and 'lat' + shapefile_path String : a string with the path to the shapefile of the ROI + Returns: pd.Dataframe() : pandas dataframe only containing coordinates within the shapefile + """ + shapefile = gpd.read_file(shapefile_path) + for lon, lat in zip(plumes['lon'], plumes['lat']): + point = Point(lon, lat) + is_within = shapefile.contains(point) + # checks if it is within any of the polygons if multiple + is_within_any = is_within.any() + if not is_within_any: + plumes = plumes[(plumes['lon'] != float(lon)) | (plumes['lat'] != float(lat))] + + return plumes + + +def rectangular_filter(plumes, LatMax, LatMin, LonMax, LonMin): + """ + Description: + Removes any plumes (coordinate) not within a given set of coordinates + arguments: + plumes pd.Dataframe() : a pandas dataframe with columns 'lon' and 'lat' + LatMax float : a float indicating the maximum latitude in the ROI + LatMin float : a float indicating the minimum latitude in the ROI + LonMax float : a float indicating the maximum longitude in the ROI + LonMin float : a float indicating the minimum longitude in the ROI + Returns: pd.Dataframe() : pandas dataframe + """ + inLat = (plumes['lat'] > LatMin) & (plumes['lat'] < LatMax) + inLon = (plumes['lon'] > LonMin) & (plumes['lon'] < LonMax) + filtered_plumes = plumes[inLat & inLon] + return filtered_plumes + +def SRON_plumes(config): + """ + Description: + Selects all the recorded methane plumes on the SRON database for the selected time frame and region + arguments: + config parsed YAML file + Returns: [[]] : list of [lat, lon] coordinates of floats + """ + + #variables from config file, specifying time period and region. + plumes = pd.DataFrame() + shapefile_path = config["ShapeFile"] + startDate = datetime.datetime.strptime(str(config["StartDate"]), "%Y%m%d") + endDate = datetime.datetime.strptime(str(config["EndDate"]), "%Y%m%d") + if endDate.year < 2023: # SRON plumes are only available beginning in 2023 + return None + custom_vectorfile = not config["CreateAutomaticRectilinearStateVectorFile"] + LatMax = config["LatMax"] + LatMin = config["LatMin"] + LonMax = config["LonMax"] + LonMin = config["LonMin"] + currentDate = startDate + + + + #calls the get_plumes function for every month in the selected time frame + while currentDate <= endDate: + if currentDate.year >= 2023: # SRON plumes are only available beginning in 2023 + p = get_plumes(str(currentDate.month), str(currentDate.year)) + plumes = pd.concat([plumes, pd.DataFrame(p)], ignore_index=True) + currentDate = currentDate + relativedelta(months=1) + + #filters through the dataset to remove any plumes outside the ROI + if custom_vectorfile: + plumes = shapefile_filter(plumes, shapefile_path) #calls function to filter through coordinates found in shapefile + else: + plumes = rectangular_filter(plumes, LatMax, LatMin, LonMax, LonMin) + + plumes_list = plumes[['lat', 'lon']].values.tolist() + return plumes_list + + def read_coordinates(coord_var): """ Description: @@ -344,9 +459,24 @@ def force_native_res_pixels(config, clusters, sensitivities): Returns: [double] : updated sensitivities """ coords = read_coordinates(config["ForcedNativeResolutionElements"]) + + if "SRON" in config["PointSourceDatasets"]: + print("Fetching plumes from SRON database...") + plumes = SRON_plumes(config) + else: + plumes = None + + if plumes is not None: + if coords is None: + coords = plumes + else: + coords.extend(plumes) + + if coords is None: # No forced pixels inputted + print(f"No forced native pixels specified or in {config['PointSourceDatasets']} dataset.") return sensitivities if config["Res"] == "0.25x0.3125": @@ -355,11 +485,24 @@ def force_native_res_pixels(config, clusters, sensitivities): elif config["Res"] == "0.5x0.625": lat_step = 0.5 lon_step = 0.625 + + for lat, lon in coords: + lon = np.floor(lon / lon_step) * lon_step + lat = np.floor(lat / lat_step) * lat_step + + + coords = sorted(set(map(tuple, coords)), reverse=True) # Remove any duplicate coordinates within the same gridcell. + coords = [list(coordinate) for coordinate in coords] + + + if len(coords) > config["NumberOfElements"]: + coords = coords[0:config["NumberOfElements"] - 1] for lat, lon in coords: binned_lon = np.floor(lon / lon_step) * lon_step binned_lat = np.floor(lat / lat_step) * lat_step + try: cluster_index = int( clusters.sel(lat=binned_lat, lon=binned_lon).values.flatten()[0] diff --git a/src/geoschem_run_scripts/ch4_run.template b/src/geoschem_run_scripts/ch4_run.template index e0b85b40..02913c6a 100755 --- a/src/geoschem_run_scripts/ch4_run.template +++ b/src/geoschem_run_scripts/ch4_run.template @@ -1,5 +1,4 @@ #!/bin/bash - ##SBATCH -N 1 ##SBATCH --mail-type=END diff --git a/src/geoschem_run_scripts/run_jacobian_simulations.sh b/src/geoschem_run_scripts/run_jacobian_simulations.sh index 58c5917e..4f56ee12 100755 --- a/src/geoschem_run_scripts/run_jacobian_simulations.sh +++ b/src/geoschem_run_scripts/run_jacobian_simulations.sh @@ -1,5 +1,4 @@ #!/bin/bash - #SBATCH -J {RunName} #SBATCH -N 1 diff --git a/src/inversion_scripts/imi_preview.py b/src/inversion_scripts/imi_preview.py index 2251b0c1..f3a4cf34 100755 --- a/src/inversion_scripts/imi_preview.py +++ b/src/inversion_scripts/imi_preview.py @@ -255,6 +255,7 @@ def imi_preview( mask=mask, only_ROI=False, ) + plt.savefig( os.path.join(preview_dir, "preview_observations.png"), bbox_inches="tight", @@ -295,6 +296,7 @@ def imi_preview( lon_bounds=None, lat_bounds=None, title="Observation density", + point_sources=config["ForcedNativeResolutionElements"], cbar_label="Number of observations", mask=mask, only_ROI=False, diff --git a/src/inversion_scripts/utils.py b/src/inversion_scripts/utils.py index 81aa3773..1a158793 100644 --- a/src/inversion_scripts/utils.py +++ b/src/inversion_scripts/utils.py @@ -6,6 +6,8 @@ import cartopy import cartopy.crs as ccrs import pickle +import csv +from matplotlib.lines import Line2D def save_obj(obj, name): @@ -113,6 +115,7 @@ def plot_field( vmin=None, vmax=None, title=None, + point_sources=None, cbar_label=None, mask=None, only_ROI=False, @@ -133,6 +136,7 @@ def plot_field( vmin : colorbar lower bound vmax : colorbar upper bound title : plot title + point_sources: plot given point sources on map cbar_label : colorbar label mask : mask for region of interest, boolean dataarray only_ROI : zero out data outside the region of interest, true or false @@ -192,6 +196,92 @@ def plot_field( # Title if title: ax.set_title(title) + + # Marks any specified high-resolution coordinates on the preview observation density map + if point_sources: + if isinstance(point_sources, list): + for coord in point_sources: + ax.plot(coord[1], coord[0], marker="x", markeredgecolor="black") + point = Line2D([0], [0], label='point source', marker='x', markersize=10, markeredgecolor='black', markerfacecolor='k', linestyle='') + ax.legend(handles=[point]) + elif isinstance(point_sources, str): + with open(point_sources, 'r') as file: + csvFile = csv.reader(file, delimiter=';') + next(csvFile) + for line in csvFile: + ax.plot(float(line[2].replace(',', '.')), float(line[1].replace(',','.')), marker="x", markeredgecolor="black") + point = Line2D([0], [0], label='point source', marker='x', markersize=10, markeredgecolor='black', markerfacecolor='k', linestyle='') + ax.legend(handles=[point]) + + +def plot_time_series( + x_data, + y_data, + line_labels, + title, + y_label, + x_label="Date", + DOFS=None, + fig_size=(15, 6), + x_rotation=45, + y_sci_notation=True, +): + """ + Function to plot inversion time series results. + + Arguments + x_data : x data datetimes to plot + y_data : list of y data to plot + line_labels : line label string for each y data + title : plot title + y_label : label for y axis + x_label : label for x axis + DOFS : DOFs for each interval + fig_size : tuple for figure size + x_rotation : rotation of x axis labels + y_sci_notation : whether to use scientific notation for y axis + """ + assert len(y_data) == len(line_labels) + plt.clf() + # Set the figure size + _, ax1 = plt.subplots(figsize=fig_size) + + # Plot emissions time series + for i in range(len(y_data)): + # only use line for moving averages + if "moving" in line_labels[i].lower(): + ax1.plot(x_data, y_data[i], label=line_labels[i]) + else: + ax1.plot( + x_data, y_data[i], linestyle="None", marker="o", label=line_labels[i] + ) + + lines = ax1.get_lines() + + # Plot DOFS time series using red + if DOFS is not None: + # Create a twin y-axis + ax2 = ax1.twinx() + ax2.plot(x_data, DOFS, linestyle="None", marker="o", color="red", label="DOFS") + ax2.set_ylabel("DOFS", color="red") + ax2.tick_params(axis="y", labelcolor="red") + ax2.set_ylim(0, 1) + # add DOFS line to legend + lines = ax1.get_lines() + ax2.get_lines() + + # use a date string for the x axis locations + plt.gca().xaxis.set_major_locator(mdates.WeekdayLocator()) + plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d")) + # tilt the x axis labels + plt.xticks(rotation=x_rotation) + # scientific notation for y axis + if y_sci_notation: + plt.ticklabel_format(style="sci", axis="y", scilimits=(0, 0)) + ax1.set_xlabel(x_label) + ax1.set_ylabel(y_label) + plt.title(title) + plt.legend(lines, [line.get_label() for line in lines]) + plt.show() def plot_time_series( diff --git a/src/notebooks/visualization_notebook.ipynb b/src/notebooks/visualization_notebook.ipynb index b8bd2bf2..0811a23b 100644 --- a/src/notebooks/visualization_notebook.ipynb +++ b/src/notebooks/visualization_notebook.ipynb @@ -233,6 +233,52 @@ " only_ROI=True, state_vector_labels=state_vector_labels, last_ROI_element=last_ROI_element)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate posterior sectoral emissions\n", + "prior = xr.load_dataset(prior_pth)\n", + "posterior = prior * scale\n", + "\n", + "emission_types = {\n", + "'SoilPost emissions' : sum_total_emissions(posterior[\"EmisCH4_SoilAbsorb\"].isel(time=0), areas, mask),\n", + "'Termites emissions' : sum_total_emissions(posterior[\"EmisCH4_Termites\"].isel(time=0), areas, mask),\n", + "'Lakes emissions' : sum_total_emissions(posterior[\"EmisCH4_Lakes\"].isel(time=0), areas, mask),\n", + "'Seeps emissions' : sum_total_emissions(posterior[\"EmisCH4_Seeps\"].isel(time=0), areas, mask),\n", + "'Wetlands emissions' : sum_total_emissions(posterior[\"EmisCH4_Wetlands\"].isel(time=0), areas, mask),\n", + "'BiomassBurn emissions' :sum_total_emissions(posterior[\"EmisCH4_BiomassBurn\"].isel(time=0), areas, mask),\n", + "'OtherAnth emissions' : sum_total_emissions(posterior[\"EmisCH4_OtherAnth\"].isel(time=0), areas, mask),\n", + "'Rice emissions' : sum_total_emissions(posterior[\"EmisCH4_Rice\"].isel(time=0), areas, mask),\n", + "'Wastewater emissions' : sum_total_emissions(posterior[\"EmisCH4_Wastewater\"].isel(time=0), areas, mask),\n", + "'Landfills emissions' : sum_total_emissions(posterior[\"EmisCH4_Landfills\"].isel(time=0), areas, mask),\n", + "'Livestock emissions' : sum_total_emissions(posterior[\"EmisCH4_Livestock\"].isel(time=0), areas, mask),\n", + "'Coal emissions' : sum_total_emissions(posterior[\"EmisCH4_Coal\"].isel(time=0), areas, mask),\n", + "'Gas emissions' : sum_total_emissions(posterior[\"EmisCH4_Gas\"].isel(time=0), areas, mask) }\n", + "\n", + "labels = []\n", + "values = []\n", + "\n", + "for subemission in emission_types:\n", + " print(subemission, \"=\", emission_types[subemission], \"Tg/y\")\n", + " if (emission_types[subemission]>0):\n", + " labels.append(subemission)\n", + " values.append(emission_types[subemission])\n", + "\n", + "\n", + "title = plt.title('CH4 emissions by sector')\n", + "title.set_ha(\"center\")\n", + "plt.gca().axis(\"equal\")\n", + "def my_autopct(pct): # Only plots the data labels for sectoral emissions that are larger than 15%\n", + " return ('%.1f%%' % pct) if pct > 15 else ''\n", + "pie = plt.pie(values, startangle=0, autopct=my_autopct)\n", + "plt.legend(pie[0],labels, bbox_to_anchor=(1,0.5), loc=\"center right\", fontsize=10, \n", + " bbox_transform=plt.gcf().transFigure)\n", + "plt.subplots_adjust(left=0.0, bottom=0.1, right=0.65)" + ] + }, { "attachments": {}, "cell_type": "markdown", From ba996c173d082893063707489b56ff3e22900793 Mon Sep 17 00:00:00 2001 From: laestrada Date: Fri, 18 Aug 2023 13:51:43 -0400 Subject: [PATCH 02/12] remove duplicate plot function --- src/inversion_scripts/utils.py | 70 ---------------------------------- 1 file changed, 70 deletions(-) diff --git a/src/inversion_scripts/utils.py b/src/inversion_scripts/utils.py index 1a158793..9a261671 100644 --- a/src/inversion_scripts/utils.py +++ b/src/inversion_scripts/utils.py @@ -284,76 +284,6 @@ def plot_time_series( plt.show() -def plot_time_series( - x_data, - y_data, - line_labels, - title, - y_label, - x_label="Date", - DOFS=None, - fig_size=(15, 6), - x_rotation=45, - y_sci_notation=True, -): - """ - Function to plot inversion time series results. - - Arguments - x_data : x data datetimes to plot - y_data : list of y data to plot - line_labels : line label string for each y data - title : plot title - y_label : label for y axis - x_label : label for x axis - DOFS : DOFs for each interval - fig_size : tuple for figure size - x_rotation : rotation of x axis labels - y_sci_notation : whether to use scientific notation for y axis - """ - assert len(y_data) == len(line_labels) - plt.clf() - # Set the figure size - _, ax1 = plt.subplots(figsize=fig_size) - - # Plot emissions time series - for i in range(len(y_data)): - # only use line for moving averages - if "moving" in line_labels[i].lower(): - ax1.plot(x_data, y_data[i], label=line_labels[i]) - else: - ax1.plot( - x_data, y_data[i], linestyle="None", marker="o", label=line_labels[i] - ) - - lines = ax1.get_lines() - - # Plot DOFS time series using red - if DOFS is not None: - # Create a twin y-axis - ax2 = ax1.twinx() - ax2.plot(x_data, DOFS, linestyle="None", marker="o", color="red", label="DOFS") - ax2.set_ylabel("DOFS", color="red") - ax2.tick_params(axis="y", labelcolor="red") - ax2.set_ylim(0, 1) - # add DOFS line to legend - lines = ax1.get_lines() + ax2.get_lines() - - # use a date string for the x axis locations - plt.gca().xaxis.set_major_locator(mdates.WeekdayLocator()) - plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d")) - # tilt the x axis labels - plt.xticks(rotation=x_rotation) - # scientific notation for y axis - if y_sci_notation: - plt.ticklabel_format(style="sci", axis="y", scilimits=(0, 0)) - ax1.set_xlabel(x_label) - ax1.set_ylabel(y_label) - plt.title(title) - plt.legend(lines, [line.get_label() for line in lines]) - plt.show() - - def filter_tropomi(tropomi_data, xlim, ylim, startdate, enddate): """ Description: From 56de23be328c0266871e1487c6e89d738b06d4f0 Mon Sep 17 00:00:00 2001 From: laestrada Date: Fri, 18 Aug 2023 14:52:09 -0400 Subject: [PATCH 03/12] start refactor of point source viz update --- .../statevector_component/aggregation.py | 57 ++---------------- src/inversion_scripts/imi_preview.py | 3 +- src/inversion_scripts/point_sources.py | 59 +++++++++++++++++++ src/inversion_scripts/utils.py | 17 ++---- 4 files changed, 70 insertions(+), 66 deletions(-) create mode 100644 src/inversion_scripts/point_sources.py diff --git a/src/components/statevector_component/aggregation.py b/src/components/statevector_component/aggregation.py index e4a6e70d..21842e66 100755 --- a/src/components/statevector_component/aggregation.py +++ b/src/components/statevector_component/aggregation.py @@ -14,6 +14,7 @@ from shapely.geometry import Point from dateutil.relativedelta import relativedelta +from src.inversion_scripts.point_sources import get_point_source_coordinates from src.inversion_scripts.imi_preview import ( estimate_averaging_kernel, map_sensitivities_to_sv, @@ -411,41 +412,6 @@ def SRON_plumes(config): return plumes_list -def read_coordinates(coord_var): - """ - Description: - Read coordinates either from a list of lists or a csv file - arguments: - coord_var [] or String : either a list of coordinates or a csv file - Returns: [[]] : list of [lat, lon] coordinates of floats - """ - - # handle path to csv file containg coordinates - if isinstance(coord_var, str): - if not coord_var.endswith(".csv"): - raise Exception( - "ForcedNativeResolutionElements expects either a .csv file or a list of lists." - ) - coords_df = pd.read_csv(coord_var) - - # check if lat and lon columns are present - if not ("lat" in coords_df.columns and "lon" in coords_df.columns): - raise Exception( - "lat or lon columns are not present in the csv file." - + " csv file must have lat and lon in header using lowercase." - ) - # select lat and lon columns and convert to list of lists - return coords_df[["lat", "lon"]].values.tolist() - - # handle list of lists - elif isinstance(coord_var, list): - return coord_var - else: - # Variable is neither a string nor a list - print("Warning: No ForcedNativeResolutionElements specified or invalid format.") - return None - - def force_native_res_pixels(config, clusters, sensitivities): """ Description: @@ -458,23 +424,9 @@ def force_native_res_pixels(config, clusters, sensitivities): cluster_pairs [(tuple)]: cluster pairings Returns: [double] : updated sensitivities """ - coords = read_coordinates(config["ForcedNativeResolutionElements"]) - - if "SRON" in config["PointSourceDatasets"]: - print("Fetching plumes from SRON database...") - plumes = SRON_plumes(config) - else: - plumes = None - - if plumes is not None: - if coords is None: - coords = plumes - else: - coords.extend(plumes) - - + coords = get_point_source_coordinates(config) - if coords is None: + if len(coords) == 0: # No forced pixels inputted print(f"No forced native pixels specified or in {config['PointSourceDatasets']} dataset.") return sensitivities @@ -491,7 +443,8 @@ def force_native_res_pixels(config, clusters, sensitivities): lat = np.floor(lat / lat_step) * lat_step - coords = sorted(set(map(tuple, coords)), reverse=True) # Remove any duplicate coordinates within the same gridcell. + # Remove any duplicate coordinates within the same gridcell. + coords = sorted(set(map(tuple, coords)), reverse=True) coords = [list(coordinate) for coordinate in coords] diff --git a/src/inversion_scripts/imi_preview.py b/src/inversion_scripts/imi_preview.py index f3a4cf34..0d09025b 100755 --- a/src/inversion_scripts/imi_preview.py +++ b/src/inversion_scripts/imi_preview.py @@ -19,6 +19,7 @@ import cartopy.crs as ccrs import colorcet as cc from joblib import Parallel, delayed +from src.inversion_scripts.point_sources import get_point_source_coordinates from src.inversion_scripts.utils import ( sum_total_emissions, count_obs_in_mask, @@ -228,6 +229,7 @@ def imi_preview( lat_bounds=None, levels=21, title="Prior emissions", + point_sources=get_point_source_coordinates(config), cbar_label="Emissions (kg km$^{-2}$ h$^{-1}$)", mask=mask, only_ROI=False, @@ -296,7 +298,6 @@ def imi_preview( lon_bounds=None, lat_bounds=None, title="Observation density", - point_sources=config["ForcedNativeResolutionElements"], cbar_label="Number of observations", mask=mask, only_ROI=False, diff --git a/src/inversion_scripts/point_sources.py b/src/inversion_scripts/point_sources.py new file mode 100644 index 00000000..d6f078fc --- /dev/null +++ b/src/inversion_scripts/point_sources.py @@ -0,0 +1,59 @@ +# python function related to point sources + +def read_point_source_csv(path): + """ + Description: + Read lat, lon coordinates either from a csv file and return as list + arguments: + path String : path to a csv file + Returns: [[]] : list of [lat, lon] coordinates of floats + """ + if not path.endswith(".csv"): + raise Exception( + "ForcedNativeResolutionElements expects either a .csv file or a list of lists." + ) + coords_df = pd.read_csv(path) + + # check if lat and lon columns are present + if not ("lat" in coords_df.columns and "lon" in coords_df.columns): + raise Exception( + "lat or lon columns are not present in the csv file." + + " csv file must have lat and lon in header using lowercase." + ) + # select lat and lon columns and convert to list of lists + return coords_df[["lat", "lon"]].values.tolist() + +def get_point_source_coordinates(config): + """ + Description: + Read point point sources from config file, csv file, or user-specified + external point source dataset + arguments: + config {} : dictionary of config variables + Returns: [[]] : list of [lat, lon] coordinates of floats + """ + coords = [] + # first we attempt to read direct, user inputted point sources + if "ForcedNativeResolutionElements" in config.keys(): + coord_var = config["ForcedNativeResolutionElements"] + # handle path to csv file containg coordinates + if isinstance(coord_var, str): + coords = read_point_source_csv(path) + # handle list of lists + elif isinstance(coord_var, list): + coords = coord_var + else: + # Variable is neither a string nor a list + print("Warning: No ForcedNativeResolutionElements specified or invalid format.") + + # then we read point sources from external datasources + plumes = [] + if "PointSourceDatasets" in config.keys(): + if "SRON" in config["PointSourceDatasets"]: + print("Fetching plumes from SRON database...") + plumes = SRON_plumes(config) + else: + print('No valid external point source datasets specified. Valid values are: "SRON"') + + # append SRON + return coords + plumes \ No newline at end of file diff --git a/src/inversion_scripts/utils.py b/src/inversion_scripts/utils.py index 9a261671..054ddf89 100644 --- a/src/inversion_scripts/utils.py +++ b/src/inversion_scripts/utils.py @@ -199,19 +199,10 @@ def plot_field( # Marks any specified high-resolution coordinates on the preview observation density map if point_sources: - if isinstance(point_sources, list): - for coord in point_sources: - ax.plot(coord[1], coord[0], marker="x", markeredgecolor="black") - point = Line2D([0], [0], label='point source', marker='x', markersize=10, markeredgecolor='black', markerfacecolor='k', linestyle='') - ax.legend(handles=[point]) - elif isinstance(point_sources, str): - with open(point_sources, 'r') as file: - csvFile = csv.reader(file, delimiter=';') - next(csvFile) - for line in csvFile: - ax.plot(float(line[2].replace(',', '.')), float(line[1].replace(',','.')), marker="x", markeredgecolor="black") - point = Line2D([0], [0], label='point source', marker='x', markersize=10, markeredgecolor='black', markerfacecolor='k', linestyle='') - ax.legend(handles=[point]) + for coord in point_sources: + ax.plot(coord[1], coord[0], marker="x", markeredgecolor="black") + point = Line2D([0], [0], label='point source', marker='x', markersize=10, markeredgecolor='black', markerfacecolor='k', linestyle='') + ax.legend(handles=[point]) def plot_time_series( From 77e110f4f5107c3cde3757c6ac58fb7ce1b70d40 Mon Sep 17 00:00:00 2001 From: laestrada Date: Fri, 18 Aug 2023 15:04:27 -0400 Subject: [PATCH 04/12] refactored to have new point_sources.py file --- .../statevector_component/aggregation.py | 132 +--------------- src/inversion_scripts/imi_preview.py | 46 +++--- src/inversion_scripts/point_sources.py | 149 +++++++++++++++++- 3 files changed, 174 insertions(+), 153 deletions(-) diff --git a/src/components/statevector_component/aggregation.py b/src/components/statevector_component/aggregation.py index 21842e66..c68fff0a 100755 --- a/src/components/statevector_component/aggregation.py +++ b/src/components/statevector_component/aggregation.py @@ -3,16 +3,8 @@ import sys import yaml -import datetime -import requests import xarray as xr import numpy as np -import pandas as pd -import geopandas as gpd -from bs4 import BeautifulSoup -import matplotlib.pyplot as plt -from shapely.geometry import Point -from dateutil.relativedelta import relativedelta from src.inversion_scripts.point_sources import get_point_source_coordinates from src.inversion_scripts.imi_preview import ( @@ -303,115 +295,6 @@ def generate_cluster_pairs(config, sensitivities): return sorted(cluster_pairs, key=lambda x: x[0]) -def get_plumes(month, year): - """ - Description: - Scrapes the SRON database for weekly methane plumes, saving each week's data as csv files - arguments: - month String : the month (number) for which to select the plumes - year String : the year (number) for which to select the plumes - Returns: pd.Dataframe() : pandas dataframe with all of the plumes detected for that month - """ - sron_url = "https://earth.sron.nl/wp-content/uploads/" #URL of the SRON database for weekly methane plumes - url = sron_url + year + "/" + month.zfill(2) - response = requests.get(url) - parser = BeautifulSoup(response.content, 'html.parser') - plume = pd.DataFrame() - for link in parser.find_all('a'): - if ('.csv' in link.get('href') and 'SRON_Weekly_Methane_Plumes' in link.get('href')): #filters through all .csv files containing "SRON_Weekly_Methane_Plumes" - csvUrl = url + "/" + link.get('href') - dates = csvUrl.split("_v")[1] - try: - rcsv = requests.get(csvUrl, allow_redirects=True) - file = f"SRON_{dates}" - open(file, 'wb').write(rcsv.content) # downloads all of the plumes from that week in a CSV file in the current directory - df = pd.read_csv(file) # reads from the csv file into a pandas dataframe - plume = plume.append(df, ignore_index=True) - except: - print(f"Warning: Unable to access data for csv file at {csvUrl}. The file may not exist or there may be a connection problem.") - return plume - - -def shapefile_filter(plumes, shapefile_path): - """ - Description: - Removes any plumes (coordinates) that are not within a given shapefile - arguments: - plumes pd.Dataframe() : a pandas dataframe with columns 'lon' and 'lat' - shapefile_path String : a string with the path to the shapefile of the ROI - Returns: pd.Dataframe() : pandas dataframe only containing coordinates within the shapefile - """ - shapefile = gpd.read_file(shapefile_path) - for lon, lat in zip(plumes['lon'], plumes['lat']): - point = Point(lon, lat) - is_within = shapefile.contains(point) - # checks if it is within any of the polygons if multiple - is_within_any = is_within.any() - if not is_within_any: - plumes = plumes[(plumes['lon'] != float(lon)) | (plumes['lat'] != float(lat))] - - return plumes - - -def rectangular_filter(plumes, LatMax, LatMin, LonMax, LonMin): - """ - Description: - Removes any plumes (coordinate) not within a given set of coordinates - arguments: - plumes pd.Dataframe() : a pandas dataframe with columns 'lon' and 'lat' - LatMax float : a float indicating the maximum latitude in the ROI - LatMin float : a float indicating the minimum latitude in the ROI - LonMax float : a float indicating the maximum longitude in the ROI - LonMin float : a float indicating the minimum longitude in the ROI - Returns: pd.Dataframe() : pandas dataframe - """ - inLat = (plumes['lat'] > LatMin) & (plumes['lat'] < LatMax) - inLon = (plumes['lon'] > LonMin) & (plumes['lon'] < LonMax) - filtered_plumes = plumes[inLat & inLon] - return filtered_plumes - -def SRON_plumes(config): - """ - Description: - Selects all the recorded methane plumes on the SRON database for the selected time frame and region - arguments: - config parsed YAML file - Returns: [[]] : list of [lat, lon] coordinates of floats - """ - - #variables from config file, specifying time period and region. - plumes = pd.DataFrame() - shapefile_path = config["ShapeFile"] - startDate = datetime.datetime.strptime(str(config["StartDate"]), "%Y%m%d") - endDate = datetime.datetime.strptime(str(config["EndDate"]), "%Y%m%d") - if endDate.year < 2023: # SRON plumes are only available beginning in 2023 - return None - custom_vectorfile = not config["CreateAutomaticRectilinearStateVectorFile"] - LatMax = config["LatMax"] - LatMin = config["LatMin"] - LonMax = config["LonMax"] - LonMin = config["LonMin"] - currentDate = startDate - - - - #calls the get_plumes function for every month in the selected time frame - while currentDate <= endDate: - if currentDate.year >= 2023: # SRON plumes are only available beginning in 2023 - p = get_plumes(str(currentDate.month), str(currentDate.year)) - plumes = pd.concat([plumes, pd.DataFrame(p)], ignore_index=True) - currentDate = currentDate + relativedelta(months=1) - - #filters through the dataset to remove any plumes outside the ROI - if custom_vectorfile: - plumes = shapefile_filter(plumes, shapefile_path) #calls function to filter through coordinates found in shapefile - else: - plumes = rectangular_filter(plumes, LatMax, LatMin, LonMax, LonMin) - - plumes_list = plumes[['lat', 'lon']].values.tolist() - return plumes_list - - def force_native_res_pixels(config, clusters, sensitivities): """ Description: @@ -428,7 +311,9 @@ def force_native_res_pixels(config, clusters, sensitivities): if len(coords) == 0: # No forced pixels inputted - print(f"No forced native pixels specified or in {config['PointSourceDatasets']} dataset.") + print( + f"No forced native pixels specified or in {config['PointSourceDatasets']} dataset." + ) return sensitivities if config["Res"] == "0.25x0.3125": @@ -437,25 +322,22 @@ def force_native_res_pixels(config, clusters, sensitivities): elif config["Res"] == "0.5x0.625": lat_step = 0.5 lon_step = 0.625 - + for lat, lon in coords: lon = np.floor(lon / lon_step) * lon_step lat = np.floor(lat / lat_step) * lat_step - - # Remove any duplicate coordinates within the same gridcell. - coords = sorted(set(map(tuple, coords)), reverse=True) + # Remove any duplicate coordinates within the same gridcell. + coords = sorted(set(map(tuple, coords)), reverse=True) coords = [list(coordinate) for coordinate in coords] - if len(coords) > config["NumberOfElements"]: - coords = coords[0:config["NumberOfElements"] - 1] + coords = coords[0 : config["NumberOfElements"] - 1] for lat, lon in coords: binned_lon = np.floor(lon / lon_step) * lon_step binned_lat = np.floor(lat / lat_step) * lat_step - try: cluster_index = int( clusters.sel(lat=binned_lat, lon=binned_lon).values.flatten()[0] diff --git a/src/inversion_scripts/imi_preview.py b/src/inversion_scripts/imi_preview.py index 0d09025b..752c7074 100755 --- a/src/inversion_scripts/imi_preview.py +++ b/src/inversion_scripts/imi_preview.py @@ -1,23 +1,23 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -#SBATCH -N 1 +# SBATCH -N 1 +import os import sys +import yaml +import time +import warnings +import datetime import numpy as np import xarray as xr import pandas as pd import matplotlib +import colorcet as cc +import cartopy.crs as ccrs matplotlib.use("Agg") import matplotlib.pyplot as plt -import yaml -import os -import datetime -import time -import warnings -import cartopy.crs as ccrs -import colorcet as cc from joblib import Parallel, delayed from src.inversion_scripts.point_sources import get_point_source_coordinates from src.inversion_scripts.utils import ( @@ -33,12 +33,13 @@ read_tropomi, read_blended, ) -import warnings warnings.filterwarnings("ignore", category=FutureWarning) -def get_TROPOMI_data(file_path, BlendedTROPOMI, xlim, ylim, startdate_np64, enddate_np64): +def get_TROPOMI_data( + file_path, BlendedTROPOMI, xlim, ylim, startdate_np64, enddate_np64 +): """ Returns a dict with the lat, lon, xch4, and albedo_swir observations extracted from the given tropomi file. Filters are applied to remove @@ -329,12 +330,16 @@ def imi_preview( bbox_inches="tight", dpi=150, ) - expectedDOFS = np.round(sum(a),5) + expectedDOFS = np.round(sum(a), 5) if expectedDOFS < config["DOFSThreshold"]: - print(f"\nExpected DOFS = {expectedDOFS} are less than DOFSThreshold = {config['DOFSThreshold']}. Exiting.\n") - print("Consider increasing the inversion period, increasing the prior error, or using another prior inventory.\n") + print( + f"\nExpected DOFS = {expectedDOFS} are less than DOFSThreshold = {config['DOFSThreshold']}. Exiting.\n" + ) + print( + "Consider increasing the inversion period, increasing the prior error, or using another prior inventory.\n" + ) # if run with sbatch this ensures the exit code is not lost. - file = open(".error_status_file.txt", 'w') + file = open(".error_status_file.txt", "w") file.write("Error Status: 1") file.close() sys.exit(1) @@ -444,7 +449,9 @@ def estimate_averaging_kernel( # Read in and filter tropomi observations (uses parallel processing) observation_dicts = Parallel(n_jobs=-1)( - delayed(get_TROPOMI_data)(file_path, BlendedTROPOMI, xlim, ylim, startdate_np64, enddate_np64) + delayed(get_TROPOMI_data)( + file_path, BlendedTROPOMI, xlim, ylim, startdate_np64, enddate_np64 + ) for file_path in tropomi_paths ) # Remove any problematic observation dicts (eg. corrupted data file) @@ -496,12 +503,11 @@ def process(i): # unpack list of tuples into individual lists emissions, L, num_obs = [list(item) for item in zip(*result)] - + if np.sum(num_obs) < 1: sys.exit("Error: No observations found in region of interest") outstring2 = f"Found {np.sum(num_obs)} observations in the region of interest" - # ---------------------------------- # Estimate information content # ---------------------------------- @@ -510,7 +516,7 @@ def process(i): emissions = np.array(emissions) m = np.array(num_obs) # Number of observations per state vector element L = np.array(L) - + # If Kalman filter mode, count observations per inversion period if config["KalmanMode"]: startday_dt = datetime.datetime.strptime(startday, "%Y%m%d") @@ -550,10 +556,10 @@ def process(i): outstring3 = f"k = {np.round(k,5)} kg-1 m2 s" outstring4 = f"a = {np.round(a,5)} \n" outstring5 = f"expectedDOFS: {np.round(sum(a),5)}" - + if config["KalmanMode"]: outstring5 += " per inversion period" - + print(outstring3) print(outstring4) print(outstring5) diff --git a/src/inversion_scripts/point_sources.py b/src/inversion_scripts/point_sources.py index d6f078fc..5ebac32c 100644 --- a/src/inversion_scripts/point_sources.py +++ b/src/inversion_scripts/point_sources.py @@ -1,4 +1,12 @@ -# python function related to point sources +# This file contains functions related to accessing point source data +import datetime +import requests +import pandas as pd +import geopandas as gpd +from bs4 import BeautifulSoup +from shapely.geometry import Point +from dateutil.relativedelta import relativedelta + def read_point_source_csv(path): """ @@ -23,10 +31,11 @@ def read_point_source_csv(path): # select lat and lon columns and convert to list of lists return coords_df[["lat", "lon"]].values.tolist() + def get_point_source_coordinates(config): """ Description: - Read point point sources from config file, csv file, or user-specified + Read point point sources from config file, csv file, or user-specified external point source dataset arguments: config {} : dictionary of config variables @@ -38,14 +47,16 @@ def get_point_source_coordinates(config): coord_var = config["ForcedNativeResolutionElements"] # handle path to csv file containg coordinates if isinstance(coord_var, str): - coords = read_point_source_csv(path) + coords = read_point_source_csv(coord_var) # handle list of lists elif isinstance(coord_var, list): coords = coord_var else: # Variable is neither a string nor a list - print("Warning: No ForcedNativeResolutionElements specified or invalid format.") - + print( + "Warning: No ForcedNativeResolutionElements specified or invalid format." + ) + # then we read point sources from external datasources plumes = [] if "PointSourceDatasets" in config.keys(): @@ -53,7 +64,129 @@ def get_point_source_coordinates(config): print("Fetching plumes from SRON database...") plumes = SRON_plumes(config) else: - print('No valid external point source datasets specified. Valid values are: "SRON"') - + print( + 'No valid external point source datasets specified. Valid values are: "SRON"' + ) + # append SRON - return coords + plumes \ No newline at end of file + return coords + plumes + + +def get_plumes(month, year): + """ + Description: + Scrapes the SRON database for weekly methane plumes, saving each week's data as csv files + arguments: + month String : the month (number) for which to select the plumes + year String : the year (number) for which to select the plumes + Returns: pd.Dataframe() : pandas dataframe with all of the plumes detected for that month + """ + sron_url = "https://earth.sron.nl/wp-content/uploads/" # URL of the SRON database for weekly methane plumes + url = sron_url + year + "/" + month.zfill(2) + response = requests.get(url) + parser = BeautifulSoup(response.content, "html.parser") + plume = pd.DataFrame() + for link in parser.find_all("a"): + if ".csv" in link.get("href") and "SRON_Weekly_Methane_Plumes" in link.get( + "href" + ): # filters through all .csv files containing "SRON_Weekly_Methane_Plumes" + csvUrl = url + "/" + link.get("href") + dates = csvUrl.split("_v")[1] + try: + rcsv = requests.get(csvUrl, allow_redirects=True) + file = f"SRON_{dates}" + open(file, "wb").write( + rcsv.content + ) # downloads all of the plumes from that week in a CSV file in the current directory + df = pd.read_csv( + file + ) # reads from the csv file into a pandas dataframe + plume = plume.append(df, ignore_index=True) + except: + print( + f"Warning: Unable to access data for csv file at {csvUrl}. The file may not exist or there may be a connection problem." + ) + return plume + + +def shapefile_filter(plumes, shapefile_path): + """ + Description: + Removes any plumes (coordinates) that are not within a given shapefile + arguments: + plumes pd.Dataframe() : a pandas dataframe with columns 'lon' and 'lat' + shapefile_path String : a string with the path to the shapefile of the ROI + Returns: pd.Dataframe() : pandas dataframe only containing coordinates within the shapefile + """ + shapefile = gpd.read_file(shapefile_path) + for lon, lat in zip(plumes["lon"], plumes["lat"]): + point = Point(lon, lat) + is_within = shapefile.contains(point) + # checks if it is within any of the polygons if multiple + is_within_any = is_within.any() + if not is_within_any: + plumes = plumes[ + (plumes["lon"] != float(lon)) | (plumes["lat"] != float(lat)) + ] + + return plumes + + +def rectangular_filter(plumes, LatMax, LatMin, LonMax, LonMin): + """ + Description: + Removes any plumes (coordinate) not within a given set of coordinates + arguments: + plumes pd.Dataframe() : a pandas dataframe with columns 'lon' and 'lat' + LatMax float : a float indicating the maximum latitude in the ROI + LatMin float : a float indicating the minimum latitude in the ROI + LonMax float : a float indicating the maximum longitude in the ROI + LonMin float : a float indicating the minimum longitude in the ROI + Returns: pd.Dataframe() : pandas dataframe + """ + inLat = (plumes["lat"] > LatMin) & (plumes["lat"] < LatMax) + inLon = (plumes["lon"] > LonMin) & (plumes["lon"] < LonMax) + filtered_plumes = plumes[inLat & inLon] + return filtered_plumes + + +def SRON_plumes(config): + """ + Description: + Selects all the recorded methane plumes on the SRON database for the selected time frame and region + arguments: + config parsed YAML file + Returns: [[]] : list of [lat, lon] coordinates of floats + """ + + # variables from config file, specifying time period and region. + plumes = pd.DataFrame() + shapefile_path = config["ShapeFile"] + startDate = datetime.datetime.strptime(str(config["StartDate"]), "%Y%m%d") + endDate = datetime.datetime.strptime(str(config["EndDate"]), "%Y%m%d") + if endDate.year < 2023: # SRON plumes are only available beginning in 2023 + return None + custom_vectorfile = not config["CreateAutomaticRectilinearStateVectorFile"] + LatMax = config["LatMax"] + LatMin = config["LatMin"] + LonMax = config["LonMax"] + LonMin = config["LonMin"] + currentDate = startDate + + # calls the get_plumes function for every month in the selected time frame + while currentDate <= endDate: + if currentDate.year >= 2023: # SRON plumes are only available beginning in 2023 + p = get_plumes(str(currentDate.month), str(currentDate.year)) + plumes = pd.concat([plumes, pd.DataFrame(p)], ignore_index=True) + currentDate = currentDate + relativedelta(months=1) + + # filters through the dataset to remove any plumes outside the ROI + if custom_vectorfile: + plumes = shapefile_filter( + plumes, shapefile_path + ) # calls function to filter through coordinates found in shapefile + else: + plumes = rectangular_filter(plumes, LatMax, LatMin, LonMax, LonMin) + + plumes_list = plumes[["lat", "lon"]].values.tolist() + return plumes_list From 36e00779650dd4ee78d5cd4db9c007cd8e6ae8d3 Mon Sep 17 00:00:00 2001 From: laestrada Date: Fri, 18 Aug 2023 15:22:42 -0400 Subject: [PATCH 05/12] upfate correct path for upload --- run_imi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_imi.sh b/run_imi.sh index 6ee7f003..e928ec75 100755 --- a/run_imi.sh +++ b/run_imi.sh @@ -197,7 +197,7 @@ cd $InversionPath cp $ConfigFile "${RunDirs}/config_${RunName}.yml" # Upload output to S3 if specified -cd $RunDirs +cd $InversionPath python src/utilities/s3_upload.py $ConfigFile exit 0 From ee6492ec59555a76fb33b0009b4abeaa9b4ff56f Mon Sep 17 00:00:00 2001 From: laestrada Date: Fri, 18 Aug 2023 15:40:36 -0400 Subject: [PATCH 06/12] archive point sources in their own directory --- src/inversion_scripts/point_sources.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/inversion_scripts/point_sources.py b/src/inversion_scripts/point_sources.py index 5ebac32c..79c18601 100644 --- a/src/inversion_scripts/point_sources.py +++ b/src/inversion_scripts/point_sources.py @@ -1,4 +1,5 @@ # This file contains functions related to accessing point source data +import os import datetime import requests import pandas as pd @@ -83,6 +84,9 @@ def get_plumes(month, year): """ sron_url = "https://earth.sron.nl/wp-content/uploads/" # URL of the SRON database for weekly methane plumes url = sron_url + year + "/" + month.zfill(2) + write_dir = "SRON_plumes" + if not os.path.exists(write_dir): + os.makedirs(write_dir) response = requests.get(url) parser = BeautifulSoup(response.content, "html.parser") plume = pd.DataFrame() @@ -94,17 +98,16 @@ def get_plumes(month, year): dates = csvUrl.split("_v")[1] try: rcsv = requests.get(csvUrl, allow_redirects=True) - file = f"SRON_{dates}" - open(file, "wb").write( - rcsv.content - ) # downloads all of the plumes from that week in a CSV file in the current directory - df = pd.read_csv( - file - ) # reads from the csv file into a pandas dataframe + file_path = f"{SRON_plumes}/SRON_{dates}" + # downloads all of the plumes from that week in a CSV file in the current directory + open(file_path, "wb").write(rcsv.content) + # reads from the csv file into a pandas dataframe + df = pd.read_csv(file_path) plume = plume.append(df, ignore_index=True) except: print( - f"Warning: Unable to access data for csv file at {csvUrl}. The file may not exist or there may be a connection problem." + f"Warning: Unable to access data for csv file at {csvUrl}. " + + "The file may not exist or there may be a connection problem." ) return plume From 166ebb806a2051b262c686d67ae6aa5d30bfe3c1 Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Fri, 18 Aug 2023 19:43:12 +0000 Subject: [PATCH 07/12] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index 171bf364..490fe55d 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -13,6 +13,7 @@ Note: This does not include variables defined in python scripts. | BCversion | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | BlendedTROPOMI | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/write_BCs/config_write_BCs.yml', 'src/components/posterior_component/posterior.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_posterior'] | | BufferDeg | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | +| buildJacobian | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | | channels | ['envs/Harvard-Cannon/imi_env.yml'] | ['envs/Harvard-Cannon/imi_env.yml'] | [] | [] | | ClusteringMethod | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | cmd | ['src/write_BCs/run_BCs.sh', 'src/components/template_component/template.sh'] | ['src/write_BCs/run_BCs.sh', 'src/components/template_component/template.sh'] | ['setup_template'] | ['setup_template'] | @@ -20,7 +21,7 @@ Note: This does not include variables defined in python scripts. | CondaFile | ['config.yml', 'src/components/setup_component/setup.sh'] | ['config.yml'] | [] | ['setup_imi'] | | config_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | | configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | -| ConfigFile | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh'] | [] | ['create_statevector', 'reduce_dimension', 'run_period', 'setup_inversion', 'run_preview'] | +| ConfigFile | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh'] | [] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_period', 'setup_inversion', 'run_notebooks', 'run_preview'] | | ConfigPath | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | | constYr | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | | copydir | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | @@ -47,8 +48,8 @@ Note: This does not include variables defined in python scripts. | EndDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | | EndMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | FetchTROPOMI | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | -| file | ['src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/utilities/find_corrupt_files.sh'] | ['src/utilities/common.sh'] | ['imi_failed'] | ['print_stats', 'imi_failed', 'ncmax', 'ncmin'] | | FILE | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | +| file | ['src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/utilities/find_corrupt_files.sh'] | ['src/utilities/common.sh'] | ['imi_failed'] | ['print_stats', 'imi_failed', 'ncmax', 'ncmin'] | | FILE_LIST | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | FirstPeriod | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | | FirstSimSwitch | ['src/inversion_scripts/run_inversion.sh', 'src/components/inversion_component/inversion.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/inversion_component/inversion.sh'] | ['run_inversion'] | ['setup_inversion', 'run_inversion', 'run_notebooks'] | @@ -212,8 +213,8 @@ Note: This does not include variables defined in python scripts. | SpinupStart | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_imi', 'activate_observations'] | | START | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | start_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| startdate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | StartDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'setup_posterior', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi', 'setup_jacobian'] | +| startdate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | StartDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | | StartMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | state_vector_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | @@ -235,5 +236,5 @@ Note: This does not include variables defined in python scripts. | x | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['run_period', 'setup_jacobian', 'run_jacobian'] | | xstr | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh'] | ['run_period', 'setup_jacobian'] | ['setup_kf', 'run_kf', 'run_period', 'setup_jacobian', 'run_jacobian'] | | xUSE | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'run_jacobian'] | -| YEAR | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | -| Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | \ No newline at end of file +| Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | +| YEAR | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | \ No newline at end of file From e25dd5a04796d4ee3515cd0069888b5eb4cdc245 Mon Sep 17 00:00:00 2001 From: laestrada Date: Fri, 18 Aug 2023 16:36:05 -0400 Subject: [PATCH 08/12] better error handling --- src/inversion_scripts/point_sources.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/inversion_scripts/point_sources.py b/src/inversion_scripts/point_sources.py index 79c18601..3c67dbbc 100644 --- a/src/inversion_scripts/point_sources.py +++ b/src/inversion_scripts/point_sources.py @@ -98,16 +98,17 @@ def get_plumes(month, year): dates = csvUrl.split("_v")[1] try: rcsv = requests.get(csvUrl, allow_redirects=True) - file_path = f"{SRON_plumes}/SRON_{dates}" + file_path = f"{write_dir}/SRON_{dates}" # downloads all of the plumes from that week in a CSV file in the current directory open(file_path, "wb").write(rcsv.content) # reads from the csv file into a pandas dataframe df = pd.read_csv(file_path) - plume = plume.append(df, ignore_index=True) - except: + plume = pd.concat([plume, df], ignore_index=True) + except Exception as err: print( f"Warning: Unable to access data for csv file at {csvUrl}. " + "The file may not exist or there may be a connection problem." + + f"\nError message: {err}" ) return plume @@ -182,7 +183,11 @@ def SRON_plumes(config): p = get_plumes(str(currentDate.month), str(currentDate.year)) plumes = pd.concat([plumes, pd.DataFrame(p)], ignore_index=True) currentDate = currentDate + relativedelta(months=1) - + + # if no plumes found in the time frame, returns an empty list + if plumes.empty: + return [] + # filters through the dataset to remove any plumes outside the ROI if custom_vectorfile: plumes = shapefile_filter( From 6a161b1336f441e17e6c16af762d6b3675f9b3f7 Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Fri, 18 Aug 2023 20:36:42 +0000 Subject: [PATCH 09/12] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index 490fe55d..45023b6d 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -20,8 +20,8 @@ Note: This does not include variables defined in python scripts. | CondaEnv | ['config.yml', 'run_imi.sh', 'src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_inversion', 'setup_imi'] | | CondaFile | ['config.yml', 'src/components/setup_component/setup.sh'] | ['config.yml'] | [] | ['setup_imi'] | | config_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | -| configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | ConfigFile | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh'] | [] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_period', 'setup_inversion', 'run_notebooks', 'run_preview'] | +| configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | ConfigPath | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | | constYr | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | | copydir | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | @@ -43,8 +43,8 @@ Note: This does not include variables defined in python scripts. | DoSpinup | ['config.yml', 'run_imi.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_posterior', 'setup_imi', 'setup_jacobian'] | | DownloadGlobalMet | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | end_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| EndDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi'] | | enddate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| EndDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi'] | | EndDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | | EndMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | FetchTROPOMI | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | @@ -108,8 +108,8 @@ Note: This does not include variables defined in python scripts. | LonMin | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | LonMinInvDomain | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh'] | ['run_posterior', 'setup_imi'] | ['setup_posterior', 'run_posterior', 'setup_inversion', 'setup_imi', 'activate_observations'] | | Lons | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_imi', 'activate_observations'] | -| Met | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | MET | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | +| Met | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | metDir | ['src/components/template_component/template.sh', 'src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'create_statevector', 'setup_imi', 'activate_observations'] | | metUC | ['src/components/template_component/template.sh', 'src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'create_statevector', 'setup_imi', 'activate_observations'] | | mm | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | @@ -163,8 +163,8 @@ Note: This does not include variables defined in python scripts. | RegionalDir | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | RemoveGlobalMet | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | RequestedTime | ['config.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_posterior', 'reduce_dimension', 'run_inversion', 'run_preview'] | -| Res | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | RES | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | +| Res | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | RESOLUTION | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | RestartDir | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | RestartDownload | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | @@ -175,8 +175,8 @@ Note: This does not include variables defined in python scripts. | RestartFilePreviewPrefix | ['config.yml', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_preview', 'setup_imi'] | | retVal | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | | ROOT | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | -| RUNDIR | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | | runDir | ['src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_spinup', 'setup_posterior', 'run_preview', 'setup_imi', 'setup_jacobian'] | ['setup_template', 'setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | +| RUNDIR | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | | RunDirs | ['run_imi.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'setup_inversion', 'run_inversion', 'run_notebooks', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | | RunFilesPath | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | RunGEOSChem | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | @@ -213,8 +213,8 @@ Note: This does not include variables defined in python scripts. | SpinupStart | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_imi', 'activate_observations'] | | START | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | start_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| StartDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'setup_posterior', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi', 'setup_jacobian'] | | startdate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| StartDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'setup_posterior', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi', 'setup_jacobian'] | | StartDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | | StartMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | state_vector_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | From 61a211648d8673ed4c442cb79c80b20586ba488d Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Fri, 18 Aug 2023 21:58:52 +0000 Subject: [PATCH 10/12] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index 45023b6d..cf596a27 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -48,8 +48,8 @@ Note: This does not include variables defined in python scripts. | EndDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | | EndMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | FetchTROPOMI | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | -| FILE | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | | file | ['src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/utilities/find_corrupt_files.sh'] | ['src/utilities/common.sh'] | ['imi_failed'] | ['print_stats', 'imi_failed', 'ncmax', 'ncmin'] | +| FILE | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | | FILE_LIST | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | FirstPeriod | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | | FirstSimSwitch | ['src/inversion_scripts/run_inversion.sh', 'src/components/inversion_component/inversion.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/inversion_component/inversion.sh'] | ['run_inversion'] | ['setup_inversion', 'run_inversion', 'run_notebooks'] | @@ -163,8 +163,8 @@ Note: This does not include variables defined in python scripts. | RegionalDir | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | RemoveGlobalMet | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | RequestedTime | ['config.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_posterior', 'reduce_dimension', 'run_inversion', 'run_preview'] | -| RES | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | Res | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | +| RES | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | RESOLUTION | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | RestartDir | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | RestartDownload | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | @@ -175,8 +175,8 @@ Note: This does not include variables defined in python scripts. | RestartFilePreviewPrefix | ['config.yml', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_preview', 'setup_imi'] | | retVal | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | | ROOT | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | -| runDir | ['src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_spinup', 'setup_posterior', 'run_preview', 'setup_imi', 'setup_jacobian'] | ['setup_template', 'setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | | RUNDIR | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | +| runDir | ['src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_spinup', 'setup_posterior', 'run_preview', 'setup_imi', 'setup_jacobian'] | ['setup_template', 'setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | | RunDirs | ['run_imi.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'setup_inversion', 'run_inversion', 'run_notebooks', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | | RunFilesPath | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | RunGEOSChem | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | @@ -213,8 +213,8 @@ Note: This does not include variables defined in python scripts. | SpinupStart | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_imi', 'activate_observations'] | | START | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | start_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| startdate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | StartDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'setup_posterior', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi', 'setup_jacobian'] | +| startdate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | StartDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | | StartMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | state_vector_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | From 1c3b81d6b44206b7cac63f404183cff818c0f5e1 Mon Sep 17 00:00:00 2001 From: laestrada Date: Fri, 18 Aug 2023 18:06:05 -0400 Subject: [PATCH 11/12] add to PointSourceDatasets to config --- config.yml | 4 ++++ envs/Harvard-Cannon/config.harvard-cannon.yml | 4 ++++ resources/containers/container_config.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/config.yml b/config.yml index a460f1e8..23e5a657 100644 --- a/config.yml +++ b/config.yml @@ -54,6 +54,10 @@ BufferDeg: 5 LandThreshold: 0.25 OffshoreEmisThreshold: 0 +## Point source datasets +## Used for visualizations and state vector clustering +PointSourceDatasets: ["SRON"] + ## Clustering Options ReducedDimensionStateVector: false ClusteringMethod: "kmeans" diff --git a/envs/Harvard-Cannon/config.harvard-cannon.yml b/envs/Harvard-Cannon/config.harvard-cannon.yml index 2973b49f..7b9269dd 100644 --- a/envs/Harvard-Cannon/config.harvard-cannon.yml +++ b/envs/Harvard-Cannon/config.harvard-cannon.yml @@ -54,6 +54,10 @@ BufferDeg: 5 LandThreshold: 0.25 OffshoreEmisThreshold: 0 +## Point source datasets +## Used for visualizations and state vector clustering +PointSourceDatasets: ["SRON"] + ## Clustering Options ReducedDimensionStateVector: false ClusteringMethod: "kmeans" diff --git a/resources/containers/container_config.yml b/resources/containers/container_config.yml index 6170389b..0d9a4d33 100644 --- a/resources/containers/container_config.yml +++ b/resources/containers/container_config.yml @@ -54,6 +54,10 @@ BufferDeg: 5 LandThreshold: 0.25 OffshoreEmisThreshold: 0 +## Point source datasets +## Used for visualizations and state vector clustering +PointSourceDatasets: ["SRON"] + ## Clustering Options ReducedDimensionStateVector: false ClusteringMethod: "kmeans" From 19953a1728f9bc5df141423129b0e6a6953f38e3 Mon Sep 17 00:00:00 2001 From: integrated_methane_inversion Continuous Integration Date: Fri, 18 Aug 2023 22:06:41 +0000 Subject: [PATCH 12/12] Continuous Integration Manifest Generation --- src/components/shell_variable_manifest.md | 205 +++++++++++----------- 1 file changed, 103 insertions(+), 102 deletions(-) diff --git a/src/components/shell_variable_manifest.md b/src/components/shell_variable_manifest.md index cf596a27..6403dec1 100644 --- a/src/components/shell_variable_manifest.md +++ b/src/components/shell_variable_manifest.md @@ -11,142 +11,143 @@ Note: This does not include variables defined in python scripts. | BCdryrun | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | BCpath | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | BCversion | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| BlendedTROPOMI | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/write_BCs/config_write_BCs.yml', 'src/components/posterior_component/posterior.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_posterior'] | +| BlendedTROPOMI | ['config.yml', 'src/components/posterior_component/posterior.sh', 'src/write_BCs/config_write_BCs.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_posterior'] | | BufferDeg | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| buildJacobian | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | +| buildJacobian | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | | channels | ['envs/Harvard-Cannon/imi_env.yml'] | ['envs/Harvard-Cannon/imi_env.yml'] | [] | [] | | ClusteringMethod | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| cmd | ['src/write_BCs/run_BCs.sh', 'src/components/template_component/template.sh'] | ['src/write_BCs/run_BCs.sh', 'src/components/template_component/template.sh'] | ['setup_template'] | ['setup_template'] | -| CondaEnv | ['config.yml', 'run_imi.sh', 'src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_inversion', 'setup_imi'] | +| cmd | ['src/components/template_component/template.sh', 'src/write_BCs/run_BCs.sh'] | ['src/components/template_component/template.sh', 'src/write_BCs/run_BCs.sh'] | ['setup_template'] | ['setup_template'] | +| CondaEnv | ['run_imi.sh', 'config.yml', 'src/components/setup_component/setup.sh', 'src/components/inversion_component/inversion.sh', 'src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi', 'run_inversion'] | | CondaFile | ['config.yml', 'src/components/setup_component/setup.sh'] | ['config.yml'] | [] | ['setup_imi'] | -| config_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | -| ConfigFile | ['run_imi.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh'] | ['run_imi.sh'] | [] | ['create_statevector', 'reduce_dimension', 'setup_kf', 'run_period', 'setup_inversion', 'run_notebooks', 'run_preview'] | +| config_path | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh'] | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh'] | ['run_preview', 'create_statevector', 'reduce_dimension'] | ['run_preview', 'create_statevector', 'reduce_dimension'] | +| ConfigFile | ['run_imi.sh', 'src/components/preview_component/preview.sh', 'src/components/kalman_component/kalman.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh'] | ['run_imi.sh'] | [] | ['run_preview', 'setup_kf', 'run_period', 'create_statevector', 'reduce_dimension', 'setup_inversion', 'run_notebooks'] | | configFile | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | ConfigPath | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | -| constYr | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | +| constYr | ['src/components/setup_component/setup.sh', 'src/components/statevector_component/statevector.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'create_statevector'] | | copydir | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | | corrupt | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | ['report'] | ['download_aws_files', 'report'] | | cpu_count | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | cpu_str | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | CreateAutomaticRectilinearStateVectorFile | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| DataPath | ['config.yml', 'src/components/template_component/template.sh', 'src/components/statevector_component/statevector.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'create_statevector'] | +| DataPath | ['config.yml', 'src/components/statevector_component/statevector.sh', 'src/components/template_component/template.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['create_statevector', 'setup_template'] | | DataPathTROPOMI | ['run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | dependencies | ['envs/Harvard-Cannon/imi_env.yml'] | ['envs/Harvard-Cannon/imi_env.yml'] | [] | [] | | DEST | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | dir | ['src/write_BCs/run_BCs.sh'] | [] | [] | [] | | dirs | ['src/write_BCs/run_BCs.sh'] | ['src/write_BCs/run_BCs.sh'] | [] | [] | | DOFSThreshold | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| DoInversion | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_kf'] | -| DoJacobian | ['config.yml', 'run_imi.sh', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_kf'] | -| DoPosterior | ['config.yml', 'run_imi.sh', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_kf'] | +| DoInversion | ['run_imi.sh', 'config.yml', 'src/components/kalman_component/kalman.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_kf'] | +| DoJacobian | ['run_imi.sh', 'config.yml', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_kf'] | +| DoPosterior | ['run_imi.sh', 'config.yml', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_kf'] | | DoPreview | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| DoSpinup | ['config.yml', 'run_imi.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_posterior', 'setup_imi', 'setup_jacobian'] | +| DoSpinup | ['run_imi.sh', 'config.yml', 'src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi', 'setup_posterior', 'setup_jacobian'] | | DownloadGlobalMet | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | end_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| enddate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | -| EndDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi'] | -| EndDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | +| EndDate | ['run_imi.sh', 'config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/template_component/template.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup', 'setup_imi', 'run_preview', 'setup_kf', 'run_posterior', 'setup_template'] | +| enddate | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| EndDate_i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh'] | ['run_period', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'run_inversion'] | | EndMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | FetchTROPOMI | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | -| file | ['src/utilities/common.sh', 'src/utilities/crop_met.sh', 'src/utilities/find_corrupt_files.sh'] | ['src/utilities/common.sh'] | ['imi_failed'] | ['print_stats', 'imi_failed', 'ncmax', 'ncmin'] | -| FILE | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | +| file | ['src/utilities/find_corrupt_files.sh', 'src/utilities/common.sh', 'src/utilities/crop_met.sh'] | ['src/utilities/common.sh'] | ['imi_failed'] | ['print_stats', 'imi_failed', 'ncmax', 'ncmin'] | +| FILE | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | | FILE_LIST | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | FirstPeriod | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | -| FirstSimSwitch | ['src/inversion_scripts/run_inversion.sh', 'src/components/inversion_component/inversion.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/inversion_component/inversion.sh'] | ['run_inversion'] | ['setup_inversion', 'run_inversion', 'run_notebooks'] | +| FirstSimSwitch | ['src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_inversion'] | ['setup_inversion', 'run_inversion', 'run_notebooks'] | | ForcedNativeResolutionElements | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | fOut | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | fs | ['src/utilities/parse_yaml.sh'] | [] | [] | ['parse_yaml'] | -| fullBCpath | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_imi', 'activate_observations'] | +| fullBCpath | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'setup_template'] | | Gamma | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | gc_enddate | ['src/write_BCs/run_BCs.sh'] | ['src/write_BCs/run_BCs.sh'] | [] | [] | -| GCClassicPath | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_imi', 'activate_observations'] | -| GCDir | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | -| GCsourcepth | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | -| GEOSChemEnv | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'run_spinup', 'run_posterior', 'run_preview', 'setup_imi', 'run_jacobian'] | +| GCClassicPath | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'setup_template'] | +| GCDir | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | +| GCsourcepth | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | +| GEOSChemEnv | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh', 'src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['src/write_BCs/config_write_BCs.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'setup_imi', 'run_preview', 'run_posterior', 'run_jacobian', 'setup_template'] | | GlobalDir | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | GOSAT | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['activate_observations'] | | GriddedPosterior | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | -| gridDir | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | -| gridRes | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | -| gridResLong | ['src/components/template_component/template.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_inversion', 'setup_imi', 'activate_observations'] | +| gridDir | ['src/components/setup_component/setup.sh', 'src/components/statevector_component/statevector.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'create_statevector'] | +| gridRes | ['src/components/setup_component/setup.sh', 'src/components/statevector_component/statevector.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'create_statevector'] | +| gridResLong | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'setup_template', 'setup_inversion'] | | HemcoDiagFile | ['src/components/statevector_component/statevector.sh'] | ['src/components/statevector_component/statevector.sh'] | ['create_statevector'] | ['create_statevector', 'reduce_dimension'] | | HOME | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | -| HourlyCH4 | ['config.yml', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'setup_posterior', 'setup_jacobian'] | -| i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/jacobian_component/jacobian.sh'] | [] | [] | ['run_posterior', 'run_period', 'run_inversion', 'run_jacobian'] | -| imidir | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| HourlyCH4 | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup', 'setup_posterior', 'setup_jacobian', 'setup_template'] | +| i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/inversion_component/inversion.sh'] | [] | [] | ['run_period', 'run_posterior', 'run_jacobian', 'run_inversion'] | +| imidir | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | indent | ['src/utilities/parse_yaml.sh'] | ['src/utilities/parse_yaml.sh'] | ['parse_yaml'] | ['parse_yaml'] | | InPath | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | -| inversion_end | ['src/components/inversion_component/inversion.sh', 'src/utilities/common.sh'] | ['src/components/inversion_component/inversion.sh'] | ['run_inversion'] | ['setup_inversion', 'run_inversion', 'run_notebooks', 'print_stats'] | -| inversion_start | ['src/components/inversion_component/inversion.sh', 'src/utilities/common.sh'] | ['src/components/inversion_component/inversion.sh'] | ['run_inversion'] | ['setup_inversion', 'run_inversion', 'run_notebooks', 'print_stats'] | +| inversion_end | ['src/utilities/common.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/inversion_component/inversion.sh'] | ['run_inversion'] | ['print_stats', 'setup_inversion', 'run_inversion', 'run_notebooks'] | +| inversion_start | ['src/utilities/common.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/inversion_component/inversion.sh'] | ['run_inversion'] | ['print_stats', 'setup_inversion', 'run_inversion', 'run_notebooks'] | | InversionDir | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | -| InversionPath | ['run_imi.sh', 'src/components/template_component/template.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'src/utilities/common.sh'] | ['run_imi.sh'] | [] | ['setup_template', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_kf', 'run_period', 'setup_inversion', 'run_preview', 'setup_imi', 'setup_jacobian', 'run_jacobian', 'imi_failed'] | +| InversionPath | ['run_imi.sh', 'src/utilities/common.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh', 'src/components/inversion_component/inversion.sh'] | ['run_imi.sh'] | [] | ['imi_failed', 'setup_imi', 'run_preview', 'setup_kf', 'run_period', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_jacobian', 'run_jacobian', 'setup_template', 'setup_inversion'] | | invPath | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | InvPeriodLength | ['src/components/template_component/template.sh'] | ['src/components/template_component/template.sh'] | ['setup_template'] | ['setup_template'] | -| isAWS | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'run_spinup', 'run_posterior', 'create_statevector', 'run_inversion', 'run_preview', 'setup_imi', 'run_jacobian'] | -| isPost | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | +| isAWS | ['run_imi.sh', 'config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'setup_imi', 'run_preview', 'run_posterior', 'create_statevector', 'run_jacobian', 'setup_template', 'run_inversion'] | +| isPost | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | | ithDates | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | | ithLine | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_period'] | ['setup_kf', 'run_kf', 'run_period'] | -| jacobian_end | ['src/components/jacobian_component/jacobian.sh', 'src/utilities/common.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['run_jacobian'] | ['setup_jacobian', 'run_jacobian', 'print_stats'] | +| jacobian_end | ['src/utilities/common.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['run_jacobian'] | ['print_stats', 'setup_jacobian', 'run_jacobian'] | | jacobian_sf | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | -| jacobian_start | ['src/components/jacobian_component/jacobian.sh', 'src/utilities/common.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['run_jacobian'] | ['setup_jacobian', 'run_jacobian', 'print_stats'] | +| jacobian_start | ['src/utilities/common.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['run_jacobian'] | ['print_stats', 'setup_jacobian', 'run_jacobian'] | | JacobianCPUs | ['config.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | JacobianDir | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | JacobianMemory | ['config.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| JacobianRunsDir | ['src/inversion_scripts/run_inversion.sh', 'src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period', 'run_jacobian'] | -| KalmanMode | ['config.yml', 'run_imi.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'run_posterior', 'setup_inversion', 'run_inversion', 'run_preview', 'run_jacobian'] | +| JacobianRunsDir | ['src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period', 'run_jacobian'] | +| KalmanMode | ['run_imi.sh', 'config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh', 'src/components/inversion_component/inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup', 'run_preview', 'run_posterior', 'run_jacobian', 'setup_template', 'setup_inversion', 'run_inversion'] | | LandCoverFile | ['src/components/statevector_component/statevector.sh'] | ['src/components/statevector_component/statevector.sh'] | ['create_statevector'] | ['create_statevector', 'reduce_dimension'] | -| LandCoverFileExtension | ['src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['create_statevector', 'setup_imi', 'activate_observations'] | +| LandCoverFileExtension | ['src/components/setup_component/setup.sh', 'src/components/statevector_component/statevector.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'create_statevector'] | | LandThreshold | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | LatMax | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| LatMaxInvDomain | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh'] | ['run_posterior', 'setup_imi'] | ['setup_posterior', 'run_posterior', 'setup_inversion', 'setup_imi', 'activate_observations'] | +| LatMaxInvDomain | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['setup_imi', 'run_posterior'] | ['setup_imi', 'activate_observations', 'setup_posterior', 'run_posterior', 'setup_inversion'] | | LatMin | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| LatMinInvDomain | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh'] | ['run_posterior', 'setup_imi'] | ['setup_posterior', 'run_posterior', 'setup_inversion', 'setup_imi', 'activate_observations'] | -| Lats | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_imi', 'activate_observations'] | -| LINENO | ['run_imi.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/jacobian_component/jacobian.sh'] | [] | [] | ['run_spinup', 'run_posterior', 'run_inversion', 'run_preview', 'run_jacobian'] | +| LatMinInvDomain | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['setup_imi', 'run_posterior'] | ['setup_imi', 'activate_observations', 'setup_posterior', 'run_posterior', 'setup_inversion'] | +| Lats | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'setup_template'] | +| LINENO | ['run_imi.sh', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/inversion_component/inversion.sh'] | [] | [] | ['run_spinup', 'run_preview', 'run_posterior', 'run_jacobian', 'run_inversion'] | | LonMax | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| LonMaxInvDomain | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh'] | ['run_posterior', 'setup_imi'] | ['setup_posterior', 'run_posterior', 'setup_inversion', 'setup_imi', 'activate_observations'] | +| LonMaxInvDomain | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['setup_imi', 'run_posterior'] | ['setup_imi', 'activate_observations', 'setup_posterior', 'run_posterior', 'setup_inversion'] | | LonMin | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| LonMinInvDomain | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh'] | ['run_posterior', 'setup_imi'] | ['setup_posterior', 'run_posterior', 'setup_inversion', 'setup_imi', 'activate_observations'] | -| Lons | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_imi', 'activate_observations'] | +| LonMinInvDomain | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['setup_imi', 'run_posterior'] | ['setup_imi', 'activate_observations', 'setup_posterior', 'run_posterior', 'setup_inversion'] | +| Lons | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'setup_template'] | | MET | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | Met | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| metDir | ['src/components/template_component/template.sh', 'src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'create_statevector', 'setup_imi', 'activate_observations'] | -| metUC | ['src/components/template_component/template.sh', 'src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'create_statevector', 'setup_imi', 'activate_observations'] | +| metDir | ['src/components/setup_component/setup.sh', 'src/components/statevector_component/statevector.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'create_statevector', 'setup_template'] | +| metUC | ['src/components/setup_component/setup.sh', 'src/components/statevector_component/statevector.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'create_statevector', 'setup_template'] | | mm | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | Month | ['src/utilities/crop_met.sh'] | [] | [] | [] | | name | ['src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/imi_env.yml'] | ['src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/imi_env.yml'] | ['setup_jacobian'] | ['setup_jacobian', 'run_jacobian'] | -| native | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_imi', 'activate_observations'] | +| native | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations', 'setup_template'] | | native_state_vector_path | ['src/components/statevector_component/statevector.sh'] | ['src/components/statevector_component/statevector.sh'] | ['reduce_dimension'] | ['create_statevector', 'reduce_dimension'] | | nBufferClusters | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| nElements | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/setup_component/setup.sh'] | ['run_posterior', 'reduce_dimension', 'setup_kf', 'setup_imi'] | ['setup_posterior', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'setup_inversion', 'setup_imi', 'activate_observations', 'setup_jacobian'] | +| nElements | ['src/components/setup_component/setup.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/setup_component/setup.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/inversion_scripts/run_inversion.sh'] | ['setup_imi', 'setup_kf', 'run_posterior', 'reduce_dimension'] | ['setup_imi', 'activate_observations', 'setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_jacobian', 'setup_inversion'] | | NestedGrid | ['config.yml', 'src/components/template_component/template.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template'] | -| NestedRegion | ['config.yml', 'src/components/template_component/template.sh', 'src/components/statevector_component/statevector.sh', 'src/components/setup_component/setup.sh', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'create_statevector', 'setup_imi'] | -| NEW | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['setup_template', 'activate_observations'] | ['setup_template', 'setup_imi', 'activate_observations'] | +| NestedRegion | ['config.yml', 'src/utilities/crop_met.sh', 'src/components/setup_component/setup.sh', 'src/components/statevector_component/statevector.sh', 'src/components/template_component/template.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/utilities/crop_met.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi', 'create_statevector', 'setup_template'] | +| NEW | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['activate_observations', 'setup_template'] | ['setup_imi', 'activate_observations', 'setup_template'] | | nPeriods | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | | NudgeFactor | ['config.yml', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_period'] | | NumberOfElements | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | ObsError | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | | OffshoreEmisThreshold | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| OLD | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['src/components/template_component/template.sh', 'src/components/setup_component/setup.sh'] | ['setup_template', 'activate_observations'] | ['setup_template', 'setup_imi', 'activate_observations'] | +| OLD | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh', 'src/components/template_component/template.sh'] | ['activate_observations', 'setup_template'] | ['setup_imi', 'activate_observations', 'setup_template'] | | output | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | -| output_log_file | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | -| OutputPath | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/inversion_component/inversion.sh', 'src/components/setup_component/setup.sh', 'src/utilities/common.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_inversion', 'setup_imi', 'imi_failed'] | -| Partition | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| output_log_file | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | +| OutputPath | ['run_imi.sh', 'config.yml', 'src/utilities/common.sh', 'src/components/setup_component/setup.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['imi_failed', 'setup_imi', 'setup_inversion'] | +| Partition | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | PeriodsFile | ['src/components/kalman_component/kalman.sh'] | ['src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | | Perturbation | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | PerturbValue | ['config.yml', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_jacobian'] | | PLANEFLIGHT | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['activate_observations'] | -| posterior_end | ['src/components/posterior_component/posterior.sh', 'src/utilities/common.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior', 'print_stats'] | -| posterior_start | ['src/components/posterior_component/posterior.sh', 'src/utilities/common.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior', 'print_stats'] | +| PointSourceDatasets | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | +| posterior_end | ['src/utilities/common.sh', 'src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['print_stats', 'setup_posterior', 'run_posterior'] | +| posterior_start | ['src/utilities/common.sh', 'src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['print_stats', 'setup_posterior', 'run_posterior'] | | PosteriorDryRun | ['config.yml', 'src/components/posterior_component/posterior.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_posterior'] | | PosteriorName | ['src/components/posterior_component/posterior.sh'] | ['src/components/posterior_component/posterior.sh'] | ['setup_posterior'] | ['setup_posterior', 'run_posterior'] | -| PosteriorRunDir | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_kf'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period'] | +| PosteriorRunDir | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_kf', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior'] | | posteriorSF | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | -| PrecomputedJacobian | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_jacobian'] | +| PrecomputedJacobian | ['config.yml', 'src/components/jacobian_component/jacobian.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_jacobian'] | | precomputedJacobianCache | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['run_jacobian'] | ['setup_jacobian', 'run_jacobian'] | | prefix | ['src/utilities/parse_yaml.sh'] | [] | [] | ['parse_yaml'] | -| PrevDir | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | -| preview_dir | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | +| PrevDir | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_posterior'] | ['setup_posterior', 'run_posterior'] | +| preview_dir | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh'] | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh'] | ['run_preview', 'reduce_dimension'] | ['run_preview', 'create_statevector', 'reduce_dimension'] | | preview_end | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | preview_file | ['src/components/preview_component/preview.sh'] | ['src/components/preview_component/preview.sh'] | ['run_preview'] | ['run_preview'] | | preview_start | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | @@ -154,7 +155,7 @@ Note: This does not include variables defined in python scripts. | PreviewEnd | ['src/components/preview_component/preview.sh'] | ['src/components/preview_component/preview.sh'] | ['run_preview'] | ['run_preview'] | | PreviewName | ['src/components/preview_component/preview.sh'] | ['src/components/preview_component/preview.sh'] | ['run_preview'] | ['run_preview'] | | PriorError | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| PriorRunDir | ['src/inversion_scripts/run_inversion.sh', 'src/components/kalman_component/kalman.sh'] | ['src/inversion_scripts/run_inversion.sh', 'src/components/kalman_component/kalman.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | +| PriorRunDir | ['src/components/kalman_component/kalman.sh', 'src/inversion_scripts/run_inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_kf'] | ['setup_kf', 'run_kf', 'run_period'] | | ProcessConstantFields | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | ProductionDryRun | ['config.yml', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_jacobian'] | | PYTHONPATH | ['run_imi.sh', 'src/components/kalman_component/kalman.sh'] | [] | [] | ['run_period'] | @@ -162,79 +163,79 @@ Note: This does not include variables defined in python scripts. | ReferenceRunDir | ['src/components/jacobian_component/jacobian.sh'] | [] | [] | ['run_jacobian'] | | RegionalDir | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | RemoveGlobalMet | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | -| RequestedTime | ['config.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_posterior', 'reduce_dimension', 'run_inversion', 'run_preview'] | -| Res | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | +| RequestedTime | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_preview', 'run_posterior', 'reduce_dimension', 'run_inversion'] | +| Res | ['config.yml', 'src/components/setup_component/setup.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | RES | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | | RESOLUTION | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | -| RestartDir | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| RestartDir | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | | RestartDownload | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| RestartFile | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_spinup', 'setup_posterior', 'setup_imi', 'setup_jacobian'] | ['setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | +| RestartFile | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_spinup', 'setup_imi', 'setup_posterior', 'setup_jacobian'] | ['setup_spinup', 'run_spinup', 'setup_imi', 'activate_observations', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'run_jacobian'] | | RestartFileFromSpinup | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_posterior', 'setup_jacobian'] | ['setup_posterior', 'run_posterior', 'setup_jacobian', 'run_jacobian'] | -| RestartFilePrefix | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup', 'setup_posterior', 'setup_imi', 'setup_jacobian'] | -| RestartFilePreview | ['src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh'] | ['src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh'] | ['run_preview', 'setup_imi'] | ['run_preview', 'setup_imi', 'activate_observations'] | -| RestartFilePreviewPrefix | ['config.yml', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_preview', 'setup_imi'] | -| retVal | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | +| RestartFilePrefix | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup', 'setup_imi', 'setup_posterior', 'setup_jacobian'] | +| RestartFilePreview | ['src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh'] | ['src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh'] | ['setup_imi', 'run_preview'] | ['setup_imi', 'activate_observations', 'run_preview'] | +| RestartFilePreviewPrefix | ['config.yml', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi', 'run_preview'] | +| retVal | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | | ROOT | ['src/components/template_component/template.sh'] | [] | [] | ['setup_template'] | -| RUNDIR | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh'] | [] | [] | -| runDir | ['src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_spinup', 'setup_posterior', 'run_preview', 'setup_imi', 'setup_jacobian'] | ['setup_template', 'setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | -| RunDirs | ['run_imi.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_kf', 'run_kf', 'run_period', 'setup_inversion', 'run_inversion', 'run_notebooks', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian', 'run_jacobian'] | +| RUNDIR | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | +| runDir | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh'] | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_spinup', 'setup_imi', 'run_preview', 'setup_posterior', 'setup_jacobian'] | ['setup_spinup', 'run_spinup', 'setup_imi', 'activate_observations', 'run_preview', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'run_jacobian', 'setup_template'] | +| RunDirs | ['run_imi.sh', 'src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh', 'src/components/inversion_component/inversion.sh'] | ['run_imi.sh', 'src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'run_spinup', 'setup_imi', 'activate_observations', 'run_preview', 'setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'create_statevector', 'reduce_dimension', 'setup_jacobian', 'run_jacobian', 'setup_template', 'setup_inversion', 'run_inversion', 'run_notebooks'] | | RunFilesPath | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | -| RunGEOSChem | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | -| RunName | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'src/utilities/common.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup', 'run_spinup', 'setup_posterior', 'run_posterior', 'run_kf', 'run_period', 'setup_inversion', 'run_preview', 'setup_imi', 'setup_jacobian', 'imi_failed'] | -| RunSetup | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| RunTemplate | ['src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_template', 'setup_spinup', 'setup_posterior', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian'] | +| RunGEOSChem | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| RunName | ['run_imi.sh', 'config.yml', 'src/utilities/common.sh', 'src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/inversion_component/inversion.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['imi_failed', 'setup_spinup', 'run_spinup', 'setup_imi', 'run_preview', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'setup_inversion'] | +| RunSetup | ['run_imi.sh', 'config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | +| RunTemplate | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_imi', 'activate_observations', 'run_preview', 'setup_posterior', 'setup_jacobian', 'setup_template'] | | s | ['src/utilities/parse_yaml.sh'] | [] | [] | ['parse_yaml'] | | S3_BUCKET | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | s3_hd_path | ['src/components/statevector_component/statevector.sh'] | ['src/components/statevector_component/statevector.sh'] | ['create_statevector'] | ['create_statevector', 'reduce_dimension'] | | s3_lc_path | ['src/components/statevector_component/statevector.sh'] | ['src/components/statevector_component/statevector.sh'] | ['create_statevector'] | ['create_statevector', 'reduce_dimension'] | | S3_PATH | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | | S3Upload | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| SafeMode | ['config.yml', 'run_imi.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| SchedulerPartition | ['config.yml', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_posterior', 'reduce_dimension', 'run_inversion', 'run_preview'] | +| SafeMode | ['run_imi.sh', 'config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | +| SchedulerPartition | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/geoschem_run_scripts/submit_jacobian_simulations_array.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_preview', 'run_posterior', 'reduce_dimension', 'run_inversion'] | | sensiCache | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | setup_end | ['run_imi.sh', 'src/utilities/common.sh'] | ['run_imi.sh'] | [] | ['print_stats'] | | setup_start | ['run_imi.sh', 'src/utilities/common.sh'] | ['run_imi.sh'] | [] | ['print_stats'] | -| SetupInversion | ['config.yml', 'run_imi.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| SetupJacobianRuns | ['config.yml', 'run_imi.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| SetupPosteriorRun | ['config.yml', 'run_imi.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | -| SetupSpinupRun | ['config.yml', 'run_imi.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | +| SetupInversion | ['run_imi.sh', 'config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | +| SetupJacobianRuns | ['run_imi.sh', 'config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | +| SetupPosteriorRun | ['run_imi.sh', 'config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | +| SetupSpinupRun | ['run_imi.sh', 'config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | SetupTemplateRundir | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | ShapeFile | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | [] | -| SimulationCPUs | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_posterior', 'reduce_dimension', 'run_inversion', 'run_preview'] | -| SimulationMemory | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'src/components/preview_component/preview.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_posterior', 'reduce_dimension', 'run_inversion', 'run_preview'] | +| SimulationCPUs | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_preview', 'run_posterior', 'reduce_dimension', 'run_inversion'] | +| SimulationMemory | ['config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/statevector_component/statevector.sh', 'src/components/inversion_component/inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['run_spinup', 'run_preview', 'run_posterior', 'reduce_dimension', 'run_inversion'] | | SLURM_ARRAY_TASK_ID | ['src/geoschem_run_scripts/run_jacobian_simulations.sh'] | [] | [] | [] | -| spinup_end | ['src/components/spinup_component/spinup.sh', 'src/utilities/common.sh'] | ['src/components/spinup_component/spinup.sh'] | ['run_spinup'] | ['setup_spinup', 'run_spinup', 'print_stats'] | -| spinup_start | ['src/components/spinup_component/spinup.sh', 'src/utilities/common.sh'] | ['src/components/spinup_component/spinup.sh'] | ['run_spinup'] | ['setup_spinup', 'run_spinup', 'print_stats'] | +| spinup_end | ['src/utilities/common.sh', 'src/components/spinup_component/spinup.sh'] | ['src/components/spinup_component/spinup.sh'] | ['run_spinup'] | ['print_stats', 'setup_spinup', 'run_spinup'] | +| spinup_start | ['src/utilities/common.sh', 'src/components/spinup_component/spinup.sh'] | ['src/components/spinup_component/spinup.sh'] | ['run_spinup'] | ['print_stats', 'setup_spinup', 'run_spinup'] | | SpinupDir | ['src/inversion_scripts/run_inversion.sh'] | ['src/inversion_scripts/run_inversion.sh'] | [] | [] | | SpinupDryrun | ['config.yml', 'src/components/spinup_component/spinup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup'] | -| SpinupEnd | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_posterior', 'setup_imi', 'activate_observations', 'setup_jacobian'] | +| SpinupEnd | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_imi', 'activate_observations', 'setup_posterior', 'setup_jacobian'] | | SpinupMonths | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi'] | | SpinupName | ['src/components/spinup_component/spinup.sh'] | ['src/components/spinup_component/spinup.sh'] | ['setup_spinup'] | ['setup_spinup', 'run_spinup'] | | SpinupStart | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_imi', 'activate_observations'] | | START | ['src/components/setup_component/setup.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_imi', 'activate_observations'] | | start_time | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | -| StartDate | ['config.yml', 'run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/template_component/template.sh', 'src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template', 'setup_spinup', 'setup_posterior', 'run_posterior', 'setup_kf', 'run_preview', 'setup_imi', 'setup_jacobian'] | -| startdate | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | -| StartDate_i | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/posterior_component/posterior.sh', 'src/components/kalman_component/kalman.sh'] | ['run_posterior', 'run_period'] | ['setup_posterior', 'run_posterior', 'setup_kf', 'run_kf', 'run_period', 'run_inversion'] | +| startdate | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| StartDate | ['run_imi.sh', 'config.yml', 'src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh', 'src/components/template_component/template.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_spinup', 'setup_imi', 'run_preview', 'setup_kf', 'setup_posterior', 'run_posterior', 'setup_jacobian', 'setup_template'] | +| StartDate_i | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh', 'src/components/inversion_component/inversion.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/posterior_component/posterior.sh'] | ['run_period', 'run_posterior'] | ['setup_kf', 'run_kf', 'run_period', 'setup_posterior', 'run_posterior', 'run_inversion'] | | StartMonth | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | -| state_vector_path | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | -| StateVectorFile | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/components/kalman_component/kalman.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/inversion_scripts/run_inversion.sh', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['setup_kf'] | ['setup_kf', 'run_kf', 'run_period', 'setup_imi'] | +| state_vector_path | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh'] | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh'] | ['run_preview', 'reduce_dimension'] | ['run_preview', 'create_statevector', 'reduce_dimension'] | +| StateVectorFile | ['config.yml', 'src/components/setup_component/setup.sh', 'src/components/kalman_component/kalman.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'src/components/kalman_component/kalman.sh', 'src/inversion_scripts/run_inversion.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['setup_kf'] | ['setup_imi', 'setup_kf', 'run_kf', 'run_period'] | | StateVectorFName | ['src/components/statevector_component/statevector.sh'] | ['src/components/statevector_component/statevector.sh'] | ['create_statevector'] | ['create_statevector', 'reduce_dimension'] | | stdout | ['run_imi.sh'] | ['run_imi.sh'] | [] | [] | | TCCON | ['config.yml', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['activate_observations'] | -| tropomi_cache | ['src/write_BCs/config_write_BCs.yml', 'src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['src/write_BCs/config_write_BCs.yml', 'src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh'] | ['reduce_dimension', 'run_preview'] | ['create_statevector', 'reduce_dimension', 'run_preview'] | -| tropomiCache | ['run_imi.sh', 'src/inversion_scripts/run_inversion.sh', 'src/components/posterior_component/posterior.sh'] | ['run_imi.sh', 'src/inversion_scripts/run_inversion.sh'] | [] | ['run_posterior'] | +| tropomi_cache | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh', 'src/write_BCs/config_write_BCs.yml'] | ['run_preview', 'reduce_dimension'] | ['run_preview', 'create_statevector', 'reduce_dimension'] | +| tropomiCache | ['run_imi.sh', 'src/components/posterior_component/posterior.sh', 'src/inversion_scripts/run_inversion.sh'] | ['run_imi.sh', 'src/inversion_scripts/run_inversion.sh'] | [] | ['run_posterior'] | | UpdateFreqDays | ['config.yml', 'src/components/kalman_component/kalman.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_kf'] | -| UseBCsForRestart | ['src/components/spinup_component/spinup.sh', 'src/components/posterior_component/posterior.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_posterior', 'run_preview', 'setup_imi', 'activate_observations', 'setup_jacobian'] | +| UseBCsForRestart | ['src/components/spinup_component/spinup.sh', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/posterior_component/posterior.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/components/setup_component/setup.sh'] | ['setup_imi'] | ['setup_spinup', 'setup_imi', 'activate_observations', 'run_preview', 'setup_posterior', 'setup_jacobian'] | | UseEmisSF | ['config.yml', 'src/components/template_component/template.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template'] | | UseOHSF | ['config.yml', 'src/components/template_component/template.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_template'] | -| UseSlurm | ['config.yml', 'src/components/statevector_component/statevector.sh', 'src/components/preview_component/preview.sh', 'src/components/setup_component/setup.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['reduce_dimension', 'run_preview', 'setup_imi'] | +| UseSlurm | ['config.yml', 'src/components/setup_component/setup.sh', 'src/components/preview_component/preview.sh', 'src/components/statevector_component/statevector.sh', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | ['config.yml', 'envs/Harvard-Cannon/config.harvard-cannon.yml'] | [] | ['setup_imi', 'run_preview', 'reduce_dimension'] | | vn | ['src/utilities/parse_yaml.sh'] | ['src/utilities/parse_yaml.sh'] | ['parse_yaml'] | ['parse_yaml'] | | w | ['src/utilities/parse_yaml.sh'] | [] | [] | ['parse_yaml'] | -| workdir | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | -| WriteBCs | ['src/write_BCs/run_BCs.sh', 'src/write_BCs/config_write_BCs.yml'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | -| x | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['run_period', 'setup_jacobian', 'run_jacobian'] | -| xstr | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh'] | ['src/geoschem_run_scripts/run_jacobian_simulations.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh'] | ['run_period', 'setup_jacobian'] | ['setup_kf', 'run_kf', 'run_period', 'setup_jacobian', 'run_jacobian'] | +| workdir | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| WriteBCs | ['src/write_BCs/config_write_BCs.yml', 'src/write_BCs/run_BCs.sh'] | ['src/write_BCs/config_write_BCs.yml'] | [] | [] | +| x | ['src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/components/jacobian_component/jacobian.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['setup_jacobian'] | ['run_period', 'setup_jacobian', 'run_jacobian'] | +| xstr | ['src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['src/components/kalman_component/kalman.sh', 'src/components/jacobian_component/jacobian.sh', 'src/geoschem_run_scripts/run_prior_simulation.sh', 'src/geoschem_run_scripts/run_jacobian_simulations.sh'] | ['run_period', 'setup_jacobian'] | ['setup_kf', 'run_kf', 'run_period', 'setup_jacobian', 'run_jacobian'] | | xUSE | ['src/components/jacobian_component/jacobian.sh'] | ['src/components/jacobian_component/jacobian.sh'] | ['setup_jacobian'] | ['setup_jacobian', 'run_jacobian'] | | Year | ['src/utilities/crop_met.sh'] | ['src/utilities/crop_met.sh'] | [] | [] | | YEAR | ['src/utilities/find_corrupt_files.sh'] | ['src/utilities/find_corrupt_files.sh'] | [] | ['download_aws_files', 'report'] | \ No newline at end of file