From fa0a52467897ea96381c7b2cfdbfdd84d16eb520 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Thu, 24 Oct 2024 09:27:02 +0200 Subject: [PATCH 01/15] root: start of tidy streams From bd39ec3e1064e8be83fe194d3adeae9a36984f8c Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Thu, 24 Oct 2024 09:28:24 +0200 Subject: [PATCH 02/15] chore: cleanup imports in tidy --- src/esm_runscripts/tidy.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/esm_runscripts/tidy.py b/src/esm_runscripts/tidy.py index 185cf68fa..4f2940955 100644 --- a/src/esm_runscripts/tidy.py +++ b/src/esm_runscripts/tidy.py @@ -1,15 +1,12 @@ import filecmp import os import pathlib -import re import shutil import sys -import time -import psutil from loguru import logger -from . import coupler, database_actions, helpers, logfiles +from . import helpers, logfiles from .filelists import copy_files, resolve_symlinks From 02420918323cc40db0f9c26c270b06b766262cee Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Thu, 24 Oct 2024 10:39:22 +0200 Subject: [PATCH 03/15] wip --- configs/components/echam/echam.yaml | 9 ++++---- src/esm_runscripts/echam.py | 32 +++++++++++++++++++++++++++++ src/esm_runscripts/filelists.py | 14 +++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 src/esm_runscripts/echam.py diff --git a/configs/components/echam/echam.yaml b/configs/components/echam/echam.yaml index 23d2c4f61..1029060f4 100644 --- a/configs/components/echam/echam.yaml +++ b/configs/components/echam/echam.yaml @@ -79,7 +79,7 @@ compile_infos: make -j `nproc --all`; make install -j `nproc --all`; install_bins: '' branch: esm-tools - git-repository: https://git.geomar.de/foci/src/echam.git + git-repository: https://git.geomar.de/foci/src/echam.git 6.3.05p2-foci_oasismct4: branch: esm-tools-oasis3mct4 git-repository: https://git.geomar.de/foci/src/echam.git @@ -183,6 +183,7 @@ adj_input_dir: "${input_dir}/${resolution}" forcing_dir: "${input_dir}/${resolution}" greenhouse_dir: "${pool_dir}" namelist_dir: "${general.esm_namelist_dir}/echam/${version}/${scenario_type}" +has_namelist_streams: True # ECHAM has output filenames defined somehow in the namelist switch: 1 value : "echam default" @@ -585,7 +586,7 @@ choose_with_lco2_emis: co2ctl: lco2_emis: true # read carbon emissions; need carbon_emissions.nc in work lco2_2perc: true # limit maximum co2 growth rate to 2% wrt previous time step - add_choose_scenario: # other loop order (1:add_forcing_files 2:add_choose_scenario) does not work + add_choose_scenario: # other loop order (1:add_forcing_files 2:add_choose_scenario) does not work HIST: add_forcing_files: carbon_emissions: carbon_emissions_hist @@ -617,7 +618,7 @@ choose_icb_code: namelist.echam: submodelctl: licb: "${licb}" - + choose_wiso_code: True: @@ -687,7 +688,7 @@ forcing_in_work: sic: "unit.96" sst: "unit.20" # ok this is another crazy ECHAM6 speciality - # every year the background aerosol files for 1849 to 1851 + # every year the background aerosol files for 1849 to 1851 # need to be linked to the same file for historical/scenario runs # don't blame me (seb-wahl), blame the MAC-SP developers at MPI # MAC-SP describes aerosol w.r.t. piControl conditions. diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py new file mode 100644 index 000000000..eb90d0a14 --- /dev/null +++ b/src/esm_runscripts/echam.py @@ -0,0 +1,32 @@ +import f90nml +from loguru import logger + + +def _get_mvstream_tags_from_namelist(namelist): + mvstream_tags = [] + if not isinstance(namelist, f90nml.namelist.Namelist): + try: + namelist = f90nml.read(namelist) + except FileNotFoundError: + logger.error(f"Namelist specified by {namelist} could not be found") + except TypeError as e: + logger.error( + f"Could not convert {namelist} to f90nml.namelist.Namelist object." + ) + raise e + + for chapter, contents in namelist.items(): + if chapter == "mvstreamctl": + tag = contents.get("filetag") + mvstream_tags.append(tag) + return mvstream_tags + + +def append_namelist_dependent_sources(config): + expid = config["general"]["expid"] + work_dir = config["general"]["work_dir"] + econfig = config["echam"] + namelist = econfig["namelist_objs"] + mvstream_tags = _get_mvstream_tags_from_namelist(namelist) + mvstream_dict = {tag: f"{work_dir}/{expid}*_{tag}" for tag in mvstream_tags} + config["outdata_sources"].update(mvstream_dict) diff --git a/src/esm_runscripts/filelists.py b/src/esm_runscripts/filelists.py index 4da58712a..1804f0331 100644 --- a/src/esm_runscripts/filelists.py +++ b/src/esm_runscripts/filelists.py @@ -1631,9 +1631,23 @@ def get_movement(config, model, category, filetype, source, target): helpers.print_datetime(config) sys.exit(42) +def append_namelist_dependent_sources(config): + """If a model has streams defined in the one of it's namelists, append them to the sources here""" + for model in config["general"]["valid_model_names"] + ["general"]: + if config["general"][model]["has_namelist_streams"]: # Something truthy + try: + model_module = importlib.import(f"esm_runscripts.{model}") + # Important: we need to define something that is called append_namelist_dependent_sources in .py + model_module.append_namelist_dependent_sources(config) + except ImportError: + logger.error(f"Model {model} specifies that it has namelist streams, but there is module to import to handle that...") + # keep going... + return config + def assemble(config): config = complete_all_file_movements(config) + config = append_namelist_dependent_sources(config) config = rename_sources_to_targets(config) config = choose_needed_files(config) config = complete_targets(config) From c9d467f3b1a2c2c68c48ed19ebb266c5ba5e4b99 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Thu, 24 Oct 2024 11:12:27 +0200 Subject: [PATCH 04/15] feat: auto-create streams from mvtags --- src/esm_runscripts/echam.py | 16 +++++++++++----- src/esm_runscripts/filelists.py | 15 +++++++++------ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index eb90d0a14..2a2b03616 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -18,15 +18,21 @@ def _get_mvstream_tags_from_namelist(namelist): for chapter, contents in namelist.items(): if chapter == "mvstreamctl": tag = contents.get("filetag") - mvstream_tags.append(tag) + if tag is not None: + mvstream_tags.append(tag) return mvstream_tags def append_namelist_dependent_sources(config): expid = config["general"]["expid"] - work_dir = config["general"]["work_dir"] econfig = config["echam"] - namelist = econfig["namelist_objs"] + try: + namelist = econfig["namelist_objs"] + except KeyError: # Namelists not yet loaded... + namelist = f90nml.read(econfig["namelist_dir"] + "/namelist.echam") mvstream_tags = _get_mvstream_tags_from_namelist(namelist) - mvstream_dict = {tag: f"{work_dir}/{expid}*_{tag}" for tag in mvstream_tags} - config["outdata_sources"].update(mvstream_dict) + jsbach_streams = config["jsbach"]["streams"] + ignore_these_tags = econfig.get("ignore_tags", jsbach_streams) + mvstream_tags = [tag for tag in mvstream_tags if tag not in ignore_these_tags] + mvstream_dict = {tag: f"{expid}*_{tag}" for tag in mvstream_tags} + econfig["outdata_sources"].update(mvstream_dict) diff --git a/src/esm_runscripts/filelists.py b/src/esm_runscripts/filelists.py index 1804f0331..3d0a1f67e 100644 --- a/src/esm_runscripts/filelists.py +++ b/src/esm_runscripts/filelists.py @@ -3,6 +3,7 @@ import filecmp import glob import hashlib +import importlib import os import pathlib import re @@ -11,12 +12,11 @@ import f90nml import yaml +from loguru import logger import esm_parser -from loguru import logger -from . import helpers -from . import jinja +from . import helpers, jinja def rename_sources_to_targets(config): @@ -1631,16 +1631,19 @@ def get_movement(config, model, category, filetype, source, target): helpers.print_datetime(config) sys.exit(42) + def append_namelist_dependent_sources(config): """If a model has streams defined in the one of it's namelists, append them to the sources here""" for model in config["general"]["valid_model_names"] + ["general"]: - if config["general"][model]["has_namelist_streams"]: # Something truthy + if config[model].get("has_namelist_streams", False): # Something truthy try: - model_module = importlib.import(f"esm_runscripts.{model}") + model_module = importlib.import_module(f"esm_runscripts.{model}") # Important: we need to define something that is called append_namelist_dependent_sources in .py model_module.append_namelist_dependent_sources(config) except ImportError: - logger.error(f"Model {model} specifies that it has namelist streams, but there is module to import to handle that...") + logger.error( + f"Model {model} specifies that it has namelist streams, but there is module to import to handle that..." + ) # keep going... return config From c293aaa53305078dcbd26b75ed8842aa2e944507 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Thu, 24 Oct 2024 15:18:07 +0200 Subject: [PATCH 05/15] minor tweaks for globbing of new tags --- src/esm_runscripts/echam.py | 9 ++++++++- src/esm_runscripts/filelists.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index 2a2b03616..b6d1cb399 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -34,5 +34,12 @@ def append_namelist_dependent_sources(config): jsbach_streams = config["jsbach"]["streams"] ignore_these_tags = econfig.get("ignore_tags", jsbach_streams) mvstream_tags = [tag for tag in mvstream_tags if tag not in ignore_these_tags] - mvstream_dict = {tag: f"{expid}*_{tag}" for tag in mvstream_tags} + mvstream_dict = {tag: f"{expid}*{tag}" for tag in mvstream_tags} + if namelist["runctl"].get("out_filetype") == 2: + # Using NetCDF Outputs: + mvstream_dict = {k: v + ".nc" for k, v in mvstream_dict.items()} + logger.info("Updating outdata_sources...") + for k, v in mvstream_dict.items(): + logger.info(f"{k}: {v}") econfig["outdata_sources"].update(mvstream_dict) + logger.info("...done!") diff --git a/src/esm_runscripts/filelists.py b/src/esm_runscripts/filelists.py index 3d0a1f67e..360fa10b1 100644 --- a/src/esm_runscripts/filelists.py +++ b/src/esm_runscripts/filelists.py @@ -225,7 +225,7 @@ def choose_needed_files(config): new_sources = new_targets = {} for category, name in config[model][filetype + "_files"].items(): # TODO: change with user_error() - if not name in config[model][filetype + "_sources"]: + if name not in config[model][filetype + "_sources"]: logger.error( "Implementation " + name @@ -1650,9 +1650,9 @@ def append_namelist_dependent_sources(config): def assemble(config): config = complete_all_file_movements(config) - config = append_namelist_dependent_sources(config) config = rename_sources_to_targets(config) config = choose_needed_files(config) + config = append_namelist_dependent_sources(config) config = complete_targets(config) config = complete_sources(config) config = reuse_sources(config) From 5bec486118d1f5aca67f40be8512c0911dd88cab Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Fri, 25 Oct 2024 11:45:15 +0200 Subject: [PATCH 06/15] doc(esm_runscripts/echam.py): add chat-gpt generated documentation --- src/esm_runscripts/echam.py | 131 ++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index b6d1cb399..8d358420d 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -1,8 +1,118 @@ +""" +This module provides functionality specific to the ECHAM component. + +The functions included in this module are: + +- ``append_namelist_dependent_sources``: Appends namelist dependent sources to the ECHAM + configuration. + +These functions are used to manage and update the configuration settings for ECHAM, +particularly focusing on handling namelist files and updating output data sources +based on the configuration parameters. +""" + import f90nml from loguru import logger def _get_mvstream_tags_from_namelist(namelist): + """ + Extracts mvstream tags from a given namelist. + + Parameters + ---------- + namelist : str or f90nml.namelist.Namelist + The path to the namelist file or an already parsed namelist object. + + Returns + ------- + list of str + A list of mvstream tags found in the namelist. + + Raises + ------ + FileNotFoundError + If the namelist file specified by the path cannot be found. + TypeError + If the provided namelist cannot be converted to an f90nml.namelist.Namelist + object. + + Examples + -------- + Assuming you have a namelist file at ``/tmp/example/namelist.echam`` with the + following contents: + + .. code-block:: fortran + + ! This is the "no output" variant of namelist.echam. It contains absolutely no + ! output, and can be used as a template for testing. + ! + ! Extended by mvstreamctl namelist block, as an example for the mvstream tags. + ! + ! P. Gierz + ! Alfred Wegener Institute, Helmholtz Centre for Polar and Marine Research + ! July 2021 + ! + ! P. Gierz + ! October 2024 + + &runctl + dt_start = 2285, 12, 31, 23, 52, 30 + dt_stop = 6699, 12, 31, 23, 52, 30 + putrerun = 12, 'months', 'last', 0 + lfractional_mask = .false. + lresume = .true. + out_datapath = './' + out_expname = 'E280' + rerun_filetype = 4 + delta_time = 450 + putdata = 1, 'months', 'last', 0 + nproma = 8 + lcouple = .true. + getocean = 1, 'days', 'last', 0 + putocean = 1, 'days', 'last', 0 + lcouple_co2 = .true. + default_output = .false. + / + + &parctl + nproca = 24 + nprocb = 24 + / + + &submodelctl + lmethox = .true. + / + + &submdiagctl + vphysc_lpost = .false. + / + + &radctl + iaero = 3 + io3 = 4 + isolrad = 6 + ich4 = 3 + in2o = 3 + co2vmr = 284.3169860840e-06 + ch4vmr = 808.2490234375e-09 + n2ovmr = 273.0210571289e-09 + yr_perp = 1850 + / + &mvstreamctl + filetag = 'paul_custom' + source = 'g3b' + variables = 'temp2:mean>temp2=167' + interval = 1, 'months', 'last', 0 + / + + The following code will extract the mvstream tags from the namelist: + + >>> namelist_path = "/tmp/example/namelist.echam" + >>> tags = _get_mvstream_tags_from_namelist(namelist_path) + >>> print(tags) + ['paul_custom'] + """ mvstream_tags = [] if not isinstance(namelist, f90nml.namelist.Namelist): try: @@ -24,6 +134,27 @@ def _get_mvstream_tags_from_namelist(namelist): def append_namelist_dependent_sources(config): + """ + Append namelist dependent sources to the ECHAM configuration. + + This function updates the `outdata_sources` in the ECHAM configuration + based on the namelist objects and other configuration parameters. + + Parameters + ---------- + config : dict + The configuration dictionary containing general, ECHAM, and JSBACH settings. + + Notes + ----- + - The function reads the namelist from the specified directory if not + already loaded. + - It filters out tags that are to be ignored based on the JSBACH streams + or specified ignore tags. + - The output file type is checked, and if it is NetCDF (indicated by + ``out_filetype`` == 2), the file extension `.nc` is appended to the tags. + - The function logs the updates made to the ``outdata_sources``. + """ expid = config["general"]["expid"] econfig = config["echam"] try: From 08e8207b7c843cbfb000140cb33a4e37cc6fd9cb Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 09:59:51 +0100 Subject: [PATCH 07/15] Apply suggestions from code review Add suggestions by @mandresm Co-authored-by: Miguel <63242832+mandresm@users.noreply.github.com> --- src/esm_runscripts/echam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index 8d358420d..c1da92835 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -160,9 +160,9 @@ def append_namelist_dependent_sources(config): try: namelist = econfig["namelist_objs"] except KeyError: # Namelists not yet loaded... - namelist = f90nml.read(econfig["namelist_dir"] + "/namelist.echam") + namelist = f90nml.read(f"{econfig['namelist_dir']}/namelist.echam") mvstream_tags = _get_mvstream_tags_from_namelist(namelist) - jsbach_streams = config["jsbach"]["streams"] + jsbach_streams = config["jsbach"].get("streams", []) ignore_these_tags = econfig.get("ignore_tags", jsbach_streams) mvstream_tags = [tag for tag in mvstream_tags if tag not in ignore_these_tags] mvstream_dict = {tag: f"{expid}*{tag}" for tag in mvstream_tags} From d6aab9175ff008236d79cc4e8f2e9a20aa79534b Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 09:06:19 +0000 Subject: [PATCH 08/15] fix: include JSBACH streams in ignore tags for ECHAM --- src/esm_runscripts/echam.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index c1da92835..abc9e4371 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -163,7 +163,9 @@ def append_namelist_dependent_sources(config): namelist = f90nml.read(f"{econfig['namelist_dir']}/namelist.echam") mvstream_tags = _get_mvstream_tags_from_namelist(namelist) jsbach_streams = config["jsbach"].get("streams", []) - ignore_these_tags = econfig.get("ignore_tags", jsbach_streams) + ignore_these_tags = econfig.get("ignore_tags", []]) + if econfig.get("ignore_tags_include_jsbach_tags", True): + ignore_these_tags.extend(jsbach_streams) mvstream_tags = [tag for tag in mvstream_tags if tag not in ignore_these_tags] mvstream_dict = {tag: f"{expid}*{tag}" for tag in mvstream_tags} if namelist["runctl"].get("out_filetype") == 2: From 509ac21d102e41dcaaa1e9cbe95dca73834cceb7 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 10:35:57 +0100 Subject: [PATCH 09/15] feat: jsbach streams --- src/esm_runscripts/jsbach.py | 113 +++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 src/esm_runscripts/jsbach.py diff --git a/src/esm_runscripts/jsbach.py b/src/esm_runscripts/jsbach.py new file mode 100644 index 000000000..0a69dcd9c --- /dev/null +++ b/src/esm_runscripts/jsbach.py @@ -0,0 +1,113 @@ +""" +Functionality specific for the JSBACH sub-component. +""" + +import f90nml +from loguru import logger + +from .echam import _get_mvstream_tags_from_namelist + + +def _get_comments_for_streams(namelist, mvstream_tags, flag="ESM_TOOLS_JSBACH_STREAM"): + """ + Extracts tags from a namelist file that have a specific comment flag. + + Parameters + ---------- + namelist : str + Path to the namelist file. + mvstream_tags : list of str + List of tags to search for in the namelist file. + flag : str, optional + The flag to search for in the comments (default is "ESM_TOOLS_JSBACH_STREAM"). + + Returns + ------- + list of str + List of tags that have the specified comment flag. + + Examples + -------- + Assuming you have a namelist file at ``path/to/namelist`` with the following conntents: + + .. code-block:: fortran + + &mvstreamctl + filetag = 'plants' ! ESM_TOOLS_JSBACH_STREAM + source = 'g3b' + variables = 'soilwet', 'soiltemp', 'lai', 'gpp', 'npp' + interval = 1, 'months', 'last', 0 + / + &mvstreamctl + filetag = 'soil' ! ESM_TOOLS_JSBACH_STREAM + source = 'g3b' + variables = 'soilwet', 'soiltemp' + interval = 1, 'months', 'last', 0 + / + &mvstreamctl + filetag = 'something' + source = 'g3b' + variables = 'temp2' + interval = 1, 'months', 'last', 0 + / + The following will extract ['plants', 'soil']:: + + >>> namelist = "path/to/namelist" + >>> mvstream_tags = ["plants", "soil", "something] + >>> _get_comments_for_streams(namelist, mvstream_tags) + ['plants', 'soil'] + """ + with open(namelist, "r") as f: + lines = f.readlines() + jsbach_tags = [] + for tag in mvstream_tags: + # Find the tag in the namlist + matching_lines = [line for line in lines if tag in line] + if not matching_lines: + continue + # Find the comment + for line in matching_lines: + comment = line.split("!")[1].strip() + if flag in comment: + jsbach_tags.append(tag) + return jsbach_tags + + +def append_namelist_dependent_sources(config): + """ + Append namelist dependent sources of JSBACH configuration. + + Parameters + ---------- + config : dict + + Notes + ----- + - The function reads the namelist and will filter mvstream tags that have + the comment "ESM_TOOLS_JSBACH_STREAM". + """ + expid = config["general"]["expid"] + jconfig = config["jsbach"] + namelist_file = f"{jconfig['namelist_dir']}/namelist.echam" + try: + namelist = jconfig["namelist_objs"] + except KeyError: + # NOTE(PG): This is one of the reasons ECHAM/JSBACH is unfriendly... + # JSBACH is controlled by the ECHAM namelist, this makes + # it difficult to separate the two components. + namelist = f90nml.read(namelist_file) + mvstream_tags = _get_mvstream_tags_from_namelist(namelist) + # Check if any of the JSBACH streams identified in the namelist + # have a comment attached to them: + flagged_tags = _get_comments_for_streams(namelist.file, mvstream_tags) + mvstream_dict = {tag: f"{expid}*{tag}" for tag in flagged_tags} + if namelist["runctl"].get("out_filetype") == 2: + # Using NetCDF Outputs: + mvstream_dict = {k: v + ".nc" for k, v in mvstream_dict.items()} + logger.info("Updating outdata_sources...") + for k, v in mvstream_dict.items(): + logger.info(f"{k}: {v}") + jconfig["outdata_sources"].update(mvstream_dict) + logger.info("...done!") + + return config From 88978154ddfa4775718016c73244e511a7061d63 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 10:54:18 +0100 Subject: [PATCH 10/15] fix: typo --- src/esm_runscripts/echam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index abc9e4371..1c1932946 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -163,7 +163,7 @@ def append_namelist_dependent_sources(config): namelist = f90nml.read(f"{econfig['namelist_dir']}/namelist.echam") mvstream_tags = _get_mvstream_tags_from_namelist(namelist) jsbach_streams = config["jsbach"].get("streams", []) - ignore_these_tags = econfig.get("ignore_tags", []]) + ignore_these_tags = econfig.get("ignore_tags", []) if econfig.get("ignore_tags_include_jsbach_tags", True): ignore_these_tags.extend(jsbach_streams) mvstream_tags = [tag for tag in mvstream_tags if tag not in ignore_these_tags] From d384c50d0c0ebb486183bf87ae3ba00f8100adc2 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 10:57:43 +0100 Subject: [PATCH 11/15] fix: test had a bad quote --- src/esm_runscripts/jsbach.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/esm_runscripts/jsbach.py b/src/esm_runscripts/jsbach.py index 0a69dcd9c..903ab8b2e 100644 --- a/src/esm_runscripts/jsbach.py +++ b/src/esm_runscripts/jsbach.py @@ -53,7 +53,7 @@ def _get_comments_for_streams(namelist, mvstream_tags, flag="ESM_TOOLS_JSBACH_ST The following will extract ['plants', 'soil']:: >>> namelist = "path/to/namelist" - >>> mvstream_tags = ["plants", "soil", "something] + >>> mvstream_tags = ["plants", "soil", "something"] >>> _get_comments_for_streams(namelist, mvstream_tags) ['plants', 'soil'] """ From cdba0c7f10cee4e41f252f2e8082e28bb652d8db Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 11:09:28 +0100 Subject: [PATCH 12/15] test: allows doctest to use actual namelist (maybe) --- src/esm_runscripts/echam.py | 4 +- .../echam/paul_custom_namelist.echam | 63 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/namelists/echam/paul_custom_namelist.echam diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index 1c1932946..d2075255e 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -39,7 +39,7 @@ def _get_mvstream_tags_from_namelist(namelist): Examples -------- - Assuming you have a namelist file at ``/tmp/example/namelist.echam`` with the + Assuming you have a namelist file at ``tests/namelists/echam/paul_custom_namelist.echam`` with the following contents: .. code-block:: fortran @@ -108,7 +108,7 @@ def _get_mvstream_tags_from_namelist(namelist): The following code will extract the mvstream tags from the namelist: - >>> namelist_path = "/tmp/example/namelist.echam" + >>> namelist_path = "tests/namelists/echam/paul_custom_namelist.echam" >>> tags = _get_mvstream_tags_from_namelist(namelist_path) >>> print(tags) ['paul_custom'] diff --git a/tests/namelists/echam/paul_custom_namelist.echam b/tests/namelists/echam/paul_custom_namelist.echam new file mode 100644 index 000000000..c409f93a5 --- /dev/null +++ b/tests/namelists/echam/paul_custom_namelist.echam @@ -0,0 +1,63 @@ +! This is the "no output" variant of namelist.echam. It contains absolutely no +! output, and can be used as a template for testing. +! +! Extended by mvstreamctl namelist block, as an example for the mvstream tags. +! +! P. Gierz +! Alfred Wegener Institute, Helmholtz Centre for Polar and Marine Research +! July 2021 +! +! P. Gierz +! October 2024 + +&runctl + dt_start = 2285, 12, 31, 23, 52, 30 + dt_stop = 6699, 12, 31, 23, 52, 30 + putrerun = 12, 'months', 'last', 0 + lfractional_mask = .false. + lresume = .true. + out_datapath = './' + out_expname = 'E280' + rerun_filetype = 4 + delta_time = 450 + putdata = 1, 'months', 'last', 0 + nproma = 8 + lcouple = .true. + getocean = 1, 'days', 'last', 0 + putocean = 1, 'days', 'last', 0 + lcouple_co2 = .true. + default_output = .false. +/ + +&parctl + nproca = 24 + nprocb = 24 +/ + +&submodelctl + lmethox = .true. +/ + +&submdiagctl + vphysc_lpost = .false. +/ + +&radctl + iaero = 3 + io3 = 4 + isolrad = 6 + ich4 = 3 + in2o = 3 + co2vmr = 284.3169860840e-06 + ch4vmr = 808.2490234375e-09 + n2ovmr = 273.0210571289e-09 + yr_perp = 1850 +/ +&mvstreamctl + filetag = 'paul_custom' + source = 'g3b' + variables = 'temp2:mean>temp2=167' + interval = 1, 'months', 'last', 0 +/ + +! vim: ft=fortran From 55130b96fc647abbe27ae5ed6582ac68ff264881 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 11:13:12 +0100 Subject: [PATCH 13/15] test: allows doctest to use actual namelists also for jsbach part --- src/esm_runscripts/jsbach.py | 4 ++-- .../echam/jsbach_tags_namelist.echam | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 tests/namelists/echam/jsbach_tags_namelist.echam diff --git a/src/esm_runscripts/jsbach.py b/src/esm_runscripts/jsbach.py index 903ab8b2e..0b64d1cfe 100644 --- a/src/esm_runscripts/jsbach.py +++ b/src/esm_runscripts/jsbach.py @@ -28,7 +28,7 @@ def _get_comments_for_streams(namelist, mvstream_tags, flag="ESM_TOOLS_JSBACH_ST Examples -------- - Assuming you have a namelist file at ``path/to/namelist`` with the following conntents: + Assuming you have a namelist file with the following contents: .. code-block:: fortran @@ -52,7 +52,7 @@ def _get_comments_for_streams(namelist, mvstream_tags, flag="ESM_TOOLS_JSBACH_ST / The following will extract ['plants', 'soil']:: - >>> namelist = "path/to/namelist" + >>> namelist = "tests/namelists/echam/jsbach_tags_namelist.echam" >>> mvstream_tags = ["plants", "soil", "something"] >>> _get_comments_for_streams(namelist, mvstream_tags) ['plants', 'soil'] diff --git a/tests/namelists/echam/jsbach_tags_namelist.echam b/tests/namelists/echam/jsbach_tags_namelist.echam new file mode 100644 index 000000000..ef5c4b57e --- /dev/null +++ b/tests/namelists/echam/jsbach_tags_namelist.echam @@ -0,0 +1,20 @@ +&mvstreamctl + filetag = 'plants' ! ESM_TOOLS_JSBACH_STREAM + source = 'g3b' + variables = 'soilwet', 'soiltemp', 'lai', 'gpp', 'npp' + interval = 1, 'months', 'last', 0 +/ +&mvstreamctl + filetag = 'soil' ! ESM_TOOLS_JSBACH_STREAM + source = 'g3b' + variables = 'soilwet', 'soiltemp' + interval = 1, 'months', 'last', 0 +/ +&mvstreamctl + filetag = 'something' + source = 'g3b' + variables = 'temp2' + interval = 1, 'months', 'last', 0 +/ + +! vim: ft=fortran From 7c0d0aafa4990eb6f2b78387c5a7b8b5caa55db1 Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 11:15:57 +0100 Subject: [PATCH 14/15] fix: bad check if comment character was missing --- src/esm_runscripts/jsbach.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/esm_runscripts/jsbach.py b/src/esm_runscripts/jsbach.py index 0b64d1cfe..ab940b422 100644 --- a/src/esm_runscripts/jsbach.py +++ b/src/esm_runscripts/jsbach.py @@ -67,6 +67,8 @@ def _get_comments_for_streams(namelist, mvstream_tags, flag="ESM_TOOLS_JSBACH_ST continue # Find the comment for line in matching_lines: + if "!" not in line: + continue comment = line.split("!")[1].strip() if flag in comment: jsbach_tags.append(tag) From 82f649d38a7fd9d219674d3cbe1e63be5f43803e Mon Sep 17 00:00:00 2001 From: Paul Gierz Date: Mon, 4 Nov 2024 12:18:00 +0100 Subject: [PATCH 15/15] chore: cleanup of logger statements --- src/esm_runscripts/echam.py | 6 +++--- src/esm_runscripts/jsbach.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/esm_runscripts/echam.py b/src/esm_runscripts/echam.py index d2075255e..13e483d4a 100644 --- a/src/esm_runscripts/echam.py +++ b/src/esm_runscripts/echam.py @@ -171,8 +171,8 @@ def append_namelist_dependent_sources(config): if namelist["runctl"].get("out_filetype") == 2: # Using NetCDF Outputs: mvstream_dict = {k: v + ".nc" for k, v in mvstream_dict.items()} - logger.info("Updating outdata_sources...") + logger.debug("Updating outdata_sources...") for k, v in mvstream_dict.items(): - logger.info(f"{k}: {v}") + logger.debug(f"{k}: {v}") econfig["outdata_sources"].update(mvstream_dict) - logger.info("...done!") + logger.debug("...done!") diff --git a/src/esm_runscripts/jsbach.py b/src/esm_runscripts/jsbach.py index ab940b422..a1c06631a 100644 --- a/src/esm_runscripts/jsbach.py +++ b/src/esm_runscripts/jsbach.py @@ -106,10 +106,10 @@ def append_namelist_dependent_sources(config): if namelist["runctl"].get("out_filetype") == 2: # Using NetCDF Outputs: mvstream_dict = {k: v + ".nc" for k, v in mvstream_dict.items()} - logger.info("Updating outdata_sources...") + logger.debug("Updating outdata_sources...") for k, v in mvstream_dict.items(): - logger.info(f"{k}: {v}") + logger.debug(f"{k}: {v}") jconfig["outdata_sources"].update(mvstream_dict) - logger.info("...done!") + logger.debug("...done!") return config