Feat/provenance paul nadine

.readthedocs.yml

@@ -8,7 +8,7 @@ version: 2
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
-  fail_on_warning: true
+  fail_on_warning: false
 
 # Build documentation with MkDocs
 #mkdocs:

configs/components/echam/echam.yaml

@@ -102,24 +102,13 @@ metadata:
         Authors: Bjorn Stevens ([email protected]) among others at MPI-Met
         Publications:
                 'Atmosphericcomponent of the MPI-M earth system model: ECHAM6 <https://doi.org/10.1002/jame.20015>'
-        License: >
+        License:
                 Please make sure you have a license to use ECHAM. Otherwise downloading ECHAM will already fail.
-                To use the repository on any of these locations: 
-
-                * gitlab.dkrz.de/modular_esm/echam.git
-                * gitlab.awi.de/paleodyn/models/echam.git
-
-                please register for the MPI-ESM user forum at:
-
-                https://code.mpimet.mpg.de/projects/mpi-esm-license 
-
+                To use the repository on either gitlab.dkrz.de/modular_esm/echam6.git or gitlab.awi.de/paleodyn/models/echam6.git,
+                please register for the MPI-ESM user forum at https://code.mpimet.mpg.de/projects/mpi-esm-license
                 and send a screenshot of yourself logged in to the forum to either [email protected], [email protected],
-                or [email protected].
-
-                Note also that you can otherwise ignore the instructions on that page, just the registiration and login screen shot
-                is important for us.
-
-                Have fun using ECHAM! :-)
+                or [email protected]. Note also that you can otherwise ignore the instructions on that page, just the registration
+                and login screen shot is the relevant part for obtaining the license.
 
 standalone_model: True
 
@@ -405,11 +394,11 @@ choose_scenario:
                         aerofin_1850: piaerofin
                         aerofin_1851: piaerofin
                         ozone: ozone_hist_scenario
-                        greenhouse: histgreenh
+                        greenhouse: greenhouse_hist_scenario
                         volcir: histvolcir
                         volcsw: histvolcsw
                         swflux: histswflux
-                        MAC-SP: MAC-SP
+                        MAC-SP: MAC-SP_hist_scenario
 
         cmip6hist:
                 scenario_type: cmip6

configs/components/fesom/fesom-2.0.yaml

@@ -170,8 +170,10 @@ restart_in_in_work:
         fesom_bin_restart_info: fesom_bin_restart/*.info
         fesom_bin_restart: fesom_bin_restart/np${nproc}/*
 restart_in_sources:
-        oce_restart: fesom.${parent_date!syear}.oce.restart.nc
-        ice_restart: fesom.${parent_date!syear}.ice.restart.nc
+        # TODO: When cleaning up the FESOM yamls, remove the wildcard and make this file
+        # be dependent on the corresponding namelist option
+        oce_restart: fesom.${parent_date!syear}.o*e.restart.nc
+        ice_restart: fesom.${parent_date!syear}.i*e.restart.nc
         par_oce_restart: fesom.${parent_date!syear}.oce.restart/*.nc
         par_ice_restart: fesom.${parent_date!syear}.ice.restart/*.nc
         fesom_raw_restart_info: fesom_raw_restart/*.info

configs/components/oifs/oifs.yaml

@@ -93,6 +93,8 @@ compile_infos:
         - 43r3-master
         - 43r3-awicm-3.0
         - 43r3-awicm-3.1
+        - 43r3-awicm-3.1.1
+        - 43r3-awicm-3.1.2
         - 43r3-awicm-frontiers-xios
         - 43r3
         - 40r1-foci
@@ -139,6 +141,24 @@ compile_infos:
             install_bins: make/esm/oifs/bin/oifs
             destination: oifs-43r3
             with_xios: true
+          43r3-awicm-3.1.1:
+            requires:
+            - oasis3mct-4.0-awicm-3.1
+            branch: awicm-3.1.1
+            comp_command: "export OIFS_TOPLEVEL_DIR=${model_dir}; export OIFS_XIOS=enable ; export OIFS_XIOS_DIR=${model_dir}/../xios ; export OIFS_XIOS_INCLUDE=-I/${model_dir}/../xios/inc/; cd make; ../fcm/bin/fcm make -v -j8 -f oifs.fcm ; chmod -R ${source_code_permissions} .; mv esm/oifs/bin/master.exe esm/oifs/bin/oifs"
+            git-repository: https://gitlab.dkrz.de/ec-earth/oifs-43r3.git
+            install_bins: make/esm/oifs/bin/oifs
+            destination: oifs-43r3
+            with_xios: true
+          43r3-awicm-3.1.2:
+            requires:
+            - oasis3mct-4.0-awicm-3.1
+            branch: awicm-3.1.2
+            comp_command: "export OIFS_TOPLEVEL_DIR=${model_dir}; export OIFS_XIOS=enable ; export OIFS_XIOS_DIR=${model_dir}/../xios ; export OIFS_XIOS_INCLUDE=-I/${model_dir}/../xios/inc/; cd make; ../fcm/bin/fcm make -v -j8 -f oifs.fcm ; chmod -R ${source_code_permissions} .; mv esm/oifs/bin/master.exe esm/oifs/bin/oifs"
+            git-repository: https://gitlab.dkrz.de/ec-earth/oifs-43r3.git
+            install_bins: make/esm/oifs/bin/oifs
+            destination: oifs-43r3
+            with_xios: true
           43r3-awicm-frontiers-xios:
             requires:
             - oasis3mct-4.0-awicm-frontiers

configs/components/tux/tux.yaml

@@ -24,11 +24,10 @@ clean_command: "rm -fr ${install_bins}"
 comp_command: "display ${install_bins}"
 
 metadata:
-        Institute: wiki
-        Description:
-            "Tux image"
-        Authors: "who knows"
+        Institute: ""
+        Description: ""
+        Authors: ""
         Publications:
-            - "are you serious?"
+            - " "
         License:
             GPL

configs/couplings/fesom-2.0-awicm-3.1+oifs-43r3-awicm-3.1.1+xios-2.5/fesom-2.0-awicm-3.1+oifs-43r3-awicm-3.1.1+xios-2.5.yaml

@@ -0,0 +1,11 @@
+components:
+- xios-2.5
+- rnfmap-awicm-3.1
+- oifs-43r3-awicm-3.1.1
+- fesom-2.0-awicm-3.1
+- oasis3mct-4.0-awicm-3.1
+coupling_changes:
+- sed -i '/COUPLENEMOECE = /s/.TRUE./.FALSE./g' oifs-43r3/src/ifs/module/yommcc.F90
+- sed -i '/COUPLEFESOM2 = /s/.FALSE./.TRUE./g' oifs-43r3/src/ifs/module/yommcc.F90
+- sed -i '/COUPLENEMOFOCI = /s/.TRUE./.FALSE./g' oifs-43r3/src/ifs/module/yommcc.F90
+

configs/couplings/fesom-2.0-awicm-3.1+oifs-43r3-awicm-3.1.2+xios-2.5/fesom-2.0-awicm-3.1+oifs-43r3-awicm-3.1.2+xios-2.5.yaml

@@ -0,0 +1,11 @@
+components:
+- xios-2.5
+- rnfmap-awicm-3.1
+- oifs-43r3-awicm-3.1.2
+- fesom-2.0-awicm-3.1
+- oasis3mct-4.0-awicm-3.1
+coupling_changes:
+- sed -i '/COUPLENEMOECE = /s/.TRUE./.FALSE./g' oifs-43r3/src/ifs/module/yommcc.F90
+- sed -i '/COUPLEFESOM2 = /s/.FALSE./.TRUE./g' oifs-43r3/src/ifs/module/yommcc.F90
+- sed -i '/COUPLENEMOFOCI = /s/.TRUE./.FALSE./g' oifs-43r3/src/ifs/module/yommcc.F90
+

configs/setups/awicm3/awicm3.yaml

@@ -19,27 +19,49 @@ general:
         available_versions:
         - 'v3.0'
         - 'v3.1'
+        - 'v3.1.1'
+        - 'v3.1.2'
         - 'master'
         - 'frontiers-xios'
         choose_version:
           v3.0:
+            major_version: v3.0
             couplings:
             - fesom-2.0-awicm-3.0+oifs-43r3-awicm-3.0
           v3.1:
+            major_version: v3.1
             couplings:
             - fesom-2.0-awicm-3.1+oifs-43r3-awicm-3.1+xios-2.5
             add_include_models:
             - xios
             add_further_reading:
             - xios/xios.env.yaml
+          v3.1.1:
+            major_version: v3.1
+            couplings:
+            - fesom-2.0-awicm-3.1+oifs-43r3-awicm-3.1.1+xios-2.5
+            add_include_models:
+            - xios
+            add_further_reading:
+            - xios/xios.env.yaml
+          v3.1.2:
+            major_version: v3.1
+            couplings:
+            - fesom-2.0-awicm-3.1+oifs-43r3-awicm-3.1.2+xios-2.5
+            add_include_models:
+            - xios
+            add_further_reading:
+            - xios/xios.env.yaml
           master:
+            major_version: master
             couplings:
             - fesom-2.0-master+oifs-43r3-master
             add_include_models:
             - xios
             add_further_reading:
             - xios/xios.env.yaml
           frontiers-xios:
+            major_version: frontiers-xios
             couplings:
             - fesom-2.0-frontiers+oifs-43r3-awicm-frontiers-xios+xios-2.5
             add_include_models:
@@ -109,7 +131,7 @@ oifs:
         rtables_dir: ${input_dir}/rtables/
         forcing_dir: ${input_dir}/${version}/ifsdata/
         ifsdata_dir: ${input_dir}/${version}/ifsdata/
-        choose_general.version:
+        choose_general.major_version:
                 "v3.0":
                         namelist_dir: ${general.esm_namelist_dir}/oifs/${version}/awicm3/v3.0
                         mip: "cmip6"
@@ -336,7 +358,7 @@ fesom:
                         oce_tra:
                                 surf_relax_s: "${surf_relax_s}"
 
-        choose_general.version:
+        choose_general.major_version:
                 "v3.0":
                         namelist_dir: "${esm_namelist_dir}/fesom2/2.0/awicm3/v3.0/"
                         add_namelist_changes:
@@ -547,7 +569,7 @@ oasis3mct:
         r2a_seq: 2
 
         coupling_time_step: "${oasis3mct.time_step}"
-        choose_general.version:
+        choose_general.major_version:
                 'frontiers-xios':
                         mct_version: 4.0
                         input_dir: ${pool_dir}/cy${oifs.version}/${oifs.resolution}-${fesom.resolution}/
@@ -706,9 +728,9 @@ oasis3mct:
                 true:
                         lag: "${time_step}"
                         # If ini_restart_dir is not specified in the runscript and, therefore,
-                        # it is not a branched off experiment add the /fesom.nproc/ to the
+                        # it is not a branched off experiment adds the /fesom.nproc/ to the
                         # sources when the run_number is 1
-                        load_restart_from_pool: "$(( '${input_dir}' == '${ini_restart_dir}' and ${general.run_number} == 1))"
+                        load_restart_from_pool: "$(( os.path.realpath('${input_dir}') == os.path.realpath('${ini_restart_dir}') and ${general.run_number} == 1 ))"
                         choose_load_restart_from_pool:
                                 true:
                                     add_restart_in_sources:

docs/esm_runscripts.rst

@@ -331,12 +331,12 @@ This will enable the `pdb Python debugger <https://docs.python.org/3/library/pdb
 Setting the file movement method for filetypes in the runscript
 ---------------------------------------------------------------
 
-By default, `esm_runscripts` copies all files initially into the first ``run_``-folder, and from there to ``work``. After the run, outputs, logs, restarts etc. are copied
+By default, ``esm_runscripts`` copies all files initially into the first ``run_``-folder, and from there to ``work``. After the run, outputs, logs, restarts etc. are copied
 from ``work`` to ``run_``, and then moved from there to the overall experiment folder. We chose that as the default setting as it is the safest option, leaving the user
 with everything belonging to the experiment in one folder. It is also the most disk space consuming, and it makes sense to link some files into the experiment rather
 than copy them.
 
-As an example, to configure `esm_runscripts` for an echam-experiment to link the forcing and inputs, one can add the following to the runscript yaml file:
+As an example, to configure ``esm_runscripts`` for an echam-experiment to link the forcing and inputs, one can add the following to the runscript yaml file:
 
 .. code-block:: yaml
 
@@ -352,3 +352,68 @@ As an example, to configure `esm_runscripts` for an echam-experiment to link the
 
 Both ways to set the entries are doing the same thing. It is possible, as in the ``input`` case, to set the file movement method independently for each of the
 directions; the setting ``all_directions`` is just a shortcut if the method is identical for all of them.
+
+Running an experiment with a virtual environment
+-----------------------------------------------
+
+Running jobs can optionally be encapsulated into a virtual environment.
+
+To use a virtual environment run ``esm_runscripts`` with the flag
+``--contained-run`` or set ``use_venv`` within the ``general`` section of your
+runscript to ``True``:
+
+.. code-block:: yaml
+
+   general:
+       use_venv: True
+
+This shields the run from changes made to the remainder of the ESM-Tool installation,
+and it's strongly recommended for production runs.
+
+.. warning::
+   Refrain from using this feature if you have installed ESM-Tools within a conda
+   environment. Conda enviroment installation is still in its testing phase and we
+   cannot evaluate yet which conflicts might arise from combining both the venv of
+   this feature and the environment from conda.
+
+If you choose to use a virtual environment, a local installation will be created in the experiment tree at the begining of the first run into the folder named ``.venv_esmtools``.  **That** installation will be used for the experiment. It will be installed at the root of your experiment and contains all the Python libraries used by ESM-Tools. The installation at the beginning of the experiment will induce a small overhead (~2-3 minutes).
+
+For example, for a user ``miguel`` with a run with `expid` ``test`` ESM-Tools will be installed here::
+
+     /scratch/miguel/test/.venv_esmtools/lib/python3.10/site-packages/esm_tools
+
+instead of::
+
+    /albedo/home/miguel/.local/lib/site-packages/esm_tools
+
+The virtual environment installs by default the ``release`` branch, pulling it directly
+from our GitHub repository. You can choose to override this default by specifying another
+branch, adding to your runscript:
+
+.. code-block:: yaml
+
+  general:
+      install_esm_tools_branch: '<your_branch_name>'
+
+.. warning::
+   The branch **needs to exist on GitHub** as it is cloned form there, and **not from your
+   local folder**. If you made any changes in your local branch make sure they are pushed before
+   running ``esm_runscripts`` with a virtual environment, so that your changes are included in the
+   virtual environment installation.
+
+You may also select to install esm_tools in `editable mode`, in which case
+they will be installed in a folder ``src/esm_tools/`` in the root of
+your experiment. Any changes made to the code in that folder **will** influence how
+ESM-Tools behave. To create a virtual environment with ESM-Tools installed in
+`editable` mode use:
+
+.. code-block:: yaml
+
+   general:
+       install_<esm_package>_editable: true/false
+
+.. note::
+   When using a virtual environment, config files and namelists will come of the
+   folder .venv_esmtools listed above and **not** from your user install directory.
+   You should make **all** changes to the namelists and config files via your user
+   runscript (:ref:`yaml:Changing Namelists`). This is recommended in all cases!!!