Merge branch 'jgfouca/scream_downstream_merge' into master (PR #6153)

Downstream merge of scream/eamxx fork into E3SM

The build system fixes on the fork are needed for E3SM.
Also updates E3SM with latest version of eamxx.

Merge remote-tracking branch 'scream/master' into jgfouca/scream_downstream_merge

Updates submodules:
* ekat
* haero
* mam4xx

EAM updates:
* Consolidate shoc_Ck nml settings
* New iop_coriolis nml setting (for nudging)
* initialize IOP information on the first restart time step
* SHOC updates
* Add wet/dry qv_sat (saturation mixing ratio), old qv_sat is still available for backwards compatibility
* P3 updates (scream version only)

This PR now includes #5925 :

Update the YAKL submodule, and update PAM submodule to work with the YAKL update.

[YAKL]
[BFB] - (except for SCREAM/EAMXX and MMF2 cases)

.github/workflows/e3sm-gh-pages.yml

@@ -15,22 +15,31 @@ concurrency:
 
 jobs:
   Build-and-Deploy-docs:
+    if: ${{ github.event.repository.name != 'scream' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           show-progress: false
           fetch-depth: 0 # Needed, or else gh-pages won't be fetched, and push rejected
-          submodules: false # speeds up clone and not building anything in submodules
+          # TODO: git rid of dependency on CIME
+          # TODO: another option to investigate is a sparse checkout. 
+          # In the scream repo, all other components do not need to be checked out. 
+          # And even in the upstream, we mainly need only components/xyz/docs (and a few more places).
+          submodules: true
       - name: Show action trigger
-        run: echo "= The job was automatically triggered by a ${{github.event_name}} event."
+        run: echo "= The job was automatically triggered by a ${{github.event_name}} event on repo ${{github.event.repository.name}}."
       - name: Set up Python 3.10
         uses: actions/[email protected]
         with:
           python-version: "3.10"
       - name: Install python deps
         run: python3 -m pip install mkdocs-material pymdown-extensions mkdocs-monorepo-plugin mdutils mkdocs-bibtex
       # build every time (PR or push to master)
+      - name: Generate EAMxx params docs
+        working-directory: components/eamxx/scripts
+        run: |
+          ./eamxx-params-docs-autogen
       - name: Build
         run: mkdocs build --strict --verbose
       # Only deploy to the main github page when there is a push to master

.github/workflows/eamxx-gh-pages.yml

@@ -0,0 +1,88 @@
+# This workflow aims to automatically rebuild eamxx documentation
+# every time the master branch is updated on github and within every PR
+
+name: EAMxx Docs
+
+on:
+  # Runs every time master branch is updated
+  push:
+    branches: [ master ]
+    # Only if docs-related files are touched
+    paths:
+      - components/eamxx/mkdocs.yml
+      - components/eamxx/docs/**
+      - components/eamxx/cime_config/namelist_defaults_scream.xml
+  # Runs every time a PR is open against master
+  pull_request:
+    branches: [ master ]
+    # Only if docs-related files are touched
+    paths:
+      - components/eamxx/mkdocs.yml
+      - components/eamxx/docs/**
+      - components/eamxx/cime_config/namelist_defaults_scream.xml
+
+  label:
+    types:
+      - created
+
+  workflow_dispatch:
+
+concurrency:
+  # Prevent 2+ copies of this workflow from running concurrently
+  group: eamxx-docs-action
+
+jobs:
+
+  eamxx-docs:
+    if: ${{ github.event.repository.name == 'scream' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          show-progress: false
+          # TODO: git rid of dependency on CIME
+          # TODO: another option to investigate is a sparse checkout. 
+          # In the scream repo, all other components do not need to be checked out. 
+          # And even in the upstream, we mainly need only components/xyz/docs (and a few more places).
+          submodules: true
+
+      - name: Show action trigger
+        run: |
+          echo "= The job was automatically triggered by a ${{github.event_name}} event."
+
+      - name: Set up Python 3.10
+        uses: actions/[email protected]
+        with:
+          python-version: "3.10"
+
+      - name: Install Python deps
+        run: |
+          pip install mkdocs pymdown-extensions mkdocs-material mdutils
+  
+      - name: Generate EAMxx params docs
+        working-directory: components/eamxx/scripts
+        run: |
+          ./eamxx-params-docs-autogen
+
+      - name: Build docs
+        working-directory: components/eamxx
+        run: |
+          mkdocs build --strict --verbose
+
+      # only deploy to the main github page when there is a push to master
+      - if: ${{ github.event_name == 'push' }}
+        name: GitHub Pages action
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          # Do not remove existing pr-preview pages
+          clean-exclude: pr-preview
+          folder: ./components/eamxx/site
+
+      # If it's a PR from within the same repo, deploy to a preview page
+      - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+        name: Preview docs
+        uses: rossjrw/pr-preview-action@v1
+        with:
+          source-dir: components/eamxx/site/

.gitignore

@@ -29,3 +29,8 @@ site
 
 # Ignore emacs backup files
 *~
+
+# Ignore mkdocs site-generated files in eamxx
+components/eamxx/site/*
+# Ignore auto-generated eamxx_params.md file
+components/eamxx/docs/common/eamxx_params.md

cime_config/allactive/config_compsets.xml

@@ -14,7 +14,7 @@
     TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_ESP%phys][_BGC%phys]
     Where for the EAM specific compsets below the following is supported
     TIME = Time period (e.g. 2000, HIST, RCP8...)
-    ATM  = [EAM, SATM, SCREAM]
+    ATM  = [EAM, EAMXX, SATM, SCREAM]
     LND  = [ELM,  SLND]
     ICE  = [MPASSI, CICE, DICE, SICE]
     OCN  = [MPASO, DOCN, SOCN]
@@ -396,6 +396,14 @@
   <lname>1850_EAM%CMIP6_ELM%SPBC_MPASSI_MPASO_MOSART_MALI%STATIC_SWAV</lname>
 </compset>
 
+<!-- EAMXX fully coupled compset -->
+
+<compset>
+  <alias>WCYCLXX2010</alias>
+  <lname>2010_EAMXX_ELM%SPBC_MPASSI_MPASO_MOSART_SGLC_SWAV</lname>
+</compset>
+
+
 <!-- OCN + CICE + GLC Only Compsets -->
 <compset>
   <alias>MPAS_LISIO_TEST</alias>

cime_config/allactive/config_pesall.xml

@@ -1853,6 +1853,29 @@
         </nthrds>
       </pes>
     </mach>
+    <mach name="frontier-scream-gpu">
+      <pes compset="any" pesize="any">
+        <comment>"crusher-gpu-scream ne30np4 and ne30np4.pg2"</comment>
+        <ntasks>
+          <ntasks_atm>-2</ntasks_atm>
+          <ntasks_lnd>-2</ntasks_lnd>
+          <ntasks_rof>-2</ntasks_rof>
+          <ntasks_ice>-2</ntasks_ice>
+          <ntasks_ocn>-2</ntasks_ocn>
+          <ntasks_glc>-2</ntasks_glc>
+          <ntasks_wav>-2</ntasks_wav>
+          <ntasks_cpl>-2</ntasks_cpl>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>7</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+      </pes>
+    </mach>
   </grid>
   <grid name="a%ne0np4_northamericax4v1.pg2_l%.+_oi%WC14to60E2r3">
     <mach name="compy">
@@ -2336,6 +2359,30 @@
         </nthrds>
       </pes>
     </mach>
+    <mach name="frontier-scream-gpu">
+      <pes compset="any" pesize="any">
+        <MAX_MPITASKS_PER_NODE>8</MAX_MPITASKS_PER_NODE>
+        <MAX_TASKS_PER_NODE>56</MAX_TASKS_PER_NODE>
+        <ntasks>
+          <ntasks_atm>-6</ntasks_atm>
+          <ntasks_lnd>-6</ntasks_lnd>
+          <ntasks_rof>-6</ntasks_rof>
+          <ntasks_ice>-6</ntasks_ice>
+          <ntasks_ocn>-6</ntasks_ocn>
+          <ntasks_glc>-6</ntasks_glc>
+          <ntasks_wav>-6</ntasks_wav>
+          <ntasks_cpl>-6</ntasks_cpl>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>1</nthrds_atm>
+          <nthrds_lnd>7</nthrds_lnd>
+          <nthrds_rof>1</nthrds_rof>
+          <nthrds_ice>1</nthrds_ice>
+          <nthrds_ocn>1</nthrds_ocn>
+          <nthrds_cpl>1</nthrds_cpl>
+        </nthrds>
+      </pes>
+    </mach>
     <mach name="pm-gpu|muller">
       <pes compset="any" pesize="any">
         <MAX_MPITASKS_PER_NODE>4</MAX_MPITASKS_PER_NODE>

cime_config/config_files.xml

@@ -127,6 +127,7 @@
       <value component="satm"                        >$SRCROOT/components/stub_comps/satm</value>
       <value component="xatm"                        >$SRCROOT/components/xcpl_comps/xatm</value>
       <value component="eam"                         >$SRCROOT/components/eam/</value>
+      <value component="eamxx"                       >$SRCROOT/components/eamxx/</value>
       <value component="scream"                      >$SRCROOT/components/eamxx/</value>
     </values>
     <group>case_comps</group>

cime_config/config_grids.xml

@@ -1446,6 +1446,16 @@
       <mask>oEC60to30v3</mask>
     </model_grid>
 
+    <model_grid alias="ne120pg2_EC30to60E2r2">
+      <grid name="atm">ne120np4.pg2</grid>
+      <grid name="lnd">ne120np4.pg2</grid>
+      <grid name="ocnice">EC30to60E2r2</grid>
+      <grid name="rof">null</grid>
+      <grid name="glc">null</grid>
+      <grid name="wav">null</grid>
+      <mask>EC30to60E2r2</mask>
+    </model_grid>
+
     <model_grid alias="ne120pg2_r05_EC30to60E2r2">
       <grid name="atm">ne120np4.pg2</grid>
       <grid name="lnd">r05</grid>
@@ -1486,6 +1496,16 @@
       <mask>oRRS18to6v3</mask>
     </model_grid>
 
+    <model_grid alias="ne256pg2_oRRS18to6v3">
+      <grid name="atm">ne256np4.pg2</grid>
+      <grid name="lnd">ne256np4.pg2</grid>
+      <grid name="ocnice">oRRS18to6v3</grid>
+      <grid name="rof">null</grid>
+      <grid name="glc">null</grid>
+      <grid name="wav">null</grid>
+      <mask>oRRS18to6v3</mask>
+    </model_grid>
+
     <model_grid alias="ne256pg2_r0125_oRRS18to6v3">
       <grid name="atm">ne256np4.pg2</grid>
       <grid name="lnd">r0125</grid>

cime_config/machines/Depends.frontier-gpu.crayclang.cmake

@@ -0,0 +1,43 @@
+set(CICE_F90
+  ice_FY.F90
+  ice_aerosol.F90
+  ice_age.F90
+  ice_atmo.F90
+  ice_blocks.F90
+  ice_calendar.F90
+  ice_diagnostics.F90
+  ice_distribution.F90
+  ice_domain.F90
+  ice_domain_size.F90
+  ice_dyn_evp.F90
+  ice_fileunits.F90
+  ice_flux.F90
+  ice_forcing.F90
+  ice_grid.F90
+  ice_history.F90
+  ice_history_fields.F90
+  ice_init.F90
+  ice_itd.F90
+  ice_kinds_mod.F90
+  ice_lvl.F90
+  ice_mechred.F90
+  ice_meltpond.F90
+  ice_ocean.F90
+  ice_orbital.F90
+  ice_probability.F90
+  ice_probability_tools.F90
+  ice_read_write.F90
+  ice_restoring.F90
+  ice_shortwave.F90
+  ice_spacecurve.F90
+  ice_state.F90
+  ice_step_mod.F90
+  ice_therm_itd.F90
+  ice_therm_vertical.F90
+  ice_transport_driver.F90
+  ice_transport_remap.F90
+  ice_work.F90)
+
+foreach(ITEM IN LISTS CICE_F90)
+  e3sm_add_flags("cice/src/source/${ITEM}" "-O0")
+endforeach()

cime_config/machines/Depends.frontier-scream-gpu.crayclang-scream.cmake

@@ -0,0 +1,53 @@
+set(REDOPT
+  ../driver-mct/main/seq_io_mod.F90
+  elm/src/biogeophys/BandDiagonalMod.F90)
+
+if (NOT DEBUG)
+  foreach(ITEM IN LISTS REDOPT)
+    e3sm_add_flags("${ITEM}" "-O1 -g")
+  endforeach()
+endif()
+
+set(CICE_F90
+  ice_FY.F90
+  ice_aerosol.F90
+  ice_age.F90
+  ice_atmo.F90
+  ice_blocks.F90
+  ice_calendar.F90
+  ice_diagnostics.F90
+  ice_distribution.F90
+  ice_domain.F90
+  ice_domain_size.F90
+  ice_dyn_evp.F90
+  ice_fileunits.F90
+  ice_flux.F90
+  ice_forcing.F90
+  ice_grid.F90
+  ice_history.F90
+  ice_history_fields.F90
+  ice_init.F90
+  ice_itd.F90
+  ice_kinds_mod.F90
+  ice_lvl.F90
+  ice_mechred.F90
+  ice_meltpond.F90
+  ice_ocean.F90
+  ice_orbital.F90
+  ice_probability.F90
+  ice_probability_tools.F90
+  ice_read_write.F90
+  ice_restoring.F90
+  ice_shortwave.F90
+  ice_spacecurve.F90
+  ice_state.F90
+  ice_step_mod.F90
+  ice_therm_itd.F90
+  ice_therm_vertical.F90
+  ice_transport_driver.F90
+  ice_transport_remap.F90
+  ice_work.F90)
+
+foreach(ITEM IN LISTS CICE_F90)
+  e3sm_add_flags("cice/src/source/${ITEM}" "-O0")
+endforeach()

cime_config/machines/cmake_macros/crayclang-scream.cmake

@@ -9,16 +9,17 @@ string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
 string(APPEND CPPDEFS_DEBUG  " -DYAKL_DEBUG")
 string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY")
-# -em (default) generates MODULENAME.mod files
-string(APPEND CMAKE_Fortran_FLAGS " -f free -N 255 -h byteswapio -em")
+# -em -ef generates modulename.mod files (lowercase), which we must have
+string(APPEND CMAKE_Fortran_FLAGS " -f free -em -ef")
 if (NOT compile_threaded)
 	# -M1077 flag used to suppress message about OpenMP directives
 	# that are ignored for non-threaded builds. (-h omp inactive)
 	# Details: `explain ftn-1077`
   string(APPEND CMAKE_Fortran_FLAGS " -M1077")
 endif()
 set(HAS_F2008_CONTIGUOUS "TRUE")
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -h byteswapio")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -ldl")
+set(E3SM_LINK_WITH_FORTRAN "TRUE")
 set(MPICC "cc")
 set(MPICXX "CC")
 set(MPIFC "ftn")

cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake

@@ -8,6 +8,5 @@ if (COMP_NAME STREQUAL elm)
   string(APPEND CMAKE_Fortran_FLAGS " -hfp0")
 endif()
 string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero")
-string(APPEND CMAKE_Fortran_FLAGS " -em -ef")
 
 set(PIO_FILESYSTEM_HINTS "gpfs")