Merge branch 'jgfouca/scream_downstream_2024_10_09' into next (PR #6675)

Scream downstream merge 2024_10_09

[BFB]

* jgfouca/scream_downstream_2024_10_09: (471 commits)
  Simply call to new abort feature in the AD code
  better comment
  Use asynchronous deep_copy in non-contig sync_views_impl
  Change the name of fail option to be self-descriptive
  EAMxx: allow non-contiguous set of cpus in test-all-scream
  Change sync_views_impl to use scratch contiguous field
  Pass data_type to other subfield impl
  Make deep_copy const
  Create contiguous helper field
  Add function to return if host == device mem space
  A quick change to recreate an aborted run mid simulation
  Update rrtmgp to latest kokkos-coversion-branch
  Remove most allocations from rrtmgp interface
  Try async deep copy for sync to host
  EAMxx: fix usage of regex when parsing rpointer file
  EAMxx: fix nc file names in some standalone tests
  EAMxx: fix usage of layout extents in field methods
  Consolidate sync_to_host_impl and sync_to_device_impl into sync_views_impl
  Add comment on need for mutable m_is_read_only
  Allow sync_to_host for read-only fields
  ...

.github/workflows/e3sm-gh-ci-cime-tests.yml

@@ -26,7 +26,7 @@ on:
 jobs:
 
   ci:
-    if: ${{ github.event.repository.name == 'e3sm' }}
+    if: false
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

.github/workflows/eamxx_default_files.yml

@@ -13,6 +13,7 @@ on:
 
 jobs:
   scream-defaults:
+    if: false
     runs-on: ubuntu-latest
     outputs:
       event_name: ${{ github.event_name }}

cime_config/config_grids.xml

@@ -1783,6 +1783,26 @@
       <mask>oRRS18to6v3</mask>
     </model_grid>
 
+    <model_grid alias="ne512pg2_ICOS10">
+      <grid name="atm">ne512np4.pg2</grid>
+      <grid name="lnd">ne512np4.pg2</grid>
+      <grid name="ocnice">ICOS10</grid>
+      <grid name="rof">r0125</grid>
+      <grid name="glc">null</grid>
+      <grid name="wav">null</grid>
+      <mask>ICOS10</mask>
+    </model_grid>
+
+    <model_grid alias="ne512pg2_ne512pg2">
+      <grid name="atm">ne512np4.pg2</grid>
+      <grid name="lnd">ne512np4.pg2</grid>
+      <grid name="ocnice">ne512np4.pg2</grid>
+      <grid name="rof">r0125</grid>
+      <grid name="glc">null</grid>
+      <grid name="wav">null</grid>
+      <mask>ICOS10</mask>
+    </model_grid>
+
     <model_grid alias="ne512np4_360x720cru_ne512np4">
       <grid name="atm">ne512np4</grid>
       <grid name="lnd">360x720cru</grid>
@@ -3082,6 +3102,8 @@
       <ny>1</ny>
       <file grid="atm|lnd" mask="oRRS18to6v3">$DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_oRRS18to6v3.200212.nc</file>
       <file grid="ice|ocn" mask="oRRS18to6v3">$DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_oRRS18to6v3.200212.nc</file>
+      <file grid="atm|lnd" mask="ICOS10">$DIN_LOC_ROOT/share/domains/domain.lnd.ne512pg2_ICOS10.240602.nc</file>
+      <file grid="ice|ocn" mask="ICOS10">$DIN_LOC_ROOT/share/domains/domain.ocn.ne512pg2_ICOS10.240602.nc</file>
       <desc>ne512np4.pg2 is Spectral Elem 6km grid w/ 2x2 FV physics grid per element:</desc>
     </domain>
 
@@ -4187,6 +4209,11 @@
       <map name="ATM2ROF_SMAPNAME">cpl/gridmaps/ne256pg2/map_ne256pg2_to_r0125_bilin.200212.nc</map>
     </gridmap>
 
+    <gridmap atm_grid="ne512np4.pg2" rof_grid="r0125">
+      <map name="ATM2ROF_FMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc</map>
+      <map name="ATM2ROF_SMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc</map>
+    </gridmap>
+
     <gridmap atm_grid="ne512np4.pg2" ocn_grid="oRRS18to6v3">
       <!-- 6km atm /  18to6 ocean, so use bilin for state -->
       <map name="ATM2OCN_FMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_oRRS18to6v3_nco.200212.nc</map>
@@ -4196,6 +4223,15 @@
       <map name="OCN2ATM_SMAPNAME">cpl/gridmaps/ne512pg2/map_oRRS18to6v3_to_ne512pg2_nco.200212.nc</map>
     </gridmap>
 
+    <gridmap atm_grid="ne512np4.pg2" ocn_grid="ICOS10">
+      <!-- 6km atm to 7.5 km ICOS10 ocean, so also use nco for state -->
+      <map name="ATM2OCN_FMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc</map>
+      <map name="ATM2OCN_SMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc</map>
+      <map name="ATM2OCN_VMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_ICOS10_nco_c240531.nc</map>
+      <map name="OCN2ATM_FMAPNAME">cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc</map>
+      <map name="OCN2ATM_SMAPNAME">cpl/gridmaps/ne512pg2/map_ICOS10_to_ne512pg2_nco_c240531.nc</map>
+    </gridmap>
+
     <gridmap atm_grid="ne512np4.pg2" lnd_grid="r0125">
       <!-- 6km atm /  13km land.  downscale atm->land, land->atm use bilinear -->
       <map name="ATM2LND_FMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.200212.nc</map>
@@ -5002,6 +5038,11 @@
       <map name="ROF2LND_FMAPNAME">cpl/gridmaps/ne256pg2/map_r0125_to_ne256pg2_mono.200212.nc</map>
     </gridmap>
 
+    <gridmap lnd_grid="ne5124np4.pg2" rof_grid="r0125">
+      <map name="LND2ROF_FMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc</map>
+      <map name="ROF2LND_FMAPNAME">cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc</map>
+    </gridmap>
+
     <gridmap lnd_grid="ne1024np4.pg2" rof_grid="r0125">
       <map name="LND2ROF_FMAPNAME">cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc</map>
       <map name="ROF2LND_FMAPNAME">cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc</map>
@@ -5082,6 +5123,11 @@
       <map name="ROF2LND_FMAPNAME">lnd/clm2/mappingdata/maps/ne240np4/map_0.5x0.5_nomask_to_ne240np4_nomask_aave_da_c121019.nc</map>
     </gridmap>
 
+    <gridmap lnd_grid="ne512np4.pg2" rof_grid="r0125">
+      <map name="LND2ROF_FMAPNAME">cpl/gridmaps/ne512pg2/map_ne512pg2_to_r0125_mono.c20240625.nc</map>
+      <map name="ROF2LND_FMAPNAME">cpl/gridmaps/ne512pg2/map_r0125_to_ne512pg2_mono.c20240625.nc</map>
+    </gridmap>
+
     <gridmap lnd_grid="ne1024np4.pg2" rof_grid="r0125">
       <map name="LND2ROF_FMAPNAME">cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc</map>
       <map name="ROF2LND_FMAPNAME">cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc</map>

cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake

@@ -5,7 +5,7 @@ set(SCC "cc")
 set(SCXX "hipcc")
 set(SFC "ftn")
 
-string(APPEND CPPDEFS " -DLINUX")
+string(APPEND CPPDEFS " -DLINUX -DSCREAM_SYSTEM_WORKAROUND=0")
 if (COMP_NAME STREQUAL gptl)
     string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
 endif()
@@ -19,7 +19,7 @@ endif()
 
 string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -f free")
 
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64 -L/opt/gcc/12.2.0/snos/lib64")
 string(APPEND CMAKE_CXX_FLAGS " -I$ENV{ROCM_PATH}/include")
 
 # Crusher: this resolves a crash in mct in docn init

cime_config/machines/cmake_macros/craygnuamdgpu.cmake

@@ -0,0 +1,42 @@
+set(MPICC "cc")
+set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc
+set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90
+set(SCC "cc")
+set(SCXX "hipcc")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU -DSCREAM_SYSTEM_WORKAROUND_P3_PART2")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+string(APPEND CMAKE_Fortran_FLAGS " -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fallow-argument-mismatch")
+
+string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow")
+string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace")
+
+string(APPEND CMAKE_C_FLAGS_RELEASE " -g -O2")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g -O2")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -g -O2")
+
+if (COMP_NAME STREQUAL csm_share)
+  string(APPEND CMAKE_C_FLAGS " -std=c99")
+endif()
+string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form")
+string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form")
+
+set(E3SM_LINK_WITH_FORTRAN "TRUE")
+string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64")
+
+if (compile_threaded)
+  string(APPEND CMAKE_C_FLAGS " -fopenmp")
+  string(APPEND CMAKE_Fortran_FLAGS " -fopenmp")
+  string(APPEND CMAKE_CXX_FLAGS " -fopenmp=libgomp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp")
+endif()
+
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off")
+
+set(USE_HIP "TRUE")
+string(APPEND CMAKE_HIP_FLAGS "$ENV{CXXFLAGS} --offload-arch=gfx90a -munsafe-fp-atomics")