Merge branch 'dev/gfdl' into pseudo_salt_fix

.github/workflows/macos-regression.yml

@@ -10,6 +10,7 @@ jobs:
     env:
       CC: gcc
       FC: gfortran
+      FMS_COMMIT: 2019.01.03
 
     defaults:
       run:

.github/workflows/macos-stencil.yml

@@ -10,6 +10,7 @@ jobs:
     env:
       CC: gcc
       FC: gfortran
+      FMS_COMMIT: 2019.01.03
 
     defaults:
       run:

.gitlab-ci.yml

@@ -32,6 +32,8 @@ p:clone:
   tags:
     - ncrc5
   script:
+    # NOTE: We could sweep any builds older than 3 days here if needed
+    #- find $HOME/ci/[0-9]* -mtime +3 -delete 2> /dev/null || true
     - .gitlab/pipeline-ci-tool.sh create-job-dir
 #.gitlab/pipeline-ci-tool.sh clean-job-dir
 
@@ -353,4 +355,5 @@ cleanup:
   before_script:
     - echo Skipping usual preamble
   script:
+    - rm -rf $HOME/ci/$CI_PIPELINE_ID
     - rm -rf $JOB_DIR

.gitlab/pipeline-ci-tool.sh

@@ -94,6 +94,13 @@ create-job-dir () {
     make -f tools/MRS/Makefile.clone clone_gfdl -j # Extras and link to datasets
     bash tools/MRS/generate_manifest.sh . tools/MRS/excluded-expts.txt > manifest.mk
     mkdir -p results
+    # Temporarily move build directory to $HOME to circumvent poor F5 performance
+    mkdir -p $HOME/ci/$CI_PIPELINE_ID/build
+    ln -s $HOME/ci/$CI_PIPELINE_ID/build build
+    # Builds need non-mangled access to src/.
+    ln -s "$(pwd)"/src $HOME/ci/$CI_PIPELINE_ID/src
+    # Static builds need access to ocean_only/
+    ln -s "$(pwd)"/ocean_only $HOME/ci/$CI_PIPELINE_ID/ocean_only
   fi
   section-end create-job-dir
 }

.testing/Makefile

@@ -20,7 +20,6 @@
 #
 # General test configuration:
 #   MPIRUN                  MPI job launcher (mpirun, srun, etc)
-#   FRAMEWORK               Model framework (fms1 or fms2)
 #   DO_REPRO_TESTS          Enable production ("repro") testing equivalence
 #   DO_REGRESSION_TESTS     Enable regression tests (usually dev/gfdl)
 #   DO_COVERAGE             Enable code coverage and generate .gcov reports
@@ -74,8 +73,11 @@ AC_SRCDIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))../ac
 # User-defined configuration
 -include config.mk
 
-# Set the infra framework
-FRAMEWORK ?= fms2
+# Set the FMS library
+FMS_COMMIT ?= 2023.03
+FMS_URL ?= https://github.com/NOAA-GFDL/FMS.git
+export FMS_COMMIT
+export FMS_URL
 
 # Set the MPI launcher here
 # TODO: This needs more automated configuration
@@ -133,9 +135,6 @@ TIME ?= time
 WORKSPACE ?= .
 
 # Set directories for build/ and work/
-#BUILD ?= $(WORKSPACE)build
-#DEPS ?= $(BUILD)/deps
-#WORK ?= $(WORKSPACE)work
 BUILD ?= $(WORKSPACE)/build
 DEPS ?= $(BUILD)/deps
 WORK ?= $(WORKSPACE)/work
@@ -207,34 +206,6 @@ else
 endif
 
 
-# List of source files to link this Makefile's dependencies to model Makefiles
-# Assumes a depth of two, and the following extensions: F90 inc c h
-# (1): Root directory
-# NOTE: extensions could be a second variable
-SOURCE = \
-  $(foreach ext,F90 inc c h,$(wildcard $(1)/*/*.$(ext) $(1)/*/*/*.$(ext)))
-
-MOM_SOURCE = \
-  $(call SOURCE,../src) \
-  $(wildcard ../config_src/drivers/solo_driver/*.F90) \
-  $(wildcard ../config_src/ext*/*/*.F90)
-
-TARGET_SOURCE = \
-  $(call SOURCE,$(BUILD)/target_codebase/src) \
-  $(wildcard $(BUILD)/target_codebase/config_src/drivers/solo_driver/*.F90) \
-  $(wildcard $(BUILD)target_codebase/config_src/ext*/*.F90)
-
-ifeq ($(FRAMEWORK), fms1)
-  MOM_SOURCE += $(wildcard ../config_src/infra/FMS1/*.F90)
-  TARGET_SOURCE += $(wildcard $(BUILD)/target_codebase/config_src/infra/FMS1/*.F90)
-else
-  MOM_SOURCE +=$(wildcard ../config_src/infra/FMS2/*.F90)
-  TARGET_SOURCE += $(wildcard $(BUILD)/target_codebase/config_src/infra/FMS2/*.F90)
-endif
-
-FMS_SOURCE = $(call SOURCE,$(DEPS)/fms/src)
-
-
 ## Rules
 
 .PHONY: all build.regressions build.prof
@@ -286,7 +257,6 @@ $(BUILD)/unit/Makefile: MOM_ENV += $(COV_FCFLAGS) $(COV_LDFLAGS)
 $(BUILD)/timing/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS)
 
 # Configure script flags
-MOM_ACFLAGS := --with-framework=$(FRAMEWORK)
 $(BUILD)/openmp/Makefile: MOM_ACFLAGS += --enable-openmp
 $(BUILD)/coupled/Makefile: MOM_ACFLAGS += --with-driver=FMS_cap
 $(BUILD)/nuopc/Makefile: MOM_ACFLAGS += --with-driver=nuopc_cap
@@ -298,11 +268,21 @@ $(BUILD)/timing/Makefile: MOM_ACFLAGS += --with-driver=timing_tests
 $(BUILD)/unit/test_%: $(BUILD)/unit/Makefile FORCE
 	cd $(@D) && $(TIME) $(MAKE) $(@F) -j
 $(BUILD)/unit/Makefile: $(foreach e,$(UNIT_EXECS),../config_src/drivers/unit_tests/$(e).F90)
+
 $(BUILD)/timing/time_%: $(BUILD)/timing/Makefile FORCE
 	cd $(@D) && $(TIME) $(MAKE) $(@F) -j
 $(BUILD)/timing/Makefile: $(foreach e,$(TIMING_EXECS),../config_src/drivers/timing_tests/$(e).F90)
+
 $(BUILD)/%/MOM6: $(BUILD)/%/Makefile FORCE
 	cd $(@D) && $(TIME) $(MAKE) $(@F) -j
+
+# Target codebase should use its own build system
+$(BUILD)/target/MOM6: $(BUILD)/target FORCE | $(TARGET_CODEBASE)
+	$(MAKE) -C $(TARGET_CODEBASE)/.testing build/symmetric/MOM6
+
+$(BUILD)/target: | $(TARGET_CODEBASE)
+	ln -s $(abspath $(TARGET_CODEBASE))/.testing/build/symmetric $@
+
 FORCE:
 
 
@@ -334,27 +314,12 @@ $(BUILD)/%/configure.ac: ../ac/configure.ac | $(BUILD)/%/
 $(BUILD)/%/m4/: ../ac/m4/ | $(BUILD)/%/
 	cp -r ../ac/m4 $(@D)
 
-ALL_EXECS = symmetric asymmetric repro openmp target opt opt_target coupled \
-  nuopc cov unit timing
+ALL_EXECS = symmetric asymmetric repro openmp opt opt_target coupled nuopc \
+  cov unit timing
 $(foreach b,$(ALL_EXECS),$(BUILD)/$(b)/):
 	mkdir -p $@
 
 # Fetch the regression target codebase
-
-$(BUILD)/target/config.status: $(BUILD)/target/configure $(DEPS)/lib/libFMS.a
-	cd $(@D) && $(MOM_ENV) ./configure -n \
-	  --srcdir=$(abspath $(BUILD))/target_codebase/ac $(MOM_ACFLAGS) \
-	|| (cat config.log && false)
-
-$(BUILD)/target/Makefile.in: | $(TARGET_CODEBASE) $(BUILD)/target/
-	cp $(TARGET_CODEBASE)/ac/Makefile.in $(@D)
-
-$(BUILD)/target/configure.ac: | $(TARGET_CODEBASE) $(BUILD)/target/
-	cp $(TARGET_CODEBASE)/ac/configure.ac $(@D)
-
-$(BUILD)/target/m4/: | $(TARGET_CODEBASE) $(BUILD)/target/
-	cp -r $(TARGET_CODEBASE)/ac/m4 $(@D)
-
 $(TARGET_CODEBASE):
 	git clone --recursive $(MOM_TARGET_URL) $@
 	cd $@ && git checkout --recurse-submodules $(MOM_TARGET_BRANCH)

.testing/README.rst

@@ -47,9 +47,11 @@ Several of the following may require configuration for particular systems.
    Name of the MPI launcher.  Often this is ``mpirun`` or ``mpiexec`` but may
    all need to run through a scheduler, e.g. ``srun`` if using Slurm.
 
-``FRAMEWORK`` (*default:* ``fms1``)
-   Select either the legacy FMS framework (``fms1``) or an FMS2 I/O compatible
-   version (``fms2``).
+``FMS_COMMIT`` (*default:* ``2023.03``)
+   Set the FMS version, either by tag or commit (as defined in ``FMS_URL``).
+
+``FMS_URL`` (*default*: ``https://github.com/NOAA-GFDL/FMS.git``)
+   Set the URL of the FMS repository.
 
 ``DO_REPRO_TESTS`` (*default:* *none*)
    Set to ``true`` to test the REPRO build and confirm equivalence of DEBUG and

.testing/tools/parse_perf.py

@@ -3,10 +3,20 @@
 import collections
 import json
 import os
+import re
 import shlex
 import subprocess
 import sys
 
+perf_scanner = re.Scanner([
+  (r'<', lambda scanner, token: token),
+  (r'>', lambda scanner, token: token),
+  (r'\(', lambda scanner, token: token),
+  (r'\)', lambda scanner, token: token),
+  (r'[ \t]+', lambda scanner, token: token),
+  (r'[^<>() \t]+', lambda scanner, token: token),
+])
+
 
 def main():
     desc = 'Parse perf.data and return in JSON format.'
@@ -58,15 +68,55 @@ def parse_perf_report(perf_data_path):
 
             # get per-symbol count
             else:
+                tokens, remainder = perf_scanner.scan(line)
+                if remainder:
+                    print('Line could not be tokenized', file=sys.stderr)
+                    print(' line:', repr(line), file=sys.stderr)
+                    print(' tokens:', tokens, file=sys.stderr)
+                    print(' remainder:', remainder, file=sys.stderr)
+                    sys.exit(os.EX_DATAERR)
+
+                # Construct record from tokens
+                # (NOTE: Not a proper grammar, just dumb bracket counting)
+                record = []
+                bracks = 0
+                parens = 0
+
+                for tok in tokens:
+                    if tok == '<':
+                        bracks += 1
+
+                    if tok == '(':
+                        parens += 1
+
+                    rec = record[-1] if record else None
+
+                    inside_bracket = rec and (bracks > 0 or parens > 0)
+                    lead_rec = tok in '<(' and rec and not rec.isspace()
+                    tail_rec = not tok.isspace() and rec and rec[-1] in '>)'
+
+                    if inside_bracket or lead_rec or tail_rec:
+                        record[-1] += tok
+                    else:
+                        record.append(tok)
+
+                    if tok == '>':
+                        bracks -= 1
+                    if tok == '(':
+                        parens -= 1
+
+                # Strip any whitespace tokens
+                record = [rec for rec in record if not rec.isspace()]
+
                 try:
-                    tokens = line.split()
-                    symbol = tokens[2]
-                    period = int(tokens[3])
-                except ValueError:
+                    symbol = record[2]
+                    period = int(record[3])
+                except:
                     print("parse_perf.py: Error extracting symbol count",
-                            file=sys.stderr)
+                          file=sys.stderr)
                     print("line:", repr(line), file=sys.stderr)
                     print("tokens:", tokens, file=sys.stderr)
+                    print("record:", record, file=sys.stderr)
                     raise
 
                 profile[event_name]['symbol'][symbol] = period

ac/configure.ac

@@ -79,18 +79,6 @@ AS_IF([test "x$with_driver" != "x"],
 # used to configure a header based on a template.
 #AC_CONFIG_HEADERS(["$MEM_LAYOUT/MOM_memory.h"])
 
-# Select the model framework (default: FMS1)
-# NOTE: We can phase this out after the FMS1 I/O has been removed from FMS and
-#   replace with a detection test.  For now, it is a user-defined switch.
-MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2
-AC_ARG_WITH([framework],
-  AS_HELP_STRING([--with-framework=fms1|fms2], [Select the model framework]))
-AS_CASE(["$with_framework"],
-  [fms1], [MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS1],
-  [fms2], [MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2],
-  [MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2]
-)
-
 
 # Explicitly assume free-form Fortran
 AC_LANG(Fortran)
@@ -220,7 +208,6 @@ AX_FC_CHECK_LIB([FMS], [fms_init], [fms_mod],
   ]
 )
 
-
 # Verify that FMS is at least 2019.01.02
 # NOTE: 2019.01.02 introduced two changes:
 #   - diag_axis_init supports an optional domain_position argument
@@ -236,6 +223,14 @@ AC_COMPILE_IFELSE(
   ]
 )
 
+# Determine the FMS IO implementation.
+AX_FC_CHECK_MODULE([fms2_io_mod], [
+  MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2
+],[
+  MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS1
+])
+
+
 # Python interpreter test
 
 # Declare the Python interpreter variable

config_src/drivers/ice_solo_driver/ice_shelf_driver.F90

@@ -26,6 +26,7 @@ program Shelf_main
   use MOM_debugging,       only : MOM_debugging_init
   use MOM_diag_mediator,   only : diag_mediator_init, diag_mediator_infrastructure_init, set_axes_info
   use MOM_diag_mediator,   only : diag_mediator_end, diag_ctrl, diag_mediator_close_registration
+  use MOM_diag_manager_infra, only : diag_manager_set_time_end_infra
   use MOM_domains,         only : MOM_infra_init, MOM_infra_end
   use MOM_domains,         only : MOM_domains_init, clone_MOM_domain, pass_var
   use MOM_dyn_horgrid,     only : dyn_horgrid_type, create_dyn_horgrid, destroy_dyn_horgrid
@@ -325,6 +326,8 @@ program Shelf_main
     Time_end = daymax
   endif
 
+  call diag_manager_set_time_end_infra (Time_end)
+
   if (Time >= Time_end) call MOM_error(FATAL, &
     "Shelf_driver: The run has been started at or after the end time of the run.")
 

config_src/drivers/nuopc_cap/mom_cap.F90

@@ -136,6 +136,7 @@ module MOM_cap_mod
 logical              :: grid_attach_area = .false.
 logical              :: use_coldstart = .true.
 logical              :: use_mommesh = .true.
+logical              :: restart_eor = .false.
 character(len=128)   :: scalar_field_name = ''
 integer              :: scalar_field_count = 0
 integer              :: scalar_field_idx_grid_nx = 0
@@ -381,6 +382,13 @@ subroutine InitializeP0(gcomp, importState, exportState, clock, rc)
     geomtype = ESMF_GEOMTYPE_GRID
   endif
 
+  ! Read end of run restart config option
+  call NUOPC_CompAttributeGet(gcomp, name="write_restart_at_endofrun", value=value, &
+                              isPresent=isPresent, isSet=isSet, rc=rc)
+  if (ChkErr(rc,__LINE__,u_FILE_u)) return
+  if (isPresent .and. isSet) then
+     if (trim(value) .eq. '.true.') restart_eor = .true.
+  end if
 
 end subroutine
 
@@ -1637,6 +1645,8 @@ subroutine ModelAdvance(gcomp, rc)
   real(8)                                :: MPI_Wtime, timers
   logical                                :: write_restart
   logical                                :: write_restartfh
+  logical                                :: write_restart_eor
+
 
   rc = ESMF_SUCCESS
   if(profile_memory) call ESMF_VMLogMemInfo("Entering MOM Model_ADVANCE: ")
@@ -1776,7 +1786,6 @@ subroutine ModelAdvance(gcomp, rc)
   !---------------
   ! Get the stop alarm
   !---------------
-
   call ESMF_ClockGetAlarm(clock, alarmname='stop_alarm', alarm=stop_alarm, rc=rc)
   if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
@@ -1807,7 +1816,18 @@ subroutine ModelAdvance(gcomp, rc)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return
     end if
 
-    if (write_restart .or. write_restartfh) then
+    write_restart_eor = .false.
+    if (restart_eor) then
+      if (ESMF_AlarmIsRinging(stop_alarm, rc=rc)) then
+         if (ChkErr(rc,__LINE__,u_FILE_u)) return
+         write_restart_eor = .true.
+         ! turn off the alarm
+         call ESMF_AlarmRingerOff(stop_alarm, rc=rc )
+         if (ChkErr(rc,__LINE__,u_FILE_u)) return
+       end if
+    end if
+
+    if (write_restart .or. write_restartfh .or. write_restart_eor) then
       ! determine restart filename
       call ESMF_ClockGetNextTime(clock, MyTime, rc=rc)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return