Merge pull request #919 from nusbaume/sparse_checkout_2_2

Add Github sparse checkout to CESM2.2 release branch

Externals_CAM.cfg

@@ -14,9 +14,10 @@ required = True
 
 [cosp2]
 local_path = src/physics/cosp2/src
-protocol = svn
-repo_url = https://github.com/CFMIP/COSPv2.0/tags/
-tag = v2.1.4cesm/src
+protocol = git
+repo_url = https://github.com/CFMIP/COSPv2.0
+sparse = ../.cosp_sparse_checkout
+tag = v2.1.4cesm
 required = True
 
 [clubb]

bld/configure

@@ -2211,13 +2211,13 @@ sub write_cosp_makefile
 
 CAM_BLD      := $cam_bld
 COSP_PATH    := $cam_dir/src/physics/cosp2
-ISCCP_PATH   := $cam_dir/src/physics/cosp2/src/simulator/icarus
-RS_PATH      := $cam_dir/src/physics/cosp2/src/simulator/quickbeam
-RT_PATH      := $cam_dir/src/physics/cosp2/src/simulator/rttov
-CS_PATH      := $cam_dir/src/physics/cosp2/src/simulator/actsim
-MISR_PATH    := $cam_dir/src/physics/cosp2/src/simulator/MISR_simulator
-MODIS_PATH   := $cam_dir/src/physics/cosp2/src/simulator/MODIS_simulator
-PARASOL_PATH := $cam_dir/src/physics/cosp2/src/simulator/parasol
+ISCCP_PATH   := $cam_dir/src/physics/cosp2/src/src/simulator/icarus
+RS_PATH      := $cam_dir/src/physics/cosp2/src/src/simulator/quickbeam
+RT_PATH      := $cam_dir/src/physics/cosp2/src/src/simulator/rttov
+CS_PATH      := $cam_dir/src/physics/cosp2/src/src/simulator/actsim
+MISR_PATH    := $cam_dir/src/physics/cosp2/src/src/simulator/MISR_simulator
+MODIS_PATH   := $cam_dir/src/physics/cosp2/src/src/simulator/MODIS_simulator
+PARASOL_PATH := $cam_dir/src/physics/cosp2/src/src/simulator/parasol
 
 EOF
 

cime_config/SystemTests/tmc.py

@@ -9,8 +9,6 @@
 import glob, gzip
 
 
-logger = logging.getLogger(__name__)
-
 class TMC(SystemTestsCommon):
 
     def __init__(self, case):
@@ -21,46 +19,39 @@ def __init__(self, case):
 
     def run_phase(self):
 
+        with self._test_status:
+            self._test_status.set_status("COMPARE_MASS", TEST_PEND_STATUS)
         self.run_indv()
-        cesmlog = ''.join(self._get_latest_cpl_logs())
-        if '.gz' == cesmlog[-3:]:
+        cpllog = ''.join(self._get_latest_cpl_logs())
+        atmlog  = cpllog.replace("cpl.log","atm.log")
+        if '.gz' == atmlog[-3:]:
             fopen = gzip.open
         else:
             fopen = open
 
-        with fopen(cesmlog, "rb") as f:
-            lines = [line.rstrip('\n') for line in f]
-        first_val = 0.0
+        f = fopen(atmlog,'r')
+        lines = f.readlines()
+        first_val = -9.0
         with self._test_status:
-            self._test_status.set_status(RUN_PHASE, TEST_PASS_STATUS)
+            self._test_status.set_status("COMPARE_MASS", TEST_PASS_STATUS)
+        use_this_tt_un = False 
         for line in lines:
-            if re.search('between DRY m=30 name=TT_UN gavg dry, wet, min, max',line):
-                if first_val == 0.0:
-                    first_val = re.findall('\s*[\d]+ name=TT_UN [^0-9]+([\S]+)',line)
-                if first_val != re.findall('\s*[\d]+ name=TT_UN [^0-9]+([\S]+)',line):
-                    with self._test_status:
-                        self._test_status.set_status(RUN_PHASE, TEST_FAIL_STATUS)
-                    comments = "CAM mass conservation test FAILED."
-                    CIME.utils.append_testlog(comments, self._orig_caseroot)
-            first_var = 1.1
-
-    def _get_latest_cesm_logs(self):
-        """
-        find and return the latest cesm log file in the run directory
-        """
-        coupler_log_path = self._case.get_value("RUNDIR")
-        cesmlogs = glob.glob(os.path.join(coupler_log_path, 'cesm*.log.*'))
-        lastcesmlogs = []
-        if cesmlogs:
-            lastcesmlogs.append(max(cesmlogs, key=os.path.getctime))
-            basename = os.path.basename(lastcesmlogs[0])
-            suffix = basename.split('.',1)[1]
-            for log in cesmlogs:
-                if log in lastcesmlogs:
-                    continue
-
-                if log.endswith(suffix):
-                    lastcesmlogs.append(log)
-
-        return lastcesmlogs
+            if re.search('vvvvv gmean_mass: before tphysbc DRY',line.decode('utf-8')): 
+                use_this_tt_un = True
+            if re.search('TT_UN ',line.decode('utf-8')) and use_this_tt_un:
+                tt_un_flt=re.findall("\d+\.\d+",line.decode('utf-8'))
+                if len(tt_un_flt) > 0:
+                    if first_val == -9.0:
+                        first_val = tt_un_flt[0]
+                    if first_val != tt_un_flt[0]:
+                        with self._test_status:
+                            self._test_status.set_status("COMPARE_MASS", TEST_FAIL_STATUS, comments="Mass Not Conserved")
+                        comments = "CAM mass conservation test FAILED."
+                        append_testlog(comments, self._orig_caseroot)
+                use_this_tt_un = False 
+        if first_val == -9.0:
+                with self._test_status:
+                    self._test_status.set_status("COMPARE_MASS", TEST_FAIL_STATUS, comments="Failed to find TT_UN in atm.log")
+                comments = "CAM mass conservation test FAILED."
+                append_testlog(comments, self._orig_caseroot)
 

doc/ChangeLog

@@ -1,5 +1,83 @@
 ===============================================================
 
+Tag name: cam_cesm2_2_rel_08
+Originator(s): nusbaume
+Date: 21 Nov 2023
+One-line Summary: Add Github sparse checkout to CESM2.2 release branch
+Github PR URL: https://github.com/ESCOMP/CAM/pull/919
+
+Purpose of changes (include the issue number and title text for each relevant GitHub issue):
+
+Use git sparse checkout for all repos, combine clubb and silhs into clubb.
+git sparse checkout has been available in manage_externals since version 1.2.1.
+
+Fixes #912 -> cosp2 svn access to github support is ending soon
+
+Describe any changes made to build system:
+
+Build path were modified in the "configure" Perl script, as well
+as in the COSP Makefile.in file, in order to account for the
+new source code paths.
+
+Describe any changes made to the namelist: N/A
+
+List any changes to the defaults for the boundary datasets: N/A
+
+Describe any substantial timing or memory changes: N/A
+
+Code reviewed by: cacraigucar, adamrher
+
+List all files eliminated:  N/A
+
+List all files added and what they do:
+
+A       src/physics/cosp2/.cosp_sparse_checkout
+ - Provides information on how to do the git sparse checkout of COSP
+
+List all existing files that have been modified, and describe the changes:
+
+M       Externals_CAM.cfg
+ - Change the Github-SVN bridge method to git sparse checkout
+
+M       bld/configure
+ - Update source code paths for CLUBB, SILHS, and COSP
+
+M       src/physics/cosp2/Makefile.in
+ - Update source code paths for relevant COSP files
+
+M       cime_config/SystemTests/tmc.py
+M       src/physics/cam/phys_gmean.F90
+ - Fix failing TMC tests on Izumi
+
+If there were any failures reported from running test_driver.sh on any test
+platform, and checkin with these failures has been OK'd by the gatekeeper,
+then copy the lines from the td.*.status files for the failed tests to the
+appropriate machine below.  All failed tests must be justified.
+
+cheyenne/intel/aux_cam:
+
+All tests have NLCOMP failures due to changes in input data paths on Cheyenne.
+
+  ERP_Ln9.ne30pg3_ne30pg3_mg17.FW2000climo.cheyenne_intel.cam-outfrq9s_wcm_ne30 (Overall: FAIL) details:
+ - expected (pre-existing) failure for waccm on SE grid
+
+izumi/nag/aux_cam:
+  DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details:
+ - pre-existing failure
+
+  ERC_D_Ln9.T5_T5_mg37.QPC4.izumi_nag.cam-outfrq3s_usecase
+  ERC_D_Ln9.f10_f10_mg37.FHS94.izumi_nag.cam-idphys
+  ERI_D_Ln18.T5_T5_mg37.QPC5.izumi_nag.cam-ghgrmp_e8
+  ERI_D_Ln18.ne5pg3_ne5pg3_mg37.FADIAB.izumi_nag.cam-outfrq3s_bwic
+  TMC_D.T5_T5_mg37.QPC5.izumi_nag.cam-ghgrmp_e8
+ - TPUTCOMP failures (although failures may be random false positives)
+
+izumi/pgi/aux_cam: PGI compiler no longer available.
+
+Summarize any changes to answers: bit-for-bit
+
+===============================================================
+
 Tag name: cam_cesm2_2_rel_07
 Originator(s): fvitt
 Date: 2 Mar 2023

src/physics/cam/phys_gmean.F90

@@ -2,43 +2,18 @@ module phys_gmean
 !-----------------------------------------------------------------------
 !
 ! Purpose:
-! Perform mixed layer global calculations for energy conservation checks.
-!
-! Methods:
-! Reproducible (nonscalable):
-!    Gather to a master processor who does all the work.
-! Reproducible (scalable):
-!    Convert to fixed point (integer representation) to enable
-!    reproducibility when using MPI collectives. Results compared with
-!    a nonreproducible (but scalable) algorithm using floating point
-!    and MPI_Allreduce to verify the results are good enough.
+! Computes global mean mass, max and min mmr, of constituents on the
+! physics decomposition. Prints diagnostics to log file.
 !
-! Author: Byron Boville from SOM code by Jim Rosinski/Bruce Briegleb
-! Modified: P. Worley to aggregate calculations (4/04)
-! Modified: J. White/P. Worley to introduce scalable algorithms;
-!           B. Eaton to remove dycore-specific dependencies and to
-!           introduce gmean_mass (10/07)
-! Modified: P. Worley to replace in-place implementation with call
-!           to repro_sum.
+! Author: B. Eaton (based on gavglook)
 !
 !-----------------------------------------------------------------------
-   use shr_kind_mod,  only: r8 => shr_kind_r8
-   use physconst,     only: pi
-   use spmd_utils,    only: masterproc, MPI_REAL8, MPI_MAX, MPI_MIN, mpicom
-   use gmean_mod,     only: gmean
-   use ppgrid,        only: pcols, begchunk, endchunk
-   use shr_reprosum_mod, only: shr_reprosum_calc, shr_reprosum_tolExceeded, &
-                            shr_reprosum_reldiffmax, shr_reprosum_recompute
-   use perf_mod
-   use cam_logfile,   only: iulog
-
    implicit none
    private
-   save
 
    public :: gmean_mass ! compute global mean mass of constituent fields on physics decomposition
 
-   CONTAINS
+CONTAINS
 
 !
 !========================================================================
@@ -54,8 +29,14 @@ subroutine gmean_mass(title, state)
 ! Author: B. Eaton (based on gavglook)
 !
 !-----------------------------------------------------------------------
-      use ppgrid,         only: pver
+      use shr_kind_mod,   only: r8 => shr_kind_r8
+      use spmd_utils,     only: MPI_REAL8, MPI_MAX, MPI_MIN
+      use spmd_utils,     only: masterproc, masterprocid, mpicom
+      use cam_logfile,    only: iulog
+      use cam_abortutils, only: endrun
+      use ppgrid,         only: pver, pcols, begchunk, endchunk
       use physconst,      only: gravit
+      use gmean_mod,      only: gmean
       use phys_grid,      only: get_ncols_p
       use physics_types,  only: physics_state
       use constituents,   only: pcnst, cnst_name
@@ -68,10 +49,9 @@ subroutine gmean_mass(title, state)
 ! Local workspace
 !
       character(len=*), parameter :: sub_name='gmean_mass: '
-
-      integer :: c, i, k, m
-      integer :: ierr
-      integer :: ncols
+      integer                     :: ick, icol, lind, im
+      integer                     :: ierr
+      integer                     :: ncols
 
       real(r8), pointer :: mass_wet(:,:,:) ! constituent masses assuming moist mmr
       real(r8), pointer :: mass_dry(:,:,:) ! constituent masses assuming dry mmr
@@ -85,35 +65,39 @@ subroutine gmean_mass(title, state)
 !-----------------------------------------------------------------------
 !
       allocate(mass_wet(pcols,begchunk:endchunk,pcnst), stat=ierr)
-      if (ierr /= 0) write(iulog,*) sub_name // 'FAIL to allocate mass_wet'
+      if (ierr /= 0) then
+         call endrun(sub_name //'FAIL to allocate mass_wet')
+      end if
 
       allocate(mass_dry(pcols,begchunk:endchunk,pcnst), stat=ierr)
-      if (ierr /= 0) write(iulog,*) sub_name // 'FAIL to allocate mass_wet'
+      if (ierr /= 0) then
+         call endrun(sub_name//'FAIL to allocate mass_wet')
+      end if
 
       mmr_max(:) = -1.e36_r8
       mmr_min(:) =  1.e36_r8
-      do m = 1, pcnst
-         do c = begchunk, endchunk
-            ncols = get_ncols_p(c)
-            do i = 1, ncols
+      do im = 1, pcnst
+         do ick = begchunk, endchunk
+            ncols = get_ncols_p(ick)
+            do icol = 1, ncols
 
                ! Compute column masses assuming both dry and wet mixing ratios
 
-               mass_wet(i,c,m) = 0.0_r8
-               do k = 1, pver
-                  mass_wet(i,c,m) = mass_wet(i,c,m) + &
-                                    state(c)%pdel(i,k)*state(c)%q(i,k,m)
-                  mmr_max(m) = max(mmr_max(m), state(c)%q(i,k,m))
-                  mmr_min(m) = min(mmr_min(m), state(c)%q(i,k,m))
+               mass_wet(icol, ick, im) = 0.0_r8
+               do lind = 1, pver
+                  mass_wet(icol,ick,im) = mass_wet(icol,ick,im) +     &
+                       state(ick)%pdel(icol,lind)*state(ick)%q(icol,lind,im)
+                  mmr_max(im) = max(mmr_max(im), state(ick)%q(icol,lind,im))
+                  mmr_min(im) = min(mmr_min(im), state(ick)%q(icol,lind,im))
                end do
-               mass_wet(i,c,m) = mass_wet(i,c,m)/gravit
+               mass_wet(icol,ick,im) = mass_wet(icol,ick,im)/gravit
 
-               mass_dry(i,c,m) = 0.0_r8
-               do k = 1, pver
-                  mass_dry(i,c,m) = mass_dry(i,c,m) + &
-                                    state(c)%pdeldry(i,k)*state(c)%q(i,k,m)
+               mass_dry(icol,ick,im) = 0.0_r8
+               do lind = 1, pver
+                  mass_dry(icol,ick,im) = mass_dry(icol,ick,im) +             &
+                       state(ick)%pdeldry(icol,lind)*state(ick)%q(icol,lind,im)
                end do
-               mass_dry(i,c,m) = mass_dry(i,c,m)/gravit
+               mass_dry(icol,ick,im) = mass_dry(icol,ick,im)/gravit
 
             end do
          end do
@@ -124,20 +108,24 @@ subroutine gmean_mass(title, state)
       call gmean(mass_dry, mass_dry_mean, pcnst)
 
       ! global min/max mmr
-      call mpi_reduce(mmr_max, mmr_max_glob, pcnst, MPI_REAL8, MPI_MAX, 0, mpicom, ierr)
-      call mpi_reduce(mmr_min, mmr_min_glob, pcnst, MPI_REAL8, MPI_MIN, 0, mpicom, ierr)
+      call mpi_reduce(mmr_max, mmr_max_glob, pcnst, MPI_REAL8, MPI_MAX,       &
+           masterprocid, mpicom, ierr)
+      call mpi_reduce(mmr_min, mmr_min_glob, pcnst, MPI_REAL8, MPI_MIN,       &
+           masterprocid, mpicom, ierr)
 
       ! report to log file
       if (masterproc) then
-
-         do m = 1, pcnst
-               write (6,66) trim(title)//' m=',m, &
-                  'name='//trim(cnst_name(m))//' gavg dry, wet, min, max ', &
-                  mass_dry_mean(m), mass_wet_mean(m), mmr_min_glob(m), mmr_max_glob(m)
-66             format (a24,i2,a36,1p,4e25.13)
+         write(iulog, *) 'vvvvv ', sub_name, trim(title), ' vvvvv'
+         write(iulog, *) 'm                  name               ',            &
+              '       gavg dry       ', '       gavg wet       ',             &
+              '       gavg min       ', '       gavg max       '
+         do im = 1, pcnst
+            write (iulog, '(i2,a36,4("  ",e20.13e2))') im,                    &
+                 trim(cnst_name(im)), mass_dry_mean(im), mass_wet_mean(im),   &
+                 mmr_min_glob(im), mmr_max_glob(im)
          end do
-
-      endif
+         write(iulog, *) '^^^^^ ', sub_name, trim(title), ' ^^^^^'
+      end if
 
       deallocate(mass_wet)
       deallocate(mass_dry)

src/physics/cosp2/.cosp_sparse_checkout

@@ -0,0 +1 @@
+/src/