Merge branch 'NOAA-EMC:develop' into feature/atmos_upp_com_in_out

.gitignore

@@ -200,6 +200,7 @@ ush/month_name.sh
 ush/imsfv3_scf2ioda.py
 ush/atparse.bash
 ush/run_bufr2ioda.py
+ush/bufr2ioda_insitu*
 
 # version files
 versions/build.ver

ci/Jenkinsfile

@@ -78,7 +78,7 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim()
                         CUSTOM_WORKSPACE = "${WORKSPACE}"
                         sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*")
@@ -267,7 +267,7 @@ pipeline {
                 }
             }
         }
-            
+
 
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
@@ -297,6 +297,6 @@ pipeline {
                     }
                 }
             }
-        }   
+        }
     }
 }

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -9,7 +9,7 @@ arguments:
   resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610
   idate: 2021032412
   edate: 2021032418
   nens: 0

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -10,7 +10,7 @@ arguments:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122106
   nens: 2

ci/cases/pr/C96_atmaerosnowDA.yaml → ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -6,12 +6,13 @@ arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATMA
   resdetatmos: 96
+  resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122012
   edate: 2021122100
-  nens: 0
+  nens: 2
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -9,7 +9,7 @@ arguments:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2024022318
   edate: 2024022400
   nens: 2
@@ -22,4 +22,5 @@ skip_ci_on_hosts:
   - gaea
   - orion
   - hercules
+  - wcoss2
 

ci/cases/pr/C96_atm3DVar.yaml

@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122106
   nens: 0

ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122118
   nens: 0

ci/scripts/utils/rocotostat.py

@@ -136,7 +136,7 @@ def rocoto_statcount(rocotostat):
     rocotostat_output = [line.split()[0:4] for line in rocotostat_output]
     rocotostat_output = [line for line in rocotostat_output if len(line) != 1]
 
-    status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE']
+    status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE', 'UNKNOWN']
 
     rocoto_status = {}
     status_counts = Counter(case for sublist in rocotostat_output for case in sublist)
@@ -214,14 +214,14 @@ def is_stalled(rocoto_status):
     elif rocoto_status['DEAD'] > 0:
         error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
         rocoto_state = 'FAIL'
-    elif 'UNAVAILABLE' in rocoto_status or 'UNKNOWN' in rocoto_status:
+    elif rocoto_status['UNAVAILABLE'] > 0 or rocoto_status['UNKNOWN'] > 0:
         rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError)
         error_return = 0
         rocoto_state = 'RUNNING'
-        if 'UNAVAILABLE' in rocoto_status:
+        if rocoto_status['UNAVAILABLE'] > 0:
             error_return = rocoto_status['UNAVAILABLE']
             rocoto_state = 'UNAVAILABLE'
-        if 'UNKNOWN' in rocoto_status:
+        if rocoto_status['UNKNOWN'] > 0:
             error_return += rocoto_status['UNKNOWN']
             rocoto_state = 'UNKNOWN'
     elif is_stalled(rocoto_status):

docs/source/init.rst

@@ -125,14 +125,14 @@ Start date = 2021032312
    ├── enkfgdas.20210323
    │   ├── 06
    │   │   ├── mem001
-   │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │   │   └── model -> ../../../gdas.20210323/06/model
    │   │   ├── mem002
-   │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │   │   └── model -> ../../../gdas.20210323/06/model
    │   │   ├── mem003
-   │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │   │   └── model -> ../../../gdas.20210323/06/model
    ...
    │   │   └── mem080
-   │   │       └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │       └── model -> ../../../gdas.20210323/06/model
    │   └── 12
    │       ├── mem001
    │       │   └── analysis
@@ -153,7 +153,7 @@ Start date = 2021032312
    │                   └── gdas.t12z.ocninc.nc -> ../../../../../gdas.20210323/12/analysis/ocean/gdas.t12z.ocninc.nc
    └── gdas.20210323
        ├── 06
-       │   └── model_data
+       │   └── model
        │       ├── atmos
        │       │   └── restart
        │       │       ├── 20210323.120000.ca_data.tile1.nc

env/AWSPW.env

@@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     NTHREADS1=${threads_per_task:-1}
     [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
-    APRUN="${launcher} -n ${ntasks}"
+    export APRUN="${launcher} -n ${ntasks}"
 else
     echo "ERROR config.resources must be sourced before sourcing AWSPW.env"
     exit 2
@@ -43,6 +43,13 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -52,6 +59,15 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}

env/AZUREPW.env

@@ -0,0 +1,55 @@
+#! /usr/bin/env bash
+
+if [[ $# -ne 1 ]]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    exit 1
+
+fi
+
+step=$1
+
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
+
+# Configure MPI environment
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+
+ulimit -s unlimited
+ulimit -a
+
+# Calculate common variables
+# Check first if the dependent variables are set
+if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
+    max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
+    NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
+    NTHREADS1=${threads_per_task:-1}
+    [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
+    [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
+    APRUN="${launcher} -n ${ntasks}"
+else
+    echo "ERROR config.resources must be sourced before sourcing AZUREPW.env"
+    exit 2
+fi
+
+if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+
+    export launcher="srun --mpi=pmi2 -l"
+
+    (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
+    (( ufs_ntasks = nnodes*tasks_per_node ))
+    # With ESMF threading, the model wants to use the full node
+    export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
+    unset nnodes ufs_ntasks
+
+elif [[ "${step}" = "post" ]]; then
+
+    export NTHREADS_NP=${NTHREADS1}
+    export APRUN_NP="${APRUN}"
+
+    export NTHREADS_DWN=${threads_per_task_dwn:-1}
+    [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
+    export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
+
+fi

env/GOOGLEPW.env

@@ -0,0 +1,55 @@
+#! /usr/bin/env bash
+
+if [[ $# -ne 1 ]]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    exit 1
+
+fi
+
+step=$1
+
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
+
+# Configure MPI environment
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+
+ulimit -s unlimited
+ulimit -a
+
+# Calculate common variables
+# Check first if the dependent variables are set
+if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
+    max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
+    NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
+    NTHREADS1=${threads_per_task:-1}
+    [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
+    [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
+    APRUN="${launcher} -n ${ntasks}"
+else
+    echo "ERROR config.resources must be sourced before sourcing GOOGLEPW.env"
+    exit 2
+fi
+
+if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+
+    export launcher="srun --mpi=pmi2 -l"
+
+    (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
+    (( ufs_ntasks = nnodes*tasks_per_node ))
+    # With ESMF threading, the model wants to use the full node
+    export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
+    unset nnodes ufs_ntasks
+
+elif [[ "${step}" = "post" ]]; then
+
+    export NTHREADS_NP=${NTHREADS1}
+    export APRUN_NP="${APRUN}"
+
+    export NTHREADS_DWN=${threads_per_task_dwn:-1}
+    [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
+    export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
+
+fi

env/HERA.env

@@ -80,7 +80,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
 elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
-    export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"    
+    export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -106,6 +106,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "marinebmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"

env/HERCULES.env

@@ -105,6 +105,14 @@ case ${step} in
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
  ;;
+ "esnowrecen")
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+ ;;
+
  "marinebmat")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

env/JET.env

@@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

env/ORION.env

@@ -97,6 +97,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

env/S4.env

@@ -68,7 +68,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
 elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
-    export APRUN_ATMENSANLFV3INC="${APRUN}"    
+    export APRUN_ATMENSANLFV3INC="${APRUN}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}