Merge branch 'develop' into feature/multi_model_build

ci/cases/yamls/soca_gfs_defaults_ci.yaml

@@ -7,3 +7,5 @@ base:
 marineanl:
   SOCA_NINNER: 1
   DO_TEST_MODE: "YES"
+marineanl:
+  JCB_ALGO_YAML_VAR: "{{ HOMEgfs }}/sorc/gdas.cd/test/gw-ci/soca/jcb-prototype_3dfgat_3DVarAOWCDA.yaml.j2"

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -5,10 +5,17 @@ base:
   DO_JEDIATMVAR: "YES"
   DO_JEDIATMENS: "YES"
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  DO_TEST_MODE: "YES"
 atmanl:
+  JCB_ALGO_YAML_VAR: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_3dvar_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_FV3INC: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_3dvar-fv3inc_ufs_hybatmDA.yaml.j2"
   LAYOUT_X_ATMANL: 4
   LAYOUT_Y_ATMANL: 4
 atmensanl:
+  JCB_ALGO_YAML_LETKF: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_OBS: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_observer_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_SOL: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_solver_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_FV3INC: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf-fv3inc_ufs_hybatmDA.yaml.j2"
   LAYOUT_X_ATMENSANL: 4
   LAYOUT_Y_ATMENSANL: 4
 esfc:

ci/platforms/config.gaea

@@ -3,6 +3,6 @@
 export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT
 export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR
 export STMP="/gpfs/f5/epic/scratch/${USER}"
-export SLURM_ACCOUNT=ufs-ard
+export HPC_ACCOUNT=ufs-ard
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.hercules

@@ -2,7 +2,7 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export HPC_ACCOUNT=nems
+export HPC_ACCOUNT=fv3-cpu
 export max_concurrent_cases=5
 export max_concurrent_pr=4
 

jobs/JGDAS_ENKF_SELECT_OBS

@@ -32,21 +32,25 @@ GPREFIX_DET="${GDUMP}.t${gcyc}z."
 export GSUFFIX=".ensmean.nc"
 
 # Generate COM variables from templates
-RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS
-MEMDIR='ensstat' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS
-declare -rx COM_ATMOS_ANALYSIS_ENS="${COM_ATMOS_ANALYSIS}"
+RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMIN_OBS:COM_OBS_TMPL
+MEMDIR='ensstat' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+declare -rx COMOUT_ATMOS_ANALYSIS_ENS="${COMOUT_ATMOS_ANALYSIS}"
 
-RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -r COM_ATMOS_ANALYSIS_DET:COM_ATMOS_ANALYSIS_TMPL
+RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -r \
+	COMOUT_ATMOS_ANALYSIS_DET:COM_ATMOS_ANALYSIS_TMPL
 
 MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
-    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL \
+    COMIN_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
+    COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL 
 
-RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -r COM_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
+RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -r \
+	COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
 
-mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}"
+mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
 
-export ATMGES_ENSMEAN="${COM_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006${GSUFFIX}"
+export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006${GSUFFIX}"
 if [[ ! -f ${ATMGES_ENSMEAN} ]]; then
     echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = ${ATMGES_ENSMEAN}"
     exit 1
@@ -61,44 +65,44 @@ status=$?
 export LEVS
 
 # Link observational data
-export PREPQC="${COM_OBS}/${OPREFIX}prepbufr"
+export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr"
 if [[ ! -f ${PREPQC} ]]; then
     echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING"
 fi
-export TCVITL="${COM_OBS}/${APREFIX_DET}syndata.tcvitals.tm00"
+export TCVITL="${COMIN_OBS}/${APREFIX_DET}syndata.tcvitals.tm00"
 if [[ ${DONST} = "YES" ]]; then
-   export NSSTBF="${COM_OBS}/${OPREFIX}nsstbufr"
+   export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr"
 fi
-export PREPQCPF="${COM_OBS}/${OPREFIX}prepbufr.acft_profiles"
+export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles"
 
 # Deterministic analysis and increment files
-export SFCANL="${COM_ATMOS_ANALYSIS_DET}/${APREFIX_DET}sfcanl.nc"
-export DTFANL="${COM_ATMOS_ANALYSIS_DET}/${APREFIX_DET}dtfanl.nc"
-export ATMANL="${COM_ATMOS_ANALYSIS_DET}/${APREFIX_DET}atmanl.nc"
-export ATMINC="${COM_ATMOS_ANALYSIS_DET}/${APREFIX_DET}atminc.nc"
+export SFCANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}sfcanl.nc"
+export DTFANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}dtfanl.nc"
+export ATMANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}atmanl.nc"
+export ATMINC="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}atminc.nc"
 
 # Guess Bias correction coefficients related to control
-export GBIAS=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias
-export GBIASPC=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc
-export GBIASAIR=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air
-export GRADSTAT=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat
+export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias
+export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc
+export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air
+export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat
 
 # Bias correction coefficients related to ensemble mean
-export ABIAS="${COM_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean"
-export ABIASPC="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean"
-export ABIASAIR="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean"
-export ABIASe="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean"
+export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean"
+export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean"
+export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean"
+export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean"
 
 # Diagnostics related to ensemble mean
-export GSISTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean"
-export CNVSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean"
-export OZNSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean"
-export RADSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean"
+export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean"
+export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean"
+export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean"
+export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean"
 
 # Select observations based on ensemble mean
 export RUN_SELECT="YES"
 export USE_SELECT="NO"
-export SELECT_OBS="${COM_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean"
+export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean"
 
 export DIAG_SUFFIX="_ensmean"
 

jobs/JGLOBAL_ATMOS_ANALYSIS

@@ -31,18 +31,20 @@ export APREFIX="${rCDUMP}.t${cyc}z."
 export GPREFIX_ENS="${GDUMP_ENS}.t${gcyc}z."
 
 # Generate COM variables from templates
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_ATMOS_ANALYSIS
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+	COMIN_OBS:COM_OBS_TMPL \
+	COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
-    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
+    COMIN_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \
+    COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
 
 MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL
+    COMIN_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL
 
-mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}"
+mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
 
-export ATMGES="${COM_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.nc"
+export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.nc"
 if [ ! -f ${ATMGES} ]; then
     echo "FATAL ERROR: FILE MISSING: ATMGES = ${ATMGES}"
     exit 1
@@ -55,7 +57,7 @@ status=$?
 
 
 if [ ${DOHYBVAR} = "YES" ]; then
-    export ATMGES_ENSMEAN="${COM_ATMOS_HISTORY_ENS_PREV}/${GPREFIX_ENS}atmf006.ensmean.nc"
+    export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_ENS_PREV}/${GPREFIX_ENS}atmf006.ensmean.nc"
     if [ ! -f ${ATMGES_ENSMEAN} ]; then
         echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = ${ATMGES_ENSMEAN}"
         exit 2
@@ -64,17 +66,17 @@ fi
 
 
 # Link observational data
-export PREPQC="${COM_OBS}/${OPREFIX}prepbufr"
+export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr"
 if [[ ! -f ${PREPQC} ]]; then
     echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING"
 fi
-export TCVITL="${COM_OBS}/${OPREFIX}syndata.tcvitals.tm00"
+export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00"
 if [[ ${DONST} = "YES" ]]; then
    if [[ ${MAKE_NSSTBUFR} == "YES" ]]; then
-      export NSSTBF="${COM_OBS}/${OPREFIX}nsstbufr"
+      export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr"
    fi
 fi
-export PREPQCPF="${COM_OBS}/${OPREFIX}prepbufr.acft_profiles"
+export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles"
 
 # Copy fix file for obsproc  # TODO: Why is this necessary?
 if [[ ${RUN} = "gfs" ]]; then
@@ -95,9 +97,9 @@ status=$?
 # Send Alerts
 ##############################################
 if [ ${SENDDBN} = YES -a ${RUN} = gdas ] ; then
-    ${DBNROOT}/bin/dbn_alert MODEL GDAS_MSC_abias ${job} ${COM_ATMOS_ANALYSIS}/${APREFIX}abias
-    ${DBNROOT}/bin/dbn_alert MODEL GDAS_MSC_abias_pc ${job} ${COM_ATMOS_ANALYSIS}/${APREFIX}abias_pc
-    ${DBNROOT}/bin/dbn_alert MODEL GDAS_MSC_abias_air ${job} ${COM_ATMOS_ANALYSIS}/${APREFIX}abias_air
+    ${DBNROOT}/bin/dbn_alert MODEL GDAS_MSC_abias ${job} ${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias
+    ${DBNROOT}/bin/dbn_alert MODEL GDAS_MSC_abias_pc ${job} ${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc
+    ${DBNROOT}/bin/dbn_alert MODEL GDAS_MSC_abias_air ${job} ${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air
 fi
 
 

jobs/JGLOBAL_ATMOS_ANALYSIS_CALC

@@ -28,16 +28,19 @@ export GPREFIX="${GDUMP}.t${gcyc}z."
 export APREFIX="${RUN}.t${cyc}z."
 export GPREFIX_ENS="${GDUMP_ENS}.t${gcyc}z."
 
-RUN=${rCDUMP} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS
+RUN=${rCDUMP} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_OBS:COM_OBS_TMPL
 
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS COM_ATMOS_RESTART
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+    COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_OBS_PREV:COM_OBS_TMPL \
-    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
+    COMIN_OBS_PREV:COM_OBS_TMPL \
+    COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
 
 
-export ATMGES="${COM_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.nc"
+export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.nc"
 if [ ! -f ${ATMGES} ]; then
     echo "FATAL ERROR: FILE MISSING: ATMGES = ${ATMGES}"
     exit 1

parm/config/gfs/config.atmanl

@@ -6,6 +6,7 @@
 echo "BEGIN: config.atmanl"
 
 export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@
+export JCB_ALGO_YAML_FV3INC=@JCB_ALGO_YAML_FV3INC@
 
 export STATICB_TYPE=@STATICB_TYPE@
 export LOCALIZATION_TYPE="bump"

parm/config/gfs/config.atmensanl

@@ -8,6 +8,7 @@ echo "BEGIN: config.atmensanl"
 export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@
 export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@
 export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@
+export JCB_ALGO_YAML_FV3INC=@JCB_ALGO_YAML_FV3INC@
 
 export INTERP_METHOD='barycentric'
 

parm/config/gfs/config.marineanl

@@ -6,6 +6,7 @@
 echo "BEGIN: config.marineanl"
 
 export JEDI_CONFIG_YAML="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2"
+export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@
 
 export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config"
 export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@

parm/config/gfs/config.resources.ORION

@@ -30,10 +30,13 @@ case ${step} in
     export walltime="00:45:00"
   ;;
   "atmensanlobs")
-    # Run on 8 nodes for memory requirement
-    export tasks_per_node=8
+    # Run on 12 nodes for memory requirement
+    export tasks_per_node=12
     export walltime="00:45:00"
   ;;
+  "atmensanlsol")
+    export tasks_per_node=12
+  ;;  
   *)
   ;;
 esac

parm/config/gfs/yaml/defaults.yaml

@@ -27,6 +27,7 @@ base:
 
 atmanl:
   JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"
+  JCB_ALGO_YAML_FV3INC: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar-fv3inc.yaml.j2"
   STATICB_TYPE: "gsibec"
   LAYOUT_X_ATMANL: 8
   LAYOUT_Y_ATMANL: 8
@@ -37,6 +38,7 @@ atmensanl:
   JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
   JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2"
   JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2"
+  JCB_ALGO_YAML_FV3INC: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2"
   LAYOUT_X_ATMENSANL: 8
   LAYOUT_Y_ATMENSANL: 8
   IO_LAYOUT_X: 1
@@ -54,6 +56,7 @@ marineanl:
   SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
   SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml"  # TODO: This is also repeated in oceanprepobs
   SOCA_NINNER: 100
+  JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/soca/marine-jcb-3dfgat.yaml.j2"
 
 prepoceanobs:
   SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"

parm/gdas/atmanl_jedi_config.yaml.j2

@@ -10,4 +10,4 @@ atmanlfv3inc:
   exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x'
   mpi_cmd: '{{ APRUN_ATMANLFV3INC }}'
   jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
-  jcb_algo: fv3jedi_fv3inc_variational
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML_FV3INC }}'

parm/gdas/atmensanl_jedi_config.yaml.j2

@@ -17,7 +17,7 @@ atmensanlfv3inc:
   exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x'
   mpi_cmd: '{{ APRUN_ATMENSANLFV3INC }}'
   jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
-  jcb_algo: fv3jedi_fv3inc_lgetkf
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML_FV3INC }}'
 atmensanlletkf:
   rundir: '{{ DATA }}'
   exe_src: '{{ EXECgfs }}/gdas.x'