Serialize only inputs/outputs of fv_dyanmics() in order to reduce dat…

…a volume (#118)

For use when generating data for resolution > c48. Only save the Grid and FVDynamics data needed to run the python fv3core miniapp. Trying to save all the data runs out of space for these large runs.

FV3/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90

@@ -139,6 +139,7 @@ module atmosphere_mod
 
 #include <fms_platform.h>
 !$ser verbatim use mpi
+!$ser verbatim use m_serialize, ONLY: fs_is_serialization_on
 !-----------------
 ! FMS modules:
 !-----------------
@@ -255,6 +256,8 @@ module atmosphere_mod
   !$ser verbatim integer::cld_amt
 #endif
   !$ser verbatim integer :: o3mr, sgs_tke
+  !$ser verbatim character(len=256) :: ser_env
+  !$ser verbatim logical :: serialize_only_driver
   integer :: mytile  = 1
   integer :: p_split = 1
   integer, allocatable :: pelist(:)
@@ -390,7 +393,9 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
    graupel = get_tracer_index (MODEL_ATMOS, 'graupel' )
   !$ser verbatim o3mr = get_tracer_index (MODEL_ATMOS, 'o3mr')
   !$ser verbatim sgs_tke = get_tracer_index (MODEL_ATMOS, 'sgs_tke')
-
+  !$ser verbatim call get_environment_variable("SER_ENV", ser_env)
+  !$ser verbatim serialize_only_driver = (index(ser_env, "ONLY_DRIVER") /= 0)
+
 #ifdef CCPP
    cld_amt = get_tracer_index (MODEL_ATMOS, 'cld_amt')
 #else
@@ -653,6 +658,7 @@ subroutine atmosphere_dynamics ( Time )
    type(time_type) :: atmos_time
    !$ser verbatim integer :: mpi_rank,ier
    !$ser verbatim real :: bdt
+   !$ser verbatim logical :: ser_on
    !$ser verbatim  call mpi_comm_rank(MPI_COMM_WORLD, mpi_rank,ier)
 
 !---- Call FV dynamics -----
@@ -701,7 +707,11 @@ subroutine atmosphere_dynamics ( Time )
      !$ser verbatim if (sgs_tke > 0) then
      !$ser data qsgs_tke=Atm(n)%q(:,:,:,sgs_tke)
      !$ser verbatim endif
-      call fv_dynamics(npx, npy, npz, nq, Atm(n)%ng, dt_atmos/real(abs(p_split)),&
+     !$ser verbatim if (serialize_only_driver) then  
+         !$ser verbatim ser_on=fs_is_serialization_on()
+         !$ser off
+     !$ser verbatim endif
+     call fv_dynamics(npx, npy, npz, nq, Atm(n)%ng, dt_atmos/real(abs(p_split)),&
                        Atm(n)%flagstruct%consv_te, Atm(n)%flagstruct%fill,       &
                        Atm(n)%flagstruct%reproduce_sum, kappa, cp_air, zvir,     &
                        Atm(n)%ptop, Atm(n)%ks, nq,                               &
@@ -720,11 +730,18 @@ subroutine atmosphere_dynamics ( Time )
                        Atm(n)%neststruct,  Atm(n)%idiag, Atm(n)%bd,              &
                        Atm(n)%parent_grid, Atm(n)%domain,Atm(n)%diss_est,        &
                        Atm(n)%lagrangian_tendency_of_hydrostatic_pressure)
-
+     !$ser verbatim if (serialize_only_driver) then        
+       !$ser verbatim if (ser_on) then 
+       !$ser on
+       !$ser verbatim endif
+     !$ser verbatim endif    
      !$ser savepoint FVDynamics-Out
      !$ser data  u=Atm(n)%u v=Atm(n)%v w=Atm(n)%w delz=Atm(n)%delz pt=Atm(n)%pt delp=Atm(n)%delp qvapor=Atm(n)%q(:,:,:,sphum) qliquid=Atm(n)%q(:,:,:,liq_wat) qice=Atm(n)%q(:,:,:,ice_wat) qrain=Atm(n)%q(:,:,:,rainwat) qsnow=Atm(n)%q(:,:,:,snowwat) qgraupel=Atm(n)%q(:,:,:,graupel) qcld=Atm(n)%q(:,:,:,cld_amt) qo3mr=Atm(n)%q(:,:,:,o3mr) ps=Atm(n)%ps pe=Atm(n)%pe pk=Atm(n)%pk peln=Atm(n)%peln pkz=Atm(n)%pkz phis=Atm(n)%phis q_con=Atm(n)%q_con omga=Atm(n)%omga ua=Atm(n)%ua va=Atm(n)%va uc=Atm(n)%uc vc=Atm(n)%vc mfxd=Atm(n)%mfx mfyd=Atm(n)%mfy cxd=Atm(n)%cx cyd=Atm(n)%cy diss_estd=Atm(n)%diss_est
      !$ser verbatim if (sgs_tke > 0) then
      !$ser data qsgs_tke=Atm(n)%q(:,:,:,sgs_tke)
+     !$ser verbatim endif
+     !$ser verbatim if (serialize_only_driver) then        
+     !$ser off                                                                                                  
      !$ser verbatim endif
       call timing_off('fv_dynamics')
 

tests/serialized_test_data_generation/Makefile

@@ -5,13 +5,12 @@ EXPERIMENT_DIR = $(CWD)/configs
 EXPERIMENT ?= c12_6ranks_standard
 # base images w/ or w/o CUDA
 CUDA ?=y
-
+SER_ENV ?= ALL
 # version number or tag (determines image tag and destination directory in cloud storage)
 # -convention: <tag>-<some large conceptual version change>.<serialization statement change>.<hotfix>
 # -version number should be increased when serialization data is expected to change
 # -omit tag (e.g. 7.1.1) for "operational" serialization data, use tag for experimental serialization data
 FORTRAN_VERSION ?= 7.2.3
-
 # docker container image setup (used for running model, see ../../README.md)
 GCR_URL = us.gcr.io/vcm-ml
 COMPILED_IMAGE = $(GCR_URL)/fv3gfs-compiled:$(FORTRAN_VERSION)-serialize-gt4pydev
@@ -67,12 +66,14 @@ run_model: ## run model using Docker image to generate serialized data
 	fi
 	mkdir -p $(DATA_DIR_HOST)
 	docker run -e USER_ID_HOST=$(USER_ID_HOST) -e GROUP_ID_HOST=$(GROUP_ID_HOST) \
+		-e SER_ENV=$(SER_ENV) \
 		--network host --rm -v $(RUN_DIR_HOST):$(RUN_DIR_CONTAINER) \
 		-v $(DATA_DIR_HOST):$(DATA_DIR_CONTAINER) $(COMPILED_IMAGE) \
 		/rundir/submit_job.sh
 	cd $(DATA_DIR_HOST) && \
 		md5sum `/bin/ls -1d * | egrep -v '^fortran_sha.txt$$|^logfile.000000.out$$|^stdout.out$$|^stderr.out$$|^env.out$$'` > md5sums.txt
 	./check_data_dir.sh $(DATA_DIR_HOST)
+	cp $(EXPERIMENT_DIR)/$(EXPERIMENT).yml $(DATA_DIR_HOST)
 
 pack_data: ## pack *.dat files into a *.tar.gz and delete them
 	@if [ -f $(TARFILE_HOST) ] ; then \

tests/serialized_test_data_generation/make_all_datasets.sh

@@ -47,6 +47,12 @@ for exp_file in ${EXPERIMENTS} ; do
   exp_name=`basename ${exp_file} .yml`
   echo "====================================================="
   echo "Generating data for ${exp_name} ..."
+  npx=`cat ${exp_file} | grep npx | sed s/npx://g | sed 's/^ *//g'`
+  if [ ${npx} -gt 48 ] ; then
+      export SER_ENV="ONLY_DRIVER"
+  else
+      export SER_ENV="ALL"
+  fi
   if [ "${VALIDATE_ONLY}" == "true" ] ; then
       EXPERIMENT=${exp_name} make generate_data validate_data
   else