diff --git a/FV3/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 b/FV3/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 index ba7fce9ab..5381079b6 100644 --- a/FV3/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 +++ b/FV3/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 @@ -139,6 +139,7 @@ module atmosphere_mod #include !$ser verbatim use mpi +!$ser verbatim use m_serialize, ONLY: fs_is_serialization_on !----------------- ! FMS modules: !----------------- @@ -255,6 +256,8 @@ module atmosphere_mod !$ser verbatim integer::cld_amt #endif !$ser verbatim integer :: o3mr, sgs_tke + !$ser verbatim character(len=256) :: ser_env + !$ser verbatim logical :: serialize_only_driver integer :: mytile = 1 integer :: p_split = 1 integer, allocatable :: pelist(:) @@ -390,7 +393,9 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area) graupel = get_tracer_index (MODEL_ATMOS, 'graupel' ) !$ser verbatim o3mr = get_tracer_index (MODEL_ATMOS, 'o3mr') !$ser verbatim sgs_tke = get_tracer_index (MODEL_ATMOS, 'sgs_tke') - + !$ser verbatim call get_environment_variable("SER_ENV", ser_env) + !$ser verbatim serialize_only_driver = (index(ser_env, "ONLY_DRIVER") /= 0) + #ifdef CCPP cld_amt = get_tracer_index (MODEL_ATMOS, 'cld_amt') #else @@ -653,6 +658,7 @@ subroutine atmosphere_dynamics ( Time ) type(time_type) :: atmos_time !$ser verbatim integer :: mpi_rank,ier !$ser verbatim real :: bdt + !$ser verbatim logical :: ser_on !$ser verbatim call mpi_comm_rank(MPI_COMM_WORLD, mpi_rank,ier) !---- Call FV dynamics ----- @@ -701,7 +707,11 @@ subroutine atmosphere_dynamics ( Time ) !$ser verbatim if (sgs_tke > 0) then !$ser data qsgs_tke=Atm(n)%q(:,:,:,sgs_tke) !$ser verbatim endif - call fv_dynamics(npx, npy, npz, nq, Atm(n)%ng, dt_atmos/real(abs(p_split)),& + !$ser verbatim if (serialize_only_driver) then + !$ser verbatim ser_on=fs_is_serialization_on() + !$ser off + !$ser verbatim endif + call fv_dynamics(npx, npy, npz, nq, Atm(n)%ng, dt_atmos/real(abs(p_split)),& Atm(n)%flagstruct%consv_te, Atm(n)%flagstruct%fill, & Atm(n)%flagstruct%reproduce_sum, kappa, cp_air, zvir, & Atm(n)%ptop, Atm(n)%ks, nq, & @@ -720,11 +730,18 @@ subroutine atmosphere_dynamics ( Time ) Atm(n)%neststruct, Atm(n)%idiag, Atm(n)%bd, & Atm(n)%parent_grid, Atm(n)%domain,Atm(n)%diss_est, & Atm(n)%lagrangian_tendency_of_hydrostatic_pressure) - + !$ser verbatim if (serialize_only_driver) then + !$ser verbatim if (ser_on) then + !$ser on + !$ser verbatim endif + !$ser verbatim endif !$ser savepoint FVDynamics-Out !$ser data u=Atm(n)%u v=Atm(n)%v w=Atm(n)%w delz=Atm(n)%delz pt=Atm(n)%pt delp=Atm(n)%delp qvapor=Atm(n)%q(:,:,:,sphum) qliquid=Atm(n)%q(:,:,:,liq_wat) qice=Atm(n)%q(:,:,:,ice_wat) qrain=Atm(n)%q(:,:,:,rainwat) qsnow=Atm(n)%q(:,:,:,snowwat) qgraupel=Atm(n)%q(:,:,:,graupel) qcld=Atm(n)%q(:,:,:,cld_amt) qo3mr=Atm(n)%q(:,:,:,o3mr) ps=Atm(n)%ps pe=Atm(n)%pe pk=Atm(n)%pk peln=Atm(n)%peln pkz=Atm(n)%pkz phis=Atm(n)%phis q_con=Atm(n)%q_con omga=Atm(n)%omga ua=Atm(n)%ua va=Atm(n)%va uc=Atm(n)%uc vc=Atm(n)%vc mfxd=Atm(n)%mfx mfyd=Atm(n)%mfy cxd=Atm(n)%cx cyd=Atm(n)%cy diss_estd=Atm(n)%diss_est !$ser verbatim if (sgs_tke > 0) then !$ser data qsgs_tke=Atm(n)%q(:,:,:,sgs_tke) + !$ser verbatim endif + !$ser verbatim if (serialize_only_driver) then + !$ser off !$ser verbatim endif call timing_off('fv_dynamics') diff --git a/tests/serialized_test_data_generation/Makefile b/tests/serialized_test_data_generation/Makefile index 5a75c8767..464152da5 100644 --- a/tests/serialized_test_data_generation/Makefile +++ b/tests/serialized_test_data_generation/Makefile @@ -5,13 +5,12 @@ EXPERIMENT_DIR = $(CWD)/configs EXPERIMENT ?= c12_6ranks_standard # base images w/ or w/o CUDA CUDA ?=y - +SER_ENV ?= ALL # version number or tag (determines image tag and destination directory in cloud storage) # -convention: -.. # -version number should be increased when serialization data is expected to change # -omit tag (e.g. 7.1.1) for "operational" serialization data, use tag for experimental serialization data FORTRAN_VERSION ?= 7.2.3 - # docker container image setup (used for running model, see ../../README.md) GCR_URL = us.gcr.io/vcm-ml COMPILED_IMAGE = $(GCR_URL)/fv3gfs-compiled:$(FORTRAN_VERSION)-serialize-gt4pydev @@ -67,12 +66,14 @@ run_model: ## run model using Docker image to generate serialized data fi mkdir -p $(DATA_DIR_HOST) docker run -e USER_ID_HOST=$(USER_ID_HOST) -e GROUP_ID_HOST=$(GROUP_ID_HOST) \ + -e SER_ENV=$(SER_ENV) \ --network host --rm -v $(RUN_DIR_HOST):$(RUN_DIR_CONTAINER) \ -v $(DATA_DIR_HOST):$(DATA_DIR_CONTAINER) $(COMPILED_IMAGE) \ /rundir/submit_job.sh cd $(DATA_DIR_HOST) && \ md5sum `/bin/ls -1d * | egrep -v '^fortran_sha.txt$$|^logfile.000000.out$$|^stdout.out$$|^stderr.out$$|^env.out$$'` > md5sums.txt ./check_data_dir.sh $(DATA_DIR_HOST) + cp $(EXPERIMENT_DIR)/$(EXPERIMENT).yml $(DATA_DIR_HOST) pack_data: ## pack *.dat files into a *.tar.gz and delete them @if [ -f $(TARFILE_HOST) ] ; then \ diff --git a/tests/serialized_test_data_generation/make_all_datasets.sh b/tests/serialized_test_data_generation/make_all_datasets.sh index 20c1a0864..84bca0061 100755 --- a/tests/serialized_test_data_generation/make_all_datasets.sh +++ b/tests/serialized_test_data_generation/make_all_datasets.sh @@ -47,6 +47,12 @@ for exp_file in ${EXPERIMENTS} ; do exp_name=`basename ${exp_file} .yml` echo "=====================================================" echo "Generating data for ${exp_name} ..." + npx=`cat ${exp_file} | grep npx | sed s/npx://g | sed 's/^ *//g'` + if [ ${npx} -gt 48 ] ; then + export SER_ENV="ONLY_DRIVER" + else + export SER_ENV="ALL" + fi if [ "${VALIDATE_ONLY}" == "true" ] ; then EXPERIMENT=${exp_name} make generate_data validate_data else