Skip to content

Commit

Permalink
Some initial infrastructure for online coarse-graining (#26)
Browse files Browse the repository at this point in the history
* Add basic example of a coarse-grained diagnostic

* Slight refactor; added error checking

* Minor cleanup

* Remove placeholder implementation of weighted_block_average; add static diagnostic fields

* Make sure fine array bounds are properly initialized

* Reorder subroutines

* Clean up static diagnostics

* Remove unneeded block_sum_3d

* Reduce things down to a minimal example

* Simplify interface some

* Shorten names somewhat

* Address Jeremy's commemnts

* Add more comments

* Address Oli's comment; a few other minor cleanups

* Move do_coarse_graining flag up a level
  • Loading branch information
spencerkclark authored Apr 17, 2020
1 parent 31fc2ee commit 0e240ab
Show file tree
Hide file tree
Showing 6 changed files with 462 additions and 3 deletions.
17 changes: 17 additions & 0 deletions FV3/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,8 @@ module atmosphere_mod
a_step, p_step, current_time_in_seconds

use mpp_domains_mod, only: mpp_get_data_domain, mpp_get_compute_domain
use coarse_graining_mod, only: coarse_graining_init
use coarse_grained_diagnostics_mod, only: fv_coarse_diag_init, fv_coarse_diag
!$ser verbatim use k_checkpoint, only: set_nz

implicit none
Expand Down Expand Up @@ -335,6 +337,11 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
if (grids_on_this_pe(n)) mytile = n
enddo

if (Atm(mytile)%flagstruct%do_coarse_graining) then
call coarse_graining_init(Atm(mytile)%flagstruct%npx, Atm(mytile)%npz, &
Atm(mytile)%layout, Atm(mytile)%bd, Atm(mytile)%coarse_graining)
endif

Atm(mytile)%Time_init = Time_init

a_step = 0
Expand Down Expand Up @@ -426,6 +433,10 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
!I've had trouble getting this to work with multiple grids at a time; worth revisiting?
call fv_diag_init(Atm(mytile:mytile), Atm(mytile)%atmos_axes, Time, npx, npy, npz, Atm(mytile)%flagstruct%p_ref)

if (Atm(mytile)%coarse_graining%do_coarse_graining) then
call fv_coarse_diag_init(Atm(mytile)%bd, Time, Atm(mytile)%atmos_axes(3), &
Atm(mytile)%atmos_axes(4), Atm(mytile)%coarse_graining)
endif
!---------- reference profile -----------
ps1 = 101325.
ps2 = 81060.
Expand Down Expand Up @@ -779,6 +790,9 @@ subroutine atmosphere_end (Time, Grid_box)
call timing_on('FV_DIAG')
call fv_diag(Atm(mytile:mytile), zvir, fv_time, Atm(mytile)%flagstruct%print_freq)
call fv_nggps_diag(Atm(mytile:mytile), zvir, fv_time)
if (Atm(mytile)%coarse_graining%do_coarse_graining) then
call fv_coarse_diag(Atm(mytile:mytile), fv_time)
endif
first_diag = .false.
call timing_off('FV_DIAG')
endif
Expand Down Expand Up @@ -1627,6 +1641,9 @@ subroutine atmosphere_state_update (Time, IPD_Data, IAU_Data, Atm_block, flip_vc
call nullify_domain()
call timing_on('FV_DIAG')
call fv_diag(Atm(mytile:mytile), zvir, fv_time, Atm(mytile)%flagstruct%print_freq)
if (Atm(mytile)%coarse_graining%do_coarse_graining) then
call fv_coarse_diag(Atm(mytile:mytile), fv_time)
endif
first_diag = .false.
call timing_off('FV_DIAG')

Expand Down
2 changes: 2 additions & 0 deletions FV3/atmos_cubed_sphere/makefile
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ SRCS_F90 = \
./model/nh_utils.F90 \
./tools/external_ic.F90 \
./tools/external_sst.F90 \
./tools/coarse_grained_diagnostics.F90 \
./tools/fv_diagnostics.F90 \
./tools/fv_eta.F90 \
./tools/fv_grid_tools.F90 \
Expand All @@ -67,6 +68,7 @@ SRCS_F90 = \
./tools/fv_surf_map.F90 \
./tools/fv_timing.F90 \
./tools/init_hydro.F90 \
./tools/coarse_graining.F90 \
./tools/sim_nc_mod.F90 \
./tools/sorted_index.F90 \
./tools/test_cases.F90 \
Expand Down
36 changes: 34 additions & 2 deletions FV3/atmos_cubed_sphere/model/fv_arrays.F90
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,12 @@ module fv_arrays_mod

end type fv_diag_type

type fv_coarse_diag_type

integer :: id_omega_coarse
integer :: n_3d_diagnostics = 1

end type fv_coarse_diag_type

!>@brief The type 'fv_grid_type' is made up of grid-dependent information from fv_grid_tools and fv_grid_utils.
!>@details It should not contain any user options (that goes in a different structure) nor data which
Expand Down Expand Up @@ -1015,10 +1021,10 @@ module fv_arrays_mod
!f1p
logical :: adj_mass_vmr = .false. !TER: This is to reproduce answers for verona patch. This default can be changed
! to .true. in the next city release if desired


logical :: do_coarse_graining = .false. ! Whether to enable online coarse-graining of restart files and diagnostics
!integer, pointer :: test_case
!real, pointer :: alpha

end type fv_flags_type

type fv_nest_BC_type_3D
Expand Down Expand Up @@ -1158,6 +1164,30 @@ module fv_arrays_mod

end type fv_grid_bounds_type

type fv_coarse_grid_bounds_type

integer :: is_coarse, ie_coarse, js_coarse, je_coarse

end type fv_coarse_grid_bounds_type

type fv_coarse_graining_type

type(fv_coarse_grid_bounds_type) :: bd
type(domain2d) :: domain
integer :: factor
integer :: nx_coarse
integer :: id_x_coarse ! diagnostic x-axis id for data on x-edges
integer :: id_y_coarse ! diagnostic y-axis id for data on y-edges
integer :: id_xt_coarse ! diagnostic x-axis id for data on x-centers
integer :: id_yt_coarse ! diagnostic y-axis id for data on y-centers
integer :: id_pfull ! diagnostic vertical axis id for data on z-centers
integer :: id_phalf ! diagnostic vertical axis id for data on z-edges
character(len=64) :: strategy ! Current valid values are: 'model_level'
logical :: do_coarse_graining = .false.
type(fv_coarse_diag_type) :: idiag ! container for coarse diagnostic ids

end type fv_coarse_graining_type

type fv_regional_bc_bounds_type

integer :: is_north ,ie_north ,js_north ,je_north &
Expand Down Expand Up @@ -1328,6 +1358,8 @@ module fv_arrays_mod

type(nudge_diag_type) :: nudge_diag

type(fv_coarse_graining_type) :: coarse_graining

end type fv_atmos_type

contains
Expand Down
4 changes: 3 additions & 1 deletion FV3/atmos_cubed_sphere/model/fv_control.F90
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ module fv_control_mod
real, pointer :: s_weight, update_blend

integer, pointer :: layout(:), io_layout(:)
logical, pointer :: do_coarse_graining

integer :: ntilesMe ! Number of tiles on this process =1 for now

Expand Down Expand Up @@ -669,7 +670,7 @@ subroutine run_setup(Atm, dt_atmos, grids_on_this_pe, p_split)
nested, twowaynest, parent_grid_num, parent_tile, nudge_qv, &
refinement, nestbctype, nestupdate, nsponge, s_weight, &
ioffset, joffset, check_negative, nudge_ic, halo_update_type, gfs_phil, agrid_vel_rst, &
do_uni_zfull, adj_mass_vmr, fac_n_spl, fhouri, regional, bc_update_interval
do_uni_zfull, adj_mass_vmr, fac_n_spl, fhouri, regional, bc_update_interval, do_coarse_graining

namelist /test_case_nml/test_case, bubble_do, alpha, nsolitons, soliton_Umax, soliton_size
#ifdef MULTI_GASES
Expand Down Expand Up @@ -1340,6 +1341,7 @@ subroutine setup_pointers(Atm)

layout => Atm%layout
io_layout => Atm%io_layout
do_coarse_graining => Atm%flagstruct%do_coarse_graining
end subroutine setup_pointers


Expand Down
135 changes: 135 additions & 0 deletions FV3/atmos_cubed_sphere/tools/coarse_grained_diagnostics.F90
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
module coarse_grained_diagnostics_mod

use diag_manager_mod, only: diag_axis_init, register_diag_field, register_static_field, send_data
use fv_arrays_mod, only: fv_atmos_type, fv_coarse_diag_type, fv_coarse_graining_type, fv_grid_bounds_type
use mpp_domains_mod, only: domain2d
use mpp_mod, only: FATAL, mpp_error
use coarse_graining_mod, only: block_sum, get_fine_array_bounds, get_coarse_array_bounds, MODEL_LEVEL, weighted_block_average
use time_manager_mod, only: time_type

implicit none
private

public :: fv_coarse_diag_init, fv_coarse_diag

integer :: tile_count = 1 ! Following fv_diagnostics.F90

contains

subroutine fv_coarse_diag_init(bd, Time, id_pfull, id_phalf, coarse_graining)
type(fv_grid_bounds_type), intent(in) :: bd
type(time_type), intent(in) :: Time
integer, intent(in) :: id_pfull, id_phalf
type(fv_coarse_graining_type), intent(inout) :: coarse_graining

integer :: is, ie, js, je, is_coarse, ie_coarse, js_coarse, je_coarse

call get_fine_array_bounds(bd, is, ie, js, je)
call get_coarse_array_bounds(coarse_graining%bd, is_coarse, ie_coarse, js_coarse, je_coarse)
call initialize_coarse_diagnostic_axes(coarse_graining%domain, coarse_graining%nx_coarse, &
coarse_graining%id_x_coarse, coarse_graining%id_y_coarse, coarse_graining%id_xt_coarse, &
coarse_graining%id_yt_coarse)

coarse_graining%id_pfull = id_pfull
coarse_graining%id_phalf = id_phalf

call register_coarse_diagnostics(coarse_graining%idiag, Time, &
coarse_graining%id_xt_coarse, coarse_graining%id_yt_coarse, id_pfull)
end subroutine fv_coarse_diag_init

subroutine initialize_coarse_diagnostic_axes(coarse_domain, &
nx_coarse, id_x_coarse, id_y_coarse, id_xt_coarse, id_yt_coarse)
type(domain2d), intent(in) :: coarse_domain
integer, intent(in) :: nx_coarse
integer, intent(inout) :: id_x_coarse, id_y_coarse, id_xt_coarse, id_yt_coarse

integer :: i, j
real, allocatable :: grid_x_coarse(:), grid_y_coarse(:), grid_xt_coarse(:), grid_yt_coarse(:)

allocate(grid_x_coarse(nx_coarse + 1))
allocate(grid_y_coarse(nx_coarse + 1))
allocate(grid_xt_coarse(nx_coarse))
allocate(grid_yt_coarse(nx_coarse))

grid_x_coarse = (/ (i, i=1, nx_coarse + 1) /)
grid_y_coarse = (/ (j, j=1, nx_coarse + 1) /)
grid_xt_coarse = (/ (i, i=1, nx_coarse) /)
grid_yt_coarse = (/ (j, j=1, nx_coarse) /)

id_x_coarse = diag_axis_init('grid_x_coarse', grid_x_coarse, &
'index', 'x', 'x-index of cell corner points', set_name='coarse_grid', &
Domain2=coarse_domain, tile_count=tile_count)
id_y_coarse = diag_axis_init('grid_y_coarse', grid_y_coarse, &
'index', 'y', 'y-index of cell corner points', set_name='coarse_grid', &
Domain2=coarse_domain, tile_count=tile_count)

id_xt_coarse = diag_axis_init('grid_xt_coarse', grid_xt_coarse, &
'index', 'x', 'x-index of cell center points', set_name='coarse_grid', &
Domain2=coarse_domain, tile_count=tile_count)
id_yt_coarse = diag_axis_init('grid_yt_coarse', grid_yt_coarse, &
'index', 'y', 'y-index of cell center points', set_name='coarse_grid', &
Domain2=coarse_domain, tile_count=tile_count)
end subroutine initialize_coarse_diagnostic_axes

subroutine register_coarse_diagnostics(idiag_coarse, Time, id_xt_coarse,&
id_yt_coarse, id_pfull)
type(fv_coarse_diag_type), intent(inout) :: idiag_coarse
type(time_type), intent(in) :: Time
integer, intent(in) :: id_xt_coarse, id_yt_coarse, id_pfull

integer :: coarse_axes_t(3)
real :: missing_value = -1.0e10 ! Following fv_diagnostics.F90

coarse_axes_t = (/ id_xt_coarse, id_yt_coarse, id_pfull /)
idiag_coarse%id_omega_coarse = register_diag_field('dynamics', &
'omega_coarse', coarse_axes_t(1:3), Time, &
'coarse-grained omega', &
'Pa/s', missing_value=missing_value)
end subroutine register_coarse_diagnostics

subroutine fv_coarse_diag(Atm, Time)
type(fv_atmos_type), intent(in), target :: Atm(:)
type(time_type), intent(in) :: Time

character(len=256) :: error_message

if (trim(Atm(tile_count)%coarse_graining%strategy) .eq. MODEL_LEVEL) then
call fv_coarse_diag_model_levels(Atm, Time)
endif
end subroutine fv_coarse_diag

subroutine fv_coarse_diag_model_levels(Atm, Time)
type(fv_atmos_type), intent(in), target :: Atm(:)
type(time_type), intent(in) :: Time

real, allocatable :: work_3d_coarse(:,:,:)
integer :: diagnostic_ids_3d(Atm(tile_count)%coarse_graining%idiag%n_3d_diagnostics)
integer :: is, ie, js, je, is_coarse, ie_coarse, js_coarse, je_coarse, npz
logical :: used

call get_diagnostic_ids_3d(Atm(tile_count)%coarse_graining%idiag, diagnostic_ids_3d)
call get_fine_array_bounds(Atm(tile_count)%bd, is, ie, js, je)
call get_coarse_array_bounds(Atm(tile_count)%coarse_graining%bd, is_coarse, ie_coarse, js_coarse, je_coarse)
npz = Atm(tile_count)%npz

if (any(diagnostic_ids_3d > 0)) then
allocate(work_3d_coarse(is_coarse:ie_coarse,js_coarse:je_coarse,1:npz))
endif

if (Atm(tile_count)%coarse_graining%idiag%id_omega_coarse > 0) then
call weighted_block_average( &
Atm(tile_count)%gridstruct%area(is:ie,js:je), &
Atm(tile_count)%omga(is:ie,js:je,1:npz), &
work_3d_coarse)
used = send_data(Atm(tile_count)%coarse_graining%idiag%id_omega_coarse, work_3d_coarse, Time)
endif
end subroutine fv_coarse_diag_model_levels

subroutine get_diagnostic_ids_3d(idiag_coarse, diagnostic_ids_3d)
type(fv_coarse_diag_type), intent(in) :: idiag_coarse
integer, intent(out) :: diagnostic_ids_3d(idiag_coarse%n_3d_diagnostics)

diagnostic_ids_3d = (/ idiag_coarse%id_omega_coarse /)
end subroutine

end module coarse_grained_diagnostics_mod
Loading

0 comments on commit 0e240ab

Please sign in to comment.