New and faster OpenMP implementation in atm_srk3 / threading of physics_get_tend

INSTALL

@@ -12,7 +12,11 @@ and the gfortran compiler. The gfortran compiler can be installed using homebrew
 (http://brew.sh), or using pre-compiled binaries from the MacOSX HPC website
 (http://hpc.soureforge.net), or it can be compiled by the user from the GNU sources.
 
-The standard clang compiler does not support OpenMP. Users wanting to compile MPAS
+Caveats:
+
+(1) MPAS cannot be compiled with gfortran-clang if GEN_F90=true.
+
+(2) The standard clang compiler does not support OpenMP. Users wanting to compile MPAS
 with OpenMP support on MacOSX will have to install the LLVM clang compiler, which is
 accomplished easiest with homebrew. Since this alternative clang compiler is not in
 the standard search/library path, the user will have to modify the call to the clang
@@ -57,3 +61,10 @@ would become
 ...
 
 assuming that the LLVM clang compiler is installed in /usr/local/opt/llvm.
+
+
+bluegene: Compiling MPAS on IBM Bluegene using the xl compilers
+----------
+All MPAS cores except the ocean compile on IBM Bluegene using the xl compilers. The ocean
+core currently does not work on IBM Bluegene. Known limitations: OPENMP must be disabled
+(OPENMP=false) for compiling, since the xl compilers do not support nested OpenMP directives.

Makefile

@@ -509,7 +509,6 @@ ifeq "$(OPENMP)" "true"
 endif #OPENMP IF
 
 ifeq "$(PRECISION)" "single"
-	FFLAGS += "-DSINGLE_PRECISION"
 	CFLAGS += "-DSINGLE_PRECISION"
 	CXXFLAGS += "-DSINGLE_PRECISION"
 	override CPPFLAGS += "-DSINGLE_PRECISION"

README.md

@@ -1,4 +1,4 @@
-MPAS-v5.0
+MPAS-v5.1
 ====
 
 The Model for Prediction Across Scales (MPAS) is a collaborative project for

src/core_atmosphere/Registry.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<registry model="mpas" core="atmosphere" core_abbrev="atm" version="5.0">
+<registry model="mpas" core="atmosphere" core_abbrev="atm" version="5.1">
 
 <!-- **************************************************************************************** -->
 <!-- ************************************** Dimensions ************************************** -->
@@ -224,16 +224,11 @@
                      description="Off-centering parameter for the vertically implicit acoustic integration"
                      possible_values="Positive real values"/>
 
-                <nml_option name="config_smdiv" type="real" default_value="0.025"
+                <nml_option name="config_smdiv" type="real" default_value="0.1"
                      units="-"
                      description="3-d divergence damping coefficient"
                      possible_values="Positive real values"/>
 
-                <nml_option name="config_smdiv_p_forward" type="real" default_value="0.1"
-                     units="-"
-                     description="Fractional forward weighting of the pressure during the acoustic step --- a form of 3D divergence damping"
-                     possible_values="Real values typically in the range 0 to 0.2"/>
-
                 <nml_option name="config_apvm_upwinding" type="real" default_value="0.5" in_defaults="false"
                      units="-"
                      description="Amount of upwinding in APVM"
@@ -1588,9 +1583,6 @@
                 <var name="rho_p_save" type="real" dimensions="nVertLevels nCells Time" units="kg m^{-3}"
                      description="predicted value rho_p, saved before acoustic steps"/>
 
-                <var name="divergence_3d" type="real" dimensions="nVertLevels nCells Time" units="kg m^{-3} s^{-1}"
-                     description="3D divergence used for acoustic filtering"/>
-
                 <var name="kdiff" type="real" dimensions="nVertLevels nCells Time" units="m^2 s^{-1}"
                      description="Smagorinsky horizontal eddy viscosity"/>
 

src/core_atmosphere/diagnostics/isobaric_diagnostics.F

@@ -553,7 +553,7 @@ subroutine interp_diagnostics(mesh, state, time_lev, diag)
            enddo
         enddo
 
-        if (NEED_TEMP .or. NEED_RELHUM .or. NEED_DEWPOINT) then
+        if (NEED_TEMP .or. NEED_RELHUM .or. NEED_DEWPOINT .or. need_mslp) then
            !calculation of temperature at cell centers:
             do iCell = 1,nCells
             do k = 1,nVertLevels

src/core_atmosphere/mpas_atm_core.F

@@ -1062,26 +1062,31 @@ subroutine atm_compute_damping_coefs(mesh, configs)
       real (kind=RKIND), pointer :: config_xnutr, config_zd
       real (kind=RKIND) :: z, zt, m1, pii
       real (kind=RKIND), dimension(:,:), pointer :: dss, zgrid
+      real (kind=RKIND), dimension(:), pointer :: meshDensity
+      real (kind=RKIND) :: dx_scale_power
 
       m1 = -1.0
       pii = acos(m1)
 
       call mpas_pool_get_dimension(mesh, 'nCells', nCells)
       call mpas_pool_get_dimension(mesh, 'nVertLevels', nVertLevels)
 
+      call mpas_pool_get_array(mesh, 'meshDensity', meshDensity)
       call mpas_pool_get_array(mesh, 'dss', dss)
       call mpas_pool_get_array(mesh, 'zgrid', zgrid)
 
       call mpas_pool_get_config(configs, 'config_zd', config_zd)
       call mpas_pool_get_config(configs, 'config_xnutr', config_xnutr)
 
+      dx_scale_power = 1.0
       dss(:,:) = 0.0
       do iCell=1,nCells
          zt = zgrid(nVertLevels+1,iCell)
          do k=1,nVertLevels
             z = 0.5*(zgrid(k,iCell) + zgrid(k+1,iCell))
             if (z > config_zd) then
                dss(k,iCell) = config_xnutr*sin(0.5*pii*(z-config_zd)/(zt-config_zd))**2.0
+               dss(k,iCell) = dss(k,iCell) / meshDensity(iCell)**(0.25*dx_scale_power)
             end if
          end do
       end do

src/core_atmosphere/physics/mpas_atmphys_interface.F

@@ -536,11 +536,6 @@ subroutine microphysics_from_MPAS(configs,mesh,state,time_lev,diag,diag_physics,
  qv   => scalars(index_qv,:,:)
  qc   => scalars(index_qc,:,:)
  qr   => scalars(index_qr,:,:)
- qi   => scalars(index_qi,:,:)
- qs   => scalars(index_qs,:,:)
- qg   => scalars(index_qg,:,:)
- ni   => scalars(index_ni,:,:)
- nr   => scalars(index_nr,:,:)
 
 !initialize variables needed in the cloud microphysics schemes:
  do j = jts, jte
@@ -567,6 +562,10 @@ subroutine microphysics_from_MPAS(configs,mesh,state,time_lev,diag,diag_physics,
  microp_select_init: select case(microp_scheme)
 
     case ("mp_thompson","mp_wsm6")
+       qi   => scalars(index_qi,:,:)
+       qs   => scalars(index_qs,:,:)
+       qg   => scalars(index_qg,:,:)
+
        do j = jts, jte
        do k = kts, kte
        do i = its, ite
@@ -580,6 +579,9 @@ subroutine microphysics_from_MPAS(configs,mesh,state,time_lev,diag,diag_physics,
     microp2_select: select case(microp_scheme)
 
        case("mp_thompson")
+          ni   => scalars(index_ni,:,:)
+          nr   => scalars(index_nr,:,:)
+
           do j = jts,jte
           do i = its,ite
              muc_p(i,j) = mu_c(i)
@@ -686,11 +688,6 @@ subroutine microphysics_to_MPAS(configs,mesh,state,time_lev,diag,diag_physics,te
  qv   => scalars(index_qv,:,:)
  qc   => scalars(index_qc,:,:)
  qr   => scalars(index_qr,:,:)
- qi   => scalars(index_qi,:,:)
- qs   => scalars(index_qs,:,:)
- qg   => scalars(index_qg,:,:)
- ni   => scalars(index_ni,:,:)
- nr   => scalars(index_nr,:,:)
 
  call mpas_pool_get_array(tend,'rt_diabatic_tend',rt_diabatic_tend)
 
@@ -743,6 +740,10 @@ subroutine microphysics_to_MPAS(configs,mesh,state,time_lev,diag,diag_physics,te
  microp_select_init: select case(microp_scheme)
 
     case ("mp_thompson","mp_wsm6")
+       qi   => scalars(index_qi,:,:)
+       qs   => scalars(index_qs,:,:)
+       qg   => scalars(index_qg,:,:)
+
        do j = jts, jte
        do k = kts, kte
        do i = its, ite
@@ -756,6 +757,9 @@ subroutine microphysics_to_MPAS(configs,mesh,state,time_lev,diag,diag_physics,te
     microp2_select: select case(microp_scheme)
 
        case("mp_thompson")
+          ni   => scalars(index_ni,:,:)
+          nr   => scalars(index_nr,:,:)
+
           do j = jts, jte
           do k = kts, kte
           do i = its, ite