From e8d601049730e8e6a94cc31d3dbdf30565f1584a Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 13 Mar 2017 02:44:31 -0500 Subject: [PATCH 01/17] introduce MOAB as external library similar to PNETCDF; it needs to have MOAB environment defined to the installation point (such as $(MOAB)/lib/moab.make exists, includes are at $(MOAB)/include) MOAB should be build with at least hdf5, parallel because it is c++, we also add -lstdc++ to the link line Maybe there is a better way --- Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Makefile b/Makefile index baceb05cb7..a2140c32a2 100644 --- a/Makefile +++ b/Makefile @@ -450,6 +450,13 @@ ifneq "$(NETCDF)" "" LIBS += $(NCLIB) endif +ifneq "$(MOAB)" "" + CPPINCLUDES += -DUSE_MOAB -I$(MOAB)/include + FCINCLUDES += -DUSE_MOAB -I$(MOAB)/include + include $(MOAB)/lib/moab.make + LIBS += ${MOAB_LIBS_LINK} -lstdc++ +endif + RM = rm -f CPP = cpp -P -traditional RANLIB = ranlib @@ -539,6 +546,12 @@ else # USE_PIO2 IF PIO_MESSAGE="Using the PIO 1.x library." endif # USE_PIO2 IF +ifneq "$(MOAB)" "" + MOAB_MESSAGE="Using MOAB library" +else # + MOAB_MESSAGE="Not using MOAB library" +endif + ifdef TIMER_LIB ifeq "$(TIMER_LIB)" "tau" override TAU=true @@ -729,6 +742,7 @@ endif @echo $(GEN_F90_MESSAGE) @echo $(TIMER_MESSAGE) @echo $(PIO_MESSAGE) + @echo $(MOAB_MESSAGE) @echo "*******************************************************************************" clean: cd src; $(MAKE) clean RM="$(RM)" CORE="$(CORE)" From 47f35faae11946e36f4d6a04f30b662f7552a2ab Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 13 Mar 2017 02:51:52 -0500 Subject: [PATCH 02/17] instance moab mesh during bootstrapping only if MOAB env var is defined TODO: when can we read vertex coordinates more efficiently? also, need to release memory allocated for reading the vertices --- src/framework/Makefile | 6 +- src/framework/mpas_bootstrapping.F | 46 ++++++- src/framework/mpas_moabmesh.F | 189 +++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 src/framework/mpas_moabmesh.F diff --git a/src/framework/Makefile b/src/framework/Makefile index 4e635bbbc9..4c2d10c5a8 100644 --- a/src/framework/Makefile +++ b/src/framework/Makefile @@ -36,6 +36,10 @@ OBJS = mpas_kind_types.o \ mpas_field_accessor.o \ mpas_log.o +ifneq "$(MOAB)" "" + OBJS += mpas_moabmesh.o +endif + all: framework $(DEPS) framework: $(OBJS) @@ -90,7 +94,7 @@ mpas_io.o: mpas_dmpar.o mpas_attlist.o mpas_log.o mpas_io_streams.o: mpas_attlist.o mpas_derived_types.o mpas_timekeeping.o mpas_io.o mpas_pool_routines.o add_field_indices.inc mpas_log.o $(DEPS) -mpas_bootstrapping.o: mpas_derived_types.o mpas_dmpar.o mpas_block_decomp.o mpas_block_creator.o mpas_sort.o mpas_timekeeping.o mpas_io_streams.o mpas_stream_manager.o random_id.o mpas_log.o $(DEPS) +mpas_bootstrapping.o: mpas_derived_types.o mpas_dmpar.o mpas_block_decomp.o mpas_block_creator.o mpas_sort.o mpas_timekeeping.o mpas_io_streams.o mpas_stream_manager.o random_id.o mpas_log.o mpas_moabmesh.o $(DEPS) mpas_io_units.o: mpas_kind_types.o diff --git a/src/framework/mpas_bootstrapping.F b/src/framework/mpas_bootstrapping.F index e87dadc077..00688baa6b 100644 --- a/src/framework/mpas_bootstrapping.F +++ b/src/framework/mpas_bootstrapping.F @@ -17,6 +17,11 @@ module mpas_bootstrapping use mpas_timekeeping use mpas_io_streams use mpas_log + use mpas_io_units + use mpas_abort, only : mpas_dmpar_global_abort +#ifdef USE_MOAB + use mpas_moabmesh +#endif integer :: readCellStart, readCellEnd, nReadCells @@ -89,6 +94,7 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! integer :: nCells, nEdges, maxEdges, nVertices, vertexDegree integer :: ierr + integer :: i1, j1, nv1 type (MPAS_IO_Handle_type) :: inputHandle character (len=StrKIND) :: c_on_a_sphere, c_is_periodic, c_parent_id, c_file_id, c_mesh_spec @@ -269,7 +275,13 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! call mpas_block_creator_build_edge_halos(nHalos, indexToCellID_Block, nEdgesOnCell_Block, nCellsSolveField, & verticesOnCell_Block, indexToVertexID_Block, cellsOnVertex_Block, & nVerticesSolveField) - +#ifdef USE_MOAB + call create_mpas_moabmesh(domain, nEdgesOnCell_Block, verticesOnCell_Block, xVertexField, yVertexField, zVertexField) +#endif + ! now need to deallocate xVertexField, etc; it is not ours +! mpas_deallocate_field(xVertexField) +! mpas_deallocate_field(yVertexField) +! mpas_deallocate_field(zVertexField) ! ! Before we can allocate blocks, we need the attributes on_a_sphere and sphere_radius so ! they can be propagated as configs to subpools @@ -774,6 +786,7 @@ subroutine mpas_io_setup_vertex_block_fields(inputHandle, nReadVertices, readVer integer, intent(in) :: nHalos integer :: i, ierr + integer :: nVertices integer, dimension(:), pointer :: readIndices ! Global vertex indices @@ -793,6 +806,37 @@ subroutine mpas_io_setup_vertex_block_fields(inputHandle, nReadVertices, readVer call mpas_dmpar_init_multihalo_exchange_list(indexToVertexID % copyList, nHalos+1) nullify(indexToVertexID % next) +! added for local + call mpas_io_inq_dim(inputHandle, 'nVertices', nVertices, ierr) +! this is not read in parallel + ! Global xVertex + allocate(xVertex) + allocate(xVertex % array(nVertices)) + call MPAS_io_inq_var(inputHandle, 'xVertex', ierr=ierr) + call mpas_io_get_var(inputHandle, 'xVertex', xVertex % array, ierr) + xVertex % dimSizes(1) = nVertices + xVertex % block => readingBlock + nullify(xVertex % next) + + ! Global yVertex + allocate(yVertex) + allocate(yVertex % array(nVertices)) + call MPAS_io_inq_var(inputHandle, 'yVertex', ierr=ierr) + call mpas_io_get_var(inputHandle, 'yVertex', yVertex % array, ierr) + yVertex % dimSizes(1) = nVertices + yVertex % block => readingBlock + nullify(yVertex % next) + + ! Global zVertex + allocate(zVertex) + allocate(zVertex % array(nVertices)) + call MPAS_io_inq_var(inputHandle, 'zVertex', ierr=ierr) + call mpas_io_get_var(inputHandle, 'zVertex', zVertex % array, ierr) + zVertex % dimSizes(1) = nVertices + zVertex % block => readingBlock + nullify(zVertex % next) + + ! Global indices of cells adjacent to each vertex ! used for determining which vertices are owned by a block, where diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F new file mode 100644 index 0000000000..8bdd1d4e80 --- /dev/null +++ b/src/framework/mpas_moabmesh.F @@ -0,0 +1,189 @@ + + +module mpas_moabmesh +! use, intrinsic :: ISO_C_BINDING + + use mpas_io_units + use mpas_derived_types, only: dm_info, domain_type + use mpas_field_routines + use mpas_sort + + implicit none +#include "moab/MOABConfig.h" + + contains + + SUBROUTINE errorout(ierr, message) + integer ierr + character*(*) message + if (ierr.ne.0) then + print *, message + call exit (1) + end if + return + end subroutine + + subroutine create_mpas_moabmesh(domain, nEdgesOnCell, verticesOnCell, & + xVertex, yVertex, zVertex ) + + type (field2DInteger), pointer :: verticesOnCell + type (field1DInteger), pointer :: nEdgesOnCell + type (field1dReal), pointer :: xVertex, yVertex, zVertex + type (domain_type), pointer :: domain + integer :: c_comm, pid, i1, j1, ic, lastvertex + character*12 appname + + integer :: num_verts_cells ! total number of vertices in the cells + integer ,allocatable , target :: all_connects(:) + integer ,allocatable , target :: moabvh(:) + integer nv, currentval + ! RKIND must be double + real(kind=RKIND), allocatable, target :: moab_vert_coords(:) + integer dimcoord, dimen, nLocalCells, maxv, nedg, mbtype, block_ID, proc_id + + character*100 outfile, wopts, localmeshfile, lnum + integer num_layers, dimgh, bridge ! for ghosting + + +! do we really need this? + integer , external :: iMOAB_InitializeFortran, iMOAB_RegisterFortranApplication, & + iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & + iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities + + integer :: ierr, num_verts_in_cells + + ierr = iMOAB_InitializeFortran() + call errorout(ierr, 'fail to initialize iMOAB') + write(stderrUnit,*) 'initialize MOAB ' + c_comm = domain % dminfo % comm + appname = 'MPAS_MESH'//CHAR(0) + + ierr = iMOAB_RegisterFortranApplication(appname, c_comm, pid) + call errorout(ierr, 'fail to register MPAS_MOAB mesh') + proc_id = domain % dminfo % my_proc_id + write(stderrUnit,*) 'MOAB MPAS app pid: ', pid, ' task:', proc_id + + nLocalCells=verticesOnCell%dimSizes(2) + maxv = verticesOnCell%dimSizes(1) + write(stderrUnit,*) ' vertsOnCell: ', maxv, nLocalCells + write(stderrUnit,*) ' nEdgesOnCell: ', nEdgesOnCell%dimSizes(1) + write(stderrUnit,*) ' xVertex: ', xVertex%dimSizes(1) + + num_verts_in_cells = nLocalCells * maxv + allocate(all_connects(num_verts_in_cells)) +! collect all vertices + j1 = 0 + do ic=1, nLocalCells + do i1 = 1, nEdgesOnCell % array(ic) + j1 = j1 + 1 + all_connects(j1) = verticesOnCell % array( i1, ic) + enddo + lastvertex = verticesOnCell % array( nEdgesOnCell % array(ic), ic) + ! pad the rest with the last vertex + do i1 = nEdgesOnCell % array(ic) + 1, maxv + j1 = j1 + 1 + all_connects(j1) = lastvertex ! repeat the last vertex (pad) + enddo + enddo + + call mpas_quicksort(num_verts_in_cells, all_connects) + ! now compress the duplicates, keep only one vertex; then do binary search + + ! allocate(moabvh(num_verts_in_cells)) ! used to compress + + ! allocate(moabconn(moab_dim_cquads)) + nv=1 + currentval = all_connects(1) + do i1=2,num_verts_in_cells + if (all_connects(i1) .ne. currentval ) then + nv=nv+1 + currentval = all_connects(i1) + all_connects(nv) = currentval + endif + enddo + write (stderrUnit,*) ' number of vertices:', nv + write (stderrUnit,100) (all_connects(j1), j1=1,nv) +100 format(10I6) + ! num local vertices = nv; moabnv will be the global ids, that will be used for resolve sharing + allocate(moabvh(nv)) + !do i1=1,nv + moabvh(:) = all_connects(1:nv) ! copy the first + !enddo + ! first create local vertices, accumulate xv, yv, zv from xVertex, ... + allocate(moab_vert_coords(3*nv)) + do i1 =1, nv + j1 = moabvh(i1) + moab_vert_coords(3*i1-2) = xVertex % array(j1) + moab_vert_coords(3*i1-1) = yVertex % array(j1) + moab_vert_coords(3*i1 ) = zVertex % array(j1) + enddo + dimcoord = 3*nv + dimen = 3 + ierr = iMOAB_CreateVertices(pid, dimcoord, dimen, moab_vert_coords) + call errorout(ierr, 'fail to create vertices') + ! build now connectivity arrays using binary search + j1 = 0 + do ic=1, nLocalCells + nedg = nEdgesOnCell % array(ic) + do i1 = 1, nedg + j1=j1+1 + all_connects(j1) = mpas_binary_search(moabvh, 1, 1, nv, verticesOnCell % array( i1, ic) ) + + enddo + lastvertex = all_connects(j1) + ! pad the rest with the last vertex + do i1 = nedg + 1, maxv + j1 = j1 + 1 + all_connects(j1) = lastvertex ! repeat the last vertex (pad) + enddo + enddo + mbtype = 4 ! polygon + + block_ID = 100 ! this will be for coarse mesh + + ierr = iMOAB_CreateElements( pid, nLocalCells, mbtype, maxv, all_connects, block_ID ); + call errorout(ierr, 'fail to create polygons') + + ierr = iMOAB_ResolveSharedEntities( pid, nv, moabvh ); + call errorout(ierr, 'fail to resolve shared entities') + +! write in serial, on each task, before ghosting + if (proc_id.lt. 10) then + write(lnum,"(I0.2)")proc_id + localmeshfile = 'owned_'//trim(lnum)// '.h5m' // CHAR(0) + wopts = CHAR(0) + ierr = iMOAB_WriteMesh(pid, localmeshfile, wopts) + call errorout(ierr, 'fail to write local mesh file') + endif + +! write out the mesh file to disk, in parallel + outfile = 'whole.h5m'//CHAR(0) + wopts = 'PARALLEL=WRITE_PART'//CHAR(0) + ierr = iMOAB_WriteMesh(pid, outfile, wopts) + call errorout(ierr, 'fail to write the mesh file') + + ! (iMOAB_AppID pid, int * ghost_dim, int *num_ghost_layers, int * bridge_dim ) + dimgh = 2 ! will ghost polygons, topological dim 2 + bridge = 0 ! use vertex as bridge + num_layers = 1 ! so far, one layer only + ierr = iMOAB_DetermineGhostEntities( pid, dimgh, num_layers, bridge) + call errorout(ierr, 'fail to determine ghosts') + + ! write in serial, on each task + if (proc_id .lt. 10) then + write(lnum,"(I0.2)")proc_id + localmeshfile = 'localmesh_'//trim(lnum)// '.h5m' // CHAR(0) + wopts = CHAR(0) + ierr = iMOAB_WriteMesh(pid, localmeshfile, wopts) + call errorout(ierr, 'fail to write local mesh file') + endif + + deallocate (moab_vert_coords) + deallocate (moabvh) + deallocate (all_connects) + end subroutine create_mpas_moabmesh + + +end module mpas_moabmesh + + From 62a04b0376de80a1b91bcc33f4f35b54ed718c27 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 23 Oct 2017 17:58:01 -0500 Subject: [PATCH 03/17] iMOAB API changes related to external comp id --- src/framework/mpas_moabmesh.F | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 8bdd1d4e80..dd3611a504 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -50,15 +50,15 @@ subroutine create_mpas_moabmesh(domain, nEdgesOnCell, verticesOnCell, & iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities - integer :: ierr, num_verts_in_cells + integer :: ierr, num_verts_in_cells, ext_comp_id ierr = iMOAB_InitializeFortran() call errorout(ierr, 'fail to initialize iMOAB') write(stderrUnit,*) 'initialize MOAB ' c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) - - ierr = iMOAB_RegisterFortranApplication(appname, c_comm, pid) + ext_comp_id = 17 ! some number less than 33 + ierr = iMOAB_RegisterFortranApplication(appname, c_comm, ext_comp_id, pid) call errorout(ierr, 'fail to register MPAS_MOAB mesh') proc_id = domain % dminfo % my_proc_id write(stderrUnit,*) 'MOAB MPAS app pid: ', pid, ' task:', proc_id From 0721963224b6c0fa7cd8bef42dc93a47635ec494 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 27 Oct 2017 15:18:31 -0500 Subject: [PATCH 04/17] fix compilations after rebase --- src/framework/mpas_bootstrapping.F | 2 -- src/framework/mpas_moabmesh.F | 18 +++++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/framework/mpas_bootstrapping.F b/src/framework/mpas_bootstrapping.F index 00688baa6b..0b286a01e1 100644 --- a/src/framework/mpas_bootstrapping.F +++ b/src/framework/mpas_bootstrapping.F @@ -17,8 +17,6 @@ module mpas_bootstrapping use mpas_timekeeping use mpas_io_streams use mpas_log - use mpas_io_units - use mpas_abort, only : mpas_dmpar_global_abort #ifdef USE_MOAB use mpas_moabmesh #endif diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index dd3611a504..5cfb8d5ff2 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -3,7 +3,7 @@ module mpas_moabmesh ! use, intrinsic :: ISO_C_BINDING - use mpas_io_units + use mpas_log use mpas_derived_types, only: dm_info, domain_type use mpas_field_routines use mpas_sort @@ -54,20 +54,20 @@ subroutine create_mpas_moabmesh(domain, nEdgesOnCell, verticesOnCell, & ierr = iMOAB_InitializeFortran() call errorout(ierr, 'fail to initialize iMOAB') - write(stderrUnit,*) 'initialize MOAB ' + call mpas_log_write('initialize MOAB ', MPAS_LOG_WARN) c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) ext_comp_id = 17 ! some number less than 33 ierr = iMOAB_RegisterFortranApplication(appname, c_comm, ext_comp_id, pid) call errorout(ierr, 'fail to register MPAS_MOAB mesh') proc_id = domain % dminfo % my_proc_id - write(stderrUnit,*) 'MOAB MPAS app pid: ', pid, ' task:', proc_id + call mpas_log_write('MOAB MPAS app pid: $i task $i ', intArgs=(/pid, proc_id/) ) nLocalCells=verticesOnCell%dimSizes(2) maxv = verticesOnCell%dimSizes(1) - write(stderrUnit,*) ' vertsOnCell: ', maxv, nLocalCells - write(stderrUnit,*) ' nEdgesOnCell: ', nEdgesOnCell%dimSizes(1) - write(stderrUnit,*) ' xVertex: ', xVertex%dimSizes(1) + call mpas_log_write(' vertsOnCell: $i $i ', intArgs=(/maxv, nLocalCells/) ) + call mpas_log_write(' nEdgesOnCell: $i ', intArgs=(/nEdgesOnCell%dimSizes(1)/) ) + call mpas_log_write(' xVertex: $i ', intArgs=(/xVertex%dimSizes(1)/) ) num_verts_in_cells = nLocalCells * maxv allocate(all_connects(num_verts_in_cells)) @@ -101,9 +101,9 @@ subroutine create_mpas_moabmesh(domain, nEdgesOnCell, verticesOnCell, & all_connects(nv) = currentval endif enddo - write (stderrUnit,*) ' number of vertices:', nv - write (stderrUnit,100) (all_connects(j1), j1=1,nv) -100 format(10I6) + call mpas_log_write(' number of vertices:: $i ', intArgs=(/nv/) ) +! write (stderrUnit,100) (all_connects(j1), j1=1,nv) +! 100 format(10I6) ! num local vertices = nv; moabnv will be the global ids, that will be used for resolve sharing allocate(moabvh(nv)) !do i1=1,nv From 2d5c354510c52726426a7239be27540d88ca83bd Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sun, 5 Nov 2017 00:28:01 -0500 Subject: [PATCH 05/17] align MPAB/moab with ACME / MOAB use "HAVE_MOAB" and "MOAB_PATH", instead of USE_MOAB and MOAB PIO , NETCDF are still showing the path --- Makefile | 10 +++++----- src/framework/Makefile | 2 +- src/framework/mpas_bootstrapping.F | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index a2140c32a2..3693ce0726 100644 --- a/Makefile +++ b/Makefile @@ -450,10 +450,10 @@ ifneq "$(NETCDF)" "" LIBS += $(NCLIB) endif -ifneq "$(MOAB)" "" - CPPINCLUDES += -DUSE_MOAB -I$(MOAB)/include - FCINCLUDES += -DUSE_MOAB -I$(MOAB)/include - include $(MOAB)/lib/moab.make +ifneq "$(MOAB_PATH)" "" + CPPINCLUDES += -DHAVE_MOAB -I$(MOAB_PATH)/include + FCINCLUDES += -DHAVE_MOAB -I$(MOAB_PATH)/include + include $(MOAB_PATH)/lib/moab.make LIBS += ${MOAB_LIBS_LINK} -lstdc++ endif @@ -546,7 +546,7 @@ else # USE_PIO2 IF PIO_MESSAGE="Using the PIO 1.x library." endif # USE_PIO2 IF -ifneq "$(MOAB)" "" +ifneq "$(MOAB_PATH)" "" MOAB_MESSAGE="Using MOAB library" else # MOAB_MESSAGE="Not using MOAB library" diff --git a/src/framework/Makefile b/src/framework/Makefile index 4c2d10c5a8..6fbea7482e 100644 --- a/src/framework/Makefile +++ b/src/framework/Makefile @@ -36,7 +36,7 @@ OBJS = mpas_kind_types.o \ mpas_field_accessor.o \ mpas_log.o -ifneq "$(MOAB)" "" +ifneq "$(MOAB_PATH)" "" OBJS += mpas_moabmesh.o endif diff --git a/src/framework/mpas_bootstrapping.F b/src/framework/mpas_bootstrapping.F index 0b286a01e1..cf16906153 100644 --- a/src/framework/mpas_bootstrapping.F +++ b/src/framework/mpas_bootstrapping.F @@ -17,7 +17,7 @@ module mpas_bootstrapping use mpas_timekeeping use mpas_io_streams use mpas_log -#ifdef USE_MOAB +#ifdef HAVE_MOAB use mpas_moabmesh #endif @@ -273,7 +273,7 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! call mpas_block_creator_build_edge_halos(nHalos, indexToCellID_Block, nEdgesOnCell_Block, nCellsSolveField, & verticesOnCell_Block, indexToVertexID_Block, cellsOnVertex_Block, & nVerticesSolveField) -#ifdef USE_MOAB +#ifdef HAVE_MOAB call create_mpas_moabmesh(domain, nEdgesOnCell_Block, verticesOnCell_Block, xVertexField, yVertexField, zVertexField) #endif ! now need to deallocate xVertexField, etc; it is not ours From 611bb3196fa87ca4bbe6e3afa79f977cfaeabf6d Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sun, 5 Nov 2017 01:34:58 -0600 Subject: [PATCH 06/17] move initialize moab outside of mesh creation before bootstrapping --- src/driver/mpas_subdriver.F | 14 ++++++++++++++ src/framework/mpas_moabmesh.F | 3 --- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/driver/mpas_subdriver.F b/src/driver/mpas_subdriver.F index fe80338ceb..5e1a12a277 100644 --- a/src/driver/mpas_subdriver.F +++ b/src/driver/mpas_subdriver.F @@ -81,6 +81,10 @@ subroutine mpas_init() character(len=StrKIND) :: iotype logical :: streamsExists integer :: mesh_iotype +#ifdef HAVE_MOAB + integer , external :: iMOAB_InitializeFortran +#endif + interface subroutine xml_stream_parser(xmlname, mgr_p, comm, ierr) bind(c) @@ -289,6 +293,16 @@ end subroutine xml_stream_get_attributes call mpas_set_timeInterval(filename_interval, timeString=filename_interval_temp, ierr=ierr) call mpas_build_stream_filename(ref_time, start_time, filename_interval, mesh_filename_temp, blockID, mesh_filename, ierr) end if + +#ifdef HAVE_MOAB + ierr = iMOAB_InitializeFortran() + if ( ierr /= 0 ) then + call mpas_log_write('cannot initialize MOAB', messageType=MPAS_LOG_CRIT) + else + call mpas_log_write(' initialized MOAB', messageType=MPAS_LOG_WARN) + end if +#endif + call mpas_log_write(' ** Attempting to bootstrap MPAS framework using stream: ' // trim(mesh_stream)) call mpas_bootstrap_framework_phase1(domain_ptr, mesh_filename, mesh_iotype) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 5cfb8d5ff2..03f39f5dfa 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -52,9 +52,6 @@ subroutine create_mpas_moabmesh(domain, nEdgesOnCell, verticesOnCell, & integer :: ierr, num_verts_in_cells, ext_comp_id - ierr = iMOAB_InitializeFortran() - call errorout(ierr, 'fail to initialize iMOAB') - call mpas_log_write('initialize MOAB ', MPAS_LOG_WARN) c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) ext_comp_id = 17 ! some number less than 33 From 7e3cd62cb8b40df2e9c8c7682f5bac2901856a6b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 6 Nov 2017 09:28:15 -0600 Subject: [PATCH 07/17] guard Vertex fields for MOAB also, clear memory for vertex fields that were created for MOAB --- src/framework/mpas_bootstrapping.F | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/framework/mpas_bootstrapping.F b/src/framework/mpas_bootstrapping.F index cf16906153..c624ddad7c 100644 --- a/src/framework/mpas_bootstrapping.F +++ b/src/framework/mpas_bootstrapping.F @@ -276,10 +276,7 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! #ifdef HAVE_MOAB call create_mpas_moabmesh(domain, nEdgesOnCell_Block, verticesOnCell_Block, xVertexField, yVertexField, zVertexField) #endif - ! now need to deallocate xVertexField, etc; it is not ours -! mpas_deallocate_field(xVertexField) -! mpas_deallocate_field(yVertexField) -! mpas_deallocate_field(zVertexField) + ! ! Before we can allocate blocks, we need the attributes on_a_sphere and sphere_radius so ! they can be propagated as configs to subpools @@ -423,6 +420,12 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! call mpas_deallocate_field(nCellsSolveField) call mpas_deallocate_field(nVerticesSolveField) call mpas_deallocate_field(nEdgesSolveField) +#ifdef HAVE_MOAB + call mpas_deallocate_field(xVertexField) + call mpas_deallocate_field(yVertexField) + call mpas_deallocate_field(zVertexField) +#endif + deallocate(local_cell_list) deallocate(block_id) @@ -804,6 +807,7 @@ subroutine mpas_io_setup_vertex_block_fields(inputHandle, nReadVertices, readVer call mpas_dmpar_init_multihalo_exchange_list(indexToVertexID % copyList, nHalos+1) nullify(indexToVertexID % next) +#ifdef HAVE_MOAB ! added for local call mpas_io_inq_dim(inputHandle, 'nVertices', nVertices, ierr) ! this is not read in parallel @@ -833,7 +837,7 @@ subroutine mpas_io_setup_vertex_block_fields(inputHandle, nReadVertices, readVer zVertex % dimSizes(1) = nVertices zVertex % block => readingBlock nullify(zVertex % next) - +#endif ! Global indices of cells adjacent to each vertex From 3b6b9ab28bd54cfa1bca69f76767cc5e1180c041 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sun, 19 Nov 2017 00:37:27 -0600 Subject: [PATCH 08/17] instantiate after core_init also, use mesh pool --- src/driver/mpas_subdriver.F | 8 + src/framework/mpas_bootstrapping.F | 47 +---- src/framework/mpas_moabmesh.F | 274 ++++++++++++++--------------- 3 files changed, 139 insertions(+), 190 deletions(-) diff --git a/src/driver/mpas_subdriver.F b/src/driver/mpas_subdriver.F index 5e1a12a277..4631c8536b 100644 --- a/src/driver/mpas_subdriver.F +++ b/src/driver/mpas_subdriver.F @@ -35,6 +35,10 @@ module mpas_subdriver use test_core_interface #endif +#ifdef HAVE_MOAB + use mpas_moabmesh +#endif + type (core_type), pointer :: corelist => null() type (dm_info), pointer :: dminfo type (domain_type), pointer :: domain_ptr @@ -347,6 +351,10 @@ end subroutine xml_stream_get_attributes call mpas_log_write('Core init failed for core '//trim(domain_ptr % core % coreName), messageType=MPAS_LOG_CRIT) end if +#ifdef HAVE_MOAB + call mpas_moab_instance(domain_ptr) +#endif + call mpas_timer_stop('initialize') end subroutine mpas_init diff --git a/src/framework/mpas_bootstrapping.F b/src/framework/mpas_bootstrapping.F index c624ddad7c..454d4b726d 100644 --- a/src/framework/mpas_bootstrapping.F +++ b/src/framework/mpas_bootstrapping.F @@ -17,10 +17,6 @@ module mpas_bootstrapping use mpas_timekeeping use mpas_io_streams use mpas_log -#ifdef HAVE_MOAB - use mpas_moabmesh -#endif - integer :: readCellStart, readCellEnd, nReadCells integer :: readEdgeStart, readEdgeEnd, nReadEdges @@ -273,9 +269,7 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! call mpas_block_creator_build_edge_halos(nHalos, indexToCellID_Block, nEdgesOnCell_Block, nCellsSolveField, & verticesOnCell_Block, indexToVertexID_Block, cellsOnVertex_Block, & nVerticesSolveField) -#ifdef HAVE_MOAB - call create_mpas_moabmesh(domain, nEdgesOnCell_Block, verticesOnCell_Block, xVertexField, yVertexField, zVertexField) -#endif + ! ! Before we can allocate blocks, we need the attributes on_a_sphere and sphere_radius so @@ -420,12 +414,6 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! call mpas_deallocate_field(nCellsSolveField) call mpas_deallocate_field(nVerticesSolveField) call mpas_deallocate_field(nEdgesSolveField) -#ifdef HAVE_MOAB - call mpas_deallocate_field(xVertexField) - call mpas_deallocate_field(yVertexField) - call mpas_deallocate_field(zVertexField) -#endif - deallocate(local_cell_list) deallocate(block_id) @@ -807,39 +795,6 @@ subroutine mpas_io_setup_vertex_block_fields(inputHandle, nReadVertices, readVer call mpas_dmpar_init_multihalo_exchange_list(indexToVertexID % copyList, nHalos+1) nullify(indexToVertexID % next) -#ifdef HAVE_MOAB -! added for local - call mpas_io_inq_dim(inputHandle, 'nVertices', nVertices, ierr) -! this is not read in parallel - ! Global xVertex - allocate(xVertex) - allocate(xVertex % array(nVertices)) - call MPAS_io_inq_var(inputHandle, 'xVertex', ierr=ierr) - call mpas_io_get_var(inputHandle, 'xVertex', xVertex % array, ierr) - xVertex % dimSizes(1) = nVertices - xVertex % block => readingBlock - nullify(xVertex % next) - - ! Global yVertex - allocate(yVertex) - allocate(yVertex % array(nVertices)) - call MPAS_io_inq_var(inputHandle, 'yVertex', ierr=ierr) - call mpas_io_get_var(inputHandle, 'yVertex', yVertex % array, ierr) - yVertex % dimSizes(1) = nVertices - yVertex % block => readingBlock - nullify(yVertex % next) - - ! Global zVertex - allocate(zVertex) - allocate(zVertex % array(nVertices)) - call MPAS_io_inq_var(inputHandle, 'zVertex', ierr=ierr) - call mpas_io_get_var(inputHandle, 'zVertex', zVertex % array, ierr) - zVertex % dimSizes(1) = nVertices - zVertex % block => readingBlock - nullify(zVertex % next) -#endif - - ! Global indices of cells adjacent to each vertex ! used for determining which vertices are owned by a block, where ! iVtx is owned iff cellsOnVertex(1,iVtx) is an owned cell diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 03f39f5dfa..8e1ddb2bdc 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -7,6 +7,9 @@ module mpas_moabmesh use mpas_derived_types, only: dm_info, domain_type use mpas_field_routines use mpas_sort + use mpas_stream_manager + use mpas_pool_routines + !use mpas_vector_operations implicit none #include "moab/MOABConfig.h" @@ -22,35 +25,37 @@ SUBROUTINE errorout(ierr, message) end if return end subroutine - - subroutine create_mpas_moabmesh(domain, nEdgesOnCell, verticesOnCell, & - xVertex, yVertex, zVertex ) - - type (field2DInteger), pointer :: verticesOnCell - type (field1DInteger), pointer :: nEdgesOnCell - type (field1dReal), pointer :: xVertex, yVertex, zVertex - type (domain_type), pointer :: domain + + subroutine mpas_moab_instance(domain) + + type (domain_type), intent(inout) :: domain + + type (block_type), pointer :: block + type (mpas_pool_type), pointer :: meshPool + integer, pointer :: nCells, nVertices, maxEdges + integer :: nVert, iCell, iVertex, nblocks + real (kind=RKIND), dimension(:), allocatable :: xv,yv,zv + integer, dimension(:,:), pointer :: verticesOnCell + integer, dimension(:), pointer :: nEdgesOnCell + integer, dimension(:), pointer :: indexToVertexID, indexToCellID + real(kind=RKIND), dimension(:), pointer :: xVertex, yVertex, zVertex + real (kind=RKIND), dimension(:), pointer :: xCell, yCell, zCell + logical, pointer :: on_a_sphere, is_periodic + real(kind=RKIND), pointer :: x_period, y_period + integer, pointer :: nCellsSolve, nEdgesSolve, nVerticesSolve + + + + integer , external :: iMOAB_RegisterFortranApplication, & + iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & + iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities integer :: c_comm, pid, i1, j1, ic, lastvertex character*12 appname - - integer :: num_verts_cells ! total number of vertices in the cells - integer ,allocatable , target :: all_connects(:) - integer ,allocatable , target :: moabvh(:) - integer nv, currentval - ! RKIND must be double + integer :: ierr, num_verts_in_cells, ext_comp_id real(kind=RKIND), allocatable, target :: moab_vert_coords(:) - integer dimcoord, dimen, nLocalCells, maxv, nedg, mbtype, block_ID, proc_id - + integer dimcoord, dimen, mbtype, block_ID, proc_id + integer ,allocatable , target :: all_connects(:) character*100 outfile, wopts, localmeshfile, lnum - integer num_layers, dimgh, bridge ! for ghosting - - -! do we really need this? - integer , external :: iMOAB_InitializeFortran, iMOAB_RegisterFortranApplication, & - iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & - iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities - - integer :: ierr, num_verts_in_cells, ext_comp_id c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) @@ -60,127 +65,108 @@ subroutine create_mpas_moabmesh(domain, nEdgesOnCell, verticesOnCell, & proc_id = domain % dminfo % my_proc_id call mpas_log_write('MOAB MPAS app pid: $i task $i ', intArgs=(/pid, proc_id/) ) - nLocalCells=verticesOnCell%dimSizes(2) - maxv = verticesOnCell%dimSizes(1) - call mpas_log_write(' vertsOnCell: $i $i ', intArgs=(/maxv, nLocalCells/) ) - call mpas_log_write(' nEdgesOnCell: $i ', intArgs=(/nEdgesOnCell%dimSizes(1)/) ) - call mpas_log_write(' xVertex: $i ', intArgs=(/xVertex%dimSizes(1)/) ) - - num_verts_in_cells = nLocalCells * maxv - allocate(all_connects(num_verts_in_cells)) -! collect all vertices - j1 = 0 - do ic=1, nLocalCells - do i1 = 1, nEdgesOnCell % array(ic) - j1 = j1 + 1 - all_connects(j1) = verticesOnCell % array( i1, ic) +! blocks should be merged if there is more than one block per task + nblocks = 0 + block => domain % blocklist + do while (associated(block)) !{{{ + nblocks = nblocks + 1 + ! allocate scratch memory + call mpas_pool_get_subpool(block % structs, 'mesh', meshPool) + call mpas_pool_get_array(meshPool, 'xVertex', xVertex) + call mpas_pool_get_array(meshPool, 'yVertex', yVertex) + call mpas_pool_get_array(meshPool, 'zVertex', zVertex) + call mpas_pool_get_dimension(meshPool, 'nVertices', nVertices) + call mpas_pool_get_dimension(meshPool, 'maxEdges', maxEdges) + call mpas_pool_get_config(meshPool, 'on_a_sphere', on_a_sphere) + call mpas_pool_get_config(meshPool, 'is_periodic', is_periodic) + call mpas_pool_get_config(meshPool, 'x_period', x_period) + call mpas_pool_get_config(meshPool, 'y_period', y_period) + call mpas_pool_get_array(meshPool, 'verticesOnCell', verticesOnCell) + call mpas_pool_get_array(meshPool, 'indexToVertexID', indexToVertexID) + call mpas_pool_get_array(meshPool, 'indexToCellID', indexToCellID) + call mpas_pool_get_dimension(meshPool, 'nCells', nCells) + call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) + call mpas_pool_get_dimension(meshPool, 'nCellsSolve', nCellsSolve) + call mpas_pool_get_dimension(meshPool, 'nEdgesSolve', nEdgesSolve) + call mpas_pool_get_dimension(meshPool, 'nVerticesSolve', nVerticesSolve) +! call mpas_pool_get_array(meshPool, 'xCell', xCell) +! call mpas_pool_get_array(meshPool, 'yCell', yCell) +! call mpas_pool_get_array(meshPool, 'zCell', zCell) + +#ifdef MPAS_DEBUG + call mpas_log_write(' number of vertices:: $i number of cells:: $i solve: v:$i c:$i', intArgs=(/nVertices, nCells, nVerticesSolve, nCellsSolve/) ) + do iVertex=1,nVertices + call mpas_log_write('i: $i indexToVertexID:: $i ', intArgs=(/iVertex, indexToVertexID(iVertex)/) ) + enddo +#endif + ! first create local vertices, nVerticesSolve + allocate(moab_vert_coords(3*nVertices)) + do i1 =1, nVertices + moab_vert_coords(3*i1-2) = xVertex(i1) + moab_vert_coords(3*i1-1) = yVertex(i1) + moab_vert_coords(3*i1 ) = zVertex(i1) + ! call mpas_log_write('i:: $i coords:: $r $r $r $r', intArgs=(/i1/), realArgs=(/moab_vert_coords(3*i1-2),moab_vert_coords(3*i1-1), moab_vert_coords(3*i1)/) ) + enddo + dimcoord = 3*nVertices + dimen = 3 + ierr = iMOAB_CreateVertices(pid, dimcoord, dimen, moab_vert_coords) + call errorout(ierr, 'fail to create vertices') + ! fill now connectivity array, nCellsSolve; fist pad to max nc + num_verts_in_cells = nCellsSolve * maxEdges + allocate(all_connects(num_verts_in_cells)) +! collect all vertices, and also pad + j1 = 0 + do ic=1, nCellsSolve + do i1 = 1, nEdgesOnCell(ic) + j1 = j1 + 1 + all_connects(j1) = verticesOnCell( i1, ic) + enddo + lastvertex = verticesOnCell( nEdgesOnCell (ic), ic) + ! pad the rest with the last vertex + do i1 = nEdgesOnCell (ic) + 1, maxEdges + j1 = j1 + 1 + all_connects(j1) = lastvertex ! repeat the last vertex (pad) + enddo + ! call mpas_log_write('cell: $i v:: $i $i $i $i $i $i', intArgs=(/ic, all_connects(j1-5), all_connects(j1-4), all_connects(j1-3), all_connects(j1-2), all_connects(j1-1), all_connects(j1)/) ) enddo - lastvertex = verticesOnCell % array( nEdgesOnCell % array(ic), ic) - ! pad the rest with the last vertex - do i1 = nEdgesOnCell % array(ic) + 1, maxv - j1 = j1 + 1 - all_connects(j1) = lastvertex ! repeat the last vertex (pad) - enddo - enddo - - call mpas_quicksort(num_verts_in_cells, all_connects) - ! now compress the duplicates, keep only one vertex; then do binary search - - ! allocate(moabvh(num_verts_in_cells)) ! used to compress - - ! allocate(moabconn(moab_dim_cquads)) - nv=1 - currentval = all_connects(1) - do i1=2,num_verts_in_cells - if (all_connects(i1) .ne. currentval ) then - nv=nv+1 - currentval = all_connects(i1) - all_connects(nv) = currentval - endif - enddo - call mpas_log_write(' number of vertices:: $i ', intArgs=(/nv/) ) -! write (stderrUnit,100) (all_connects(j1), j1=1,nv) -! 100 format(10I6) - ! num local vertices = nv; moabnv will be the global ids, that will be used for resolve sharing - allocate(moabvh(nv)) - !do i1=1,nv - moabvh(:) = all_connects(1:nv) ! copy the first - !enddo - ! first create local vertices, accumulate xv, yv, zv from xVertex, ... - allocate(moab_vert_coords(3*nv)) - do i1 =1, nv - j1 = moabvh(i1) - moab_vert_coords(3*i1-2) = xVertex % array(j1) - moab_vert_coords(3*i1-1) = yVertex % array(j1) - moab_vert_coords(3*i1 ) = zVertex % array(j1) - enddo - dimcoord = 3*nv - dimen = 3 - ierr = iMOAB_CreateVertices(pid, dimcoord, dimen, moab_vert_coords) - call errorout(ierr, 'fail to create vertices') - ! build now connectivity arrays using binary search - j1 = 0 - do ic=1, nLocalCells - nedg = nEdgesOnCell % array(ic) - do i1 = 1, nedg - j1=j1+1 - all_connects(j1) = mpas_binary_search(moabvh, 1, 1, nv, verticesOnCell % array( i1, ic) ) - - enddo - lastvertex = all_connects(j1) - ! pad the rest with the last vertex - do i1 = nedg + 1, maxv - j1 = j1 + 1 - all_connects(j1) = lastvertex ! repeat the last vertex (pad) - enddo - enddo - mbtype = 4 ! polygon - - block_ID = 100 ! this will be for coarse mesh - - ierr = iMOAB_CreateElements( pid, nLocalCells, mbtype, maxv, all_connects, block_ID ); - call errorout(ierr, 'fail to create polygons') - - ierr = iMOAB_ResolveSharedEntities( pid, nv, moabvh ); - call errorout(ierr, 'fail to resolve shared entities') - -! write in serial, on each task, before ghosting - if (proc_id.lt. 10) then - write(lnum,"(I0.2)")proc_id - localmeshfile = 'owned_'//trim(lnum)// '.h5m' // CHAR(0) - wopts = CHAR(0) - ierr = iMOAB_WriteMesh(pid, localmeshfile, wopts) - call errorout(ierr, 'fail to write local mesh file') - endif - -! write out the mesh file to disk, in parallel - outfile = 'whole.h5m'//CHAR(0) - wopts = 'PARALLEL=WRITE_PART'//CHAR(0) - ierr = iMOAB_WriteMesh(pid, outfile, wopts) - call errorout(ierr, 'fail to write the mesh file') - - ! (iMOAB_AppID pid, int * ghost_dim, int *num_ghost_layers, int * bridge_dim ) - dimgh = 2 ! will ghost polygons, topological dim 2 - bridge = 0 ! use vertex as bridge - num_layers = 1 ! so far, one layer only - ierr = iMOAB_DetermineGhostEntities( pid, dimgh, num_layers, bridge) - call errorout(ierr, 'fail to determine ghosts') - - ! write in serial, on each task - if (proc_id .lt. 10) then - write(lnum,"(I0.2)")proc_id - localmeshfile = 'localmesh_'//trim(lnum)// '.h5m' // CHAR(0) - wopts = CHAR(0) - ierr = iMOAB_WriteMesh(pid, localmeshfile, wopts) - call errorout(ierr, 'fail to write local mesh file') - endif - - deallocate (moab_vert_coords) - deallocate (moabvh) - deallocate (all_connects) - end subroutine create_mpas_moabmesh - + mbtype = 4 ! polygon + + block_ID = 100 + nblocks ! this will be for coarse mesh + + ierr = iMOAB_CreateElements( pid, nCellsSolve, mbtype, maxEdges, all_connects, block_ID ); + call errorout(ierr, 'fail to create polygons') + ! get next block +#ifdef MPAS_DEBUG + if (proc_id.lt. 5) then + write(lnum,"(I0.2)")proc_id + localmeshfile = 'owned_'//trim(lnum)// '.h5m' // CHAR(0) + wopts = CHAR(0) + ierr = iMOAB_WriteMesh(pid, localmeshfile, wopts) + call errorout(ierr, 'fail to write local mesh file') + endif +#endif + ierr = iMOAB_ResolveSharedEntities( pid, nVertices, indexToVertexID ); + call errorout(ierr, 'fail to resolve shared entities') + + deallocate (moab_vert_coords) + deallocate (all_connects) + block => block % next + + end do !}}} + + + if (nblocks .ne. 1) then + call errorout(1, 'more than one block per task') + endif + ! write out the mesh file to disk, in parallel + outfile = 'whole.h5m'//CHAR(0) + wopts = 'PARALLEL=WRITE_PART'//CHAR(0) + ierr = iMOAB_WriteMesh(pid, outfile, wopts) + call errorout(ierr, 'fail to write the mesh file') + + + end subroutine mpas_moab_instance end module mpas_moabmesh From 98b03ce1ead9b1025ef9d4767a6a60a9cb112cb3 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Sun, 19 Nov 2017 00:42:15 -0600 Subject: [PATCH 09/17] remove debug print bootstrap is not changed anymore --- src/framework/Makefile | 2 +- src/framework/mpas_bootstrapping.F | 5 ++--- src/framework/mpas_moabmesh.F | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/framework/Makefile b/src/framework/Makefile index 6fbea7482e..4a05b1c320 100644 --- a/src/framework/Makefile +++ b/src/framework/Makefile @@ -94,7 +94,7 @@ mpas_io.o: mpas_dmpar.o mpas_attlist.o mpas_log.o mpas_io_streams.o: mpas_attlist.o mpas_derived_types.o mpas_timekeeping.o mpas_io.o mpas_pool_routines.o add_field_indices.inc mpas_log.o $(DEPS) -mpas_bootstrapping.o: mpas_derived_types.o mpas_dmpar.o mpas_block_decomp.o mpas_block_creator.o mpas_sort.o mpas_timekeeping.o mpas_io_streams.o mpas_stream_manager.o random_id.o mpas_log.o mpas_moabmesh.o $(DEPS) +mpas_bootstrapping.o: mpas_derived_types.o mpas_dmpar.o mpas_block_decomp.o mpas_block_creator.o mpas_sort.o mpas_timekeeping.o mpas_io_streams.o mpas_stream_manager.o random_id.o mpas_log.o $(DEPS) mpas_io_units.o: mpas_kind_types.o diff --git a/src/framework/mpas_bootstrapping.F b/src/framework/mpas_bootstrapping.F index 454d4b726d..e87dadc077 100644 --- a/src/framework/mpas_bootstrapping.F +++ b/src/framework/mpas_bootstrapping.F @@ -18,6 +18,7 @@ module mpas_bootstrapping use mpas_io_streams use mpas_log + integer :: readCellStart, readCellEnd, nReadCells integer :: readEdgeStart, readEdgeEnd, nReadEdges integer :: readVertexStart, readVertexEnd, nReadVertices @@ -88,7 +89,6 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! integer :: nCells, nEdges, maxEdges, nVertices, vertexDegree integer :: ierr - integer :: i1, j1, nv1 type (MPAS_IO_Handle_type) :: inputHandle character (len=StrKIND) :: c_on_a_sphere, c_is_periodic, c_parent_id, c_file_id, c_mesh_spec @@ -270,7 +270,6 @@ subroutine mpas_bootstrap_framework_phase1(domain, mesh_filename, mesh_iotype) ! verticesOnCell_Block, indexToVertexID_Block, cellsOnVertex_Block, & nVerticesSolveField) - ! ! Before we can allocate blocks, we need the attributes on_a_sphere and sphere_radius so ! they can be propagated as configs to subpools @@ -775,7 +774,6 @@ subroutine mpas_io_setup_vertex_block_fields(inputHandle, nReadVertices, readVer integer, intent(in) :: nHalos integer :: i, ierr - integer :: nVertices integer, dimension(:), pointer :: readIndices ! Global vertex indices @@ -795,6 +793,7 @@ subroutine mpas_io_setup_vertex_block_fields(inputHandle, nReadVertices, readVer call mpas_dmpar_init_multihalo_exchange_list(indexToVertexID % copyList, nHalos+1) nullify(indexToVertexID % next) + ! Global indices of cells adjacent to each vertex ! used for determining which vertices are owned by a block, where ! iVtx is owned iff cellsOnVertex(1,iVtx) is an owned cell diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 8e1ddb2bdc..9a7820a792 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -95,9 +95,6 @@ subroutine mpas_moab_instance(domain) #ifdef MPAS_DEBUG call mpas_log_write(' number of vertices:: $i number of cells:: $i solve: v:$i c:$i', intArgs=(/nVertices, nCells, nVerticesSolve, nCellsSolve/) ) - do iVertex=1,nVertices - call mpas_log_write('i: $i indexToVertexID:: $i ', intArgs=(/iVertex, indexToVertexID(iVertex)/) ) - enddo #endif ! first create local vertices, nVerticesSolve allocate(moab_vert_coords(3*nVertices)) From d1891e527db3eee4ccf1dfff806253a5d1ce1542 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 20 Nov 2017 16:38:06 -0600 Subject: [PATCH 10/17] for ACME build, this Makefile.in.ACME is used will make sure MOAB paths are defined correctly --- src/Makefile.in.ACME | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Makefile.in.ACME b/src/Makefile.in.ACME index 866ee1fedf..d6b87f859e 100644 --- a/src/Makefile.in.ACME +++ b/src/Makefile.in.ACME @@ -37,6 +37,16 @@ else ESMFDIR = noesmf endif +# Set MOAB info if it is being used +ifeq ($(strip $(USE_MOAB)), TRUE) + ifdef MOAB_PATH + CPPDEFS += -DHAVE_MOAB + MOAB_INCLUDES = -I$(MOAB_PATH)/include + else + $(error MOAB_PATH must be defined when USE_MOAB is TRUE) + endif +endif + RM = rm -f CPP = cpp -P -traditional FC=$(MPIFC) @@ -48,9 +58,9 @@ FILE_OFFSET = -DOFFSET64BIT override CFLAGS += -DMPAS_NO_LOG_REDIRECT -DMPAS_NO_ESMF_INIT -DMPAS_ESM_SHR_CONST -DMPAS_PERF_MOD_TIMERS override FFLAGS += -DMPAS_NO_LOG_REDIRECT -DMPAS_NO_ESMF_INIT -DMPAS_ESM_SHR_CONST -DMPAS_PERF_MOD_TIMERS override CPPFLAGS += $(CPPDEFS) $(MODEL_FORMULATION) $(FILE_OFFSET) $(ZOLTAN_DEFINE) -DMPAS_NO_LOG_REDIRECT -DMPAS_NO_ESMF_INIT -DMPAS_ESM_SHR_CONST -D_MPI -DMPAS_NAMELIST_SUFFIX=$(NAMELIST_SUFFIX) -DMPAS_EXE_NAME=$(EXE_NAME) -DMPAS_PERF_MOD_TIMERS -override CPPINCLUDES += -I$(EXEROOT)/$(COMPONENT)/source/inc -I$(INSTALL_SHAREDPATH)/include -I$(INSTALL_SHAREDPATH)/$(COMP_INTERFACE)/$(ESMFDIR)/$(NINST_VALUE)/csm_share -I$(NETCDF)/include -I$(PIO) -I$(PNETCDF)/include +override CPPINCLUDES += -I$(EXEROOT)/$(COMPONENT)/source/inc -I$(INSTALL_SHAREDPATH)/include -I$(INSTALL_SHAREDPATH)/$(COMP_INTERFACE)/$(ESMFDIR)/$(NINST_VALUE)/csm_share -I$(NETCDF)/include -I$(PIO) -I$(PNETCDF)/include $(MOAB_INCLUDES) override FCINCLUDES += -I$(EXEROOT)/$(COMPONENT)/source/inc -I$(INSTALL_SHAREDPATH)/include -I$(INSTALL_SHAREDPATH)/$(COMP_INTERFACE)/$(ESMFDIR)/$(NINST_VALUE)/csm_share -I$(NETCDF)/include -I$(PIO) -I$(PNETCDF)/include -LIBS += -L$(PIO) -L$(PNETCDF)/lib -L$(NETCDF)/lib -L$(LIBROOT) -L$(INSTALL_SHAREDPATH)/lib -lpio -lpnetcdf -lnetcdf +LIBS += $(IMESH_LIBS) -L$(PIO) -L$(PNETCDF)/lib -L$(NETCDF)/lib -L$(LIBROOT) -L$(INSTALL_SHAREDPATH)/lib -lpio -lpnetcdf -lnetcdf ifneq (,$(findstring FORTRANUNDERSCORE, $(CPPFLAGS))) ifeq (,$(findstring DUNDERSCORE, $(CPPFLAGS))) From 7b6bab8a01b19800ac19fece777e15faf13fab34 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Mon, 20 Nov 2017 19:00:18 -0600 Subject: [PATCH 11/17] export MOAB pid for consumption by the coupler --- src/framework/mpas_moabmesh.F | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 9a7820a792..9089ab5b0f 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -14,6 +14,8 @@ module mpas_moabmesh implicit none #include "moab/MOABConfig.h" + integer, public :: MPOID ! app id on moab side, for MPAS ocean + contains SUBROUTINE errorout(ierr, message) @@ -33,7 +35,7 @@ subroutine mpas_moab_instance(domain) type (block_type), pointer :: block type (mpas_pool_type), pointer :: meshPool integer, pointer :: nCells, nVertices, maxEdges - integer :: nVert, iCell, iVertex, nblocks + integer :: pid, nblocks real (kind=RKIND), dimension(:), allocatable :: xv,yv,zv integer, dimension(:,:), pointer :: verticesOnCell integer, dimension(:), pointer :: nEdgesOnCell @@ -49,7 +51,7 @@ subroutine mpas_moab_instance(domain) integer , external :: iMOAB_RegisterFortranApplication, & iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities - integer :: c_comm, pid, i1, j1, ic, lastvertex + integer :: c_comm, i1, j1, ic, lastvertex character*12 appname integer :: ierr, num_verts_in_cells, ext_comp_id real(kind=RKIND), allocatable, target :: moab_vert_coords(:) @@ -59,8 +61,10 @@ subroutine mpas_moab_instance(domain) c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) - ext_comp_id = 17 ! some number less than 33 + MPOID = -1 ! initialized negative , so we know if this register passes + ext_comp_id = 17 ! some number less than 33 ! maybe this should be input ierr = iMOAB_RegisterFortranApplication(appname, c_comm, ext_comp_id, pid) + MPOID = pid ! this is exported, need for send to work call errorout(ierr, 'fail to register MPAS_MOAB mesh') proc_id = domain % dminfo % my_proc_id call mpas_log_write('MOAB MPAS app pid: $i task $i ', intArgs=(/pid, proc_id/) ) From 1a6f065be30752a5e86d5c9d2949f63d94af05e8 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 1 Dec 2017 02:15:44 -0600 Subject: [PATCH 12/17] set global id for elements and vertices some ghost vertices are duplicated and not resolved, as resolving nvolves only boundary vertices --- src/framework/mpas_moabmesh.F | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 9089ab5b0f..b5f2f6abb7 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -50,14 +50,16 @@ subroutine mpas_moab_instance(domain) integer , external :: iMOAB_RegisterFortranApplication, & iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & - iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities + iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities, & + iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage integer :: c_comm, i1, j1, ic, lastvertex character*12 appname integer :: ierr, num_verts_in_cells, ext_comp_id real(kind=RKIND), allocatable, target :: moab_vert_coords(:) integer dimcoord, dimen, mbtype, block_ID, proc_id integer ,allocatable , target :: all_connects(:) - character*100 outfile, wopts, localmeshfile, lnum + character*100 outfile, wopts, localmeshfile, lnum, tagname + integer tagtype, numco, tag_sto_len, ent_type, tagindex c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) @@ -137,6 +139,21 @@ subroutine mpas_moab_instance(domain) ierr = iMOAB_CreateElements( pid, nCellsSolve, mbtype, maxEdges, all_connects, block_ID ); call errorout(ierr, 'fail to create polygons') +! set the global id for vertices +! first, retrieve the tag + tagname='GLOBAL_ID'//CHAR(0) + tagtype = 0 ! dense, integer + numco = 1 + ierr = iMOAB_DefineTagStorage(pid, tagname, tagtype, numco, tagindex ) + call errorout(ierr, 'fail to get global id tag') +! now set the values + ent_type = 0 ! vertex type + ierr = iMOAB_SetIntTagStorage ( pid, tagname, nVertices , ent_type, indexToVertexID ) + call errorout(ierr, 'fail to set global id tag for vertices') + ! set global id tag for elements + ent_type = 1 ! now set the global id tag on elements + ierr = iMOAB_SetIntTagStorage ( pid, tagname, nCellsSolve, ent_type, indexToCellID) + call errorout(ierr, 'fail to set global id tag for polygons') ! get next block #ifdef MPAS_DEBUG if (proc_id.lt. 5) then From ccb91bfa646e3e0fef8125ee5dfb417a602952e1 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 1 Dec 2017 04:03:44 -0600 Subject: [PATCH 13/17] create only necessary vertices before, halo vertices were created, and they were not necessary also, the correct number of vertices are recovered --- src/framework/mpas_moabmesh.F | 88 +++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index b5f2f6abb7..e940bfaccd 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -36,7 +36,6 @@ subroutine mpas_moab_instance(domain) type (mpas_pool_type), pointer :: meshPool integer, pointer :: nCells, nVertices, maxEdges integer :: pid, nblocks - real (kind=RKIND), dimension(:), allocatable :: xv,yv,zv integer, dimension(:,:), pointer :: verticesOnCell integer, dimension(:), pointer :: nEdgesOnCell integer, dimension(:), pointer :: indexToVertexID, indexToCellID @@ -45,8 +44,6 @@ subroutine mpas_moab_instance(domain) logical, pointer :: on_a_sphere, is_periodic real(kind=RKIND), pointer :: x_period, y_period integer, pointer :: nCellsSolve, nEdgesSolve, nVerticesSolve - - integer , external :: iMOAB_RegisterFortranApplication, & iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & @@ -56,10 +53,11 @@ subroutine mpas_moab_instance(domain) character*12 appname integer :: ierr, num_verts_in_cells, ext_comp_id real(kind=RKIND), allocatable, target :: moab_vert_coords(:) + integer, allocatable, target :: indexUsed(:), invMap(:) integer dimcoord, dimen, mbtype, block_ID, proc_id integer ,allocatable , target :: all_connects(:) character*100 outfile, wopts, localmeshfile, lnum, tagname - integer tagtype, numco, tag_sto_len, ent_type, tagindex + integer tagtype, numco, tag_sto_len, ent_type, tagindex, currentVertex c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) @@ -102,37 +100,54 @@ subroutine mpas_moab_instance(domain) #ifdef MPAS_DEBUG call mpas_log_write(' number of vertices:: $i number of cells:: $i solve: v:$i c:$i', intArgs=(/nVertices, nCells, nVerticesSolve, nCellsSolve/) ) #endif - ! first create local vertices, nVerticesSolve - allocate(moab_vert_coords(3*nVertices)) - do i1 =1, nVertices - moab_vert_coords(3*i1-2) = xVertex(i1) - moab_vert_coords(3*i1-1) = yVertex(i1) - moab_vert_coords(3*i1 ) = zVertex(i1) - ! call mpas_log_write('i:: $i coords:: $r $r $r $r', intArgs=(/i1/), realArgs=(/moab_vert_coords(3*i1-2),moab_vert_coords(3*i1-1), moab_vert_coords(3*i1)/) ) - enddo - dimcoord = 3*nVertices - dimen = 3 - ierr = iMOAB_CreateVertices(pid, dimcoord, dimen, moab_vert_coords) - call errorout(ierr, 'fail to create vertices') + !! + allocate(indexUsed(nVertices), invMap(nVertices) ) ! conservative, invMap should be smaller + indexUsed = 0 ! not used yet + invMap = 0 ! fill now connectivity array, nCellsSolve; fist pad to max nc - num_verts_in_cells = nCellsSolve * maxEdges - allocate(all_connects(num_verts_in_cells)) + num_verts_in_cells = nCellsSolve * maxEdges + allocate(all_connects(num_verts_in_cells)) ! collect all vertices, and also pad - j1 = 0 - do ic=1, nCellsSolve - do i1 = 1, nEdgesOnCell(ic) - j1 = j1 + 1 - all_connects(j1) = verticesOnCell( i1, ic) - enddo - lastvertex = verticesOnCell( nEdgesOnCell (ic), ic) - ! pad the rest with the last vertex - do i1 = nEdgesOnCell (ic) + 1, maxEdges - j1 = j1 + 1 - all_connects(j1) = lastvertex ! repeat the last vertex (pad) - enddo - ! call mpas_log_write('cell: $i v:: $i $i $i $i $i $i', intArgs=(/ic, all_connects(j1-5), all_connects(j1-4), all_connects(j1-3), all_connects(j1-2), all_connects(j1-1), all_connects(j1)/) ) + j1 = 0 + do ic=1, nCellsSolve + do i1 = 1, nEdgesOnCell(ic) + j1 = j1 + 1 + all_connects(j1) = verticesOnCell( i1, ic) + indexUsed(all_connects(j1)) = 1 + enddo + lastvertex = verticesOnCell( nEdgesOnCell (ic), ic) + ! pad the rest with the last vertex + do i1 = nEdgesOnCell (ic) + 1, maxEdges + j1 = j1 + 1 + all_connects(j1) = lastvertex ! repeat the last vertex (pad) + enddo + ! call mpas_log_write('cell: $i v:: $i $i $i $i $i $i', intArgs=(/ic, all_connects(j1-5), all_connects(j1-4), all_connects(j1-3), all_connects(j1-2), all_connects(j1-1), all_connects(j1)/) ) enddo + currentVertex = 0 + do i1 = 1, nVertices + if (indexUsed(i1) > 0) then + currentVertex = currentVertex + 1 + indexUsed(i1) = currentVertex + invMap(currentVertex) = i1 + endif + enddo + !! convert all_connects to indexUsed + do i1 = 1, num_verts_in_cells + all_connects(i1) = indexUsed( all_connects(i1) ) + enddo + allocate(moab_vert_coords(3*currentVertex)) + do i1 =1, currentVertex + moab_vert_coords(3*i1-2) = xVertex(invMap(i1)) + moab_vert_coords(3*i1-1) = yVertex(invMap(i1)) + moab_vert_coords(3*i1 ) = zVertex(invMap(i1)) + ! call mpas_log_write('i:: $i coords:: $r $r $r $r', intArgs=(/i1/), realArgs=(/moab_vert_coords(3*i1-2),moab_vert_coords(3*i1-1), moab_vert_coords(3*i1)/) ) + enddo + dimcoord = 3*currentVertex + dimen = 3 + ierr = iMOAB_CreateVertices(pid, dimcoord, dimen, moab_vert_coords) + call errorout(ierr, 'fail to create vertices') +! so we know we used only currentvertex vertices in the pool (the rest are in halo) mbtype = 4 ! polygon block_ID = 100 + nblocks ! this will be for coarse mesh @@ -147,8 +162,11 @@ subroutine mpas_moab_instance(domain) ierr = iMOAB_DefineTagStorage(pid, tagname, tagtype, numco, tagindex ) call errorout(ierr, 'fail to get global id tag') ! now set the values - ent_type = 0 ! vertex type - ierr = iMOAB_SetIntTagStorage ( pid, tagname, nVertices , ent_type, indexToVertexID ) + ent_type = 0 ! vertex type! reuse + do i1 = 1, currentVertex + invMap(i1) = indexToVertexID (invMap(i1)) + enddo + ierr = iMOAB_SetIntTagStorage ( pid, tagname, currentVertex , ent_type, invMap ) call errorout(ierr, 'fail to set global id tag for vertices') ! set global id tag for elements ent_type = 1 ! now set the global id tag on elements @@ -164,11 +182,13 @@ subroutine mpas_moab_instance(domain) call errorout(ierr, 'fail to write local mesh file') endif #endif - ierr = iMOAB_ResolveSharedEntities( pid, nVertices, indexToVertexID ); + ierr = iMOAB_ResolveSharedEntities( pid, currentVertex, invMap ); call errorout(ierr, 'fail to resolve shared entities') deallocate (moab_vert_coords) deallocate (all_connects) + deallocate (indexUsed) + deallocate (invMap) block => block % next end do !}}} From 76b7f8f777dfca7cda7ddd44d56a4ede10b919f2 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 1 Dec 2017 13:34:17 -0600 Subject: [PATCH 14/17] be more explicit with local ids from GLOBAL_ID --- src/framework/mpas_moabmesh.F | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index e940bfaccd..7d7ad56432 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -53,7 +53,7 @@ subroutine mpas_moab_instance(domain) character*12 appname integer :: ierr, num_verts_in_cells, ext_comp_id real(kind=RKIND), allocatable, target :: moab_vert_coords(:) - integer, allocatable, target :: indexUsed(:), invMap(:) + integer, allocatable, target :: indexUsed(:), invMap(:), localIds(:) integer dimcoord, dimen, mbtype, block_ID, proc_id integer ,allocatable , target :: all_connects(:) character*100 outfile, wopts, localmeshfile, lnum, tagname @@ -97,12 +97,10 @@ subroutine mpas_moab_instance(domain) ! call mpas_pool_get_array(meshPool, 'yCell', yCell) ! call mpas_pool_get_array(meshPool, 'zCell', zCell) -#ifdef MPAS_DEBUG - call mpas_log_write(' number of vertices:: $i number of cells:: $i solve: v:$i c:$i', intArgs=(/nVertices, nCells, nVerticesSolve, nCellsSolve/) ) -#endif + call mpas_log_write(' MOAB instance: number of vertices:: $i number of cells:: $i solve: v:$i c:$i', intArgs=(/nVertices, nCells, nVerticesSolve, nCellsSolve/) ) !! allocate(indexUsed(nVertices), invMap(nVertices) ) ! conservative, invMap should be smaller - indexUsed = 0 ! not used yet + indexUsed = 0 invMap = 0 ! fill now connectivity array, nCellsSolve; fist pad to max nc num_verts_in_cells = nCellsSolve * maxEdges @@ -147,10 +145,11 @@ subroutine mpas_moab_instance(domain) dimen = 3 ierr = iMOAB_CreateVertices(pid, dimcoord, dimen, moab_vert_coords) call errorout(ierr, 'fail to create vertices') + call mpas_log_write(' MOAB instance: created $i vertices on local proc $i ',intArgs=(/currentVertex, proc_id/)) ! so we know we used only currentvertex vertices in the pool (the rest are in halo) mbtype = 4 ! polygon - block_ID = 100 + nblocks ! this will be for coarse mesh + block_ID = 100*proc_id + nblocks ! we should have only one block right now ierr = iMOAB_CreateElements( pid, nCellsSolve, mbtype, maxEdges, all_connects, block_ID ); call errorout(ierr, 'fail to create polygons') @@ -162,11 +161,12 @@ subroutine mpas_moab_instance(domain) ierr = iMOAB_DefineTagStorage(pid, tagname, tagtype, numco, tagindex ) call errorout(ierr, 'fail to get global id tag') ! now set the values - ent_type = 0 ! vertex type! reuse + ent_type = 0 ! vertex type + allocate(localIds(currentVertex)) do i1 = 1, currentVertex - invMap(i1) = indexToVertexID (invMap(i1)) + localIds(i1) = indexToVertexID (invMap(i1)) enddo - ierr = iMOAB_SetIntTagStorage ( pid, tagname, currentVertex , ent_type, invMap ) + ierr = iMOAB_SetIntTagStorage ( pid, tagname, currentVertex , ent_type, localIds ) call errorout(ierr, 'fail to set global id tag for vertices') ! set global id tag for elements ent_type = 1 ! now set the global id tag on elements @@ -176,19 +176,20 @@ subroutine mpas_moab_instance(domain) #ifdef MPAS_DEBUG if (proc_id.lt. 5) then write(lnum,"(I0.2)")proc_id - localmeshfile = 'owned_'//trim(lnum)// '.h5m' // CHAR(0) + localmeshfile = 'ownedOcn_'//trim(lnum)// '.h5m' // CHAR(0) wopts = CHAR(0) ierr = iMOAB_WriteMesh(pid, localmeshfile, wopts) call errorout(ierr, 'fail to write local mesh file') endif #endif - ierr = iMOAB_ResolveSharedEntities( pid, currentVertex, invMap ); + ierr = iMOAB_ResolveSharedEntities( pid, currentVertex, localIds ); call errorout(ierr, 'fail to resolve shared entities') deallocate (moab_vert_coords) deallocate (all_connects) deallocate (indexUsed) deallocate (invMap) + deallocate (localIds) block => block % next end do !}}} @@ -198,7 +199,7 @@ subroutine mpas_moab_instance(domain) call errorout(1, 'more than one block per task') endif ! write out the mesh file to disk, in parallel - outfile = 'whole.h5m'//CHAR(0) + outfile = 'wholeMPAS.h5m'//CHAR(0) wopts = 'PARALLEL=WRITE_PART'//CHAR(0) ierr = iMOAB_WriteMesh(pid, outfile, wopts) call errorout(ierr, 'fail to write the mesh file') From e8744732904f84c0b4a6ebd2c24a6c2f30264410 Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 8 Dec 2017 17:40:58 -0600 Subject: [PATCH 15/17] need to update mesh info for atmosphere mesh, this was implicit because I called ghosting; ghosting was not necessary, but update mesh info is --- src/framework/mpas_moabmesh.F | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 7d7ad56432..81449b1f37 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -48,7 +48,8 @@ subroutine mpas_moab_instance(domain) integer , external :: iMOAB_RegisterFortranApplication, & iMOAB_CreateVertices, iMOAB_WriteMesh, iMOAB_CreateElements, & iMOAB_ResolveSharedEntities, iMOAB_DetermineGhostEntities, & - iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage + iMOAB_DefineTagStorage, iMOAB_SetIntTagStorage , & + iMOAB_UpdateMeshInfo integer :: c_comm, i1, j1, ic, lastvertex character*12 appname integer :: ierr, num_verts_in_cells, ext_comp_id @@ -198,6 +199,8 @@ subroutine mpas_moab_instance(domain) if (nblocks .ne. 1) then call errorout(1, 'more than one block per task') endif + ierr = iMOAB_UpdateMeshInfo(pid) + call errorout(ierr, 'fail to update mesh info') ! write out the mesh file to disk, in parallel outfile = 'wholeMPAS.h5m'//CHAR(0) wopts = 'PARALLEL=WRITE_PART'//CHAR(0) From cfbebcd117ccdc01cb427409d1b083d90b0a455b Mon Sep 17 00:00:00 2001 From: Iulian Grindeanu Date: Fri, 26 Jan 2018 12:48:36 -0600 Subject: [PATCH 16/17] move APP ids to seq_comm_mct module mpas ocean will use MOAB_MPAS_INSIDE_E3SM if inside E3SM --- src/Makefile.in.ACME | 2 +- src/framework/mpas_moabmesh.F | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Makefile.in.ACME b/src/Makefile.in.ACME index d6b87f859e..b8ca28e513 100644 --- a/src/Makefile.in.ACME +++ b/src/Makefile.in.ACME @@ -40,7 +40,7 @@ endif # Set MOAB info if it is being used ifeq ($(strip $(USE_MOAB)), TRUE) ifdef MOAB_PATH - CPPDEFS += -DHAVE_MOAB + CPPDEFS += -DHAVE_MOAB -DMOAB_MPAS_INSIDE_E3SM MOAB_INCLUDES = -I$(MOAB_PATH)/include else $(error MOAB_PATH must be defined when USE_MOAB is TRUE) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 81449b1f37..0f380d76a8 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -10,11 +10,14 @@ module mpas_moabmesh use mpas_stream_manager use mpas_pool_routines !use mpas_vector_operations - - implicit none #include "moab/MOABConfig.h" +#ifdef MOAB_MPAS_INSIDE_E3SM + use seq_comm_mct, only: MPOID +#else integer, public :: MPOID ! app id on moab side, for MPAS ocean +#endif + implicit none contains From 8179807cd1b74f9b71baf0044c2848545f555394 Mon Sep 17 00:00:00 2001 From: Iulian Date: Fri, 15 Jun 2018 12:56:57 -0500 Subject: [PATCH 17/17] pass ocean comp id from seq_comm --- src/framework/mpas_moabmesh.F | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/framework/mpas_moabmesh.F b/src/framework/mpas_moabmesh.F index 0f380d76a8..dc7ef18f83 100644 --- a/src/framework/mpas_moabmesh.F +++ b/src/framework/mpas_moabmesh.F @@ -13,7 +13,7 @@ module mpas_moabmesh #include "moab/MOABConfig.h" #ifdef MOAB_MPAS_INSIDE_E3SM - use seq_comm_mct, only: MPOID + use seq_comm_mct, only: MPOID, OCNID #else integer, public :: MPOID ! app id on moab side, for MPAS ocean #endif @@ -66,7 +66,11 @@ subroutine mpas_moab_instance(domain) c_comm = domain % dminfo % comm appname = 'MPAS_MESH'//CHAR(0) MPOID = -1 ! initialized negative , so we know if this register passes +#ifdef MOAB_MPAS_INSIDE_E3SM + ext_comp_id = OCNID(1) ! should be 17 +#else ext_comp_id = 17 ! some number less than 33 ! maybe this should be input +#endif ierr = iMOAB_RegisterFortranApplication(appname, c_comm, ext_comp_id, pid) MPOID = pid ! this is exported, need for send to work call errorout(ierr, 'fail to register MPAS_MOAB mesh')