Skip to content

Commit

Permalink
Merge branch 'iulian787/move_semoab' (PR E3SM-Project#6369)
Browse files Browse the repository at this point in the history 
semoab_mod.F90 file is moved from
components/eam/src/dynamics/se folder to
components/homme/src/share folder.

It belongs naturally there, as it involves just grid routines from homme base code

also, it should prepare for SCREAM-moab connection later on

fix also some complaints about non-allocated arrays when there are no cells on some dynamic tasks (when number of tasks is larger than number of spectral cells in homme)

[BFB]
  • Loading branch information
@rljacob
rljacob authored May 8, 2024
2 parents 58f1c6e + 592df47 commit 4fab3c8
Showing 1 changed file with 42 additions and 39 deletions.
81 changes: 42 additions & 39 deletions ...onents/eam/src/dynamics/se/semoab_mod.F90 → components/homme/src/share/semoab_mod.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,10 @@ module semoab_mod

use dimensions_mod, only: nelem, ne, np, nelemd, nlev
use element_mod, only : element_t
use parallel_mod, only : parallel_t
use parallel_mod, only : parallel_t, abortmp

use m_MergeSorts, only: IndexSet, IndexSort

use cam_grid_support, only: iMap
use cam_abortutils, only : endrun
use edgetype_mod, only: edgedescriptor_t
use gridgraph_mod, only: gridvertex_t

Expand Down Expand Up @@ -209,8 +207,10 @@ subroutine create_moab_meshes(par, elem)
if ( nelemd > 0 ) then
allocate(moab_vert_coords(3*nverts) )
allocate(vdone(nverts))
vdone = 0;
else
allocate(vdone(1)) ! will be passed to iMOAB_ResolveSharedEnts, and compilers are complaining about non-allocated arrays
endif
vdone = 0;
if ( nelemd > 0 ) currentval = gdofv( indx(1)) ! start over to identify coordinates of the vertices

do ix=1,moab_dim_cquads
Expand Down Expand Up @@ -251,7 +251,7 @@ subroutine create_moab_meshes(par, elem)
if ( nelemd > 0 ) then
ierr = iMOAB_CreateVertices(MHFID, dimcoord, dimen, moab_vert_coords)
if (ierr > 0 ) &
call endrun('Error: fail to create MOAB vertices ')
call abortmp('Error: fail to create MOAB vertices ')
endif
!!num_el = nelemd2
mbtype = 3 ! quadrilateral
Expand All @@ -261,7 +261,7 @@ subroutine create_moab_meshes(par, elem)
if ( nelemd > 0 ) then
ierr = iMOAB_CreateElements( MHFID, nelemd2, mbtype, nve, moabconn, block_ID );
if (ierr > 0 ) &
call endrun('Error: fail to create MOAB elements')
call abortmp('Error: fail to create MOAB elements')
endif
! nverts: num vertices; vdone will store now the markers used in global resolve
! for this particular problem, markers are the global dofs at corner nodes
Expand All @@ -272,18 +272,19 @@ subroutine create_moab_meshes(par, elem)
numco = 1
ierr = iMOAB_DefineTagStorage(MHFID, tagname, tagtype, numco, tagindex )
if (ierr > 0 ) &
call endrun('Error: fail to retrieve global id tag')
call abortmp('Error: fail to retrieve global id tag')
! now set the values
ent_type = 0 ! vertex type
if ( nverts > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHFID, tagname, nverts , ent_type, vdone)
if (ierr > 0 ) &
call endrun('Error: fail to set marker id tag for vertices')
call abortmp('Error: fail to set marker id tag for vertices')
endif

! we need to call this even when no mesh locally, it involves a collective
ierr = iMOAB_ResolveSharedEntities( MHFID, nverts, vdone );
if (ierr > 0 ) &
call endrun('Error: fail to resolve shared entities')
call abortmp('Error: fail to resolve shared entities')

if ( nelemd > 0) then
vdone = -1 ! reuse vdone for the new tag, GLOBAL_ID (actual tag that we want to store global dof )
Expand All @@ -294,7 +295,7 @@ subroutine create_moab_meshes(par, elem)
newtagg='GLOBAL_ID'//C_NULL_CHAR
ierr = iMOAB_DefineTagStorage(MHFID, newtagg, tagtype, numco, tagindex )
if (ierr > 0 ) &
call endrun('Error: fail to create new GDOF tag')
call abortmp('Error: fail to create new GDOF tag')
do ie=1,nelemd
do ii=1,elem(ie)%idxp%NumUniquePts
i=elem(ie)%idxp%ia(ii)
Expand All @@ -310,19 +311,19 @@ subroutine create_moab_meshes(par, elem)
if ( nverts > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHFID, newtagg, nverts , ent_type, vdone)
if (ierr > 0 ) &
call endrun('Error: fail to set global dof tag for vertices')
call abortmp('Error: fail to set global dof tag for vertices')
endif

ierr = iMOAB_ReduceTagsMax ( MHFID, tagindex, ent_type)
if (ierr > 0 ) &
call endrun('Error: fail to reduce max tag')
call abortmp('Error: fail to reduce max tag')

! set global id tag for elements
ent_type = 1 ! now set the global id tag on elements
if ( nelemd2 > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHFID, newtagg, nelemd2 , ent_type, elemids)
if (ierr > 0 ) &
call endrun('Error: fail to set global id tag for elements')
call abortmp('Error: fail to set global id tag for elements')
endif

! now, after reduction, we can get the actual global ids for each vertex in the fine mesh
Expand All @@ -333,18 +334,18 @@ subroutine create_moab_meshes(par, elem)
allocate(vgids(nverts))
ierr = iMOAB_GetIntTagStorage ( MHFID, newtagg, nverts , ent_type, vgids)
if (ierr > 0 ) &
call endrun('Error: fail to retrieve GLOBAL ID on each task')
call abortmp('Error: fail to retrieve GLOBAL ID on each task')
endif
ierr = iMOAB_UpdateMeshInfo(MHFID)
if (ierr > 0 ) &
call endrun('Error: fail to update mesh info')
call abortmp('Error: fail to update mesh info')
#ifdef MOABDEBUG
! write out the mesh file to disk, in parallel
outfile = 'wholeFineATM.h5m'//C_NULL_CHAR
wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR
ierr = iMOAB_WriteMesh(MHFID, outfile, wopts)
if (ierr > 0 ) &
call endrun('Error: fail to write the mesh file')
call abortmp('Error: fail to write the mesh file')
#endif


Expand Down Expand Up @@ -422,7 +423,7 @@ subroutine create_moab_meshes(par, elem)
if ( nverts_c > 0 ) then
ierr = iMOAB_CreateVertices(MHID, dimcoord, dimen, moab_vert_coords)
if (ierr > 0 ) &
call endrun('Error: fail to create MOAB vertices ')
call abortmp('Error: fail to create MOAB vertices ')
endif
! num_el = nelemd
mbtype = 3 ! quadrilateral
Expand All @@ -432,7 +433,7 @@ subroutine create_moab_meshes(par, elem)
if ( nelemd > 0 ) then
ierr = iMOAB_CreateElements( MHID, nelemd, mbtype, nve, moabconn_c, block_ID );
if (ierr > 0 ) &
call endrun('Error: fail to create MOAB elements')
call abortmp('Error: fail to create MOAB elements')
endif
! idx: num vertices; vdone will store now the markers used in global resolve
! for this particular problem, markers are the global dofs at corner nodes
Expand All @@ -443,36 +444,36 @@ subroutine create_moab_meshes(par, elem)
numco = 1
ierr = iMOAB_DefineTagStorage(MHID, tagname, tagtype, numco, tagindex )
if (ierr > 0 ) &
call endrun('Error: fail to retrieve GDOFV id tag')
call abortmp('Error: fail to retrieve GDOFV id tag')
ierr = iMOAB_DefineTagStorage(MHID, newtagg, tagtype, numco, tagindex )
if (ierr > 0 ) &
call endrun('Error: fail to retrieve GLOBAL_ID tag on coarse mesh')
call abortmp('Error: fail to retrieve GLOBAL_ID tag on coarse mesh')
! now set the values
ent_type = 0 ! vertex type
if ( nverts_c > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHID, tagname, nverts_c , ent_type, vdone_c)
if (ierr > 0 ) &
call endrun('Error: fail to set GDOFV tag for vertices')
call abortmp('Error: fail to set GDOFV tag for vertices')
endif
! set global id tag for coarse elements, too; they will start at nets=1, end at nete=nelemd
ent_type = 1 ! now set the global id tag on elements
if ( nelemd > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHID, newtagg, nelemd , ent_type, elemids)
if (ierr > 0 ) &
call endrun('Error: fail to set global id tag for vertices')
call abortmp('Error: fail to set global id tag for vertices')
endif

ierr = iMOAB_ResolveSharedEntities( MHID, idx, vdone_c );
if (ierr > 0 ) &
call endrun('Error: fail to resolve shared entities')
call abortmp('Error: fail to resolve shared entities')

! global dofs are the GLL points are set for each element
tagname='GLOBAL_DOFS'//C_NULL_CHAR
tagtype = 0 ! dense, integer
numco = np*np ! usually, it is 16; each element will have the dofs in order
ierr = iMOAB_DefineTagStorage(MHID, tagname, tagtype, numco, tagindex )
if (ierr > 0 ) &
call endrun('Error: fail to create global DOFS tag')
call abortmp('Error: fail to create global DOFS tag')
! now set the values
! set global dofs tag for coarse elements, too; they will start at nets=1, end at nete=nelemd
ent_type = 1 ! now set the global id tag on elements
Expand All @@ -499,27 +500,27 @@ subroutine create_moab_meshes(par, elem)
if ( nelemd > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHID, tagname, numvals, ent_type, gdofel)
if (ierr > 0 ) &
call endrun('Error: fail to set globalDOFs tag for coarse elements')
call abortmp('Error: fail to set globalDOFs tag for coarse elements')
endif

! set the global ids for coarse vertices the same as corresponding fine vertices
ent_type = 0 ! vertex type
if ( nverts_c > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHID, newtagg, nverts_c , ent_type, vdone_c)
if (ierr > 0 ) &
call endrun('Error: fail to set GLOBAL_DOFS tag values')
call abortmp('Error: fail to set GLOBAL_DOFS tag values')
endif

ierr = iMOAB_UpdateMeshInfo(MHID)
if (ierr > 0 ) &
call endrun('Error: fail to update mesh info')
call abortmp('Error: fail to update mesh info')
#ifdef MOABDEBUG
! write out the mesh file to disk, in parallel
outfile = 'wholeATM.h5m'//C_NULL_CHAR
wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR
ierr = iMOAB_WriteMesh(MHID, outfile, wopts)
if (ierr > 0 ) &
call endrun('Error: fail to write the mesh file')
call abortmp('Error: fail to write the mesh file')
#endif

if (fv_nphys > 0 ) then
Expand Down Expand Up @@ -629,6 +630,8 @@ subroutine create_moab_meshes(par, elem)
! reuse moab_vert_coords for coordinates of pg mesh
! the first nverts_c coords are the same as coarse mesh; but we do create new
allocate(vdone_pg(nverts_pg))
else
allocate(vdone_pg(1))
endif
do iv = 1, nverts_c
vdone_pg(iv) = vdone_c(iv) ! also the coordinates will be the same !!
Expand Down Expand Up @@ -737,7 +740,7 @@ subroutine create_moab_meshes(par, elem)
if ( nverts_pg > 0 ) then
ierr = iMOAB_CreateVertices(MHPGID, dimcoord, dimen, moab_vert_coords)
if (ierr > 0 ) &
call endrun('Error: fail to create MOAB vertices ')
call abortmp('Error: fail to create MOAB vertices ')
endif
! num_el = nelem_pg *
mbtype = 3 ! quadrilateral
Expand All @@ -747,61 +750,60 @@ subroutine create_moab_meshes(par, elem)
if ( nelem_pg > 0 ) then
ierr = iMOAB_CreateElements( MHPGID, nelem_pg, mbtype, nve, moabconn_pg, block_ID );
if (ierr > 0 ) &
call endrun('Error: fail to create MOAB elements')
call abortmp('Error: fail to create MOAB elements')
endif
tagname='GLOBAL_ID'//C_NULL_CHAR
tagtype = 0 ! dense, integer
numco = 1
ierr = iMOAB_DefineTagStorage(MHPGID, tagname, tagtype, numco, tagindex )
if (ierr > 0 ) &
call endrun('Error: fail to retrieve GLOBAL id tag')
call abortmp('Error: fail to retrieve GLOBAL id tag')

! now set the values
ent_type = 0 ! vertex type
if ( nverts_pg > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHPGID, tagname, nverts_pg , ent_type, vdone_pg)
if (ierr > 0 ) &
call endrun('Error: fail to set global id tag for vertices')
call abortmp('Error: fail to set global id tag for vertices')
endif
! set global id tag for pg2 elements, too; they will start at nets=1, end at nete=nelemd*4
ent_type = 1 ! now set the global id tag on elements
if ( nelem_pg > 0 ) then
ierr = iMOAB_SetIntTagStorage ( MHPGID, tagname, nelem_pg , ent_type, elemids)
if (ierr > 0 ) &
call endrun('Error: fail to set global id tag for edges')
call abortmp('Error: fail to set global id tag for edges')
endif

! this involves a collective, vdone_pg can be empty
ierr = iMOAB_ResolveSharedEntities( MHPGID, nverts_pg, vdone_pg );
if (ierr > 0 ) &
call endrun('Error: fail to resolve shared ents for pg2 mesh')
call abortmp('Error: fail to resolve shared ents for pg2 mesh')

ierr = iMOAB_UpdateMeshInfo(MHPGID)
if (ierr > 0 ) &
call endrun('Error: fail to update mesh info for pg2 mesh')
call abortmp('Error: fail to update mesh info for pg2 mesh')
#ifdef MOABDEBUG
! write out the mesh file to disk, in parallel
outfile = 'wholeATM_PG2.h5m'//C_NULL_CHAR
wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR
ierr = iMOAB_WriteMesh(MHPGID, outfile, wopts)
if (ierr > 0 ) &
call endrun('Error: fail to write the mesh file')
call abortmp('Error: fail to write the mesh file')
#endif
endif ! only valid for pg == 2
if ( nelemd > 0 ) then
deallocate (local_cell_gids)
deallocate (indx_cell)
deallocate (edge) !
deallocate(moabconn_pg) ! connectivity
deallocate(vdone_pg)
endif

deallocate(vdone_pg) ! this is now always allocated/deallocated, even for no mesh here
endif

! deallocate
if ( nelemd > 0 ) then
deallocate(moabvh)
deallocate(moabconn) ! do not keep it anymore, we are not setting another tag on fine mesh
deallocate(vdone)
deallocate(gdofel)
deallocate(indx)
deallocate(elemids)
Expand All @@ -810,6 +812,7 @@ subroutine create_moab_meshes(par, elem)
deallocate(moabconn_c)
deallocate(vdone_c)
endif
deallocate(vdone) ! we are always allocating this now
! end copy

end subroutine create_moab_meshes
Expand Down

0 comments on commit 4fab3c8

Please sign in to comment.