Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Umpire memory manager for GPU pool memory allocation #943

Draft
wants to merge 13 commits into
base: vlasiator_gpu
Choose a base branch
from
Draft

Add Umpire memory manager for GPU pool memory allocation #943

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
b7fd0da
Implement Umpire support for Lumi+HIP
hokkanen Apr 8, 2024
c78a55e
Umpire works for device memory, but fails for Unified Mem
hokkanen Apr 8, 2024
c77b30b
Update arch_device_hip.h and comment out failing UM alloc/free
hokkanen Apr 8, 2024
252b7ba
Add Umpire for async allocs and pinned allocs, managed still disabled
hokkanen Apr 9, 2024
88a11de
Fix arch_device_hip.h Umpire stuff
hokkanen Apr 9, 2024
738b263
Fix too large Umpire allocation size
hokkanen Apr 9, 2024
3a14535
Add threadsafe allocators, and shared free function
hokkanen Apr 9, 2024
eb4b58c
Revert "Add threadsafe allocators, and shared free function"
hokkanen Apr 9, 2024
0a162cd
Update Makefiles
hokkanen Apr 10, 2024
dce427f
Add error handling into gpu_base.hpp
hokkanen Apr 10, 2024
fc4abee
Merge branch 'vlasiator_gpu' of https://github.com/fmihpc/vlasiator i…
hokkanen Jun 4, 2024
ea47708
Add Umpire into Mahti Cuda Makefile
hokkanen Jun 4, 2024
2c6f915
Changed the only two raw managed gpuFree calls to gpuFreeManaged
markusbattarbee Jun 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion MAKE/Makefile.lumi_hipcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ LIB_MPI = -lmpi ${PE_MPICH_GTL_LIBS_amd_gfx90a}
CXXFLAGS += -DPAPI_MEM
testpackage: CXXFLAGS += -DPAPI_MEM

#======== Allocator =========
#======== Jemalloc =========
#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it
#Note: jemalloc not supported with GPUs
Expand All @@ -64,6 +64,10 @@ testpackage: CXXFLAGS += -DPAPI_MEM
#-DNO_WRITE_AT_ALL: Define to disable write at all to
# avoid memleak (much slower IO)

#======== Umpire =========
CXXFLAGS += -DUSE_UMPIRE
testpackage: CXXFLAGS += -DUSE_UMPIRE

#======== Libraries ===========
# Select the base directory based on which project you are using:
# LUMILAPIO
Expand Down Expand Up @@ -91,6 +95,9 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv
INC_PROFILE = -isystem $(LIBRARY_PREFIX)/phiprof/include -D_ROCTX -I${ROCM_PATH}/include
LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib -Wl,-rpath=${ROCM_PATH}/lib -lroctx64 -lroctracer64

INC_UMPIRE = -isystem $(LIBRARY_PREFIX)/umpire/include
LIB_UMPIRE = -L$(LIBRARY_PREFIX)/umpire/lib -L$(LIBRARY_PREFIX)/umpire/lib64 -lcamp -lumpire -lfmt

#header libraries

INC_EIGEN = -isystem ./submodules/eigen/
Expand Down
9 changes: 8 additions & 1 deletion MAKE/Makefile.mahti_cuda
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,17 @@ INC_CUDA = -isystem /appl/spack/v020/install-tree/gcc-10.4.0/cuda-12.1.1-2ppwzf/
CXXFLAGS += -DPAPI_MEM
testpackage: CXXFLAGS += -DPAPI_MEM

#======== Allocator =========
#======== Jemalloc =========
#Use jemalloc instead of system malloc to reduce memory fragmentation https://github.com/jemalloc/jemalloc
#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it
#NOTE: jemalloc not supported with GPUs
#CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
#testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE

#======== Umpire =========
CXXFLAGS += -DUSE_UMPIRE
testpackage: CXXFLAGS += -DUSE_UMPIRE

#======== Libraries ===========

LIBRARY_PREFIX = /projappl/project_2004522/libraries/gcc-10.4.0/openmpi-4.1.5-cuda/cuda-12.1.1/
Expand All @@ -90,6 +94,9 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Xlinker=-rpath=$(LIBRARY_PREFIX)/vls
INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include
LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -Xlinker=-rpath=$(LIBRARY_PREFIX)/phiprof/lib

INC_UMPIRE = -isystem $(LIBRARY_PREFIX)/umpire/include
LIB_UMPIRE = -L$(LIBRARY_PREFIX)/umpire/lib -L$(LIBRARY_PREFIX)/umpire/lib64 -lcamp -lumpire -lfmt

#======== Header-only Libraries ===========

INC_EIGEN = -isystem ./submodules/eigen
Expand Down
5 changes: 5 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ COMPFLAGS += ${INC_PROFILE}
#use jemalloc
COMPFLAGS += ${INC_JEMALLOC}

#use umpire
COMPFLAGS += ${INC_UMPIRE}

#define precision
COMPFLAGS += -D${FP_PRECISION}

Expand Down Expand Up @@ -165,6 +168,8 @@ LIBS += ${LIB_PROFILE}
LIBS += ${LIB_VLSV}
LIBS += ${LIB_JEMALLOC}
LIBS += ${LIB_PAPI}
LIBS += ${LIB_UMPIRE}


# Define common dependencies
DEPS_COMMON = common.h common.cpp definitions.h mpiconversion.h logger.h object_wrapper.h
Expand Down
Loading