Enable BML to use hypre's GPU-capable sparse matrix routines

* Enables sparse matrix multiplication and addition on GPUs
* Uses openMP offload to transfer BML data to hypre
* Current implementation supports NVIDIA GPUs only
* Supports only one precision build of hypre (single  of double)

CMakeLists.txt

@@ -229,6 +229,7 @@ set(BML_ROCSPARSE FALSE CACHE BOOL "Whether to compile with rocSPARSE support")
 set(BML_CUDA FALSE CACHE BOOL "Whether to compile with CUDA support")
 set(BML_CUSOLVER FALSE CACHE BOOL "Whether to compile with cuSolver support")
 set(BML_MAGMA FALSE CACHE BOOL "Whether to use MAGMA library")
+set(BML_HYPRE FALSE CACHE BOOL "Whether to use HYPRE library")
 set(BML_OMP_OFFLOAD OFF CACHE BOOL "Compile with OpenMP GPU Offload support")
 set(BML_OFFLOAD_ARCH "NVIDIA" CACHE STRING "Offload architecture")
 set_property(CACHE BML_OFFLOAD_ARCH PROPERTY STRINGS "NVIDIA" "AMD")
@@ -306,7 +307,7 @@ if(BML_OMP_OFFLOAD)
   endif()
 endif()
 
-set(BML_USE_DEVICE (BML_CUDA OR BML_MAGMA OR BML_CUSOLVER OR BML_ROCSOLVER OR BML_CUSPARSE OR BML_ROCSPARSE))
+set(BML_USE_DEVICE (BML_CUDA OR BML_MAGMA OR BML_CUSOLVER OR BML_ROCSOLVER OR BML_CUSPARSE OR BML_ROCSPARSE OR BML_HYPRE))
 if(BML_USE_DEVICE)
   message(STATUS "Will use device libraries for some solvers")
   if (CUDAToolkit_FOUND)
@@ -390,6 +391,53 @@ if(BML_MAGMA)
   endif()
 endif()
 
+if(BML_HYPRE)
+  message(STATUS "Search for HYPRE...")
+  find_package(HYPRE REQUIRED)
+
+  if (${HYPRE_FOUND})
+    add_definitions(-DBML_USE_HYPRE)
+    message(STATUS
+      "HYPRE was found:\n"
+      "  HYPRE_INCLUDE_DIRS: ${HYPRE_INCLUDE_DIRS}\n"
+      "  HYPRE_LIBRARY_DIRS: ${HYPRE_LIBRARY_DIRS}\n"
+      "  HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}"
+    )
+    include_directories(${HYPRE_INCLUDE_DIRS})
+
+    list(APPEND LINK_LIBRARIES "-L${HYPRE_LIBRARY_DIRS}")
+    list(APPEND LINK_LIBRARIES ${HYPRE_LIBRARIES})
+
+    if (CUDAToolkit_FOUND)
+       list(APPEND LINK_LIBRARIES "-L${CUDAToolkit_LIBRARY_DIR}")
+       list(APPEND LINK_LIBRARIES ${CUDA_cublas_LIBRARY})
+       list(APPEND LINK_LIBRARIES ${CUDA_cudart_LIBRARY})
+       include_directories(${CUDAToolkit_INCLUDE_DIR})
+
+#      add_definitions(-DBML_USE_CUSPARSE)
+      list(APPEND LINK_LIBRARIES ${CUDA_cusparse_LIBRARY})
+      list(APPEND LINK_LIBRARIES ${CUDA_curand_LIBRARY})
+    elseif(hip_FOUND)
+      find_package(rocblas REQUIRED)
+      list(APPEND LINK_LIBRARIES ${rocblas_LIBRARIES})
+      include_directories(${rocblas_INCLUDE_DIRS})
+
+#      add_definitions(-DBML_USE_rocSPARSE)
+      find_package(rocsparse REQUIRED)
+      include_directories(${rocsparse_INCLUDE_DIRS})
+      list(APPEND LINK_LIBRARIES ${rocsparse_LIBRARIES})
+      find_package(rocrand REQUIRED)
+      list(APPEND LINK_LIBRARIES ${rocrand_LIBRARIES})
+
+      include_directories(${hip_INCLUDE_DIRS})
+    endif()
+    list(APPEND LINK_LIBRARIES -lstdc++)
+    message(STATUS "LINK_LIBRARIES: ${LINK_LIBRARIES}")
+  else()
+    message(FATAL_ERROR "HYPRE was not found")
+  endif()
+endif()
+
 set(BML_ELPA FALSE CACHE BOOL "Whether to use ELPA library")
 if(BML_ELPA)
   message(STATUS "Search for ELPA in directory ${ELPA_DIR}\n")

build.sh

@@ -89,6 +89,7 @@ EOF
     echo "BML_MAGMA                  Build with MAGMA            (default is ${BML_MAGMA})"
     echo "BML_CUSOLVER               Build with cuSOLVER         (default is ${BML_CUSOLVER})"
     echo "BML_CUSPARSE               Build with cuSPARSE         (default is ${BML_CUSPARSE})"
+    echo "BML_HYPRE                  Build with HYPRE            (default is ${BML_HYPRE})"
     echo "BML_ROCSOLVER              Build with rocSOLVER        (default is ${BML_ROCSOLVER})"
     echo "BML_ROCSPARSE              Build with rocSPARSE        (default is ${BML_ROCSPARSE})"
     echo "BML_SYEVD                  Build with SYEVD            (default is ${BML_SYEVD})"
@@ -144,6 +145,7 @@ set_defaults() {
     : ${BML_MAGMA:=no}
     : ${BML_CUSOLVER:=no}
     : ${BML_CUSPARSE:=no}
+    : ${BML_HYPRE:=no}
     : ${BML_ROCSOLVER:=no}
     : ${BML_ROCSPARSE:=no}
     : ${BML_SYEVD:=yes}
@@ -205,8 +207,8 @@ is_enabled() {
 }
 
 sanity_check() {
-    if (is_enabled ${BML_CUSPARSE} || is_enabled ${BML_ROCSPARSE}) && ! is_enabled ${BML_OMP_OFFLOAD}; then
-        die "In order to enable BML_CUSPARSE or BML_ROCSPARSE, BML_OMP_OFFLOAD needs to be enabled as well"
+    if (is_enabled ${BML_CUSPARSE} || is_enabled ${BML_ROCSPARSE} || is_enabled ${BML_HYPRE}) && ! is_enabled ${BML_OMP_OFFLOAD}; then
+        die "In order to enable BML_CUSPARSE or BML_ROCSPARSE or BML_HYPRE, BML_OMP_OFFLOAD needs to be enabled as well"
     fi
 }
 
@@ -256,6 +258,7 @@ configure() {
         -DBML_MAGMA="${BML_MAGMA}" \
         -DBML_CUSOLVER="${BML_CUSOLVER}" \
         -DBML_CUSPARSE="${BML_CUSPARSE}" \
+        -DBML_HYPRE="${BML_HYPRE}" \
         -DBML_ROCSOLVER="${BML_ROCSOLVER}" \
         -DBML_ROCSPARSE="${BML_ROCSPARSE}" \
         -DBML_SYEVD="${BML_SYEVD}" \

cmake/FindHYPRE.cmake

@@ -0,0 +1,36 @@
+# - Find the HYPRE library
+#
+# Usage:
+#   find_package(HYPRE [REQUIRED] [QUIET] )
+#
+# It sets the following variables:
+#   HYPRE_FOUND               ... true if HYPRE is found on the system
+#   HYPRE_LIBRARY_DIRS        ... full path to HYPRE library
+#   HYPRE_INCLUDE_DIRS        ... HYPRE include directory
+#   HYPRE_LIBRARIES           ... HYPRE libraries
+#
+# The following variables will be checked by the function
+#   HYPRE_USE_STATIC_LIBS     ... if true, only static libraries are found
+#   HYPRE_ROOT                ... if set, the libraries are exclusively searched
+#                                 under this path
+
+#If environment variable HYPRE_ROOT is specified, it has same effect as HYPRE_ROOT
+if( NOT HYPRE_ROOT AND NOT $ENV{HYPRE_ROOT} STREQUAL "" )
+    set( HYPRE_ROOT $ENV{HYPRE_ROOT} )
+    # set library directories
+    set(HYPRE_LIBRARY_DIRS ${HYPRE_ROOT}/lib)
+    # set include directories
+    set(HYPRE_INCLUDE_DIRS ${HYPRE_ROOT}/include)
+    # set libraries
+    find_library(
+        HYPRE_LIBRARIES
+        NAMES "HYPRE"
+        PATHS ${HYPRE_ROOT}
+        PATH_SUFFIXES "lib"
+        NO_DEFAULT_PATH
+    )
+    set(HYPRE_FOUND TRUE)
+else()
+    set(HYPRE_FOUND FALSE)
+endif()
+

scripts/build_lassen_xl_offload.sh → scripts/build_lassen_xl_offload_whypre.sh

@@ -2,28 +2,36 @@
 
 # Make sure all the paths are correct
 
-source setenv_lassen_offload.sh
+source scripts/setenv_lassen_offload.sh
 
 rm -r build
 rm -r install
 
 MY_PATH=$(pwd)
 
-export CC=${CC:=xlc-gpu}
+HYPRE_INSTALL_PATH="/usr/WS1/osei/soft/CoPA/lassen/gpu/fork/bml-hypre/hypre/src/hypre"
+
+export CC=${CC:=xlc++-gpu}
 export FC=${FC:=xlf2003-gpu}
 export CXX=${CXX:=xlc++-gpu}
 export BLAS_VENDOR=${BLAS_VENDOR:=Auto}
 export BML_OPENMP=${BML_OPENMP:=yes}
 export BML_OMP_OFFLOAD=${BML_OMP_OFFLOAD:=yes}
-export BML_CUSPARSE=${BML_CUSPARSE:=yes}
+export BML_OFFLOAD_ARCH=${BML_OFFLOAD_ARCH:=NVIDIA}
+export BML_CUSPARSE=${BML_CUSPARSE:=no}
+export BML_HYPRE=${BML_HYPRE:=yes}
+export HYPRE_ROOT=${HYPRE_INSTALL_PATH}
 export BML_COMPLEX=${BML_COMPLEX:=no}
 export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install"}
 export BML_TESTING=${BML_TESTING:=yes}
 export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
 export EXTRA_CFLAGS=${EXTRA_CFLAGS:=""}
 export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:=""}
 #export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-lm -L/usr/tce/packages/xl/xl-2021.03.11/xlC/16.1.1/lib -libmc++"}
-export BLAS_LIBRARIES=${BLAS_LIBRARIES:="-L${LAPACK_DIR} -llapack -lblas"}
+export BLAS_LIBRARIES=${BLAS_LIBRARIES:="-L${ESSLLIBDIR64} -lesslsmp"}
+export LAPACK_LIBRARIES=${LAPACK_LIBRARIES:="-L${LAPACK_DIR} -llapack"}
+
+export CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR=$CUDA_HOME}
 
 ./build.sh configure
 

scripts/build_olcf_summit_gnu_offload_openblas_whypre.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+module load cmake
+module load cuda
+module load gcc/11.2.0
+module load openblas
+
+export CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR="/sw/summit/cuda/11.0.3"}
+
+rm -r build
+rm -r install
+
+MY_PATH=$(pwd)
+
+# change this to path of hypre installation.
+# build hypre with: ./configure --with-cuda --without-MPI CUCC=nvcc
+# using gcc-9 compilers.
+HYPRE_INSTALL_PATH="/ccs/home/osei/soft/CoPA/with-hypre/hypre/src/hypre"
+
+#get jsrun with full path
+JSRUN=$(which jsrun)
+echo ${JSRUN}
+
+export CC=${CC:=gcc}
+export FC=${FC:=gfortran}
+export CXX=${CXX:=g++}
+export BLAS_VENDOR=${BLAS_VENDOR:=OpenBLAS}
+export BML_OPENMP=${BML_OPENMP:=yes}
+export BML_OMP_OFFLOAD=${BML_OMP_OFFLOAD:=yes}
+export BML_HYPRE=${BML_HYPRE:=yes}
+export HYPRE_ROOT=${HYPRE_INSTALL_PATH}
+export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install"}
+export BML_TESTING=${BML_TESTING:=yes}
+export BML_COMPLEX=${BML_COMPLEX:=no}
+export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
+export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-fopenmp -latomic -lm"}
+export BML_CUSPARSE=${BML_CUSPARSE:=no}
+export BML_COMPLEX=${BML_COMPLEX:=no}
+export BML_SYEVD=${BML_SYEVD:=no}
+
+#use jsrun to run tests on a compute node
+export BML_NONMPI_PRECOMMAND=${BML_NONMPI_PRECOMMAND:=${JSRUN}}
+export BML_NONMPI_PRECOMMAND_ARGS=${BML_NONMPI_PRECOMMAND_ARGS:="-n1;-a1;-g1;-c7"}
+
+./build.sh install

scripts/setenv_lassen_offload.sh

@@ -1,10 +1,8 @@
 #!/bin/bash
 
 #module purge
-module load cmake
-module load xl/2021.03.11-cuda-11.2.0
-module load cuda/11.2.0
+module load cmake/3.23.1
+module load xl/2022.03.10-cuda-11.8.0
+module load cuda/11.8.0
 module load lapack/3.9.0-xl-2020.11.12
-#module load essl
-export CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR="/usr/tce/packages/cuda/cuda-11.2.0"}
-
+module load essl

src/C-interface/bml_logger.c

@@ -192,4 +192,7 @@ bml_print_version(
 #ifdef BML_USE_ROCSPARSE
     fprintf(stdout, "BML uses AMD rocSparse\n");
 #endif
+#ifdef BML_USE_HYPRE
+    fprintf(stdout, "BML uses hypre library\n");
+#endif
 }

src/C-interface/ellpack/bml_add_ellpack.h

@@ -200,4 +200,40 @@ void bml_add_rocsparse_ellpack_double_complex(
     double beta,
     double threshold);
 #endif
+#if defined(BML_USE_HYPRE)
+void bml_add_hypre_ellpack(
+    bml_matrix_ellpack_t * A,
+    bml_matrix_ellpack_t * B,
+    double alpha,
+    double beta,
+    double threshold);
+
+void bml_add_hypre_ellpack_single_real(
+    bml_matrix_ellpack_t * A,
+    bml_matrix_ellpack_t * B,
+    double alpha,
+    double beta,
+    double threshold);
+
+void bml_add_hypre_ellpack_double_real(
+    bml_matrix_ellpack_t * A,
+    bml_matrix_ellpack_t * B,
+    double alpha,
+    double beta,
+    double threshold);
+
+void bml_add_hypre_ellpack_single_complex(
+    bml_matrix_ellpack_t * A,
+    bml_matrix_ellpack_t * B,
+    double alpha,
+    double beta,
+    double threshold);
+
+void bml_add_hypre_ellpack_double_complex(
+    bml_matrix_ellpack_t * A,
+    bml_matrix_ellpack_t * B,
+    double alpha,
+    double beta,
+    double threshold);
+#endif
 #endif