Merge branch 'abstractions-cuda-dev' into 'develop'

Abstract away CUDA API from main Re::Solve classes

Closes #35

See merge request ecpcitest/exasgd/resolve!37

examples/r_KLU_GLU.cpp

@@ -2,12 +2,14 @@
 #include <iostream>
 
 #include <resolve/matrix/Coo.hpp>
+#include <resolve/matrix/Csr.hpp>
 #include <resolve/vector/Vector.hpp>
 #include <resolve/matrix/io.hpp>
 #include <resolve/matrix/MatrixHandler.hpp>
 #include <resolve/vector/VectorHandler.hpp>
 #include <resolve/LinSolverDirectKLU.hpp>
 #include <resolve/LinSolverDirectCuSolverGLU.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 
 int main(int argc, char *argv[])
 {

examples/r_KLU_GLU_matrix_values_update.cpp

@@ -4,10 +4,12 @@
 #include <resolve/vector/Vector.hpp>
 #include <resolve/matrix/io.hpp>
 #include <resolve/matrix/Coo.hpp>
+#include <resolve/matrix/Csr.hpp>
 #include <resolve/matrix/MatrixHandler.hpp>
 #include <resolve/vector/VectorHandler.hpp>
 #include <resolve/LinSolverDirectKLU.hpp>
 #include <resolve/LinSolverDirectCuSolverGLU.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 
 // this updates the matrix values to simulate what CFD/optimization software does. 
 

examples/r_KLU_KLU.cpp

@@ -9,6 +9,7 @@
 #include <resolve/matrix/MatrixHandler.hpp>
 #include <resolve/vector/VectorHandler.hpp>
 #include <resolve/LinSolverDirectKLU.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 
 
 int main(int argc, char *argv[])

examples/r_KLU_KLU_standalone.cpp

@@ -9,6 +9,7 @@
 #include <resolve/matrix/MatrixHandler.hpp>
 #include <resolve/vector/VectorHandler.hpp>
 #include <resolve/LinSolverDirectKLU.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 
 
 int main(int argc, char *argv[])

examples/r_KLU_rf.cpp

@@ -2,12 +2,15 @@
 #include <iostream>
 
 #include <resolve/matrix/Coo.hpp>
+#include <resolve/matrix/Csr.hpp>
+#include <resolve/matrix/Csc.hpp>
 #include <resolve/vector/Vector.hpp>
 #include <resolve/matrix/io.hpp>
 #include <resolve/matrix/MatrixHandler.hpp>
 #include <resolve/vector/VectorHandler.hpp>
 #include <resolve/LinSolverDirectKLU.hpp>
 #include <resolve/LinSolverDirectCuSolverRf.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 
 int main(int argc, char *argv[] )
 {

examples/r_KLU_rf_FGMRES.cpp

@@ -2,13 +2,16 @@
 #include <iostream>
 
 #include <resolve/matrix/Coo.hpp>
+#include <resolve/matrix/Csr.hpp>
+#include <resolve/matrix/Csc.hpp>
 #include <resolve/vector/Vector.hpp>
 #include <resolve/matrix/io.hpp>
 #include <resolve/matrix/MatrixHandler.hpp>
 #include <resolve/vector/VectorHandler.hpp>
 #include <resolve/LinSolverDirectKLU.hpp>
 #include <resolve/LinSolverDirectCuSolverRf.hpp>
 #include <resolve/LinSolverIterativeFGMRES.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 
 int main(int argc, char *argv[])
 {

examples/r_KLU_rf_FGMRES_reuse_factorization.cpp

@@ -2,13 +2,16 @@
 #include <iostream>
 #include <resolve/GramSchmidt.hpp>
 #include <resolve/matrix/Coo.hpp>
+#include <resolve/matrix/Csr.hpp>
+#include <resolve/matrix/Csc.hpp>
 #include <resolve/vector/Vector.hpp>
 #include <resolve/matrix/io.hpp>
 #include <resolve/matrix/MatrixHandler.hpp>
 #include <resolve/vector/VectorHandler.hpp>
 #include <resolve/LinSolverDirectKLU.hpp>
 #include <resolve/LinSolverDirectCuSolverRf.hpp>
 #include <resolve/LinSolverIterativeFGMRES.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 
 int main(int argc, char *argv[])
 {

resolve/CMakeLists.txt

@@ -6,25 +6,22 @@
 
 ]]
 
-add_subdirectory(vector)
-add_subdirectory(matrix)
 add_subdirectory(utilities)
 
 set(ReSolve_SRC
-    # VectorHandler.cpp
     LinSolver.cpp
     LinSolverDirectKLU.cpp
+    LinAlgWorkspace.cpp
+    LinSolverIterativeFGMRES.cpp
+    GramSchmidt.cpp
+    LinSolverDirectCuSolverGLU.cpp
+    LinSolverDirectCuSolverRf.cpp
 )
 
 
 set(ReSolve_SRC_CUDA
-    # Vector.cpp
-    LinAlgWorkspace.cpp
-    LinSolverDirectCuSolverRf.cpp
-    LinSolverDirectCuSolverGLU.cpp
-    LinSolverIterativeFGMRES.cpp
     cudaKernels.cu
-    GramSchmidt.cpp
+    memoryUtils.cu
 )
 
 set(ReSolve_HEADER_INSTALL
@@ -40,25 +37,39 @@ set(ReSolve_HEADER_INSTALL
     RefactorizationSolver.hpp
     SystemSolver.hpp
     GramSchmidt.hpp
+    memoryUtils.hpp
 )
 
 set_source_files_properties(${ReSolve_SRC_CUDA} PROPERTIES LANGUAGE CUDA)
 
+# First create CUDA backend (this should really be CUDA _API_ backend, separate backend will be needed for CUDA SDK)
+add_library(resolve_backend_cuda SHARED ${ReSolve_SRC_CUDA})
+target_link_libraries(resolve_backend_cuda PUBLIC resolve_cuda)
+target_include_directories(resolve_backend_cuda INTERFACE
+    $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>
+    $<INSTALL_INTERFACE:include>
+)
+
+# Next build vector and matrix objects that may use this backend.
+add_subdirectory(vector)
+add_subdirectory(matrix)
+
+
 # Build shared library ReSolve
 add_library(resolve_tpl INTERFACE)
 target_link_libraries(resolve_tpl INTERFACE resolve_cuda KLU)
 
-install(TARGETS resolve_matrix resolve_vector resolve_logger resolve_tpl EXPORT ReSolveTargets)
+install(TARGETS resolve_matrix resolve_vector resolve_backend_cuda resolve_logger resolve_tpl EXPORT ReSolveTargets)
 
-add_library(ReSolve SHARED ${ReSolve_SRC} ${ReSolve_SRC_CUDA})
+add_library(ReSolve SHARED ${ReSolve_SRC})
 
 target_include_directories(ReSolve INTERFACE 
     $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>
     $<INSTALL_INTERFACE:include>
 )
 
 # TODO: Make this PRIVATE dependency (requires refactoring ReSolve code)
-target_link_libraries(ReSolve PUBLIC resolve_matrix resolve_vector resolve_logger resolve_tpl)
+target_link_libraries(ReSolve PUBLIC resolve_matrix resolve_vector resolve_backend_cuda resolve_logger resolve_tpl)
 
 install(TARGETS ReSolve
         EXPORT ReSolveTargets

resolve/GramSchmidt.cpp

@@ -1,5 +1,6 @@
 #include <iostream>
 #include <cassert>
+#include <cmath>
 
 #include <resolve/utilities/logger/Logger.hpp>
 #include <resolve/vector/Vector.hpp>

resolve/LinAlgWorkspace.cpp

@@ -1,3 +1,4 @@
+#include <resolve/memoryUtils.hpp>
 #include "LinAlgWorkspace.hpp"
 
 namespace ReSolve
@@ -13,18 +14,18 @@ namespace ReSolve
   LinAlgWorkspaceCUDA::LinAlgWorkspaceCUDA()
   {
     handle_cusolversp_ = nullptr;
-    handle_cusparse_ = nullptr;
-    handle_cublas_ = nullptr;
-    buffer_spmv_ = nullptr;
-    buffer_1norm_ = nullptr;    
+    handle_cusparse_   = nullptr;
+    handle_cublas_     = nullptr;
+    buffer_spmv_       = nullptr;
+    buffer_1norm_      = nullptr;    
 
     matvec_setup_done_ = false;
   }
 
   LinAlgWorkspaceCUDA::~LinAlgWorkspaceCUDA()
   {
-    if (buffer_spmv_ != nullptr) cudaFree(buffer_spmv_);
-    if (buffer_1norm_ != nullptr) cudaFree(buffer_1norm_);
+    if (buffer_spmv_ != nullptr)  deleteOnDevice(buffer_spmv_);
+    if (buffer_1norm_ != nullptr) deleteOnDevice(buffer_1norm_);
     cusparseDestroy(handle_cusparse_);
     cusolverSpDestroy(handle_cusolversp_);
     cublasDestroy(handle_cublas_);

resolve/LinAlgWorkspace.hpp

@@ -1,8 +1,8 @@
-#include <cuda_runtime.h>
+#pragma once
+
 #include "cublas_v2.h"
 #include "cusparse.h"
 #include "cusolverSp.h"
-#pragma once
 
 namespace ReSolve
 {
@@ -63,4 +63,19 @@ namespace ReSolve
 
       bool matvec_setup_done_; //check if setup is done for matvec i.e. if buffer is allocated, csr structure is set etc.
   };
+
+  /// @brief  Workspace factory
+  /// @param[in] memspace memory space ID 
+  /// @return pointer to the linear algebra workspace
+  inline LinAlgWorkspace* createLinAlgWorkspace(std::string memspace)
+  {
+    if (memspace == "cuda") {
+      LinAlgWorkspaceCUDA* workspace = new LinAlgWorkspaceCUDA();
+      workspace->initializeHandles();
+      return workspace;
+    } 
+    // If not CUDA, return default
+    return (new LinAlgWorkspace());
+  }
+
 }

resolve/LinSolver.cpp

@@ -1,3 +1,4 @@
+#include <resolve/matrix/Sparse.hpp>
 #include "LinSolver.hpp"
 
 

resolve/LinSolver.hpp

@@ -1,13 +1,27 @@
 #pragma once
 #include <string>
 #include "Common.hpp"
-#include <resolve/matrix/Sparse.hpp>
-#include <resolve/vector/Vector.hpp>
-#include <resolve/matrix/MatrixHandler.hpp>
-#include <resolve/vector/VectorHandler.hpp>
 
 namespace ReSolve 
 {
+  // Forward declaration of vector::Vector class
+  namespace vector
+  {
+    class Vector;
+  }
+
+  // Forward declaration of VectorHandler class
+  class VectorHandler;
+
+  // Forward declaration of matrix::Sparse class
+  namespace matrix
+  {
+    class Sparse;
+  }
+
+  // Forward declaration of MatrixHandler class
+  class MatrixHandler;
+
   class LinSolver 
   {
     protected:
@@ -25,8 +39,8 @@ namespace ReSolve
       real_type* rhs_;
       real_type* sol_;
 
-      MatrixHandler *matrix_handler_;
-      VectorHandler *vector_handler_;
+      MatrixHandler* matrix_handler_;
+      VectorHandler* vector_handler_;
   };
 
   class LinSolverDirect : public LinSolver 
@@ -60,6 +74,5 @@ namespace ReSolve
       ~LinSolverIterative();
 
       virtual int  solve(vector_type* rhs, vector_type* init_guess);
-
   };
 }

resolve/LinSolverDirectCuSolverGLU.cpp

@@ -1,5 +1,9 @@
-#include "LinSolverDirectCuSolverGLU.hpp"
+#include <cstring> // includes memcpy
 #include <vector>
+#include <resolve/memoryUtils.hpp>
+#include <resolve/vector/Vector.hpp>
+#include <resolve/matrix/Csr.hpp>
+#include "LinSolverDirectCuSolverGLU.hpp"
 
 namespace ReSolve
 {
@@ -10,7 +14,7 @@ namespace ReSolve
 
   LinSolverDirectCuSolverGLU::~LinSolverDirectCuSolverGLU()
   {
-    cudaFree(glu_buffer_);
+    deleteOnDevice(glu_buffer_);
     cusparseDestroyMatDescr(descr_M_);
     cusparseDestroyMatDescr(descr_A_);
     cusolverSpDestroyGluInfo(info_M_);
@@ -60,7 +64,7 @@ namespace ReSolve
     status_cusolver_ = cusolverSpDgluBufferSize(handle_cusolversp_, info_M_, &buffer_size);
     error_sum += status_cusolver_; 
 
-    cudaMalloc((void**)&glu_buffer_, buffer_size);
+    allocateBufferOnDevice(&glu_buffer_, buffer_size);
 
     status_cusolver_ = cusolverSpDgluAnalysis(handle_cusolversp_, info_M_, glu_buffer_);
     error_sum += status_cusolver_; 

resolve/LinSolverDirectCuSolverGLU.hpp

@@ -1,11 +1,23 @@
 #pragma once
 #include "Common.hpp"
-#include <resolve/matrix/Sparse.hpp>
+#include <resolve/LinAlgWorkspace.hpp>
 #include "LinSolver.hpp"
 #include "cusolver_defs.hpp"
 
 namespace ReSolve 
 {
+  // Forward declaration of vector::Vector class
+  namespace vector
+  {
+    class Vector;
+  }
+
+  // Forward declaration of matrix::Sparse class
+  namespace matrix
+  {
+    class Sparse;
+  }
+
   class LinSolverDirectCuSolverGLU : public LinSolverDirect 
   {
     using vector_type = vector::Vector;