From f34b05acd3b35fa48ac6151ac5ff5666e4f72e0a Mon Sep 17 00:00:00 2001
From: Nicholson Koukpaizan <72402802+nkoukpaizan@users.noreply.github.com>
Date: Mon, 18 Dec 2023 09:32:39 -0500
Subject: [PATCH] CUDA build system based on LLVM clang (#103)

---
 CMakeLists.txt                  |  5 ++++
 CMakePresets.json               | 49 +++++++++++++++++++++------------
 buildsystem/ascent-clang-env.sh |  7 +++++
 buildsystem/build.sh            |  3 ++
 4 files changed, 46 insertions(+), 18 deletions(-)
 create mode 100644 buildsystem/ascent-clang-env.sh

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6717f280..b0c96c90 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,6 +29,7 @@ option(RESOLVE_USE_HIP  "Use HIP language and ROCm library" OFF)
 option(RESOLVE_USE_GPU  "Use GPU device for computations" OFF)
 mark_as_advanced(FORCE RESOLVE_USE_GPU)
 
+option(RESOLVE_USE_ASAN "Use LLVM address sanitizer" OFF)
 option(RESOLVE_USE_DOXYGEN "Use Doxygen to generate Re::Solve documentation" ON)
 set(RESOLVE_CTEST_OUTPUT_DIR ${PROJECT_BINARY_DIR} CACHE PATH "Directory where CTest outputs are saved")
 
@@ -105,6 +106,10 @@ else()
   message(STATUS "Not using HIP")
 endif(RESOLVE_USE_HIP)
 
+if (RESOLVE_USE_ASAN)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
+endif()
 
 # The binary dir is already a global include directory
 configure_file(
diff --git a/CMakePresets.json b/CMakePresets.json
index 02f57d3c..f83a5b5e 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -26,9 +26,7 @@
             "installDir": "${sourceDir}/install",
             "generator": "Unix Makefiles",
             "cacheVariables": {
-                "RESOLVE_USE_HIP": "ON",
-                "RESOLVE_USE_CUDA": "OFF",
-                "RESOLVE_USE_GPU": "ON"
+                "RESOLVE_USE_HIP": "ON"
             }
         },
         {
@@ -39,16 +37,31 @@
             "installDir": "${sourceDir}/install",
             "generator": "Unix Makefiles"
         },
-	    {
+	{
             "name": "ascent",
             "inherits": "cuda",
             "displayName": "Ascent Build",
             "description": "Custom changes specific for Ascent",
             "cacheVariables": {
-                  "CMAKE_C_COMPILER": "$env{OLCF_GCC_ROOT}/bin/gcc",
-                  "CMAKE_CXX_COMPILER": "$env{OLCF_GCC_ROOT}/bin/g++",
-		  "CMAKE_CUDA_ARCHITECTURES": "70"
-	          }
+                "CMAKE_C_COMPILER": "$env{OLCF_GCC_ROOT}/bin/gcc",
+                "CMAKE_CXX_COMPILER": "$env{OLCF_GCC_ROOT}/bin/g++",
+		"CMAKE_CUDA_ARCHITECTURES": "70"
+	    }
+        },
+	{
+            "name": "ascent-clang",
+            "inherits": "cuda",
+            "displayName": "Ascent Build with Clang",
+            "description": "Custom changes specific for Ascent with Clang",
+            "cacheVariables": {
+                "CMAKE_C_COMPILER": "$env{OLCF_LLVM_ROOT}/bin/clang",
+                "CMAKE_CXX_COMPILER": "$env{OLCF_LLVM_ROOT}/bin/clang++",
+                "CMAKE_CUDA_COMPILER": "nvcc",
+                "CMAKE_CUDA_HOST_COMPILER": "$env{OLCF_LLVM_ROOT}/bin/clang++",
+                "CMAKE_CUDA_FLAGS": "-allow-unsupported-compiler",
+                "RESOLVE_USE_ASAN": "OFF",
+		"CMAKE_CUDA_ARCHITECTURES": "70"
+	    }
         },
         {
             "name": "deception",
@@ -72,16 +85,16 @@
             }
         },
         {
-              "name": "crusher",
-	      "inherits": "rocm",
-              "displayName": "Crusher Build",
-              "description": "Custom changes specific for Crusher",
-	      "cacheVariables": {
-                  "CMAKE_HIP_ARCHITECTURES": "gfx90a",
-                  "CMAKE_BUILD_TYPE": "Debug",
-		  "CMAKE_C_COMPILER": "/opt/rocm-5.6.0/llvm/bin/clang",
-                  "CMAKE_CXX_COMPILER": "/opt/rocm-5.6.0/llvm/bin/clang++"
-	          }  
+            "name": "crusher",
+	    "inherits": "rocm",
+            "displayName": "Crusher Build",
+            "description": "Custom changes specific for Crusher",
+	    "cacheVariables": {
+                "CMAKE_HIP_ARCHITECTURES": "gfx90a",
+                "CMAKE_BUILD_TYPE": "Debug",
+	        "CMAKE_C_COMPILER": "/opt/rocm-5.6.0/llvm/bin/clang",
+                "CMAKE_CXX_COMPILER": "/opt/rocm-5.6.0/llvm/bin/clang++"
+	    }  
 	}
     ]
 }
diff --git a/buildsystem/ascent-clang-env.sh b/buildsystem/ascent-clang-env.sh
new file mode 100644
index 00000000..0e027cc3
--- /dev/null
+++ b/buildsystem/ascent-clang-env.sh
@@ -0,0 +1,7 @@
+# Load compiler...
+module load llvm/12.0.0
+# Load deps directly
+module load cuda/11.4.2
+module load cmake/3.22.2
+module load suite-sparse/5.10.1
+# Load spack deps
diff --git a/buildsystem/build.sh b/buildsystem/build.sh
index 57d29e34..69c2b960 100755
--- a/buildsystem/build.sh
+++ b/buildsystem/build.sh
@@ -150,6 +150,9 @@ case $MY_CLUSTER in
   dl*|deception|*fat*)
     export MY_CLUSTER=deception
     ;;
+  ascent-clang*)
+    export MY_CLUSTER=ascent-clang
+    ;;
   ascent*)
     export MY_CLUSTER=ascent
     ;;