Merge pull request #30 from ORNL/hip-first-take-dev

Add HIP/ROCm support to ReSolve

.gitlab/pnnl/base.gitlab-ci.yml

@@ -369,4 +369,4 @@ stages:
   variables:
     WORKDIR_SUFFIX: "x86_64-clang-hip-build"
     MY_CLUSTER: "incline"
-    SLURM_ARGS: " --exclusive --ntasks=3 "
+    SLURM_ARGS: " -N 1 --ntasks=3 "

.gitlab/pnnl/incline.gitlab-ci.yml

@@ -3,15 +3,15 @@ Incline Build:
     - .cluster_build
     - .incline
   variables:
-    SCRIPT_ARGS: " --build-only " #--job=clang-hip "
+    SCRIPT_ARGS: " --build-only "
 
 Incline Test:
   extends:
     - .cluster_test
     - .incline
   variables:
-    SCRIPT_ARGS: " --test-only " #--job=clang-hip "
-    CTESTARGS: " --timeout 240 --output-on-failure -LE incline-skip "
+    SCRIPT_ARGS: " --test-only "
+    CTESTARGS: " --timeout 240 --output-on-failure "
   needs: ['Incline Build']
 
 Incline Module Rebuild:

CMakeLists.txt

@@ -23,39 +23,31 @@ endif()
 
 option(RESOLVE_TEST_WITH_BSUB "Use `jsrun` instead of `mpirun` commands when running tests" OFF)
 option(RESOLVE_USE_KLU  "Use KLU, AMD and COLAMD libraries from SuiteSparse" ON)
-option(RESOLVE_USE_GPU  "Use GPU device for computations" ON)
-option(RESOLVE_USE_CUDA "Use CUDA language and SDK" ON)
-set(RESOLVE_CTEST_OUTPUT_DIR ${PROJECT_BINARY_DIR} CACHE PATH "Directory where CTest outputs are saved")
+option(RESOLVE_USE_CUDA "Use CUDA language and SDK" OFF)
+option(RESOLVE_USE_HIP  "Use HIP language and ROCm library" OFF)
+
+option(RESOLVE_USE_GPU  "Use GPU device for computations" OFF)
+mark_as_advanced(FORCE RESOLVE_USE_GPU)
 
 if(RESOLVE_USE_CUDA)
-  set(RESOLVE_USE_GPU On CACHE BOOL "Using GPU!" FORCE)
-else()
-  set(RESOLVE_USE_GPU Off CACHE BOOL "Using GPU!" FORCE)
+  set(RESOLVE_USE_GPU ON CACHE BOOL "Using CUDA GPU!" FORCE)
 endif()
 
+if(RESOLVE_USE_HIP)
+  set(RESOLVE_USE_GPU ON CACHE BOOL "Using HIP GPU!" FORCE)
+endif()
 
+# MacOS specific things
 set(CMAKE_MACOSX_RPATH 1)
-# set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-# https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/RPATH-handling#always-full-rpath
-# use, i.e. don't skip the full RPATH for the build tree
-#set(CMAKE_SKIP_BUILD_RPATH FALSE)
 
-# when building, don't use the install RPATH already
-# (but later on when installing)
+# Install with RPATH but do not build with it
 set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
-
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-
 set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/lib)
-#list(APPEND CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/lib)
 
 # Add CMake sources from `cmake` dir
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
-# Including clang-format cmake files to do automatic checking of formating
-# TODO: Set up clang-format
-#include(./cmake/clang-format)
-
 if (RESOLVE_USE_KLU)
   include(FindKLU)
   if(NOT KLU_LIBRARY)
@@ -89,22 +81,25 @@ else()
   message(STATUS "Not using CUDA")
 endif()
 
+if(RESOLVE_USE_HIP)
+  enable_language(HIP)
+  check_language(HIP)
+  include(ReSolveFindHipLibraries)
+else()
+  message(STATUS "Not using HIP")
+endif(RESOLVE_USE_HIP)
+
+
 # The binary dir is already a global include directory
 configure_file(
   ${CMAKE_SOURCE_DIR}/resolve/resolve_defs.hpp.in
   ${CMAKE_BINARY_DIR}/resolve/resolve_defs.hpp)
-
-# include build directory for Fortran name mangling header
-include_directories(${CMAKE_BINARY_DIR})
-
 install(
   FILES ${CMAKE_BINARY_DIR}/resolve/resolve_defs.hpp
   DESTINATION include/resolve
   )
 
 
-include_directories(${CMAKE_SOURCE_DIR})
-
 # Enable testing
 enable_testing()
 
@@ -141,4 +136,5 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ReSolveConfig.cmake"
 add_subdirectory(examples)
 
 # Add tests
+set(RESOLVE_CTEST_OUTPUT_DIR ${PROJECT_BINARY_DIR} CACHE PATH "Directory where CTest outputs are saved")
 add_subdirectory(tests)

CMakePresets.json

@@ -12,21 +12,31 @@
             "description": "Base config to build with CUDA",
             "binaryDir": "${sourceDir}/build",
             "installDir": "${sourceDir}/install",
-            "generator": "Unix Makefiles"
+            "generator": "Unix Makefiles",
+            "cacheVariables": {
+                "RESOLVE_USE_CUDA": "ON"
+            }
         },
         {
-            "name": "cpu",
-            "displayName": "CPU only build",
-            "description": "Base config to build without GPUs",
+            "name": "rocm",
+            "displayName": "ROCM  build",
+            "description": "Base config to build with ROCM",
             "binaryDir": "${sourceDir}/build",
             "installDir": "${sourceDir}/install",
             "generator": "Unix Makefiles",
             "cacheVariables": {
-                "RESOLVE_USE_CUDA": "OFF",
-                "RESOLVE_USE_GPU": "OFF"
+                "RESOLVE_USE_HIP": "ON"
             }
         },
-	      {
+        {
+            "name": "cpu",
+            "displayName": "CPU only build",
+            "description": "Base config to build without GPUs",
+            "binaryDir": "${sourceDir}/build",
+            "installDir": "${sourceDir}/install",
+            "generator": "Unix Makefiles"
+        },
+	    {
             "name": "ascent",
             "inherits": "cuda",
             "displayName": "Ascent Build",
@@ -44,9 +54,18 @@
         },
         {
             "name": "incline",
-            "inherits": "cpu",
-            "displayName": "Incline CPU only Build",
-            "description": "Custom changes specific for Incline"
+            "inherits": "rocm",
+            "displayName": "Incline Build with rocm",
+            "description": "Custom changes specific for Incline",
+            "cacheVariables": {
+                "CMAKE_HIP_ARCHITECTURES" : "gfx908",
+                "CMAKE_BUILD_TYPE" : "Debug"
+            },
+            "environment": {
+                "CC" : "clang",
+                "CXX" : "clang++",
+                "FC" : "gfortran"
+            }
         }
 
     ]

buildsystem/incline-env.sh

@@ -1,5 +1,15 @@
+#!/bin/bash
+
+# Load system rocm
 source /etc/profile.d/modules.sh
 module purge
 module load gcc/8.4.0
-module load rocm/5.3.0
+
+# These are necessary in order to see GPUs with sbatch
+unset ROCR_VISIBLE_DEVICES
+unset CUDA_VISIBLE_DEVICES
+unset GPU_DEVICE_ORDINAL
+
+# Load spack generated modules
 source ./buildsystem/spack/incline/modules/dependencies.sh
+

buildsystem/spack/incline/env.sh

@@ -3,19 +3,22 @@
 source /etc/profile.d/modules.sh
 module purge
 
-# Load system python
+# Load system python and gcc
 module load python/miniconda4.12
 source /share/apps/python/miniconda4.12/etc/profile.d/conda.sh
+module load gcc/8.4.0
 
 # Define environment variables for where spack stores key files
 # For now, SPACK_INSTALL is the path where everything spack related is installed
 # If you want to modify the module install path, edit the spack.yaml manually
 BASE=/qfs/projects/exasgd/resolve/spack-ci
 export SPACK_INSTALL=$BASE/install
+export SPACK_MIRROR=$BASE/../$(whoami)/spack-mirror
 export SPACK_CACHE=$BASE/../$(whoami)/spack-cache
 export SPACK_DISABLE_LOCAL_CONFIG=1
-export SPACK_PYTHON=$(which python)
+export SPACK_PYTHON=$(which python3)
 
 export tempdir=$SPACK_CACHE
 export TMP=$SPACK_CACHE
 export TMPDIR=$SPACK_CACHE
+

buildsystem/spack/incline/install.sh

@@ -9,6 +9,9 @@
 #SBATCH -e spack_install.%J.output
 #SBTACH -t 240
 
+export HTTPS_PROXY=http://proxy01.pnl.gov:3128
+export https_proxy=http://proxy01.pnl.gov:3128
+
 exit() {
   # Clear all trap handlers so this isn't echo'ed multiple times, potentially
   # throwing off the CI script watching for this output
@@ -44,6 +47,9 @@ cleanup() {
 export MY_CLUSTER=incline
 . buildsystem/load-spack.sh &&
 spack develop --no-clone --path=$(pwd) resolve@develop &&
+spack concretize -f &&
+spack install -j 64 llvm-amdgpu &&
+spack load llvm-amdgpu &&
 ./buildsystem/configure-modules.sh 64