Spack cpu build

.github/workflows/spack_cpu_build.yaml

@@ -0,0 +1,137 @@
+# https://spack.readthedocs.io/en/latest/binary_caches.html#spack-build-cache-for-github-actions
+name: Spack Ubunutu x86_64 Buildcache
+
+env:
+  SPACK_COLOR: always
+  REGISTRY: ghcr.io/ornl
+  # Our repo name contains upper case characters, so we can't use ${{ github.repository }}
+  IMAGE_NAME: resolve
+  USERNAME: resolve-bot
+  BASE_VERSION: ubuntu-22.04-fortran
+
+# Until we remove the need to clone submodules to build, this should on be in PRs
+on: [pull_request]
+
+jobs:
+  base_image_build:
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+      contents: read
+
+    name: Build Custom Base Image
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Once we move submodule deps into spack, we can do some more builds
+          # Also need to change build script to use spack from base image
+          submodules: true
+
+      # Need to build custom base image with gfortran
+      - name: Create Dockerfile heredoc
+        run: |
+          cat << EOF > Dockerfile
+          FROM ubuntu:22.04
+          RUN apt-get update && \
+              apt-get install -y --no-install-recommends \
+                software-properties-common \
+                gpg-agent \
+                openssh-client \
+                openssh-server \
+              && rm -rf /var/lib/apt/lists/*
+          RUN add-apt-repository ppa:ubuntu-toolchain-r/test && \
+              apt-get install -y --no-install-recommends \
+                gcc \
+                libstdc++6 \
+              && rm -rf /var/lib/apt/lists/*
+          EOF
+
+      # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.USERNAME }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          labels: org.opencontainers.image.version=${{ env.BASE_VERSION }}
+
+      - name: Build and push Docker base image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  resolve_spack_builds:
+    needs: base_image_build
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+      contents: read
+
+    strategy:
+      matrix:
+        # Minimal Build(s) - GHCR mirror speeds these up a lot!
+        spack_spec:
+          - resolve@develop+klu
+          - resolve@develop~klu 
+
+    name: Build ExaGO with Spack
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Once we move submodule deps into spack, we can do some more builds
+          # Also need to change build script to use spack from base image
+          submodules: true
+
+      - name: Setup Spack
+        run: echo "$PWD/buildsystem/spack/spack/bin" >> "$GITHUB_PATH"
+
+      - name: Create heredoc spack.yaml
+        run: |
+          cat << EOF > spack.yaml
+          spack:
+            specs:
+            - ${{ matrix.spack_spec }} target=x86_64_v2
+            concretizer:
+              reuse: dependencies
+            config:
+              install_tree:
+                root: /opt/spack
+                padded_length: 128
+            mirrors:
+              local-buildcache: oci://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+              spack: https://binaries.spack.io/develop
+
+      - name: Configure GHCR mirror
+        run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
+
+      - name: Trust keys
+        run: spack -e . buildcache keys --install --trust
+
+      - name: Find external packages
+        run: spack -e . external find --all
+
+      - name: Spack develop exago
+        run: spack -e . develop --path=$(pwd) resolve@develop
+
+      - name: Concretize
+        run: spack -e . concretize
+
+      - name: Install
+        run: spack -e . install --no-check-signature
+
+      # Push with force to override existing binaries...
+      - name: Push to binaries to buildcache
+        run: |
+          spack -e . buildcache push --force --base-image ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BASE_VERSION }} --unsigned --update-index local-buildcache
+        if: ${{ !cancelled() }}

examples/CMakeLists.txt

@@ -6,17 +6,19 @@
 
 ]]
 
-# Build example with KLU factorization and KLU refactorization
-add_executable(klu_klu.exe r_KLU_KLU.cpp)
-target_link_libraries(klu_klu.exe PRIVATE ReSolve)
+if(RESOLVE_USE_KLU)
+  # Build example with KLU factorization and KLU refactorization
+  add_executable(klu_klu.exe r_KLU_KLU.cpp)
+  target_link_libraries(klu_klu.exe PRIVATE ReSolve)
 
-# Read ONE matrix,solve with KLU
-add_executable(klu_klu_standalone.exe r_KLU_KLU_standalone.cpp)
-target_link_libraries(klu_klu_standalone.exe PRIVATE ReSolve)
+  # Read ONE matrix,solve with KLU
+  add_executable(klu_klu_standalone.exe r_KLU_KLU_standalone.cpp)
+  target_link_libraries(klu_klu_standalone.exe PRIVATE ReSolve)
 
-# Build example with a configurable system solver
-add_executable(system.exe r_SysSolver.cpp)
-target_link_libraries(system.exe PRIVATE ReSolve)
+  # Build example with a configurable system solver
+  add_executable(system.exe r_SysSolver.cpp)
+  target_link_libraries(system.exe PRIVATE ReSolve)
+endif(RESOLVE_USE_KLU)
 
 # Create CUDA examples
 if(RESOLVE_USE_CUDA)
@@ -81,15 +83,19 @@ if(RESOLVE_USE_HIP)
   target_link_libraries(system_hip_fgmres.exe PRIVATE ReSolve)
 endif(RESOLVE_USE_HIP)
 
+set(installable_executables "")
+
 # Install all examples in bin directory
-set(installable_executables klu_klu.exe klu_klu_standalone.exe system.exe)
+if(RESOLVE_USE_KLU)
+  list(APPEND installable_executables klu_klu.exe klu_klu_standalone.exe system.exe)
+endif()
 
 if(RESOLVE_USE_CUDA)
-  set(installable_executables ${installable_executables} klu_glu.exe klu_rf.exe klu_rf_fgmres.exe klu_glu_values_update.exe gmres_cusparse_rand.exe klu_cusolverrf_check_redo.exe)        
+  list(APPEND installable_executables klu_glu.exe klu_rf.exe klu_rf_fgmres.exe  klu_glu_values_update.exe gmres_cusparse_rand.exe klu_cusolverrf_check_redo.exe)      
 endif(RESOLVE_USE_CUDA)
 
 if(RESOLVE_USE_HIP)
-  set(installable_executables ${installable_executables} klu_rocsolverrf.exe klu_rocsolverrf_fgmres.exe klu_rocsolverrf_check_redo.exe gmres_rocsparse_rand.exe)        
+  list(APPEND installable_executables klu_rocsolverrf.exe klu_rocsolverrf_fgmres.exe klu_rocsolverrf_check_redo.exe gmres_rocsparse_rand.exe)     
 endif(RESOLVE_USE_HIP)
 
 install(TARGETS ${installable_executables} 
@@ -102,13 +108,14 @@ set(CONSUMER_PATH ${CMAKE_INSTALL_PREFIX}/share/examples)
 install(PROGRAMS test.sh DESTINATION ${CONSUMER_PATH})
 
 # Select consumer app
-# TODO - have an outer loop that adds a unique consumer test for each backend supproted
 if(RESOLVE_USE_CUDA)
   set(RESOLVE_CONSUMER_APP "testKLU_Rf_FGMRES.cpp")
 elseif(RESOLVE_USE_HIP)
   set(RESOLVE_CONSUMER_APP "testKLU_RocSolver.cpp")
-else()
+elseif(RESOLVE_USE_KLU)
   set(RESOLVE_CONSUMER_APP "testKLU.cpp")
+else()
+  set(RESOLVE_CONSUMER_APP "testVersion.cpp")
 endif()
 
 # Install directory with example on how to consume ReSolve

resolve/CMakeLists.txt

@@ -11,26 +11,30 @@ add_subdirectory(utilities)
 # C++ files
 set(ReSolve_SRC
     LinSolver.cpp
-    LinSolverDirectKLU.cpp
-    SystemSolver.cpp
+    GramSchmidt.cpp
+    LinSolverIterativeFGMRES.cpp
 )
 
 # Temporary until there is CPU-only option for FGMRES
 set(ReSolve_GPU_SRC
-    GramSchmidt.cpp
-    LinSolverIterativeFGMRES.cpp
     LinSolverIterativeRandFGMRES.cpp
     RandSketchingManager.cpp
     RandSketchingCountSketch.cpp
     RandSketchingFWHT.cpp
-    )
+)
+
+set(ReSolve_KLU_SRC
+  LinSolverDirectKLU.cpp
+  SystemSolver.cpp
+)
 
 # C++ code that links to CUDA SDK libraries
 set(ReSolve_CUDASDK_SRC
     LinSolverDirectCuSolverGLU.cpp
     LinSolverDirectCuSolverRf.cpp
     LinSolverDirectCuSparseILU0.cpp
 )
+
 # HIP files
 set(ReSolve_ROCM_SRC
   LinSolverDirectRocSolverRf.cpp
@@ -41,18 +45,30 @@ set(ReSolve_HEADER_INSTALL
     Common.hpp
     cusolver_defs.hpp
     LinSolver.hpp
-    LinSolverDirectCuSolverGLU.hpp
-    LinSolverDirectCuSolverRf.hpp
-    LinSolverDirectRocSolverRf.hpp
-    LinSolverDirectRocSparseILU0.hpp
-    LinSolverDirectCuSparseILU0.hpp
-    LinSolverDirectKLU.hpp
     LinSolverIterativeFGMRES.hpp
     RefactorizationSolver.hpp
     SystemSolver.hpp
     GramSchmidt.hpp
-    MemoryUtils.hpp
-    RandSketchingManager.hpp)
+    MemoryUtils.hpp)
+
+set(ReSolve_KLU_HEADER_INSTALL
+  LinSolverDirectKLU.hpp
+)
+
+set(ReSolve_GPU_HEADER_INSTALL
+    RandSketchingManager.hpp
+)
+
+set(ReSolve_CUDA_HEADER_INSTALL
+    LinSolverDirectCuSolverGLU.hpp
+    LinSolverDirectCuSolverRf.hpp
+    LinSolverDirectCuSparseILU0.hpp
+)
+
+set(ReSolve_ROCM_HEADER_INSTALL
+  LinSolverDirectRocSolverRf.hpp
+  LinSolverDirectRocSparseILU0.hpp
+)
 
 # Now, build workspaces
 add_subdirectory(workspace)
@@ -65,8 +81,10 @@ add_subdirectory(matrix)
 add_library(resolve_tpl INTERFACE)
 
 if(RESOLVE_USE_KLU)
-  target_link_libraries(resolve_tpl INTERFACE KLU)    
-endif(RESOLVE_USE_KLU)
+  target_link_libraries(resolve_tpl INTERFACE KLU)
+  list(APPEND ReSolve_SRC ${ReSolve_KLU_SRC})
+  list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_KLU_HEADER_INSTALL})
+endif()
 
 set(ReSolve_Targets_List
     resolve_matrix
@@ -78,29 +96,32 @@ set(ReSolve_Targets_List
 
 # Temporary until there is CPU-only option for FGMRES
 if(RESOLVE_USE_GPU)
-  set(ReSolve_SRC ${ReSolve_SRC} ${ReSolve_GPU_SRC})
+  list(APPEND ReSolve_SRC ${ReSolve_GPU_SRC})
+  list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_GPU_HEADER_INSTALL})
 endif()
 
 # If CUDA support is enabled add CUDA SDK specific code and dependencies
 if(RESOLVE_USE_CUDA)
   add_subdirectory(cuda)
   target_link_libraries(resolve_tpl INTERFACE resolve_cuda)
-  set(ReSolve_SRC ${ReSolve_SRC} ${ReSolve_CUDASDK_SRC})
-  set(ReSolve_Targets_List ${ReSolve_Targets_List} resolve_backend_cuda)
+  list(APPEND ReSolve_SRC ${ReSolve_CUDASDK_SRC})
+  list(APPEND ReSolve_Targets_List resolve_backend_cuda)
+  list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_CUDA_HEADER_INSTALL})
 endif()
 
 # If HIP support is enabled add HIP SDK specific code and dependencies
 if(RESOLVE_USE_HIP)
   add_subdirectory(hip)
   target_link_libraries(resolve_tpl INTERFACE resolve_hip)
-  set(ReSolve_SRC ${ReSolve_SRC} ${ReSolve_ROCM_SRC})
-  set(ReSolve_Targets_List ${ReSolve_Targets_List} resolve_backend_hip)
+  list(APPEND ReSolve_SRC ${ReSolve_ROCM_SRC})
+  list(APPEND ReSolve_Targets_List resolve_backend_hip)
+  list(APPEND ReSolve_HEADER_INSTALL ${ReSolve_ROCM_HEADER_INSTALL})
 endif()
 
 # If no GPU support is enabled, link to dummy device backend
 if(NOT RESOLVE_USE_GPU)
   add_subdirectory(cpu)
-  set(ReSolve_Targets_List ${ReSolve_Targets_List} resolve_backend_cpu)
+  list(APPEND ReSolve_Targets_List resolve_backend_cpu)
 endif()
 
 # Set installable targets

resolve/SystemSolver.cpp

@@ -1,9 +1,12 @@
 #include <cassert>
+#include <cmath>
 
 #include <resolve/matrix/Sparse.hpp>
 #include <resolve/vector/Vector.hpp>
 
+#ifdef RESOLVE_USE_KLU
 #include <resolve/LinSolverDirectKLU.hpp>
+#endif
 
 #ifdef RESOLVE_USE_CUDA
 #include <resolve/workspace/LinAlgWorkspaceCUDA.hpp>

resolve/matrix/CMakeLists.txt

@@ -37,26 +37,18 @@ set(Matrix_HEADER_INSTALL
     MatrixHandler.hpp
 )
 
-# Add CUDA matrix handler if CUDA support is enabled
-if(RESOLVE_USE_CUDA)
-  set(Matrix_SRC ${Matrix_SRC} ${Matrix_CUDASDK_SRC})
-endif()
-
-if(RESOLVE_USE_HIP)
-  set(Matrix_SRC ${Matrix_SRC} ${Matrix_ROCM_SRC})
-endif()
-
-
 # Build shared library ReSolve::matrix
 add_library(resolve_matrix SHARED ${Matrix_SRC})
 target_link_libraries(resolve_matrix PRIVATE resolve_logger resolve_vector)
 
 # Link to CUDA ReSolve backend if CUDA is support enabled
 if (RESOLVE_USE_CUDA)
+  target_sources(resolve_matrix PRIVATE ${Matrix_CUDASDK_SRC})
   target_link_libraries(resolve_matrix PUBLIC resolve_backend_cuda)
 endif()
 
 if (RESOLVE_USE_HIP)
+  target_sources(resolve_matrix PRIVATE ${Matrix_ROCM_SRC})
   target_link_libraries(resolve_matrix PUBLIC resolve_backend_hip)
 endif()
 
@@ -65,19 +57,9 @@ if (NOT RESOLVE_USE_GPU)
   target_link_libraries(resolve_matrix PUBLIC resolve_backend_cpu)
 endif()
 
-
 target_include_directories(resolve_matrix INTERFACE
     $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>
     $<INSTALL_INTERFACE:include>
 )
 
-# # TODO: Make this PRIVATE dependency (requires refactoring ReSolve code)
-# target_link_libraries(ReSolve PUBLIC resolve_tpl)
-
-# install(TARGETS ReSolve
-#         EXPORT ReSolveTargets
-#         ARCHIVE DESTINATION lib
-#         LIBRARY DESTINATION lib)
-# install include headers
 install(FILES ${Matrix_HEADER_INSTALL} DESTINATION include/resolve/matrix)
-