Spack package implementation with Ascent rebuild pipeline

.gitignore

@@ -6,5 +6,7 @@ install/*
 .ubuild.json
 *__pycache__*
 *nvblas.conf
-spack-env/
-
+spack-env-*/
+spack-build-*
+spack_install.*
+spack-configure-*

.gitlab-ci.yml

@@ -1,54 +1,97 @@
+stages:
+  - pre-build
+  - build
+
+.rules:
+  rules:
+      # --
+      # Rule to fall back on...
+    - &default
+      when: always
+      # -- 
+      # Only run when the commit **DOES NOT** contains "[ascent-rebuild]" 
+    - &rule_no_ascent_rebuild
+      if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/'
+      when: never 
+    # Only run when the commit **DOES NOT** contains "[ascent-test]"
+    - &rule_no_ascent_test
+      if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/'
+      when: never
+    # Only run when the commit **DOES** contains "[ascent-test]"
+    - &rule_yes_ascent_test
+      if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/'
+      when: always
 
 # Ascent Variables
 .ornl_environment_template:
   variables:
     SCHEDULER_PARAMETERS: "-P CSC359 -nnodes 1 -W 120"
     WORKDIR: /gpfs/wolf/proj-shared/csc359/ci/${CI_PIPELINE_ID}
 
-.SVC-Account-Cleanup:
-  stage: .pre
+Ascent-Build-Test:
+  stage: build
+  rules:
+    - *rule_no_ascent_rebuild
+    - *rule_yes_ascent_test
+    - *default
   tags:
-    - batch 
-  variables:
-    SCHEDULER_PARAMETERS: "-P CSC359 -nnodes 1 -W 120"
+    - batch
+  extends: .ornl_environment_template
   script:
-    - export WORKDIR="/gpfs/wolf/proj-shared/csc359/ci/"
-    # clears directory of files more than 1 hour/60 minutes old
-    - find $WORKDIR -D -type d -mindepth 1 -mmin +60 -prune -print -exec rm -rf {} \; || true
-    - ls -hal $WORKDIR    
-  allow_failure: true
-  resource_group: cleanup
+    - set -xv
+    - mkdir -p "$WORKDIR"
+    - cp -r . "$WORKDIR"
+    - cd "$WORKDIR"
+    - export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install
+    - MY_CLUSTER=ascent ./buildsystem/build.sh 
+    - res=$?
+    - exit $res
+  after_script:
+    - cd "$WORKDIR/.."
+    - rm -rf "$WORKDIR"
 
-.resolve_build_script:
+Ascent-Module-Init:
+  extends: .ornl_environment_template
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+  rules:
+    - if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/'
+      when: always
+  stage: pre-build
+  tags:
+    - nobatch
   script:
-    # Do _not_ clean up WORKDIR as files are needed for testing
-    - |
-      mkdir -p "$WORKDIR"
-      cp -r . "$WORKDIR"
-      cd "$WORKDIR"
-      export srcdir=$WORKDIR builddir=$WORKDIR/build installdir=$WORKDIR/install
-      mkdir build
-      ls
-      ./buildsystem/build.sh 
-      chmod -R +X ./build
-      source buildsystem/ascent-env.sh
-      cd build
-      ctest -vv
-      make install
-      make test_install
-
-cleanup:
- stage: .pre
- extends: .SVC-Account-Cleanup
+    - set -xv
+    - mkdir -p "$WORKDIR"
+    - cp -r . "$WORKDIR"
+    - cd "$WORKDIR"
+    - MY_CLUSTER=ascent ./buildsystem/init-mirror.sh
+    - res=$?
+    - exit $res
 
- # For Ascent CI
-Ascent-Build-Test:
+Ascent-Module-Build:
+  needs: ["Ascent-Module-Init"]
+  rules:
+    - if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/'
+      when: always
+  extends: .ornl_environment_template
   stage: build
   tags:
     - batch
-  extends:
-    - .resolve_build_script
-    - .ornl_environment_template
+  script:
+    - cd "$WORKDIR"
+    - git checkout -b ${CI_COMMIT_REF_NAME}
+    - ./buildsystem/spack/ascent/install.sh
+    - git add ./buildsystem/spack/ascent/modules/*
+    - git config user.name "spack-auto-module"
+    - git config user.email "[email protected]"
+    - git commit -m "Update Ascent spack built tcl modules"
+    - git remote set-url origin https://gitlab-ci-token:${SPACK_GIT_TOKEN}@github.com/ORNL/ReSolve.git
+    - git pull --rebase origin ${CI_COMMIT_REF_NAME}
+    - git push origin ${CI_COMMIT_REF_NAME}
+  after_script:
+    - cd "$WORKDIR/.."
+    - rm -rf "$WORKDIR"
 
 .report-status:
   variables:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "buildsystem/spack/spack"]
+	path = buildsystem/spack/spack
+	url = https://github.com/cameronrutherford/spack.git

CMakeLists.txt

@@ -6,7 +6,7 @@
 
 ]]
 
-cmake_minimum_required(VERSION 3.20)
+cmake_minimum_required(VERSION 3.22)
 
 # Adds version settings and set variable CMAKE_PROJECT_VERSION
 project(ReSolve VERSION "0.1.0")

CMakePresets.json

@@ -1,17 +1,35 @@
 {
-  "version": 3,
-  "cmakeMinimumRequired": {
-    "major": 3,
-    "minor": 22
-  },
-  "configurePresets": [
-    {
-      "name": "cluster",
-      "binaryDir": "${sourceDir}/build",
-      "installDir": "${sourceDir}/install",
-      "generator": "Unix Makefiles",
-      "displayName": "cluster configuration",
-      "description": "This build is used for cluster like deception"
-      }
+    "version": 3,
+    "cmakeMinimumRequired": {
+        "major": 3,
+        "minor": 22,
+        "patch": 0
+    },
+    "configurePresets": [
+        {
+            "name": "cuda",
+            "displayName": "CUDA Build",
+            "description": "Base config to build with CUDA",
+            "binaryDir": "${sourceDir}/build",
+            "installDir": "${sourceDir}/install",
+            "generator": "Unix Makefiles"
+        },
+	{
+	      "name": "ascent",
+	      "inherits": "cuda",
+	      "displayName": "Ascent Build",
+	      "description": "Custom changes specific for Ascent",
+	      "cacheVariables": {
+	            "CMAKE_C_COMPILER": "$env{OLCF_GCC_ROOT}/bin/gcc",
+	            "CMAKE_CXX_COMPILER": "$env{OLCF_GCC_ROOT}/bin/g++"
+	      }
+	},
+	{
+	      "name": "deception",
+	      "inherits": "cuda",
+	      "displayName": "Deception Build",
+	      "description": "Custom changes specific for Deception"
+	}
+
     ]
-}
+}

buildsystem/ascent-env.sh

@@ -1,7 +1,5 @@
+# Load compiler...
 module load gcc/10.2
-module load cuda/11.4.2
-module load cmake/3.22.2
+# Load spack deps
+source ./buildsystem/spack/ascent/modules/dependencies.sh
 
-
-# load module so we can use suitesparse
-module load suite-sparse/5.10.1

buildsystem/build.sh

@@ -8,14 +8,11 @@ fi
 export SRCDIR=${SRCDIR:-$PWD}
 export BUILDDIR=${BUILDDIR:-$PWD/build}
 export INSTALLDIR=${INSTALLDIR:-$PWD/install}
+export MAKEARGS=${MAKEARGS:-"-j"}
+export CTESTARGS=${CTESTARGS:-"--output-on-failure"}
 export BUILD_VERBOSE=0
-
-echo "Paths:"
-echo "Source dir: $SRCDIR"
-echo "Build dir: $BUILDDIR"
-echo "Install dir: $INSTALLDIR"
-echo "Path to buildsystem script: $SRCDIR/buildsystem/build.sh"
-cd $SRCDIR
+export BUILD=${BUILD:-1}
+export TEST=${TEST:-1}
 
 usage() {
   echo "Usage: ./buildsystem/build.sh [options]
@@ -29,13 +26,34 @@ Long Description:
 
 Clusters:
 
-  By default, this script will run on Ascent. 
+  Each cluster has tcl modules that need to be loaded before invoking CMake. These
+  are configured through an environment variable \"MY_CLUSTER\". Current available
+  clusters for configuration are:
+
+    - deception
+    - ascent 
+
+  Run \`export MY_CLUSTER=deception\` or invoke the build script with:
+
+    \`MY_CLUSTER=ascent ./buildsystem/build.sh\`
 
 --------------------------------------------------------------------------------
 
 Options:
 
-  Not Applicable at the moment.
+  --build-only      Only run the build stage of the script. This is useful for
+                    local development.
+
+  --test-only       Only run the test stage of the script. This should be ran
+                    before every push to the repository or pull/merge request.
+                    This run takes a significant amound of time. If you omit
+                    the --*-only options and just run a particular job, tests
+                    will also be ran.
+
+  --verbose        Print all executed commands to the terminal. This is useful 
+                   for debugging, but it will be disabled in CI by default to 
+                   prevent hitting the job log limit.
+
 --------------------------------------------------------------------------------
 
 See Resolve's latest developer guidelines for more information on developing
@@ -46,36 +64,100 @@ Resolve: https://code.ornl.gov/ecpcitest/exasgd/resolve/-/blob/develop/CONTRIBUT
 "
 }
 
+while [[ $# -gt 0 ]]; do
+  case $1 in
+  --build-only)
+    export TEST=0 BUILD=1
+    shift
+    ;;
+  --test-only)
+    export TEST=1 BUILD=0
+    shift
+    ;;
+  --help|-h)
+    usage
+    exit 0
+    ;;
+  --verbose|-v)
+    export BUILD_VERBOSE=1
+    shift
+    ;;
+  *)
+    echo "Argument $1 not recognized."
+    usage
+    exit 1
+    ;;
+  esac
+done
+
 if [ $BUILD_VERBOSE == 1 ]
 then
   # verbose mode: print out all shell functions
   set -xv
-else
-  # don't print out all shell functions 
-  set +x
 fi
 
 module purge
 
-varfile="$SRCDIR/buildsystem/ascent-env.sh"
+varfile="${SRCDIR}/buildsystem/${MY_CLUSTER}-env.sh"
+
+if [[ ! -v MY_CLUSTER ]]
+then
+  echo "MY_CLUSTER" unset && exit 1
+fi
 
 if [[ -f "$varfile" ]]; then
-  source "$varfile"
-  echo Sourced system-specific variables for ascent
+  source $varfile || { echo "Could not source $varfile"; exit 1; }
+else
+  echo "No cluster variable file configured for ${MY_CLUSTER}. Try one of:\n"
+  echo "deception, ascent." && exit 1
 fi
 
+echo "Paths:"
+echo "Source dir: $SRCDIR"
+echo "Build dir: $BUILDDIR"
+echo "Install dir: $INSTALLDIR"
+echo "Path to buildsystem script: $SRCDIR/buildsystem/build.sh"
+
 module list
 
-mkdir -p build
+if [[ $BUILD -eq 1 ]]; then
+  [ -d $BUILDDIR ] && rm -rf $BUILDDIR
+  mkdir -p $BUILDDIR
 
-rm -rf build/*
+  [ -d $INSTALLDIR ] && rm -rf $INSTALLDIR
+  mkdir -p $INSTALLDIR
 
-# Utlizes CMakePresets.json file to set cmake variables such as CMAKE_INSTALL_PREFIX
-# preset cluster = CMAKE_INSTALL_PREFIX = /install
-cmake -B build --preset cluster && \
+  pushd $BUILDDIR
 
-cmake --build build
+  echo
+  echo Configuring
+  echo
+  eval "cmake -S .. --preset ${MY_CLUSTER}" || exit 1
 
-exit $?
+  echo
+  echo Building
+  echo
+  make $MAKEARGS || exit 1
 
+  echo
+  echo Installing
+  echo
+  make install || exit 1
+  popd
+fi
+
+if [[ $TEST -eq 1 ]]; then
+  pushd $BUILDDIR
+  echo
+  echo Testing
+  echo
+  ctest $CTESTARGS || exit 1
+  popd
+fi
+
+if [ $BUILD_VERBOSE == 1 ]
+then
+  set +xv
+fi
 
+exit 0