Spack module rebuild pipelines for Incline and Deception

.gitlab-ci.yml

@@ -21,19 +21,15 @@ stages:
     - &rule_yes_ascent_test
       if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/'
       when: always
-      # -- 
-      # Only run when the commit **DOES NOT** contains "[deception-rebuild]" 
+    # -- 
+    # Only run when the commit **DOES NOT** contains "[deception-rebuild]"
     - &rule_no_deception_rebuild
       if: '$CI_COMMIT_TITLE =~ /\[deception-rebuild\]/'
-      when: never 
-    # Only run when the commit **DOES NOT** contains "[deception-test]"
-    - &rule_no_deception_test
-      if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/'
       when: never
-    # Only run when the commit **DOES** contains "[deception-test]"
-    - &rule_yes_deception_test
-      if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/'
-      when: always
+      # Only run when the commit **DOES NOT** contains "[incline-rebuild]"
+    - &rule_no_incline_rebuild
+      if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/'
+      when: never
 
 # Ascent Variables
 .ornl_environment_template:
@@ -46,6 +42,10 @@ Ascent-Build-Test:
   rules:
     - *rule_no_ascent_rebuild
     - *rule_yes_ascent_test
+    - *rule_no_deception_rebuild
+    - *rule_no_deception_test
+    - *rule_no_incline_rebuild
+    - *rule_no_incline_test
     - *default
   tags:
     - batch

.gitlab/pnnl/base.gitlab-ci.yml

@@ -12,6 +12,14 @@ include:
     - &default
       when: always
       # -- 
+    # Only run when the commit **DOES NOT** contains "[deception-rebuild]"
+    - &rule_no_deception_rebuild
+      if: '$CI_COMMIT_TITLE =~ /\[deception-rebuild\]/'
+      when: never
+      # Only run when the commit **DOES NOT** contains "[incline-rebuild]"
+    - &rule_no_incline_rebuild
+      if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/'
+      when: never
     # Only run when the commit **DOES NOT** contains "[deception-test]"
     - &rule_no_deception_test
       if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/'
@@ -77,6 +85,14 @@ stages:
     paths:
       - partition
 
+.module_rebuild:
+  needs: []
+  retry: 1
+  stage: build
+  extends:
+    - .pnnl_tags_template
+    - .pnnl_module_script_template
+
 .pnnl_after_script_template:
   after_script:
     - |
@@ -162,6 +178,82 @@ stages:
       rm "$WORKDIR/jobid_${jobid}"
       exit $res
 
+.pnnl_module_script_template:
+  script:
+    - |
+      set -x
+
+      # Configure git before making changes
+      git checkout -b ${CI_COMMIT_REF_NAME}
+
+      export WORKDIR="$HOME/gitlab/${CI_PIPELINE_ID}/${WORKDIR_SUFFIX}"
+
+      mkdir -p "$WORKDIR"
+      cp -r . "$WORKDIR"
+
+      cd "$WORKDIR"
+
+      # jobid used in pnnl_after_script_template to cancel job if cancelled or
+      # timed out by gitlab through the UI
+      jobid=$(sbatch $WORKDIR/buildsystem/spack/$MY_CLUSTER/install.sh)
+      export jobid=$(echo $jobid | cut -f4 -d ' ')
+
+      # Unique jobid filename for this job
+      echo $jobid > "$WORKDIR/jobid_${jobid}"
+
+      # This is hard coded into sbatch script
+      outfile=spack_install.$jobid.output
+
+      [ -f $outfile ] && rm $outfile
+      touch $outfile
+
+      tail -f $outfile &
+      tailpid=$!
+
+      res=1
+      while :;
+      do
+        if [[ "$(awk 'BEGIN{i=0}/BUILD_STATUS/{i++}END{print i}' $outfile)" != "0" ]]; then
+          kill $tailpid
+          echo 'Last tail of build $outfile:'
+          tail -n 200 $outfile
+          res=$(grep BUILD_STATUS $outfile | tail -n 1 | cut -f2 -d':')
+          break
+        fi
+        sleep 60
+      done
+
+      if [ "$res" -ne "0" ]; then
+        echo "Finished batch job with exit code: $res"
+        rm "$WORKDIR/jobid_${jobid}"
+        rm -rf $WORKDIR
+        set +x
+        # We want to exit early if the spack build didn't succeed.
+        exit $res;
+      fi
+
+      # Modules are in /buildsystem/spack/$MY_CLUSTER/modules
+      # Assuming MY_CLUSTER is lower case
+      git add ./buildsystem/spack/${MY_CLUSTER}/modules/*
+
+      # Have to configure username to be able to add commits
+      git config user.name "spack-auto-module"
+      git config user.email "[email protected]"
+
+      # Add commit to MR branch - message can be changed
+      git commit -m "Update ${MY_CLUSTER} spack built tcl modules - [${MY_CLUSTER}-test]"
+
+      # Re-target GitHub as our remote
+      git remote set-url origin https://gitlab-ci-token:${SPACK_GIT_TOKEN}@github.com/ORNL/ReSolve.git 
+
+      # Do a rebase incase another pipeline has pushed since build started
+      git pull --rebase origin ${CI_COMMIT_REF_NAME}
+      git push origin ${CI_COMMIT_REF_NAME}
+
+      rm -rf $WORKDIR
+      set +x
+      exit $res
+
 .report-job:
   retry: 1
   image: mrnonz/alpine-git-curl:alpine3.16
@@ -253,6 +345,8 @@ stages:
 
 .deception:
   rules:
+    - *rule_no_deception_rebuild
+    - *rule_no_incline_rebuild
     - *rule_yes_deception_test
     - *rule_no_incline_test
     - *default
@@ -263,6 +357,8 @@ stages:
 
 .incline:
   rules:
+    - *rule_no_deception_rebuild
+    - *rule_no_incline_rebuild
     - *rule_no_deception_test
     - *rule_yes_incline_test
     - *default

.gitlab/pnnl/deception.gitlab-ci.yml

@@ -14,6 +14,15 @@ Deception Test:
     SCRIPT_ARGS: " --test-only "
   needs: ['Deception Build']
 
+Deception Module Rebuild:
+  extends: .module_rebuild
+  rules:
+    - if: '$CI_COMMIT_TITLE =~ /\[deception-rebuild\]/'
+      when: always
+  variables:
+    MY_CLUSTER: "deception"
+    WORKDIR_SUFFIX: "spack_deception"
+
 pending:
   variables:
     MY_CLUSTER: "Deception"

.gitlab/pnnl/incline.gitlab-ci.yml

@@ -14,6 +14,15 @@ Incline Test:
     CTESTARGS: " --timeout 240 --output-on-failure -LE incline-skip "
   needs: ['Incline Build']
 
+Incline Module Rebuild:
+  extends: .module_rebuild
+  rules:
+    - if: $CI_COMMIT_TITLE =~ /\[incline-rebuild\]/
+      when: always
+  variables:
+    MY_CLUSTER: "incline"
+    WORKDIR_SUFFIX: "spack_incline"
+
 pending:
   variables:
     MY_CLUSTER: "Incline"

buildsystem/configure-modules.sh

@@ -27,10 +27,16 @@ echo Arch for module path being used is $arch && \
 echo module use -a $SPACK_INSTALL/modules/$arch &> $base/modules/dependencies.sh && \
 spack module tcl loads -r -x resolve resolve &>> $base/modules/dependencies.sh
 
+exit_code=$?
+
 # This makes the module and installation location globally readable which isn't ideal.
 # Sticking to this avoids permission issues for other group members, but you
 # should make sure that the install location is in a folder that is only
 # accessible to group members
 # Since we use this in CI and other users will create files we cannot chmod,
 # We need to allow this command to fail
 chmod -R ugo+wrx $SPACK_INSTALL/modules > /dev/null 2>&1
+
+# Should still change permissions before exiting, but should also return the exit
+# code of spack related code
+exit $exit_code

buildsystem/spack/deception/install.sh

@@ -8,7 +8,43 @@
 #SBATCH -e spack_install.%J.output
 #SBTACH -t 240
 
+exit() {
+  # Clear all trap handlers so this isn't echo'ed multiple times, potentially
+  # throwing off the CI script watching for this output
+  trap - `seq 1 31`
+
+  # If called without an argument, assume not an error
+  local ec=${1:-0}
+
+  # Echo the snippet the CI script is looking for
+  echo BUILD_STATUS:${ec}
+
+  # Actually exit with that code, although it won't matter in most cases, as CI
+  # is only looking for the string 'BUILD_STATUS:N'
+  builtin exit ${ec}
+}
+
+# This will be the catch-all trap handler after arguments are parsed.
+cleanup() {
+  # Clear all trap handlers
+  trap - `seq 1 31`
+
+  # When 'trap' is invoked, each signal handler will be a curried version of
+  # this function which has the first argument bound to the signal it's catching
+  local sig=$1
+
+  echo
+  echo Exit code $2 caught in build script triggered by signal ${sig}.
+  echo
+
+  exit $2
+}
+
 export MY_CLUSTER=deception
 . buildsystem/load-spack.sh &&
 spack develop --no-clone --path=$(pwd) resolve@develop &&
 ./buildsystem/configure-modules.sh 64
+
+EXIT_CODE=$?
+# Required to trigger trap handler
+exit $EXIT_CODE

buildsystem/spack/deception/modules/dependencies.sh

@@ -53,7 +53,7 @@ module load texinfo/7.0.3-gcc-9.1.0-oygvjgv
 module load mpfr/4.2.0-gcc-9.1.0-3jeasnm
 # openblas@=0.3.23%gcc@=9.1.0~bignuma~consistent_fpcsr+fortran~ilp64+locking+pic+shared build_system=makefile symbol_suffix=none threads=none arch=linux-centos7-zen2
 module load openblas/0.3.23-gcc-9.1.0-4zbtr2v
-# suite-sparse@=5.13.0%gcc@=9.1.0+cuda~graphblas~openmp+pic build_system=generic arch=linux-centos7-zen2
-module load suite-sparse/5.13.0-gcc-9.1.0-pszalbf
-# resolve@=develop%gcc@=9.1.0+cuda~ipo+klu build_system=cmake build_type=Release cuda_arch=60,70,75,80 dev_path=/people/ruth521/projects/resolve generator=make arch=linux-centos7-zen2
-## module load resolve/develop-gcc-9.1.0-x6mhz5t
+# suite-sparse@=5.13.0%gcc@=9.1.0~cuda~graphblas~openmp+pic build_system=generic arch=linux-centos7-zen2
+module load suite-sparse/5.13.0-gcc-9.1.0-ajdnyum
+# resolve@=develop%gcc@=9.1.0+cuda~ipo+klu build_system=cmake build_type=Release cuda_arch=60,70,75,80 dev_path=/people/svcexasgd/gitlab/24142/spack_deception generator=make arch=linux-centos7-zen2
+## module load resolve/develop-gcc-9.1.0-5de222u

buildsystem/spack/incline/install.sh

@@ -9,8 +9,44 @@
 #SBATCH -e spack_install.%J.output
 #SBTACH -t 240
 
+exit() {
+  # Clear all trap handlers so this isn't echo'ed multiple times, potentially
+  # throwing off the CI script watching for this output
+  trap - `seq 1 31`
+
+  # If called without an argument, assume not an error
+  local ec=${1:-0}
+
+  # Echo the snippet the CI script is looking for
+  echo BUILD_STATUS:${ec}
+
+  # Actually exit with that code, although it won't matter in most cases, as CI
+  # is only looking for the string 'BUILD_STATUS:N'
+  builtin exit ${ec}
+}
+
+# This will be the catch-all trap handler after arguments are parsed.
+cleanup() {
+  # Clear all trap handlers
+  trap - `seq 1 31`
+
+  # When 'trap' is invoked, each signal handler will be a curried version of
+  # this function which has the first argument bound to the signal it's catching
+  local sig=$1
+
+  echo
+  echo Exit code $2 caught in build script triggered by signal ${sig}.
+  echo
+
+  exit $2
+}
+
 export MY_CLUSTER=incline
 . buildsystem/load-spack.sh &&
 spack develop --no-clone --path=$(pwd) resolve@develop &&
 ./buildsystem/configure-modules.sh 64
 
+
+EXIT_CODE=$?
+# Required to trigger trap handler
+exit $EXIT_CODE

buildsystem/spack/incline/modules/dependencies.sh

@@ -17,8 +17,8 @@ module load curl/8.4.0-gcc-8.4.0-g2rrs23
 module load ncurses/6.4-gcc-8.4.0-jt7rpqq
 # cmake@=3.27.7%gcc@=8.4.0~doc+ncurses+ownlibs build_system=generic build_type=Release arch=linux-centos7-zen
 module load cmake/3.27.7-gcc-8.4.0-tu2rruq
-# gmake@=4.4.1%gcc@=8.4.0~guile build_system=autotools arch=linux-centos7-zen
-module load gmake/4.4.1-gcc-8.4.0-f23wik2
+# gmake@=4.4.1%gcc@=8.4.0~guile build_system=generic arch=linux-centos7-zen
+module load gmake/4.4.1-gcc-8.4.0-l7nyr34
 # libiconv@=1.17%gcc@=8.4.0 build_system=autotools libs=shared,static arch=linux-centos7-zen
 module load libiconv/1.17-gcc-8.4.0-wfdnlg6
 # diffutils@=3.9%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
@@ -35,6 +35,8 @@ module load automake/1.16.5-gcc-8.4.0-tgloywk
 module load libtool/2.4.7-gcc-8.4.0-gs6gyy3
 # gmp@=6.2.1%gcc@=8.4.0+cxx build_system=autotools libs=shared,static patches=69ad2e2 arch=linux-centos7-zen
 module load gmp/6.2.1-gcc-8.4.0-ythx4o2
+# gmake@=4.4.1%gcc@=8.4.0~guile build_system=autotools arch=linux-centos7-zen
+module load gmake/4.4.1-gcc-8.4.0-f23wik2
 # metis@=5.1.0%gcc@=8.4.0~gdb~int64~ipo~real64+shared build_system=cmake build_type=Release generator=make patches=4991da9,93a7903,b1225da arch=linux-centos7-zen
 module load metis/5.1.0-gcc-8.4.0-gsllf6a
 # autoconf-archive@=2023.02.20%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
@@ -61,5 +63,5 @@ module load mpfr/4.2.0-gcc-8.4.0-cjhi2el
 module load openblas/0.3.24-gcc-8.4.0-4ei4hpg
 # suite-sparse@=5.13.0%gcc@=8.4.0~cuda~graphblas~openmp+pic build_system=generic arch=linux-centos7-zen
 module load suite-sparse/5.13.0-gcc-8.4.0-ivey23b
-# resolve@=develop%gcc@=8.4.0~cuda~ipo+klu build_system=cmake build_type=Release dev_path=/people/ruth521/projects/resolve generator=make arch=linux-centos7-zen
-## module load resolve/develop-gcc-8.4.0-ugoj3p3
+# resolve@=develop%gcc@=8.4.0~cuda~ipo+klu build_system=cmake build_type=Release dev_path=/people/svcexasgd/gitlab/24143/spack_incline generator=make arch=linux-centos7-zen
+## module load resolve/develop-gcc-8.4.0-l7tspub