allow rebuild pipeline

.gitlab/pnnl/base.gitlab-ci.yml

@@ -12,6 +12,10 @@ include:
     - &default
       when: always
       # -- 
+    # Only run when the commit **DOES NOT** contains "[incline-rebuild]"
+    - &rule_no_incline_rebuild
+      if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/'
+      when: never
     # Only run when the commit **DOES NOT** contains "[deception-test]"
     - &rule_no_deception_test
       if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/'
@@ -77,6 +81,14 @@ stages:
     paths:
       - partition
 
+.module_rebuild:
+  needs: []
+  retry: 1
+  stage: build
+  extends:
+    - .pnnl_tags_template
+    - .pnnl_module_script_template
+
 .pnnl_after_script_template:
   after_script:
     - |
@@ -162,6 +174,82 @@ stages:
       rm "$WORKDIR/jobid_${jobid}"
       exit $res
 
+.pnnl_module_script_template:
+  script:
+    - |
+      set -x
+
+      # Configure git before making changes
+      git checkout -b ${CI_COMMIT_REF_NAME}
+
+      export WORKDIR="$HOME/gitlab/${CI_PIPELINE_ID}/${WORKDIR_SUFFIX}"
+
+      mkdir -p "$WORKDIR"
+      cp -r . "$WORKDIR"
+
+      cd "$WORKDIR"
+
+      # jobid used in pnnl_after_script_template to cancel job if cancelled or
+      # timed out by gitlab through the UI
+      jobid=$(sbatch $WORKDIR/buildsystem/spack/$MY_CLUSTER/sbatch.sh)
+      export jobid=$(echo $jobid | cut -f4 -d ' ')
+
+      # Unique jobid filename for this job
+      echo $jobid > "$WORKDIR/jobid_${jobid}"
+
+      # This is hard coded into sbatch script
+      outfile=spack_install.$jobid.output
+
+      [ -f $outfile ] && rm $outfile
+      touch $outfile
+
+      tail -f $outfile &
+      tailpid=$!
+ 
+      res=1
+      while :;
+      do
+        if [[ "$(awk 'BEGIN{i=0}/BUILD_STATUS/{i++}END{print i}' $outfile)" != "0" ]]; then
+          kill $tailpid
+          echo 'Last tail of build $outfile:'
+          tail -n 200 $outfile
+          res=$(grep BUILD_STATUS $outfile | tail -n 1 | cut -f2 -d':')
+          break
+        fi
+        sleep 60
+      done
+
+      if [ "$res" -ne "0" ]; then
+        echo "Finished batch job with exit code: $res"
+        rm "$WORKDIR/jobid_${jobid}"
+        rm -rf $WORKDIR
+        set +x
+        # We want to exit early if the spack build didn't succeed.
+        exit $res;
+      fi
+
+      # Modules are in /buildsystem/spack/$MY_CLUSTER/modules
+      # Assuming MY_CLUSTER is lower case
+      git add ./buildsystem/spack/${MY_CLUSTER}/modules/*
+
+      # Have to configure username to be able to add commits
+      git config user.name "spack-auto-module"
+      git config user.email "[email protected]"
+
+      # Add commit to MR branch - message can be changed
+      git commit -m "Update ${MY_CLUSTER} spack built tcl modules - [${MY_CLUSTER}-test]"
+
+      # Re-target GitHub as our remote
+      git remote set-url origin https://gitlab-ci-token:${SPACK_GIT_TOKEN}@github.com/ORNL/ReSolve.git
+
+      # Do a rebase incase another pipeline has pushed since build started
+      git pull --rebase origin ${CI_COMMIT_REF_NAME}
+      git push origin ${CI_COMMIT_REF_NAME}
+
+      rm -rf $WORKDIR
+      set +x
+      exit $res
+
 .report-job:
   retry: 1
   image: mrnonz/alpine-git-curl:alpine3.16
@@ -253,6 +341,7 @@ stages:
 
 .deception:
   rules:
+    - *rule_no_incline_rebuild
     - *rule_yes_deception_test
     - *rule_no_incline_test
     - *default
@@ -263,6 +352,7 @@ stages:
 
 .incline:
   rules:
+    - *rule_no_incline_rebuild
     - *rule_no_deception_test
     - *rule_yes_incline_test
     - *default

.gitlab/pnnl/incline.gitlab-ci.yml

@@ -3,17 +3,26 @@ Incline Build:
     - .cluster_build
     - .incline
   variables:
-    SCRIPT_ARGS: " --build-only --job=clang-hip "
+    SCRIPT_ARGS: " --build-only " #--job=clang-hip "
 
 Incline Test:
   extends:
     - .cluster_test
     - .incline
   variables:
-    SCRIPT_ARGS: " --test-only --job=clang-hip "
+    SCRIPT_ARGS: " --test-only " #--job=clang-hip "
     CTESTARGS: " --timeout 240 --output-on-failure -LE incline-skip "
   needs: ['Incline Build']
 
+Incline Module Rebuild:
+  extends: .module_rebuild
+  rules:
+    - if: $CI_COMMIT_TITLE =~ /\[incline-rebuild\]/
+      when: always
+  variables:
+    MY_CLUSTER: "incline"
+    WORKDIR_SUFFIX: "spack_incline"
+
 pending:
   variables:
     MY_CLUSTER: "Incline"

buildsystem/build.sh

@@ -64,6 +64,7 @@ Clusters:
 
     - deception
     - ascent 
+    - incline
 
   Run \`export MY_CLUSTER=deception\` or invoke the build script with:
 
@@ -136,6 +137,22 @@ fi
 
 module purge
 
+# Correctly identify clusters based on hostname
+#case $MY_CLUSTER in
+#  incline*|dmi*)
+#    export MY_CLUSTER=incline
+#    ;;
+#  dl*|deception|*fat*)
+#    export MY_CLUSTER=deception
+#    ;;
+#  ascent*)
+#    export MY_CLUSTER=ascent
+#    ;;
+#  *)
+#    echo "Cluster $MY_CLUSTER not identified - you'll have to set relevant variables manually."
+#    ;;
+#esac
+
 varfile="${SRCDIR}/buildsystem/${MY_CLUSTER}-env.sh"
 
 if [[ ! -v MY_CLUSTER ]]
@@ -147,7 +164,7 @@ if [[ -f "$varfile" ]]; then
   source $varfile || { echo "Could not source $varfile"; exit 1; }
 else
   echo "No cluster variable file configured for ${MY_CLUSTER}. Try one of:\n"
-  echo "deception, ascent." && exit 1
+  echo "deception, incline, ascent." && exit 1
 fi
 
 echo "Paths:"

buildsystem/spack/incline/sbatch.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+#SBATCH -A exasgd
+#SBATCH -p incline
+#SBATCH -N 1
+#SBATCH --exclusive
+#SBATCH -J resolve_spack
+#SBATCH -o spack_install.%J.output
+#SBATCH -e spack_install.%J.output
+#SBTACH -t 240
+
+exit() {
+  # Clear all trap handlers so this isn't echo'ed multiple times, potentially
+  # throwing off the CI script watching for this output
+  trap - `seq 1 31`
+
+  # If called without an argument, assume not an error
+  local ec=${1:-0}
+
+  # Echo the snippet the CI script is looking for
+  echo BUILD_STATUS:${ec}
+
+  # Actually exit with that code, although it won't matter in most cases, as CI
+  # is only looking for the string 'BUILD_STATUS:N'
+  builtin exit ${ec}
+}
+
+# This will be the catch-all trap handler after arguments are parsed.
+cleanup() {
+  # Clear all trap handlers
+  trap - `seq 1 31`
+
+  # When 'trap' is invoked, each signal handler will be a curried version of
+  # this function which has the first argument bound to the signal it's catching
+  local sig=$1
+
+  echo
+  echo Exit code $2 caught in build script triggered by signal ${sig}.
+  echo
+
+  exit $2
+}
+
+# Assuming that you already have a binary mirror configured (if you need it)
+export MY_CLUSTER=incline
+#cp /qfs/projects/exasgd/src/coinhsl-archive-2019.05.21.tar.gz . &&
+. buildsystem/spack/spack/load_spack.sh &&
+spack develop --no-clone --path=$(pwd) resolve@develop &&
+# git submodule update --init --recursive &&
+# cd - &&
+buildsystem/spack/configure-modules.sh 24
+
+EXIT_CODE=$?
+# Required to trigger trap handler
+exit $EXIT_CODE