From f04e01e7a951556f918771d314d5e081b3c6b6ee Mon Sep 17 00:00:00 2001 From: Jaelyn Litzinger Date: Wed, 25 Oct 2023 17:30:54 -0700 Subject: [PATCH] allow rebuild pipeline --- .gitlab/pnnl/base.gitlab-ci.yml | 90 +++++++++++++++++++++++++++++ .gitlab/pnnl/incline.gitlab-ci.yml | 13 ++++- buildsystem/build.sh | 19 +++++- buildsystem/spack/incline/sbatch.sh | 55 ++++++++++++++++++ 4 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 buildsystem/spack/incline/sbatch.sh diff --git a/.gitlab/pnnl/base.gitlab-ci.yml b/.gitlab/pnnl/base.gitlab-ci.yml index 5fbd29d0c..749b6efe5 100644 --- a/.gitlab/pnnl/base.gitlab-ci.yml +++ b/.gitlab/pnnl/base.gitlab-ci.yml @@ -12,6 +12,10 @@ include: - &default when: always # -- + # Only run when the commit **DOES NOT** contains "[incline-rebuild]" + - &rule_no_incline_rebuild + if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/' + when: never # Only run when the commit **DOES NOT** contains "[deception-test]" - &rule_no_deception_test if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/' @@ -77,6 +81,14 @@ stages: paths: - partition +.module_rebuild: + needs: [] + retry: 1 + stage: build + extends: + - .pnnl_tags_template + - .pnnl_module_script_template + .pnnl_after_script_template: after_script: - | @@ -162,6 +174,82 @@ stages: rm "$WORKDIR/jobid_${jobid}" exit $res +.pnnl_module_script_template: + script: + - | + set -x + + # Configure git before making changes + git checkout -b ${CI_COMMIT_REF_NAME} + + export WORKDIR="$HOME/gitlab/${CI_PIPELINE_ID}/${WORKDIR_SUFFIX}" + + mkdir -p "$WORKDIR" + cp -r . "$WORKDIR" + + cd "$WORKDIR" + + # jobid used in pnnl_after_script_template to cancel job if cancelled or + # timed out by gitlab through the UI + jobid=$(sbatch $WORKDIR/buildsystem/spack/$MY_CLUSTER/sbatch.sh) + export jobid=$(echo $jobid | cut -f4 -d ' ') + + # Unique jobid filename for this job + echo $jobid > "$WORKDIR/jobid_${jobid}" + + # This is hard coded into sbatch script + outfile=spack_install.$jobid.output + + [ -f $outfile ] && rm $outfile + touch $outfile + + tail -f $outfile & + tailpid=$! + + res=1 + while :; + do + if [[ "$(awk 'BEGIN{i=0}/BUILD_STATUS/{i++}END{print i}' $outfile)" != "0" ]]; then + kill $tailpid + echo 'Last tail of build $outfile:' + tail -n 200 $outfile + res=$(grep BUILD_STATUS $outfile | tail -n 1 | cut -f2 -d':') + break + fi + sleep 60 + done + + if [ "$res" -ne "0" ]; then + echo "Finished batch job with exit code: $res" + rm "$WORKDIR/jobid_${jobid}" + rm -rf $WORKDIR + set +x + # We want to exit early if the spack build didn't succeed. + exit $res; + fi + + # Modules are in /buildsystem/spack/$MY_CLUSTER/modules + # Assuming MY_CLUSTER is lower case + git add ./buildsystem/spack/${MY_CLUSTER}/modules/* + + # Have to configure username to be able to add commits + git config user.name "spack-auto-module" + git config user.email "spack.bot@no-reply.com" + + # Add commit to MR branch - message can be changed + git commit -m "Update ${MY_CLUSTER} spack built tcl modules - [${MY_CLUSTER}-test]" + + # Re-target GitHub as our remote + git remote set-url origin https://gitlab-ci-token:${SPACK_GIT_TOKEN}@github.com/ORNL/ReSolve.git + + # Do a rebase incase another pipeline has pushed since build started + git pull --rebase origin ${CI_COMMIT_REF_NAME} + git push origin ${CI_COMMIT_REF_NAME} + + rm -rf $WORKDIR + set +x + exit $res + .report-job: retry: 1 image: mrnonz/alpine-git-curl:alpine3.16 @@ -253,6 +341,7 @@ stages: .deception: rules: + - *rule_no_incline_rebuild - *rule_yes_deception_test - *rule_no_incline_test - *default @@ -263,6 +352,7 @@ stages: .incline: rules: + - *rule_no_incline_rebuild - *rule_no_deception_test - *rule_yes_incline_test - *default diff --git a/.gitlab/pnnl/incline.gitlab-ci.yml b/.gitlab/pnnl/incline.gitlab-ci.yml index b66703962..e594fa292 100644 --- a/.gitlab/pnnl/incline.gitlab-ci.yml +++ b/.gitlab/pnnl/incline.gitlab-ci.yml @@ -3,17 +3,26 @@ Incline Build: - .cluster_build - .incline variables: - SCRIPT_ARGS: " --build-only --job=clang-hip " + SCRIPT_ARGS: " --build-only " #--job=clang-hip " Incline Test: extends: - .cluster_test - .incline variables: - SCRIPT_ARGS: " --test-only --job=clang-hip " + SCRIPT_ARGS: " --test-only " #--job=clang-hip " CTESTARGS: " --timeout 240 --output-on-failure -LE incline-skip " needs: ['Incline Build'] +Incline Module Rebuild: + extends: .module_rebuild + rules: + - if: $CI_COMMIT_TITLE =~ /\[incline-rebuild\]/ + when: always + variables: + MY_CLUSTER: "incline" + WORKDIR_SUFFIX: "spack_incline" + pending: variables: MY_CLUSTER: "Incline" diff --git a/buildsystem/build.sh b/buildsystem/build.sh index e6c5e79f2..240d50752 100755 --- a/buildsystem/build.sh +++ b/buildsystem/build.sh @@ -64,6 +64,7 @@ Clusters: - deception - ascent + - incline Run \`export MY_CLUSTER=deception\` or invoke the build script with: @@ -136,6 +137,22 @@ fi module purge +# Correctly identify clusters based on hostname +#case $MY_CLUSTER in +# incline*|dmi*) +# export MY_CLUSTER=incline +# ;; +# dl*|deception|*fat*) +# export MY_CLUSTER=deception +# ;; +# ascent*) +# export MY_CLUSTER=ascent +# ;; +# *) +# echo "Cluster $MY_CLUSTER not identified - you'll have to set relevant variables manually." +# ;; +#esac + varfile="${SRCDIR}/buildsystem/${MY_CLUSTER}-env.sh" if [[ ! -v MY_CLUSTER ]] @@ -147,7 +164,7 @@ if [[ -f "$varfile" ]]; then source $varfile || { echo "Could not source $varfile"; exit 1; } else echo "No cluster variable file configured for ${MY_CLUSTER}. Try one of:\n" - echo "deception, ascent." && exit 1 + echo "deception, incline, ascent." && exit 1 fi echo "Paths:" diff --git a/buildsystem/spack/incline/sbatch.sh b/buildsystem/spack/incline/sbatch.sh new file mode 100644 index 000000000..9ff9df1e2 --- /dev/null +++ b/buildsystem/spack/incline/sbatch.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +#SBATCH -A exasgd +#SBATCH -p incline +#SBATCH -N 1 +#SBATCH --exclusive +#SBATCH -J resolve_spack +#SBATCH -o spack_install.%J.output +#SBATCH -e spack_install.%J.output +#SBTACH -t 240 + +exit() { + # Clear all trap handlers so this isn't echo'ed multiple times, potentially + # throwing off the CI script watching for this output + trap - `seq 1 31` + + # If called without an argument, assume not an error + local ec=${1:-0} + + # Echo the snippet the CI script is looking for + echo BUILD_STATUS:${ec} + + # Actually exit with that code, although it won't matter in most cases, as CI + # is only looking for the string 'BUILD_STATUS:N' + builtin exit ${ec} +} + +# This will be the catch-all trap handler after arguments are parsed. +cleanup() { + # Clear all trap handlers + trap - `seq 1 31` + + # When 'trap' is invoked, each signal handler will be a curried version of + # this function which has the first argument bound to the signal it's catching + local sig=$1 + + echo + echo Exit code $2 caught in build script triggered by signal ${sig}. + echo + + exit $2 +} + +# Assuming that you already have a binary mirror configured (if you need it) +export MY_CLUSTER=incline +#cp /qfs/projects/exasgd/src/coinhsl-archive-2019.05.21.tar.gz . && +. buildsystem/spack/spack/load_spack.sh && +spack develop --no-clone --path=$(pwd) resolve@develop && +# git submodule update --init --recursive && +# cd - && +buildsystem/spack/configure-modules.sh 24 + +EXIT_CODE=$? +# Required to trigger trap handler +exit $EXIT_CODE \ No newline at end of file