From 15a8f5ec489d56d1892c00ba8400947a37ffce7a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 3 Dec 2024 21:05:15 +0000 Subject: [PATCH 01/24] Initial commit --- ci/driver_new.sh | 210 +++++++++++++++++++++++++++++++++++++++++++++++ ci/hera.sh | 1 + ci/orion.sh | 1 + 3 files changed, 212 insertions(+) create mode 100755 ci/driver_new.sh diff --git a/ci/driver_new.sh b/ci/driver_new.sh new file mode 100755 index 000000000..19c08913f --- /dev/null +++ b/ci/driver_new.sh @@ -0,0 +1,210 @@ +#!/bin/bash --login + +echo "Starting automated testing at $(date)" + +my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" +echo "Set my_dir ${my_dir}" + +# ============================================================================== +usage() { + set +x + echo + echo "Usage: $0 -t -h" + echo + echo " -t target/machine script is running on DEFAULT: $(hostname)" + echo " -h display this message and quit" + echo " -w run workflow tests on $(hostname)" + echo + exit 1 +} + +# ============================================================================== +# First, set up runtime environment + +export TARGET="$(hostname)" + +TEST_WORKFLOW=0 +while getopts "t:h:w" opt; do + case $opt in + t) + TARGET=$OPTARG + ;; + h|\?|:) + usage + ;; + w) + TEST_WORKFLOW=1 + ;; + esac +done + +echo "Running automated testing on $TARGET" + +case ${TARGET} in + hera | orion) + source $MODULESHOME/init/sh + source $my_dir/${TARGET}.sh + module purge + module use $GDAS_MODULE_USE + module load GDAS/$TARGET + module list + ;; + *) + echo "Unsupported platform. Exiting with error." + exit 1 + ;; +esac + +# ============================================================================== +# set things that depend on whether running workflow tests or not +gdasapp_url="https://github.com/NOAA-EMC/GDASApp.git" +if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Testing GDASApp inside the Global Workflow" + + CI_LABEL="${GDAS_CI_HOST}-GW-RT" + OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list_gw + PR_TEST_DIR=$GDAS_CI_ROOT/workflow/PR + RUN_CI_SCRIPT=run_gw_ci.sh + BASE_REPO=global-workflow + + # Default Global Workflow repo and branch if no companion PR found + workflow_url="https://github.com/NOAA-EMC/global-workflow.git" + workflow_branch="develop" +else + echo "Testing stand-alone GDASApp" + + CI_LABEL="${GDAS_CI_HOST}-RT" + OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list + PR_TEST_DIR=$GDAS_CI_ROOT/PR + RUN_CI_SCRIPT=run_ci.sh + BASE_REPO=GDASApp +fi + +# ============================================================================== +# pull on the repo and get list of open PRs + +cd $GDAS_CI_ROOT/repo + +gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $OPEN_PR_LIST_DIR + +open_pr=`cat $OPEN_PR_LIST_DIR | wc -l` +if (( $open_pr == 0 )); then + echo "No open PRs with ${CI_LABEL}, exit." + echo "Finished automated testing at $(date)" + exit +fi + +open_pr_list=$(cat $OPEN_PR_LIST_DIR) + +# ============================================================================== +# clone, checkout, build, test, etc. +# loop through all open PRs +for pr in $open_pr_list; do + echo " " + echo "Starting processing of pull request #${pr} at $(date)" + + # get the branch name used for the PR + gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") + + # get additional branch informatio + branch_owner=$(gh pr view $pr --repo ${gdasapp_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $pr --repo ${gdasapp_url} --json headRepository --jq '.headRepository.name') + pr_assignees=$(gh pr view $pr --repo ${gdasapp_url} --json assignees --jq '.assignees[].login') + + # check if any assignee is authorized to run CI + rc=1 + for str in ${pr_assignees[@]}; do + grep $str $AUTHORIZED_USERS_FILE > /dev/null + if (( rc != 0 )); then + rc=$? + fi + if (( rc == 0 )); then + echo "Authorized user $str assigned to this PR" + fi + done + + # Authorized to run CI + if (( rc == 0 )); then + echo "CI authorized. Running CI..." + + # update PR label + gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running + + echo "GDASApp URL: $gdasapp_url" + echo "GDASApp branch Name: $gdasapp_branch" + + if [[ $TEST_WORKFLOW == 1 ]]; then + # check for a companion PR in the global-workflow + companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) + + if [ -n "$companion_pr_exists" ]; then + # get the PR number + companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') + + # extract the necessary info + branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') + + # Construct fork URL. Update workflow branch name + workflow_url="https://github.com/$branch_owner/$branch_name.git" + workflow_branch=$gdasapp_branch + fi + + echo "Found companion Global Workflow PR #${companion_pr}!" + echo "Global Workflow URL: $workflow_url" + echo "Global Workflow branch name: $workflow_branch" + fi + + # create PR specific directory + if [ -d $PR_TEST_DIR/$pr ]; then + rm -rf $PR_TEST_DIR/$pr + fi + mkdir -p $PR_TEST_DIR/$pr + cd $PR_TEST_DIR/$pr + pwd + + # clone copy of repo + if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Cloning Global Workflow branch $workflow_branch from $workflow_url at $(date)" + git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url + cd global-workflow/sorc/gdas.cd + else + echo "Cloning GDASApp branch $workflow_branch at $(date)" + git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url + cd GDASApp + fi + pwd + + # checkout GDASApp pull request + gh pr checkout $pr + git submodule update --init --recursive + + # get commit hash + commit=$(git log --pretty=format:'%h' -n 1) + echo "$commit" > $PR_TEST_DIR/$pr/commit + + # run build and testing command + echo "Running $RUN_CI_SCRIPT for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)" + $my_dir/$RUN_CI_SCRIPT -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit} + ci_status=$? + echo "Finished running $RUN_CI_SCRIPT with ci_status ${ci_status} at $(date)" + + gh pr comment $pr --repo ${gdasapp_url} --body-file $PR_TEST_DIR/$pr/output_${commit} + if [ $ci_status -eq 0 ]; then + gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + else + gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + fi + + # Not authorized to run CI + else + echo "No authorized users assigned to this PR. Aborting CI..." + fi + + echo "Finished processing Pull Request #{pr} at $(date)" +done + +# ============================================================================== +# scrub working directory for older files +find $PR_TEST_DIR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; +echo "Finished automated testing at $(date)" diff --git a/ci/hera.sh b/ci/hera.sh index 8a9518f8b..c9c5f892c 100644 --- a/ci/hera.sh +++ b/ci/hera.sh @@ -7,3 +7,4 @@ export SBATCH_ACCOUNT=$SLURM_ACCOUNT export SLURM_QOS=debug export PATH=$PATH:/home/role.jedipara/bin export NTASKS_TESTS=12 +export AUTHORIZED_USERS_FILE=/scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users diff --git a/ci/orion.sh b/ci/orion.sh index 8c2cf9a5f..424e31b43 100644 --- a/ci/orion.sh +++ b/ci/orion.sh @@ -9,3 +9,4 @@ export SLURM_EXCLUSIVE=user export OMP_NUM_THREADS=1 ulimit -s unlimited export NTASKS_TESTS=12 +export AUTHORIZED_USERS_FILE='foo' From 97fd67746e1211b45f37baaee626f5894e01c456 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 4 Dec 2024 14:55:05 +0000 Subject: [PATCH 02/24] Consolidate GW and standalone GDASApp CI scripts --- ci/driver.sh | 176 +++++++++++++++++++++++++++++++++++------------- ci/gw_driver.sh | 169 ---------------------------------------------- ci/run_ci.sh | 53 ++++++++++++--- ci/run_gw_ci.sh | 80 ---------------------- 4 files changed, 175 insertions(+), 303 deletions(-) delete mode 100755 ci/gw_driver.sh delete mode 100755 ci/run_gw_ci.sh diff --git a/ci/driver.sh b/ci/driver.sh index 933b9223b..c7c707389 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -1,6 +1,6 @@ #!/bin/bash --login -echo "Start at $(date)" +echo "Starting automated testing at $(date)" my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" echo "Set my_dir ${my_dir}" @@ -13,6 +13,7 @@ usage() { echo echo " -t target/machine script is running on DEFAULT: $(hostname)" echo " -h display this message and quit" + echo " -w run workflow tests on $(hostname)" echo exit 1 } @@ -22,7 +23,8 @@ usage() { export TARGET="$(hostname)" -while getopts "t:h" opt; do +TEST_WORKFLOW=0 +while getopts "t:h:w" opt; do case $opt in t) TARGET=$OPTARG @@ -30,12 +32,16 @@ while getopts "t:h" opt; do h|\?|:) usage ;; + w) + TEST_WORKFLOW=1 + ;; esac done +echo "Running automated testing on $TARGET" + case ${TARGET} in hera | orion) - echo "Running Automated Testing on $TARGET" source $MODULESHOME/init/sh source $my_dir/${TARGET}.sh module purge @@ -49,106 +55,184 @@ case ${TARGET} in ;; esac +CI_TESTS=("C96C48_hybatmDA" + "C96C48_ufs_hybatmDA" + "C96C48_hybatmaerosnowDA" + "C48mx500_3DVarAOWCDA" + "C48mx500_hybAOWCDA" + "C384mx025_3DVarAOWCDA") + +# ============================================================================== +# set things that depend on whether running workflow tests or not +gdasapp_url="https://github.com/NOAA-EMC/GDASApp.git" +if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Testing GDASApp inside the Global Workflow" + + CI_LABEL="${GDAS_CI_HOST}-GW-RT" + OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list_gw + PR_TEST_DIR=$GDAS_CI_ROOT/workflow/PR + BASE_REPO=global-workflow + + # Default Global Workflow repo and branch if no companion PR found + workflow_url="https://github.com/NOAA-EMC/global-workflow.git" + workflow_branch="develop" +else + echo "Testing stand-alone GDASApp" + + CI_LABEL="${GDAS_CI_HOST}-RT" + OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list + PR_TEST_DIR=$GDAS_CI_ROOT/PR + BASE_REPO=GDASApp +fi + +# Test +CI_LABEL="hera-driver-test" + # ============================================================================== # pull on the repo and get list of open PRs + cd $GDAS_CI_ROOT/repo -CI_LABEL="${GDAS_CI_HOST}-RT" -gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list -open_pr=`cat $GDAS_CI_ROOT/open_pr_list | wc -l` +gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $OPEN_PR_LIST_DIR + +open_pr=`cat $OPEN_PR_LIST_DIR | wc -l` if (( $open_pr == 0 )); then echo "No open PRs with ${CI_LABEL}, exit." - echo "Finish at $(date)" + echo "Finished automated testing at $(date)" exit fi -open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) +open_pr_list=$(cat $OPEN_PR_LIST_DIR) # ============================================================================== # clone, checkout, build, test, etc. -repo_url="https://github.com/NOAA-EMC/GDASApp.git" # loop through all open PRs for pr in $open_pr_list; do echo " " - echo "Start processing Pull Request #${pr} at $(date)" + echo "Starting processing of pull request #${pr} at $(date)" # get the branch name used for the PR gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - # get additional branch information - branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') - pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') - + # get additional branch informatio + branch_owner=$(gh pr view $pr --repo ${gdasapp_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $pr --repo ${gdasapp_url} --json headRepository --jq '.headRepository.name') + pr_assignees=$(gh pr view $pr --repo ${gdasapp_url} --json assignees --jq '.assignees[].login') + # check if any assignee is authorized to run CI - authorized_by="" + rc=1 for str in ${pr_assignees[@]}; do - grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users - rc=$? + grep $str $AUTHORIZED_USERS_FILE > /dev/null + if (( rc != 0 )); then + rc=$? + fi if (( rc == 0 )); then - authorized_by=${str} - echo "FOUND MATCH $str, rc $rc" - break + echo "Authorized user $str assigned to this PR" fi done # Authorized to run CI if (( rc == 0 )); then - echo "Run CI" + echo "CI authorized. Running CI..." # update PR label gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - - # construct the fork URL - gdasapp_url="https://github.com/$branch_owner/${branch_name}.git" - + echo "GDASApp URL: $gdasapp_url" echo "GDASApp branch Name: $gdasapp_branch" - echo "CI authorized by $authorized_by at $(date)" + + if [[ $TEST_WORKFLOW == 1 ]]; then + # check for a companion PR in the global-workflow + companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) + + if [ -n "$companion_pr_exists" ]; then + # get the PR number + companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') + + # extract the necessary info + branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') + + # Construct fork URL. Update workflow branch name + workflow_url="https://github.com/$branch_owner/$branch_name.git" + workflow_branch=$gdasapp_branch + fi + + echo "Found companion Global Workflow PR #${companion_pr}!" + echo "Global Workflow URL: $workflow_url" + echo "Global Workflow branch name: $workflow_branch" + fi - # create PR specific directory - if [ -d $GDAS_CI_ROOT/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/PR/$pr + # create PR specific directory + if [ -d $PR_TEST_DIR/$pr ]; then + rm -rf $PR_TEST_DIR/$pr fi - mkdir -p $GDAS_CI_ROOT/PR/$pr - cd $GDAS_CI_ROOT/PR/$pr + mkdir -p $PR_TEST_DIR/$pr + cd $PR_TEST_DIR/$pr pwd # clone copy of repo - git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url - cd GDASApp + if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Cloning Global Workflow branch $workflow_branch from $workflow_url at $(date)" + git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url + cd global-workflow/sorc/gdas.cd + else + echo "Cloning GDASApp branch $workflow_branch at $(date)" + git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url + cd GDASApp + fi pwd # checkout GDASApp pull request - git pull gh pr checkout $pr git submodule update --init --recursive # get commit hash commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit + echo "$commit" > $PR_TEST_DIR/$pr/commit + + # get ci tests from PR description and convert into a regular expressions to included and exclude + branch_body=$(gh pr view $pr --repo ${gdasapp_url} --json body --jq '.body') + ci_checklist=$(echo "$branch_body" | grep '\[x\]') + ci_include_regex="gdasapp" + ci_exclude_regex="" + for ci_test in ${CI_TESTS[@]}; do + if echo "$ci_checklist" | grep -q "$ci_test"; then + ci_include_regex+="${ci_include_regex:+|}$ci_test" + else + ci_exclude_regex+="${ci_exclude_regex:+|}$ci_test" + fi + done # run build and testing command - echo "Execute $my_dir/run_ci.sh for $GDAS_CI_ROOT/PR/$pr/GDASApp at $(date)" - $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} + echo "Running run_ci.sh for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)" + run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit} -R $ci_include_regex" + if [ -n "$ci_exclude_regex" ]; then + run_ci_cmd+=" -E $ci_exclude_regex" + fi + if [[ TEST_WORKFLOW == 1 ]]; then + run_ci_cmd+=" -w" + fi + $run_ci_cmd ci_status=$? - echo "After run_ci.sh with ci_status ${ci_status} at $(date)" - gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} + echo "Finished running run_ci.sh with ci_status ${ci_status} at $(date)" + + gh pr comment $pr --repo ${gdasapp_url} --body-file $PR_TEST_DIR/$pr/output_${commit} if [ $ci_status -eq 0 ]; then - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed else - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed fi # Not authorized to run CI else - echo "Do NOT run CI" + echo "No authorized users assigned to this PR. Aborting CI..." fi - echo "Finish processing Pull Request #{pr} at $(date)" + echo "Finished processing Pull Request #{pr} at $(date)" done # ============================================================================== # scrub working directory for older files -find $GDAS_CI_ROOT/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; -echo "Finish at $(date)" +find $PR_TEST_DIR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; +echo "Finished automated testing at $(date)" diff --git a/ci/gw_driver.sh b/ci/gw_driver.sh deleted file mode 100755 index c40ff4026..000000000 --- a/ci/gw_driver.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash --login - -echo "Start at $(date)" - -my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" -echo "Set my_dir ${my_dir}" - -# ============================================================================== -usage() { - set +x - echo - echo "Usage: $0 -t -h" - echo - echo " -t target/machine script is running on DEFAULT: $(hostname)" - echo " -h display this message and quit" - echo - exit 1 -} - -# ============================================================================== -# First, set up runtime environment - -export TARGET="$(hostname)" - -while getopts "t:h" opt; do - case $opt in - t) - TARGET=$OPTARG - ;; - h|\?|:) - usage - ;; - esac -done - -case ${TARGET} in - hera | orion) - echo "Running Automated GW Testing on $TARGET" - source $MODULESHOME/init/sh - source $my_dir/${TARGET}.sh - module purge - module use $GDAS_MODULE_USE - module load GDAS/$TARGET - module list - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac - -# ============================================================================== -# pull on the repo and get list of open PRs -cd $GDAS_CI_ROOT/repo -CI_LABEL="${GDAS_CI_HOST}-GW-RT" -gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list_gw - -open_pr=`cat $GDAS_CI_ROOT/open_pr_list_gw | wc -l` -if (( $open_pr == 0 )); then - echo "No open PRs with ${CI_LABEL}, exit." - echo "Finish at $(date)" - exit -fi - -open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list_gw) - -# ============================================================================== -# clone, checkout, build, test, etc. -repo_url="https://github.com/NOAA-EMC/GDASApp.git" -workflow_url="https://github.com/NOAA-EMC/global-workflow.git" -workflow_branch="develop" -# loop through all open PRs -for pr in $open_pr_list; do - echo " " - echo "Start processing Pull Request #${pr} at $(date)" - - # get the branch name used for the PR - gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - - # get additional branch information - branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') - pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') - - # check if any assignee is authorized to run CI - authorized_by="" - for str in ${pr_assignees[@]}; do - grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users - rc=$? - if (( rc == 0 )); then - authorized_by=${str} - echo "FOUND MATCH $str, rc $rc" - break - fi - done - - # Authorized to run CI - if (( rc == 0 )); then - echo "Run CI" - - # update PR label - gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - - # check for a companion PR in the global-workflow - companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) - if [ -n "$companion_pr_exists" ]; then - # get the PR number - companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') - - # extract the necessary info - branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') - - # Construct fork URL. Update workflow branch name - workflow_url="https://github.com/$branch_owner/$branch_name.git" - workflow_branch=$gdasapp_branch - - fi - - echo "Workflow URL: $workflow_url" - echo "Workflow branch name: $workflow_branch" - echo "GDASApp branch name: $gdasapp_branch" - echo "CI authorized by $authorized_by at $(date)" - - # create PR specific directory - if [ -d $GDAS_CI_ROOT/workflow/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/workflow/PR/$pr - fi - mkdir -p $GDAS_CI_ROOT/workflow/PR/$pr - cd $GDAS_CI_ROOT/workflow/PR/$pr - pwd - - # clone global workflow develop branch - git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url - - # checkout GDASApp pull request - cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd - git pull - gh pr checkout $pr - git submodule update --init --recursive - - # get commit hash - commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/workflow/PR/$pr/commit - - # run build and testing command - echo "Execute $my_dir/run_gw_ci.sh for $GDAS_CI_ROOT/PR/workflow/PR/$pr/global-workflow at $(date)" - $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - ci_status=$? - echo "After run_gw_ci.sh with ci_status ${ci_status} at $(date)" - gh pr comment $pr --body-file $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - if [ $ci_status -eq 0 ]; then - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed - else - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed - fi - - # Not authorized to run CI - else - echo "Do NOT run CI" - fi - - echo "Finish processing Pull Request #{pr} at $(date)" -done - -# ============================================================================== -# scrub working directory for older files -find $GDAS_CI_ROOT/workflow/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; -echo "Finish at $(date)" diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 917f70e90..869817612 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -15,6 +15,9 @@ usage() { } # ============================================================================== +TEST_WORKFLOW=0 +ci_regex_include="gdasapp" +ci_regex_exclude="" while getopts "d:o:h" opt; do case $opt in d) @@ -23,12 +26,40 @@ while getopts "d:o:h" opt; do o) outfile=$OPTARG ;; + w) + TEST_WORKFLOW=1 + ;; + R) + ci_regex_include=$OPTARG + ;; + E) + ci_regex_exclude=$OPTARG + ;; h|\?|:) usage ;; esac done +if [[ $TEST_WORKFLOW == 1 ]]; then + export WORKFLOW_BUILD="ON" + + workflow_dir=$repo_dir + gdasapp_dir=$workflow_dir/sorc/gdas.cd + + build_cmd_dir=$workflow_dir/sorc + build_cmd="./build_all.sh -u &>> log.build" + build_dir=$workflow_dir/build +else + export BUILD_JOBS=8 + + gdasapp_dir=$repodir + + build_cmd_dir=$gdasapp_dir + build_cmd="./build.sh -t $TARGET &>> log.build" + build_dir=$gdasapp_dir/build +fi + # ============================================================================== # start output file echo "Automated GDASApp Testing Results:" > $outfile @@ -38,11 +69,10 @@ echo "Start: $(date) on $(hostname)" >> $outfile echo "---------------------------------------------------" >> $outfile # ============================================================================== # run build script -cd $repodir +cd $build_cmd_dir module purge -export BUILD_JOBS=8 rm -rf log.build -./build.sh -t $TARGET &>> log.build +$build_cmd build_status=$? if [ $build_status -eq 0 ]; then echo "Build: *SUCCESS*" >> $outfile @@ -50,18 +80,25 @@ if [ $build_status -eq 0 ]; then else echo "Build: *FAILED*" >> $outfile echo "Build: Failed at $(date)" >> $outfile - echo "Build: see output at $repodir/log.build" >> $outfile + echo "Build: see output at $build_cmd_dir/log.build" >> $outfile echo '```' >> $outfile exit $build_status fi +if [[ TEST_WORKFLOW == 1 ]]; then + ./link_workflow.sh +fi # ============================================================================== # run ctests -cd $repodir/build -module use $GDAS_MODULE_USE +cd $gdasapp_dir/build +module use $gdasapp_dir/modulefiles module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest -j${NTASKS_TESTS} -R gdasapp --output-on-failure &>> log.ctest +ctest_cmd="ctest -j${NTASKS_TESTS} -R $ci_regex_include" +if [ -n "$ci_regex_exclude" ]; then + ctest_cmd+=" -E $ci_regex_exclude" +fi +$ctest_cmd --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then @@ -73,7 +110,7 @@ else echo "Tests: Failed at $(date)" >> $outfile echo "Tests: $npassed" >> $outfile cat log.ctest | grep "(Failed)" >> $outfile - echo "Tests: see output at $repodir/build/log.ctest" >> $outfile + echo "Tests: see output at $gdasapp_dir/build/log.ctest" >> $outfile fi echo '```' >> $outfile exit $ctest_status diff --git a/ci/run_gw_ci.sh b/ci/run_gw_ci.sh deleted file mode 100755 index d3b7de9d0..000000000 --- a/ci/run_gw_ci.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -set -u - -# ============================================================================== -usage() { - set +x - echo - echo "Usage: $0 -d -o -h" - echo - echo " -d Run build and ctest for clone in " - echo " -o Path to output message detailing results of CI tests" - echo " -h display this message and quit" - echo - exit 1 -} - -# ============================================================================== -while getopts "d:o:h" opt; do - case $opt in - d) - repodir=$OPTARG - ;; - o) - outfile=$OPTARG - ;; - h|\?|:) - usage - ;; - esac -done - -# ============================================================================== -# start output file -echo "Automated GW GDASApp Testing Results:" > $outfile -echo "Machine: ${TARGET}" >> $outfile -echo '```' >> $outfile -echo "Start: $(date) on $(hostname)" >> $outfile -echo "---------------------------------------------------" >> $outfile -# ============================================================================== -# run build and link as part of the workflow -export WORKFLOW_BUILD="ON" -cd $repodir/sorc -module purge -rm -rf log.build -./build_all.sh -u &>> log.build -build_status=$? -if [ $build_status -eq 0 ]; then - echo "Build: *SUCCESS*" >> $outfile - echo "Build: Completed at $(date)" >> $outfile -else - echo "Build: *FAILED*" >> $outfile - echo "Build: Failed at $(date)" >> $outfile - echo "Build: see output at $repodir/sorc/log.build" >> $outfile - echo '```' >> $outfile - exit $build_status -fi -./link_workflow.sh -# ============================================================================== -# run ctests -cd $repodir/sorc/gdas.cd/build -module use $repodir/sorc/gdas.cd/modulefiles -module load GDAS/$TARGET -echo "---------------------------------------------------" >> $outfile -rm -rf log.ctest -ctest -j${NTASKS_TESTS} -R gdasapp --output-on-failure &>> log.ctest -ctest_status=$? -npassed=$(cat log.ctest | grep "tests passed") -if [ $ctest_status -eq 0 ]; then - echo "Tests: *SUCCESS*" >> $outfile - echo "Tests: Completed at $(date)" >> $outfile - echo "Tests: $npassed" >> $outfile -else - echo "Tests: *Failed*" >> $outfile - echo "Tests: Failed at $(date)" >> $outfile - echo "Tests: $npassed" >> $outfile - cat log.ctest | grep "(Failed)" >> $outfile - echo "Tests: see output at $repodir/sorc/gdas.cd/build/log.ctest" >> $outfile -fi -echo '```' >> $outfile -exit $ctest_status From e4aff0a8e94e311d60bff7a91fcddedb0e992360 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 4 Dec 2024 14:56:24 +0000 Subject: [PATCH 03/24] Remove test line --- ci/driver.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/driver.sh b/ci/driver.sh index c7c707389..02cb8b533 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -85,9 +85,6 @@ else BASE_REPO=GDASApp fi -# Test -CI_LABEL="hera-driver-test" - # ============================================================================== # pull on the repo and get list of open PRs From 576526da8550191a9213a838ef30c29ce7b4084d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 13:19:58 +0000 Subject: [PATCH 04/24] Update --- ci/driver.sh | 40 ++++++++++++++++++++-------------------- ci/run_ci.sh | 34 +++++++++++++++++++++------------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/ci/driver.sh b/ci/driver.sh index 02cb8b533..e7a4563a3 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -55,6 +55,8 @@ case ${TARGET} in ;; esac +# ============================================================================== +# set list of available CI tests to run on the Global Workflow CI_TESTS=("C96C48_hybatmDA" "C96C48_ufs_hybatmDA" "C96C48_hybatmaerosnowDA" @@ -188,27 +190,25 @@ for pr in $open_pr_list; do commit=$(git log --pretty=format:'%h' -n 1) echo "$commit" > $PR_TEST_DIR/$pr/commit - # get ci tests from PR description and convert into a regular expressions to included and exclude - branch_body=$(gh pr view $pr --repo ${gdasapp_url} --json body --jq '.body') - ci_checklist=$(echo "$branch_body" | grep '\[x\]') - ci_include_regex="gdasapp" - ci_exclude_regex="" - for ci_test in ${CI_TESTS[@]}; do - if echo "$ci_checklist" | grep -q "$ci_test"; then - ci_include_regex+="${ci_include_regex:+|}$ci_test" - else - ci_exclude_regex+="${ci_exclude_regex:+|}$ci_test" - fi - done - # run build and testing command echo "Running run_ci.sh for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)" - run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit} -R $ci_include_regex" - if [ -n "$ci_exclude_regex" ]; then - run_ci_cmd+=" -E $ci_exclude_regex" - fi - if [[ TEST_WORKFLOW == 1 ]]; then - run_ci_cmd+=" -w" + run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit} -R gdasapp" + if [[ $TEST_WORKFLOW == 1 ]]; then + # get ci tests from PR description and convert into a regular expressions to be exclude + branch_body=$(gh pr view $pr --repo ${gdasapp_url} --json body --jq '.body') + ci_checklist=$(echo "$branch_body" | grep '\[x\]') + ctest_regex_exclude="" + for ci_test in ${CI_TESTS[@]}; do + if ! echo "$ci_checklist" | grep -q "$ci_test"; then + ctest_regex_exclude+="${ctest_regex_exclude:+|}$ci_test" + fi + done + + # setup run_ci.sh arguments to exclude chosen CI tests + run_ci_cmd+=" -w" + if [ -n "$ctest_regex_exclude" ]; then + run_ci_cmd+=" -E $ctest_regex_exclude" + fi fi $run_ci_cmd ci_status=$? @@ -226,7 +226,7 @@ for pr in $open_pr_list; do echo "No authorized users assigned to this PR. Aborting CI..." fi - echo "Finished processing Pull Request #{pr} at $(date)" + echo "Finished processing Pull Request #${pr} at $(date)" done # ============================================================================== diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 869817612..b9808ffe3 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -9,6 +9,9 @@ usage() { echo echo " -d Run build and ctest for clone in " echo " -o Path to output message detailing results of CI tests" + echo " -w Test GDASApp within the Global Workflow" + echo " -R Regular expression of CTests to include" + echo " -E Regular expression of CTests to exclude" echo " -h display this message and quit" echo exit 1 @@ -16,9 +19,9 @@ usage() { # ============================================================================== TEST_WORKFLOW=0 -ci_regex_include="gdasapp" -ci_regex_exclude="" -while getopts "d:o:h" opt; do +ctest_regex_include="" +ctest_regex_exclude="" +while getopts "d:o:h:R:E:w" opt; do case $opt in d) repodir=$OPTARG @@ -26,15 +29,15 @@ while getopts "d:o:h" opt; do o) outfile=$OPTARG ;; - w) - TEST_WORKFLOW=1 - ;; R) - ci_regex_include=$OPTARG + ctest_regex_include+=$OPTARG ;; E) - ci_regex_exclude=$OPTARG + ctest_regex_exclude+=$OPTARG ;; + w) + TEST_WORKFLOW=1 + ;; h|\?|:) usage ;; @@ -44,7 +47,7 @@ done if [[ $TEST_WORKFLOW == 1 ]]; then export WORKFLOW_BUILD="ON" - workflow_dir=$repo_dir + workflow_dir=$repodir gdasapp_dir=$workflow_dir/sorc/gdas.cd build_cmd_dir=$workflow_dir/sorc @@ -84,7 +87,7 @@ else echo '```' >> $outfile exit $build_status fi -if [[ TEST_WORKFLOW == 1 ]]; then +if [[ $TEST_WORKFLOW == 1 ]]; then ./link_workflow.sh fi # ============================================================================== @@ -94,11 +97,16 @@ module use $gdasapp_dir/modulefiles module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest_cmd="ctest -j${NTASKS_TESTS} -R $ci_regex_include" -if [ -n "$ci_regex_exclude" ]; then - ctest_cmd+=" -E $ci_regex_exclude" +ctest_cmd="ctest -j${NTASKS_TESTS}" +if [ -n "$ctest_regex_include" ]; then + ctest_cmd+=" -R $ctest_regex_include" +fi +if [ -n "$ctest_regex_exclude" ]; then + ctest_cmd+=" -E $ctest_regex_exclude" fi +pwd $ctest_cmd --output-on-failure &>> log.ctest +echo "Tests: $ctest_cmd" >> $outfile ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then From 6ad7811f1e05e2eea595d26e0bb09a7270ec3d06 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 13:34:55 +0000 Subject: [PATCH 05/24] Move CI test list to its own file --- ci/ci_tests.sh | 7 +++++++ ci/driver.sh | 7 +------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 ci/ci_tests.sh diff --git a/ci/ci_tests.sh b/ci/ci_tests.sh new file mode 100644 index 000000000..e180995e1 --- /dev/null +++ b/ci/ci_tests.sh @@ -0,0 +1,7 @@ +CI_TESTS=("atm_jjob" + "C96C48_ufs_hybatmDA" + "C96C48_hybatmaerosnowDA" + "C48mx500_3DVarAOWCDA" + "C48mx500_hybAOWCDA" + "C384mx025_3DVarAOWCDA" + "C96C48_hybatmDA") diff --git a/ci/driver.sh b/ci/driver.sh index e7a4563a3..3ef8ca5ef 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -57,12 +57,7 @@ esac # ============================================================================== # set list of available CI tests to run on the Global Workflow -CI_TESTS=("C96C48_hybatmDA" - "C96C48_ufs_hybatmDA" - "C96C48_hybatmaerosnowDA" - "C48mx500_3DVarAOWCDA" - "C48mx500_hybAOWCDA" - "C384mx025_3DVarAOWCDA") +source $my_dir/ci_tests.sh # ============================================================================== # set things that depend on whether running workflow tests or not From 75e95dfa52af410d161eee437656e94a9d071957 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 13:53:05 +0000 Subject: [PATCH 06/24] Clean up --- ci/driver.sh | 6 +- ci/driver_new.sh | 210 ----------------------------------------------- ci/run_ci.sh | 12 +-- 3 files changed, 5 insertions(+), 223 deletions(-) delete mode 100755 ci/driver_new.sh diff --git a/ci/driver.sh b/ci/driver.sh index 3ef8ca5ef..b9dbb7e5c 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -187,9 +187,9 @@ for pr in $open_pr_list; do # run build and testing command echo "Running run_ci.sh for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)" - run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit} -R gdasapp" + run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit}" if [[ $TEST_WORKFLOW == 1 ]]; then - # get ci tests from PR description and convert into a regular expressions to be exclude + # get ci tests from PR description and convert into a regular expressions to be excluded branch_body=$(gh pr view $pr --repo ${gdasapp_url} --json body --jq '.body') ci_checklist=$(echo "$branch_body" | grep '\[x\]') ctest_regex_exclude="" @@ -199,7 +199,7 @@ for pr in $open_pr_list; do fi done - # setup run_ci.sh arguments to exclude chosen CI tests + # setup run_ci.sh arguments to test in the Global Workflow and exclude chosen CI tests run_ci_cmd+=" -w" if [ -n "$ctest_regex_exclude" ]; then run_ci_cmd+=" -E $ctest_regex_exclude" diff --git a/ci/driver_new.sh b/ci/driver_new.sh deleted file mode 100755 index 19c08913f..000000000 --- a/ci/driver_new.sh +++ /dev/null @@ -1,210 +0,0 @@ -#!/bin/bash --login - -echo "Starting automated testing at $(date)" - -my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" -echo "Set my_dir ${my_dir}" - -# ============================================================================== -usage() { - set +x - echo - echo "Usage: $0 -t -h" - echo - echo " -t target/machine script is running on DEFAULT: $(hostname)" - echo " -h display this message and quit" - echo " -w run workflow tests on $(hostname)" - echo - exit 1 -} - -# ============================================================================== -# First, set up runtime environment - -export TARGET="$(hostname)" - -TEST_WORKFLOW=0 -while getopts "t:h:w" opt; do - case $opt in - t) - TARGET=$OPTARG - ;; - h|\?|:) - usage - ;; - w) - TEST_WORKFLOW=1 - ;; - esac -done - -echo "Running automated testing on $TARGET" - -case ${TARGET} in - hera | orion) - source $MODULESHOME/init/sh - source $my_dir/${TARGET}.sh - module purge - module use $GDAS_MODULE_USE - module load GDAS/$TARGET - module list - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac - -# ============================================================================== -# set things that depend on whether running workflow tests or not -gdasapp_url="https://github.com/NOAA-EMC/GDASApp.git" -if [[ $TEST_WORKFLOW == 1 ]]; then - echo "Testing GDASApp inside the Global Workflow" - - CI_LABEL="${GDAS_CI_HOST}-GW-RT" - OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list_gw - PR_TEST_DIR=$GDAS_CI_ROOT/workflow/PR - RUN_CI_SCRIPT=run_gw_ci.sh - BASE_REPO=global-workflow - - # Default Global Workflow repo and branch if no companion PR found - workflow_url="https://github.com/NOAA-EMC/global-workflow.git" - workflow_branch="develop" -else - echo "Testing stand-alone GDASApp" - - CI_LABEL="${GDAS_CI_HOST}-RT" - OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list - PR_TEST_DIR=$GDAS_CI_ROOT/PR - RUN_CI_SCRIPT=run_ci.sh - BASE_REPO=GDASApp -fi - -# ============================================================================== -# pull on the repo and get list of open PRs - -cd $GDAS_CI_ROOT/repo - -gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $OPEN_PR_LIST_DIR - -open_pr=`cat $OPEN_PR_LIST_DIR | wc -l` -if (( $open_pr == 0 )); then - echo "No open PRs with ${CI_LABEL}, exit." - echo "Finished automated testing at $(date)" - exit -fi - -open_pr_list=$(cat $OPEN_PR_LIST_DIR) - -# ============================================================================== -# clone, checkout, build, test, etc. -# loop through all open PRs -for pr in $open_pr_list; do - echo " " - echo "Starting processing of pull request #${pr} at $(date)" - - # get the branch name used for the PR - gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - - # get additional branch informatio - branch_owner=$(gh pr view $pr --repo ${gdasapp_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $pr --repo ${gdasapp_url} --json headRepository --jq '.headRepository.name') - pr_assignees=$(gh pr view $pr --repo ${gdasapp_url} --json assignees --jq '.assignees[].login') - - # check if any assignee is authorized to run CI - rc=1 - for str in ${pr_assignees[@]}; do - grep $str $AUTHORIZED_USERS_FILE > /dev/null - if (( rc != 0 )); then - rc=$? - fi - if (( rc == 0 )); then - echo "Authorized user $str assigned to this PR" - fi - done - - # Authorized to run CI - if (( rc == 0 )); then - echo "CI authorized. Running CI..." - - # update PR label - gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - - echo "GDASApp URL: $gdasapp_url" - echo "GDASApp branch Name: $gdasapp_branch" - - if [[ $TEST_WORKFLOW == 1 ]]; then - # check for a companion PR in the global-workflow - companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) - - if [ -n "$companion_pr_exists" ]; then - # get the PR number - companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') - - # extract the necessary info - branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') - - # Construct fork URL. Update workflow branch name - workflow_url="https://github.com/$branch_owner/$branch_name.git" - workflow_branch=$gdasapp_branch - fi - - echo "Found companion Global Workflow PR #${companion_pr}!" - echo "Global Workflow URL: $workflow_url" - echo "Global Workflow branch name: $workflow_branch" - fi - - # create PR specific directory - if [ -d $PR_TEST_DIR/$pr ]; then - rm -rf $PR_TEST_DIR/$pr - fi - mkdir -p $PR_TEST_DIR/$pr - cd $PR_TEST_DIR/$pr - pwd - - # clone copy of repo - if [[ $TEST_WORKFLOW == 1 ]]; then - echo "Cloning Global Workflow branch $workflow_branch from $workflow_url at $(date)" - git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url - cd global-workflow/sorc/gdas.cd - else - echo "Cloning GDASApp branch $workflow_branch at $(date)" - git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url - cd GDASApp - fi - pwd - - # checkout GDASApp pull request - gh pr checkout $pr - git submodule update --init --recursive - - # get commit hash - commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $PR_TEST_DIR/$pr/commit - - # run build and testing command - echo "Running $RUN_CI_SCRIPT for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)" - $my_dir/$RUN_CI_SCRIPT -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit} - ci_status=$? - echo "Finished running $RUN_CI_SCRIPT with ci_status ${ci_status} at $(date)" - - gh pr comment $pr --repo ${gdasapp_url} --body-file $PR_TEST_DIR/$pr/output_${commit} - if [ $ci_status -eq 0 ]; then - gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed - else - gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed - fi - - # Not authorized to run CI - else - echo "No authorized users assigned to this PR. Aborting CI..." - fi - - echo "Finished processing Pull Request #{pr} at $(date)" -done - -# ============================================================================== -# scrub working directory for older files -find $PR_TEST_DIR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; -echo "Finished automated testing at $(date)" diff --git a/ci/run_ci.sh b/ci/run_ci.sh index b9808ffe3..73c60ad3b 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -10,7 +10,6 @@ usage() { echo " -d Run build and ctest for clone in " echo " -o Path to output message detailing results of CI tests" echo " -w Test GDASApp within the Global Workflow" - echo " -R Regular expression of CTests to include" echo " -E Regular expression of CTests to exclude" echo " -h display this message and quit" echo @@ -19,9 +18,8 @@ usage() { # ============================================================================== TEST_WORKFLOW=0 -ctest_regex_include="" ctest_regex_exclude="" -while getopts "d:o:h:R:E:w" opt; do +while getopts "d:o:h:E:w" opt; do case $opt in d) repodir=$OPTARG @@ -29,9 +27,6 @@ while getopts "d:o:h:R:E:w" opt; do o) outfile=$OPTARG ;; - R) - ctest_regex_include+=$OPTARG - ;; E) ctest_regex_exclude+=$OPTARG ;; @@ -97,10 +92,7 @@ module use $gdasapp_dir/modulefiles module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest_cmd="ctest -j${NTASKS_TESTS}" -if [ -n "$ctest_regex_include" ]; then - ctest_cmd+=" -R $ctest_regex_include" -fi +ctest_cmd="ctest -j${NTASKS_TESTS} -R gdasapp" if [ -n "$ctest_regex_exclude" ]; then ctest_cmd+=" -E $ctest_regex_exclude" fi From 65239657834e9296261810e02c3bffe03140482b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 13:55:08 +0000 Subject: [PATCH 07/24] Add "jjob" to first jjob ctest name --- test/atm/global-workflow/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index daa5b8edf..2270dae33 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -1,7 +1,7 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -ecbuild_add_test(TARGET test_gdasapp_setup_atm_cycled_exp +ecbuild_add_test(TARGET test_gdasapp_setup_atm_jjob_cycled_exp TYPE SCRIPT COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} @@ -12,7 +12,7 @@ ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_init COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_init.sh ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun - TEST_DEPENDS test_gdasapp_setup_atm_cycled_exp) + TEST_DEPENDS test_gdasapp_setup_atm_jjob_cycled_exp) ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_run TYPE SCRIPT From 13eae53f30f8870750fb819363d3ac1fd25beef2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 14:07:32 +0000 Subject: [PATCH 08/24] Add orion authorized users --- ci/orion.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/orion.sh b/ci/orion.sh index 424e31b43..1b2707970 100644 --- a/ci/orion.sh +++ b/ci/orion.sh @@ -9,4 +9,4 @@ export SLURM_EXCLUSIVE=user export OMP_NUM_THREADS=1 ulimit -s unlimited export NTASKS_TESTS=12 -export AUTHORIZED_USERS_FILE='foo' +export AUTHORIZED_USERS_FILE='/work2/noaa/da/role-da/CI/GDASApp/authorized_users' From 6ab9d97531837bbeef17e9fb57fb17077ba0d1df Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 16:37:18 +0000 Subject: [PATCH 09/24] Add pull request template --- .github/pull_request_template.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..23368c804 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,25 @@ +# Description + + + +# Companion PRs + + + +# Issues + + + +# Automated CI tests to run in Global Workflow + +- [ ] atm_jjob +- [ ] C96C48_ufs_hybatmDA +- [ ] C96C48_hybatmaerosnowDA +- [ ] C48mx500_3DVarAOWCDA +- [ ] C48mx500_hybAOWCDA +- [ ] C384mx025_3DVarAOWCDA +- [ ] C96C48_hybatmDA From 3af926b9305bc9c1943ed54d1ca83e3d4687cb1f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 16:45:12 +0000 Subject: [PATCH 10/24] Update stable_driver.sh to use new run_ci.sh -w command --- ci/stable_driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 4172d6659..4f1175da2 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -71,7 +71,7 @@ $my_dir/../ush/submodules/update_develop.sh $gdasdir # ============================================================================== # run the automated testing -$my_dir/run_gw_ci.sh -d $stableroot/$datestr/global-workflow -o $stableroot/$datestr/output +$my_dir/run_ci.sh -d $stableroot/$datestr/global-workflow -o $stableroot/$datestr/output -w ci_status=$? total=0 if [ $ci_status -eq 0 ]; then From 09eecd16373895bbc13186879087259a2020da7d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 5 Dec 2024 22:15:41 +0000 Subject: [PATCH 11/24] Add hercules option --- ci/driver.sh | 2 +- ci/hercules.sh | 13 +++++++++++++ ci/orion.sh | 5 +++-- 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 ci/hercules.sh diff --git a/ci/driver.sh b/ci/driver.sh index b9dbb7e5c..e66929350 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -41,7 +41,7 @@ done echo "Running automated testing on $TARGET" case ${TARGET} in - hera | orion) + hera | orion | hercules) source $MODULESHOME/init/sh source $my_dir/${TARGET}.sh module purge diff --git a/ci/hercules.sh b/ci/hercules.sh new file mode 100644 index 000000000..663bc44c6 --- /dev/null +++ b/ci/hercules.sh @@ -0,0 +1,13 @@ +export GDAS_CI_ROOT=/work2/noaa/da/role-da/CI/hercules/GDASApp +export GDAS_CI_HOST='hercules' +export GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles +export SLURM_ACCOUNT=da-cpu +export SALLOC_ACCOUNT=$SLURM_ACCOUNT +export SBATCH_ACCOUNT=$SLURM_ACCOUNT +export SLURM_QOS=debug +export SLURM_EXCLUSIVE=user +export OMP_NUM_THREADS=1 +ulimit -s unlimited +export PATH=$PATH:/home/role-da/bin +export NTASKS_TESTS=12 +export AUTHORIZED_USERS_FILE=$GDAS_CI_ROOT/authorized_users diff --git a/ci/orion.sh b/ci/orion.sh index 1b2707970..9d0674688 100644 --- a/ci/orion.sh +++ b/ci/orion.sh @@ -1,4 +1,4 @@ -export GDAS_CI_ROOT=/work2/noaa/stmp/cmartin/CI/GDASApp +export GDAS_CI_ROOT=/work2/noaa/da/role-da/CI/orion/GDASApp export GDAS_CI_HOST='orion' export GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles export SLURM_ACCOUNT=da-cpu @@ -8,5 +8,6 @@ export SLURM_QOS=debug export SLURM_EXCLUSIVE=user export OMP_NUM_THREADS=1 ulimit -s unlimited +export PATH=$PATH:/home/role-da/bin export NTASKS_TESTS=12 -export AUTHORIZED_USERS_FILE='/work2/noaa/da/role-da/CI/GDASApp/authorized_users' +export AUTHORIZED_USERS_FILE=$GDAS_CI_ROOT/authorized_users From 11826f579b325d36595323e0c4abb75d6fbab82e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 6 Dec 2024 14:30:46 +0000 Subject: [PATCH 12/24] Remove c384 CI test --- .github/pull_request_template.md | 1 - ci/ci_tests.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 23368c804..04832590e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -21,5 +21,4 @@ Refs NOAA-EMC/repo#5678 - [ ] C96C48_hybatmaerosnowDA - [ ] C48mx500_3DVarAOWCDA - [ ] C48mx500_hybAOWCDA -- [ ] C384mx025_3DVarAOWCDA - [ ] C96C48_hybatmDA diff --git a/ci/ci_tests.sh b/ci/ci_tests.sh index e180995e1..43e8eb2fb 100644 --- a/ci/ci_tests.sh +++ b/ci/ci_tests.sh @@ -3,5 +3,4 @@ CI_TESTS=("atm_jjob" "C96C48_hybatmaerosnowDA" "C48mx500_3DVarAOWCDA" "C48mx500_hybAOWCDA" - "C384mx025_3DVarAOWCDA" "C96C48_hybatmDA") From f8ad5b98357986268e32dccf17942f9aad641796 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 6 Dec 2024 16:26:38 +0000 Subject: [PATCH 13/24] Fix bug is ctest success status --- ci/run_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 73c60ad3b..9579ac850 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -97,8 +97,8 @@ if [ -n "$ctest_regex_exclude" ]; then ctest_cmd+=" -E $ctest_regex_exclude" fi pwd -$ctest_cmd --output-on-failure &>> log.ctest echo "Tests: $ctest_cmd" >> $outfile +$ctest_cmd --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then From 355446bb13924457e6f7776a953f01532f8ee698 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 6 Dec 2024 17:41:28 +0000 Subject: [PATCH 14/24] Specify which CI (GW or standalone) in emcbot output --- ci/run_ci.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 9579ac850..4897a9a80 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -60,7 +60,11 @@ fi # ============================================================================== # start output file -echo "Automated GDASApp Testing Results:" > $outfile +if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Automated GW-GDASApp Testing Results:" > $outfile +else + echo "Automated GDASApp Testing Results:" > $outfile +fi echo "Machine: ${TARGET}" >> $outfile echo '```' >> $outfile echo "Start: $(date) on $(hostname)" >> $outfile From 12e1028fb133d8a734626be0b46f5452cd2c9425 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 9 Dec 2024 16:29:45 +0000 Subject: [PATCH 15/24] Fix typo --- test/atm/global-workflow/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index 2270dae33..1741a20f6 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -40,7 +40,7 @@ ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_init COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init.sh ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun - TEST_DEPENDS test_gdasapp_setup_atm_cycled_exp) + TEST_DEPENDS test_gdasapp_setup_atm_jjob_cycled_exp) ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_letkf TYPE SCRIPT From 2f19a24a27899c752f97c0ce66917fb3595eab73 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 9 Dec 2024 16:54:05 +0000 Subject: [PATCH 16/24] Set timeout time and include timed-out tests in emcbot output --- ci/run_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 4897a9a80..3a78880c7 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -102,7 +102,7 @@ if [ -n "$ctest_regex_exclude" ]; then fi pwd echo "Tests: $ctest_cmd" >> $outfile -$ctest_cmd --output-on-failure &>> log.ctest +$ctest_cmd --timeout 7200 --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then @@ -114,6 +114,7 @@ else echo "Tests: Failed at $(date)" >> $outfile echo "Tests: $npassed" >> $outfile cat log.ctest | grep "(Failed)" >> $outfile + cat log.ctest | grep "(Timeout))" >> $outfile echo "Tests: see output at $gdasapp_dir/build/log.ctest" >> $outfile fi echo '```' >> $outfile From 37d8c6fb4671355a832af005b88be5640af2c7c4 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 9 Dec 2024 17:14:16 +0000 Subject: [PATCH 17/24] Fix typo --- ci/run_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 3a78880c7..fd90b3557 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -114,7 +114,7 @@ else echo "Tests: Failed at $(date)" >> $outfile echo "Tests: $npassed" >> $outfile cat log.ctest | grep "(Failed)" >> $outfile - cat log.ctest | grep "(Timeout))" >> $outfile + cat log.ctest | grep "(Timeout)" >> $outfile echo "Tests: see output at $gdasapp_dir/build/log.ctest" >> $outfile fi echo '```' >> $outfile From 33539c7764c53b04785fdd37025f0f3b8cacadd3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 9 Dec 2024 18:32:58 +0000 Subject: [PATCH 18/24] Make sure to build GSI on build --- ci/run_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index fd90b3557..e92d943ae 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -46,7 +46,7 @@ if [[ $TEST_WORKFLOW == 1 ]]; then gdasapp_dir=$workflow_dir/sorc/gdas.cd build_cmd_dir=$workflow_dir/sorc - build_cmd="./build_all.sh -u &>> log.build" + build_cmd="./build_all.sh -ug &>> log.build" build_dir=$workflow_dir/build else export BUILD_JOBS=8 From b208ad2309f718932de18de1d96dbc5c2d58e48a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 9 Dec 2024 22:31:00 +0000 Subject: [PATCH 19/24] Incorporate Russ' suggestions for stable_driver.sh --- ci/stable_driver.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 4f1175da2..05e406fae 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -53,6 +53,7 @@ repo_url="https://github.com/NOAA-EMC/GDASApp.git" workflow_url="https://github.com/NOAA-EMC/global-workflow.git" stableroot=$GDAS_CI_ROOT/stable +[[ -d $stableroot/$datestr ]] && rm -rf $stableroot/$datestr mkdir -p $stableroot/$datestr cd $stableroot/$datestr @@ -63,11 +64,12 @@ git clone --recursive $workflow_url cd $stableroot/$datestr/global-workflow/sorc/gdas.cd git checkout develop git pull +git submodule update --init --recursive # ============================================================================== # update the hashes to the most recent gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd -$my_dir/../ush/submodules/update_develop.sh $gdasdir +$gdasdir/ush/submodules/update_develop.sh $gdasdir # ============================================================================== # run the automated testing @@ -115,19 +117,26 @@ if [ $ci_status -eq 0 ]; then if [ $total -ne 0 ]; then echo "Unable to push" >> $stableroot/$datestr/output fi + # send email + PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov David.New@noaa.gov" + BODY=$stableroot/$datestr/output_stable_nightly if [ $total -ne 0 ]; then - echo "Issue merging with develop. please manually fix" - PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov" SUBJECT="Problem updating feature/stable-nightly branch of GDASApp" - BODY=$stableroot/$datestr/output_stable_nightly cat > $BODY << EOF Problem updating feature/stable-nightly branch of GDASApp. Please check $stableroot/$datestr/GDASApp EOF - mail -r "Darth Vader - NOAA Affiliate " -s "$SUBJECT" "$PEOPLE" < $BODY + else - echo "Stable branch updated" + SUBJECT="Success updating feature/stable-nightly branch of GDASApp" + cat > $BODY << EOF +feature/stable-nightly branch of GDASApp updated successfully. See $stableroot/$datestr/GDASApp for details. + +EOF + fi + echo $SUBJECT + mail -r "Darth Vader - NOAA Affiliate " -s "$SUBJECT" "$PEOPLE" < $BODY else # do nothing echo "Testing failed, stable branch will not be updated" From 21bc13e1adc4e92850e4bf4d5faa062df9120fc9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 10 Dec 2024 21:11:01 +0000 Subject: [PATCH 20/24] Send email if testing/building in stable driver fails --- ci/stable_driver.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 05e406fae..f15c08e6a 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -71,6 +71,11 @@ git submodule update --init --recursive gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd $gdasdir/ush/submodules/update_develop.sh $gdasdir +# ============================================================================== +# email information +PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov David.New@noaa.gov" +BODY=$stableroot/$datestr/output_stable_nightly + # ============================================================================== # run the automated testing $my_dir/run_ci.sh -d $stableroot/$datestr/global-workflow -o $stableroot/$datestr/output -w @@ -118,8 +123,6 @@ if [ $ci_status -eq 0 ]; then echo "Unable to push" >> $stableroot/$datestr/output fi # send email - PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov David.New@noaa.gov" - BODY=$stableroot/$datestr/output_stable_nightly if [ $total -ne 0 ]; then SUBJECT="Problem updating feature/stable-nightly branch of GDASApp" cat > $BODY << EOF @@ -135,12 +138,16 @@ feature/stable-nightly branch of GDASApp updated successfully. See $stableroot/$ EOF fi - echo $SUBJECT - mail -r "Darth Vader - NOAA Affiliate " -s "$SUBJECT" "$PEOPLE" < $BODY else # do nothing - echo "Testing failed, stable branch will not be updated" + SUBJECT="Testing or building of feature/stable-nightly branch of GDASApp failed" + cat > $BODY << EOF +Testing or building of feature/stable-nightly branch of GDASApp failed. Please check $stableroot/$datestr/GDASApp. + +EOF fi +echo $SUBJECT +mail -r "Darth Vader - NOAA Affiliate " -s "$SUBJECT" "$PEOPLE" < $BODY # ============================================================================== # publish some information to RZDM for quick viewing # THIS IS A TODO FOR NOW From 4f6ed12d1830ad4a844e76021267aa5851cc9699 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 11 Dec 2024 17:59:19 +0000 Subject: [PATCH 21/24] intial commit --- ci/stable_driver.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index f15c08e6a..57f57230d 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -73,8 +73,8 @@ $gdasdir/ush/submodules/update_develop.sh $gdasdir # ============================================================================== # email information -PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov David.New@noaa.gov" -BODY=$stableroot/$datestr/output_stable_nightly +PEOPLE="David.New@noaa.gov" +BODY=$stableroot/$datestr/stable_nightly # ============================================================================== # run the automated testing @@ -126,14 +126,14 @@ if [ $ci_status -eq 0 ]; then if [ $total -ne 0 ]; then SUBJECT="Problem updating feature/stable-nightly branch of GDASApp" cat > $BODY << EOF -Problem updating feature/stable-nightly branch of GDASApp. Please check $stableroot/$datestr/GDASApp +Problem updating feature/stable-nightly branch of GDASApp. Please check $stableroot/$datestr/global-workflow EOF else SUBJECT="Success updating feature/stable-nightly branch of GDASApp" cat > $BODY << EOF -feature/stable-nightly branch of GDASApp updated successfully. See $stableroot/$datestr/GDASApp for details. +feature/stable-nightly branch of GDASApp updated successfully. See $stableroot/$datestr/global-workflow for details. EOF @@ -142,7 +142,7 @@ else # do nothing SUBJECT="Testing or building of feature/stable-nightly branch of GDASApp failed" cat > $BODY << EOF -Testing or building of feature/stable-nightly branch of GDASApp failed. Please check $stableroot/$datestr/GDASApp. +Testing or building of feature/stable-nightly branch of GDASApp failed. Please check $stableroot/$datestr/global-workflow. EOF fi From eb9b07f1fc86c366dbcfa624de0657d60f583426 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 11 Dec 2024 22:19:02 +0000 Subject: [PATCH 22/24] Final touch --- ci/stable_driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 57f57230d..9e8e85e35 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -73,7 +73,7 @@ $gdasdir/ush/submodules/update_develop.sh $gdasdir # ============================================================================== # email information -PEOPLE="David.New@noaa.gov" +PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov David.New@noaa.gov" BODY=$stableroot/$datestr/stable_nightly # ============================================================================== From c74dd4fa86805a725cfbeacd637040a232f35a86 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 12 Dec 2024 21:51:54 +0000 Subject: [PATCH 23/24] Remove caution --- ci/stable_driver.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 9e8e85e35..86f416974 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -113,7 +113,6 @@ if [ $ci_status -eq 0 ]; then fi git diff-index --quiet HEAD || git commit -m "Update to new stable build on $datestr" total=$(($total+$?)) - caution="" if [ $total -ne 0 ]; then echo "Unable to commit" >> $stableroot/$datestr/output fi From e9223df01a38caf5583c6a67f24c55a29dcd4183 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:33:38 -0500 Subject: [PATCH 24/24] Update stable_driver.sh --- ci/stable_driver.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 86f416974..ca51f6f7c 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -121,7 +121,6 @@ if [ $ci_status -eq 0 ]; then if [ $total -ne 0 ]; then echo "Unable to push" >> $stableroot/$datestr/output fi - # send email if [ $total -ne 0 ]; then SUBJECT="Problem updating feature/stable-nightly branch of GDASApp" cat > $BODY << EOF