diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..04832590e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,24 @@ +# Description + + + +# Companion PRs + + + +# Issues + + + +# Automated CI tests to run in Global Workflow + +- [ ] atm_jjob +- [ ] C96C48_ufs_hybatmDA +- [ ] C96C48_hybatmaerosnowDA +- [ ] C48mx500_3DVarAOWCDA +- [ ] C48mx500_hybAOWCDA +- [ ] C96C48_hybatmDA diff --git a/ci/ci_tests.sh b/ci/ci_tests.sh new file mode 100644 index 000000000..43e8eb2fb --- /dev/null +++ b/ci/ci_tests.sh @@ -0,0 +1,6 @@ +CI_TESTS=("atm_jjob" + "C96C48_ufs_hybatmDA" + "C96C48_hybatmaerosnowDA" + "C48mx500_3DVarAOWCDA" + "C48mx500_hybAOWCDA" + "C96C48_hybatmDA") diff --git a/ci/driver.sh b/ci/driver.sh index 933b9223b..e66929350 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -1,6 +1,6 @@ #!/bin/bash --login -echo "Start at $(date)" +echo "Starting automated testing at $(date)" my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" echo "Set my_dir ${my_dir}" @@ -13,6 +13,7 @@ usage() { echo echo " -t target/machine script is running on DEFAULT: $(hostname)" echo " -h display this message and quit" + echo " -w run workflow tests on $(hostname)" echo exit 1 } @@ -22,7 +23,8 @@ usage() { export TARGET="$(hostname)" -while getopts "t:h" opt; do +TEST_WORKFLOW=0 +while getopts "t:h:w" opt; do case $opt in t) TARGET=$OPTARG @@ -30,12 +32,16 @@ while getopts "t:h" opt; do h|\?|:) usage ;; + w) + TEST_WORKFLOW=1 + ;; esac done +echo "Running automated testing on $TARGET" + case ${TARGET} in - hera | orion) - echo "Running Automated Testing on $TARGET" + hera | orion | hercules) source $MODULESHOME/init/sh source $my_dir/${TARGET}.sh module purge @@ -49,106 +55,176 @@ case ${TARGET} in ;; esac +# ============================================================================== +# set list of available CI tests to run on the Global Workflow +source $my_dir/ci_tests.sh + +# ============================================================================== +# set things that depend on whether running workflow tests or not +gdasapp_url="https://github.com/NOAA-EMC/GDASApp.git" +if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Testing GDASApp inside the Global Workflow" + + CI_LABEL="${GDAS_CI_HOST}-GW-RT" + OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list_gw + PR_TEST_DIR=$GDAS_CI_ROOT/workflow/PR + BASE_REPO=global-workflow + + # Default Global Workflow repo and branch if no companion PR found + workflow_url="https://github.com/NOAA-EMC/global-workflow.git" + workflow_branch="develop" +else + echo "Testing stand-alone GDASApp" + + CI_LABEL="${GDAS_CI_HOST}-RT" + OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list + PR_TEST_DIR=$GDAS_CI_ROOT/PR + BASE_REPO=GDASApp +fi + # ============================================================================== # pull on the repo and get list of open PRs + cd $GDAS_CI_ROOT/repo -CI_LABEL="${GDAS_CI_HOST}-RT" -gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list -open_pr=`cat $GDAS_CI_ROOT/open_pr_list | wc -l` +gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $OPEN_PR_LIST_DIR + +open_pr=`cat $OPEN_PR_LIST_DIR | wc -l` if (( $open_pr == 0 )); then echo "No open PRs with ${CI_LABEL}, exit." - echo "Finish at $(date)" + echo "Finished automated testing at $(date)" exit fi -open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list) +open_pr_list=$(cat $OPEN_PR_LIST_DIR) # ============================================================================== # clone, checkout, build, test, etc. -repo_url="https://github.com/NOAA-EMC/GDASApp.git" # loop through all open PRs for pr in $open_pr_list; do echo " " - echo "Start processing Pull Request #${pr} at $(date)" + echo "Starting processing of pull request #${pr} at $(date)" # get the branch name used for the PR gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - # get additional branch information - branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') - pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') - + # get additional branch informatio + branch_owner=$(gh pr view $pr --repo ${gdasapp_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $pr --repo ${gdasapp_url} --json headRepository --jq '.headRepository.name') + pr_assignees=$(gh pr view $pr --repo ${gdasapp_url} --json assignees --jq '.assignees[].login') + # check if any assignee is authorized to run CI - authorized_by="" + rc=1 for str in ${pr_assignees[@]}; do - grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users - rc=$? + grep $str $AUTHORIZED_USERS_FILE > /dev/null + if (( rc != 0 )); then + rc=$? + fi if (( rc == 0 )); then - authorized_by=${str} - echo "FOUND MATCH $str, rc $rc" - break + echo "Authorized user $str assigned to this PR" fi done # Authorized to run CI if (( rc == 0 )); then - echo "Run CI" + echo "CI authorized. Running CI..." # update PR label gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - - # construct the fork URL - gdasapp_url="https://github.com/$branch_owner/${branch_name}.git" - + echo "GDASApp URL: $gdasapp_url" echo "GDASApp branch Name: $gdasapp_branch" - echo "CI authorized by $authorized_by at $(date)" + + if [[ $TEST_WORKFLOW == 1 ]]; then + # check for a companion PR in the global-workflow + companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) + + if [ -n "$companion_pr_exists" ]; then + # get the PR number + companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') + + # extract the necessary info + branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') + branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') + + # Construct fork URL. Update workflow branch name + workflow_url="https://github.com/$branch_owner/$branch_name.git" + workflow_branch=$gdasapp_branch + fi + + echo "Found companion Global Workflow PR #${companion_pr}!" + echo "Global Workflow URL: $workflow_url" + echo "Global Workflow branch name: $workflow_branch" + fi - # create PR specific directory - if [ -d $GDAS_CI_ROOT/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/PR/$pr + # create PR specific directory + if [ -d $PR_TEST_DIR/$pr ]; then + rm -rf $PR_TEST_DIR/$pr fi - mkdir -p $GDAS_CI_ROOT/PR/$pr - cd $GDAS_CI_ROOT/PR/$pr + mkdir -p $PR_TEST_DIR/$pr + cd $PR_TEST_DIR/$pr pwd # clone copy of repo - git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url - cd GDASApp + if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Cloning Global Workflow branch $workflow_branch from $workflow_url at $(date)" + git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url + cd global-workflow/sorc/gdas.cd + else + echo "Cloning GDASApp branch $workflow_branch at $(date)" + git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url + cd GDASApp + fi pwd # checkout GDASApp pull request - git pull gh pr checkout $pr git submodule update --init --recursive # get commit hash commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit + echo "$commit" > $PR_TEST_DIR/$pr/commit # run build and testing command - echo "Execute $my_dir/run_ci.sh for $GDAS_CI_ROOT/PR/$pr/GDASApp at $(date)" - $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit} + echo "Running run_ci.sh for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)" + run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit}" + if [[ $TEST_WORKFLOW == 1 ]]; then + # get ci tests from PR description and convert into a regular expressions to be excluded + branch_body=$(gh pr view $pr --repo ${gdasapp_url} --json body --jq '.body') + ci_checklist=$(echo "$branch_body" | grep '\[x\]') + ctest_regex_exclude="" + for ci_test in ${CI_TESTS[@]}; do + if ! echo "$ci_checklist" | grep -q "$ci_test"; then + ctest_regex_exclude+="${ctest_regex_exclude:+|}$ci_test" + fi + done + + # setup run_ci.sh arguments to test in the Global Workflow and exclude chosen CI tests + run_ci_cmd+=" -w" + if [ -n "$ctest_regex_exclude" ]; then + run_ci_cmd+=" -E $ctest_regex_exclude" + fi + fi + $run_ci_cmd ci_status=$? - echo "After run_ci.sh with ci_status ${ci_status} at $(date)" - gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit} + echo "Finished running run_ci.sh with ci_status ${ci_status} at $(date)" + + gh pr comment $pr --repo ${gdasapp_url} --body-file $PR_TEST_DIR/$pr/output_${commit} if [ $ci_status -eq 0 ]; then - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed + gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed else - gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed + gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed fi # Not authorized to run CI else - echo "Do NOT run CI" + echo "No authorized users assigned to this PR. Aborting CI..." fi - echo "Finish processing Pull Request #{pr} at $(date)" + echo "Finished processing Pull Request #${pr} at $(date)" done # ============================================================================== # scrub working directory for older files -find $GDAS_CI_ROOT/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; -echo "Finish at $(date)" +find $PR_TEST_DIR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; +echo "Finished automated testing at $(date)" diff --git a/ci/gw_driver.sh b/ci/gw_driver.sh deleted file mode 100755 index c40ff4026..000000000 --- a/ci/gw_driver.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash --login - -echo "Start at $(date)" - -my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" -echo "Set my_dir ${my_dir}" - -# ============================================================================== -usage() { - set +x - echo - echo "Usage: $0 -t -h" - echo - echo " -t target/machine script is running on DEFAULT: $(hostname)" - echo " -h display this message and quit" - echo - exit 1 -} - -# ============================================================================== -# First, set up runtime environment - -export TARGET="$(hostname)" - -while getopts "t:h" opt; do - case $opt in - t) - TARGET=$OPTARG - ;; - h|\?|:) - usage - ;; - esac -done - -case ${TARGET} in - hera | orion) - echo "Running Automated GW Testing on $TARGET" - source $MODULESHOME/init/sh - source $my_dir/${TARGET}.sh - module purge - module use $GDAS_MODULE_USE - module load GDAS/$TARGET - module list - ;; - *) - echo "Unsupported platform. Exiting with error." - exit 1 - ;; -esac - -# ============================================================================== -# pull on the repo and get list of open PRs -cd $GDAS_CI_ROOT/repo -CI_LABEL="${GDAS_CI_HOST}-GW-RT" -gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list_gw - -open_pr=`cat $GDAS_CI_ROOT/open_pr_list_gw | wc -l` -if (( $open_pr == 0 )); then - echo "No open PRs with ${CI_LABEL}, exit." - echo "Finish at $(date)" - exit -fi - -open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list_gw) - -# ============================================================================== -# clone, checkout, build, test, etc. -repo_url="https://github.com/NOAA-EMC/GDASApp.git" -workflow_url="https://github.com/NOAA-EMC/global-workflow.git" -workflow_branch="develop" -# loop through all open PRs -for pr in $open_pr_list; do - echo " " - echo "Start processing Pull Request #${pr} at $(date)" - - # get the branch name used for the PR - gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName") - - # get additional branch information - branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name') - pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login') - - # check if any assignee is authorized to run CI - authorized_by="" - for str in ${pr_assignees[@]}; do - grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users - rc=$? - if (( rc == 0 )); then - authorized_by=${str} - echo "FOUND MATCH $str, rc $rc" - break - fi - done - - # Authorized to run CI - if (( rc == 0 )); then - echo "Run CI" - - # update PR label - gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running - - # check for a companion PR in the global-workflow - companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open) - if [ -n "$companion_pr_exists" ]; then - # get the PR number - companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}') - - # extract the necessary info - branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login') - branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name') - - # Construct fork URL. Update workflow branch name - workflow_url="https://github.com/$branch_owner/$branch_name.git" - workflow_branch=$gdasapp_branch - - fi - - echo "Workflow URL: $workflow_url" - echo "Workflow branch name: $workflow_branch" - echo "GDASApp branch name: $gdasapp_branch" - echo "CI authorized by $authorized_by at $(date)" - - # create PR specific directory - if [ -d $GDAS_CI_ROOT/workflow/PR/$pr ]; then - rm -rf $GDAS_CI_ROOT/workflow/PR/$pr - fi - mkdir -p $GDAS_CI_ROOT/workflow/PR/$pr - cd $GDAS_CI_ROOT/workflow/PR/$pr - pwd - - # clone global workflow develop branch - git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url - - # checkout GDASApp pull request - cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd - git pull - gh pr checkout $pr - git submodule update --init --recursive - - # get commit hash - commit=$(git log --pretty=format:'%h' -n 1) - echo "$commit" > $GDAS_CI_ROOT/workflow/PR/$pr/commit - - # run build and testing command - echo "Execute $my_dir/run_gw_ci.sh for $GDAS_CI_ROOT/PR/workflow/PR/$pr/global-workflow at $(date)" - $my_dir/run_gw_ci.sh -d $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow -o $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - ci_status=$? - echo "After run_gw_ci.sh with ci_status ${ci_status} at $(date)" - gh pr comment $pr --body-file $GDAS_CI_ROOT/workflow/PR/$pr/output_${commit} - if [ $ci_status -eq 0 ]; then - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed - else - gh pr edit $pr --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed - fi - - # Not authorized to run CI - else - echo "Do NOT run CI" - fi - - echo "Finish processing Pull Request #{pr} at $(date)" -done - -# ============================================================================== -# scrub working directory for older files -find $GDAS_CI_ROOT/workflow/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \; -echo "Finish at $(date)" diff --git a/ci/hera.sh b/ci/hera.sh index 8a9518f8b..c9c5f892c 100644 --- a/ci/hera.sh +++ b/ci/hera.sh @@ -7,3 +7,4 @@ export SBATCH_ACCOUNT=$SLURM_ACCOUNT export SLURM_QOS=debug export PATH=$PATH:/home/role.jedipara/bin export NTASKS_TESTS=12 +export AUTHORIZED_USERS_FILE=/scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users diff --git a/ci/hercules.sh b/ci/hercules.sh new file mode 100644 index 000000000..663bc44c6 --- /dev/null +++ b/ci/hercules.sh @@ -0,0 +1,13 @@ +export GDAS_CI_ROOT=/work2/noaa/da/role-da/CI/hercules/GDASApp +export GDAS_CI_HOST='hercules' +export GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles +export SLURM_ACCOUNT=da-cpu +export SALLOC_ACCOUNT=$SLURM_ACCOUNT +export SBATCH_ACCOUNT=$SLURM_ACCOUNT +export SLURM_QOS=debug +export SLURM_EXCLUSIVE=user +export OMP_NUM_THREADS=1 +ulimit -s unlimited +export PATH=$PATH:/home/role-da/bin +export NTASKS_TESTS=12 +export AUTHORIZED_USERS_FILE=$GDAS_CI_ROOT/authorized_users diff --git a/ci/orion.sh b/ci/orion.sh index 8c2cf9a5f..9d0674688 100644 --- a/ci/orion.sh +++ b/ci/orion.sh @@ -1,4 +1,4 @@ -export GDAS_CI_ROOT=/work2/noaa/stmp/cmartin/CI/GDASApp +export GDAS_CI_ROOT=/work2/noaa/da/role-da/CI/orion/GDASApp export GDAS_CI_HOST='orion' export GDAS_MODULE_USE=$GDAS_CI_ROOT/repo/modulefiles export SLURM_ACCOUNT=da-cpu @@ -8,4 +8,6 @@ export SLURM_QOS=debug export SLURM_EXCLUSIVE=user export OMP_NUM_THREADS=1 ulimit -s unlimited +export PATH=$PATH:/home/role-da/bin export NTASKS_TESTS=12 +export AUTHORIZED_USERS_FILE=$GDAS_CI_ROOT/authorized_users diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 917f70e90..e92d943ae 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -9,13 +9,17 @@ usage() { echo echo " -d Run build and ctest for clone in " echo " -o Path to output message detailing results of CI tests" + echo " -w Test GDASApp within the Global Workflow" + echo " -E Regular expression of CTests to exclude" echo " -h display this message and quit" echo exit 1 } # ============================================================================== -while getopts "d:o:h" opt; do +TEST_WORKFLOW=0 +ctest_regex_exclude="" +while getopts "d:o:h:E:w" opt; do case $opt in d) repodir=$OPTARG @@ -23,26 +27,54 @@ while getopts "d:o:h" opt; do o) outfile=$OPTARG ;; + E) + ctest_regex_exclude+=$OPTARG + ;; + w) + TEST_WORKFLOW=1 + ;; h|\?|:) usage ;; esac done +if [[ $TEST_WORKFLOW == 1 ]]; then + export WORKFLOW_BUILD="ON" + + workflow_dir=$repodir + gdasapp_dir=$workflow_dir/sorc/gdas.cd + + build_cmd_dir=$workflow_dir/sorc + build_cmd="./build_all.sh -ug &>> log.build" + build_dir=$workflow_dir/build +else + export BUILD_JOBS=8 + + gdasapp_dir=$repodir + + build_cmd_dir=$gdasapp_dir + build_cmd="./build.sh -t $TARGET &>> log.build" + build_dir=$gdasapp_dir/build +fi + # ============================================================================== # start output file -echo "Automated GDASApp Testing Results:" > $outfile +if [[ $TEST_WORKFLOW == 1 ]]; then + echo "Automated GW-GDASApp Testing Results:" > $outfile +else + echo "Automated GDASApp Testing Results:" > $outfile +fi echo "Machine: ${TARGET}" >> $outfile echo '```' >> $outfile echo "Start: $(date) on $(hostname)" >> $outfile echo "---------------------------------------------------" >> $outfile # ============================================================================== # run build script -cd $repodir +cd $build_cmd_dir module purge -export BUILD_JOBS=8 rm -rf log.build -./build.sh -t $TARGET &>> log.build +$build_cmd build_status=$? if [ $build_status -eq 0 ]; then echo "Build: *SUCCESS*" >> $outfile @@ -50,18 +82,27 @@ if [ $build_status -eq 0 ]; then else echo "Build: *FAILED*" >> $outfile echo "Build: Failed at $(date)" >> $outfile - echo "Build: see output at $repodir/log.build" >> $outfile + echo "Build: see output at $build_cmd_dir/log.build" >> $outfile echo '```' >> $outfile exit $build_status fi +if [[ $TEST_WORKFLOW == 1 ]]; then + ./link_workflow.sh +fi # ============================================================================== # run ctests -cd $repodir/build -module use $GDAS_MODULE_USE +cd $gdasapp_dir/build +module use $gdasapp_dir/modulefiles module load GDAS/$TARGET echo "---------------------------------------------------" >> $outfile rm -rf log.ctest -ctest -j${NTASKS_TESTS} -R gdasapp --output-on-failure &>> log.ctest +ctest_cmd="ctest -j${NTASKS_TESTS} -R gdasapp" +if [ -n "$ctest_regex_exclude" ]; then + ctest_cmd+=" -E $ctest_regex_exclude" +fi +pwd +echo "Tests: $ctest_cmd" >> $outfile +$ctest_cmd --timeout 7200 --output-on-failure &>> log.ctest ctest_status=$? npassed=$(cat log.ctest | grep "tests passed") if [ $ctest_status -eq 0 ]; then @@ -73,7 +114,8 @@ else echo "Tests: Failed at $(date)" >> $outfile echo "Tests: $npassed" >> $outfile cat log.ctest | grep "(Failed)" >> $outfile - echo "Tests: see output at $repodir/build/log.ctest" >> $outfile + cat log.ctest | grep "(Timeout)" >> $outfile + echo "Tests: see output at $gdasapp_dir/build/log.ctest" >> $outfile fi echo '```' >> $outfile exit $ctest_status diff --git a/ci/run_gw_ci.sh b/ci/run_gw_ci.sh deleted file mode 100755 index d3b7de9d0..000000000 --- a/ci/run_gw_ci.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -set -u - -# ============================================================================== -usage() { - set +x - echo - echo "Usage: $0 -d -o -h" - echo - echo " -d Run build and ctest for clone in " - echo " -o Path to output message detailing results of CI tests" - echo " -h display this message and quit" - echo - exit 1 -} - -# ============================================================================== -while getopts "d:o:h" opt; do - case $opt in - d) - repodir=$OPTARG - ;; - o) - outfile=$OPTARG - ;; - h|\?|:) - usage - ;; - esac -done - -# ============================================================================== -# start output file -echo "Automated GW GDASApp Testing Results:" > $outfile -echo "Machine: ${TARGET}" >> $outfile -echo '```' >> $outfile -echo "Start: $(date) on $(hostname)" >> $outfile -echo "---------------------------------------------------" >> $outfile -# ============================================================================== -# run build and link as part of the workflow -export WORKFLOW_BUILD="ON" -cd $repodir/sorc -module purge -rm -rf log.build -./build_all.sh -u &>> log.build -build_status=$? -if [ $build_status -eq 0 ]; then - echo "Build: *SUCCESS*" >> $outfile - echo "Build: Completed at $(date)" >> $outfile -else - echo "Build: *FAILED*" >> $outfile - echo "Build: Failed at $(date)" >> $outfile - echo "Build: see output at $repodir/sorc/log.build" >> $outfile - echo '```' >> $outfile - exit $build_status -fi -./link_workflow.sh -# ============================================================================== -# run ctests -cd $repodir/sorc/gdas.cd/build -module use $repodir/sorc/gdas.cd/modulefiles -module load GDAS/$TARGET -echo "---------------------------------------------------" >> $outfile -rm -rf log.ctest -ctest -j${NTASKS_TESTS} -R gdasapp --output-on-failure &>> log.ctest -ctest_status=$? -npassed=$(cat log.ctest | grep "tests passed") -if [ $ctest_status -eq 0 ]; then - echo "Tests: *SUCCESS*" >> $outfile - echo "Tests: Completed at $(date)" >> $outfile - echo "Tests: $npassed" >> $outfile -else - echo "Tests: *Failed*" >> $outfile - echo "Tests: Failed at $(date)" >> $outfile - echo "Tests: $npassed" >> $outfile - cat log.ctest | grep "(Failed)" >> $outfile - echo "Tests: see output at $repodir/sorc/gdas.cd/build/log.ctest" >> $outfile -fi -echo '```' >> $outfile -exit $ctest_status diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 4172d6659..ca51f6f7c 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -53,6 +53,7 @@ repo_url="https://github.com/NOAA-EMC/GDASApp.git" workflow_url="https://github.com/NOAA-EMC/global-workflow.git" stableroot=$GDAS_CI_ROOT/stable +[[ -d $stableroot/$datestr ]] && rm -rf $stableroot/$datestr mkdir -p $stableroot/$datestr cd $stableroot/$datestr @@ -63,15 +64,21 @@ git clone --recursive $workflow_url cd $stableroot/$datestr/global-workflow/sorc/gdas.cd git checkout develop git pull +git submodule update --init --recursive # ============================================================================== # update the hashes to the most recent gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd -$my_dir/../ush/submodules/update_develop.sh $gdasdir +$gdasdir/ush/submodules/update_develop.sh $gdasdir + +# ============================================================================== +# email information +PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov David.New@noaa.gov" +BODY=$stableroot/$datestr/stable_nightly # ============================================================================== # run the automated testing -$my_dir/run_gw_ci.sh -d $stableroot/$datestr/global-workflow -o $stableroot/$datestr/output +$my_dir/run_ci.sh -d $stableroot/$datestr/global-workflow -o $stableroot/$datestr/output -w ci_status=$? total=0 if [ $ci_status -eq 0 ]; then @@ -106,7 +113,6 @@ if [ $ci_status -eq 0 ]; then fi git diff-index --quiet HEAD || git commit -m "Update to new stable build on $datestr" total=$(($total+$?)) - caution="" if [ $total -ne 0 ]; then echo "Unable to commit" >> $stableroot/$datestr/output fi @@ -116,22 +122,30 @@ if [ $ci_status -eq 0 ]; then echo "Unable to push" >> $stableroot/$datestr/output fi if [ $total -ne 0 ]; then - echo "Issue merging with develop. please manually fix" - PEOPLE="Cory.R.Martin@noaa.gov Russ.Treadon@noaa.gov Guillaume.Vernieres@noaa.gov" SUBJECT="Problem updating feature/stable-nightly branch of GDASApp" - BODY=$stableroot/$datestr/output_stable_nightly cat > $BODY << EOF -Problem updating feature/stable-nightly branch of GDASApp. Please check $stableroot/$datestr/GDASApp +Problem updating feature/stable-nightly branch of GDASApp. Please check $stableroot/$datestr/global-workflow EOF - mail -r "Darth Vader - NOAA Affiliate " -s "$SUBJECT" "$PEOPLE" < $BODY + else - echo "Stable branch updated" + SUBJECT="Success updating feature/stable-nightly branch of GDASApp" + cat > $BODY << EOF +feature/stable-nightly branch of GDASApp updated successfully. See $stableroot/$datestr/global-workflow for details. + +EOF + fi else # do nothing - echo "Testing failed, stable branch will not be updated" + SUBJECT="Testing or building of feature/stable-nightly branch of GDASApp failed" + cat > $BODY << EOF +Testing or building of feature/stable-nightly branch of GDASApp failed. Please check $stableroot/$datestr/global-workflow. + +EOF fi +echo $SUBJECT +mail -r "Darth Vader - NOAA Affiliate " -s "$SUBJECT" "$PEOPLE" < $BODY # ============================================================================== # publish some information to RZDM for quick viewing # THIS IS A TODO FOR NOW diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index daa5b8edf..1741a20f6 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -1,7 +1,7 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -ecbuild_add_test(TARGET test_gdasapp_setup_atm_cycled_exp +ecbuild_add_test(TARGET test_gdasapp_setup_atm_jjob_cycled_exp TYPE SCRIPT COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} @@ -12,7 +12,7 @@ ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_init COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_init.sh ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun - TEST_DEPENDS test_gdasapp_setup_atm_cycled_exp) + TEST_DEPENDS test_gdasapp_setup_atm_jjob_cycled_exp) ecbuild_add_test(TARGET test_gdasapp_atm_jjob_var_run TYPE SCRIPT @@ -40,7 +40,7 @@ ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_init COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init.sh ARGS ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun - TEST_DEPENDS test_gdasapp_setup_atm_cycled_exp) + TEST_DEPENDS test_gdasapp_setup_atm_jjob_cycled_exp) ecbuild_add_test(TARGET test_gdasapp_atm_jjob_ens_letkf TYPE SCRIPT