Update CI driver

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+# Description
+
+<!-- Enter PR description here. -->
+
+# Companion PRs
+
+<!-- Enter links to any companion PRs here. -->
+
+# Issues
+
+<!-- Enter any issues referenced or resolved by this PR here. Use keywords "Resolves" or "Refs".
+Resolves #1234
+Refs #4321
+Refs NOAA-EMC/repo#5678
+-->
+
+# Automated CI tests to run in Global Workflow
+<!-- Which Global Workflow CI tests are required to adequately test this PR? -->
+- [ ] atm_jjob <!-- JEDI atm single cycle DA !-->
+- [ ] C96C48_ufs_hybatmDA <!-- JEDI atm cycled DA !-->
+- [ ] C96C48_hybatmaerosnowDA  <!-- JEDI aero/snow cycled DA !-->
+- [ ] C48mx500_3DVarAOWCDA <!-- JEDI low-res marine 3DVar cycled DA  !-->
+- [ ] C48mx500_hybAOWCDA <!-- JEDI marine hybrid envar cycled DA !-->
+- [ ] C96C48_hybatmDA <!-- GSI atm cycled DA !-->

ci/ci_tests.sh

@@ -0,0 +1,6 @@
+CI_TESTS=("atm_jjob"
+          "C96C48_ufs_hybatmDA"
+          "C96C48_hybatmaerosnowDA"
+          "C48mx500_3DVarAOWCDA"
+          "C48mx500_hybAOWCDA"
+          "C96C48_hybatmDA")

ci/driver.sh

@@ -1,6 +1,6 @@
 #!/bin/bash --login
 
-echo "Start at $(date)"
+echo "Starting automated testing at $(date)"
 
 my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
 echo "Set my_dir ${my_dir}"
@@ -13,6 +13,7 @@ usage() {
   echo
   echo "  -t  target/machine script is running on    DEFAULT: $(hostname)"
   echo "  -h  display this message and quit"
+  echo "  -w  run workflow tests on $(hostname)"
   echo
   exit 1
 }
@@ -22,20 +23,25 @@ usage() {
 
 export TARGET="$(hostname)"
 
-while getopts "t:h" opt; do
+TEST_WORKFLOW=0
+while getopts "t:h:w" opt; do
   case $opt in
     t)
       TARGET=$OPTARG
       ;;
     h|\?|:)
       usage
       ;;
+    w)
+      TEST_WORKFLOW=1
+      ;;  
   esac
 done
 
+echo "Running automated testing on $TARGET"
+
 case ${TARGET} in
-  hera | orion)
-    echo "Running Automated Testing on $TARGET"
+  hera | orion | hercules)
     source $MODULESHOME/init/sh
     source $my_dir/${TARGET}.sh
     module purge
@@ -49,106 +55,176 @@ case ${TARGET} in
     ;;
 esac
 
+# ==============================================================================
+# set list of available CI tests to run on the Global Workflow
+source $my_dir/ci_tests.sh
+
+# ==============================================================================
+# set things that depend on whether running workflow tests or not
+gdasapp_url="https://github.com/NOAA-EMC/GDASApp.git"
+if [[ $TEST_WORKFLOW == 1 ]]; then
+  echo "Testing GDASApp inside the Global Workflow"
+
+  CI_LABEL="${GDAS_CI_HOST}-GW-RT"
+  OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list_gw
+  PR_TEST_DIR=$GDAS_CI_ROOT/workflow/PR
+  BASE_REPO=global-workflow
+
+  # Default Global Workflow repo and branch if no companion PR found
+  workflow_url="https://github.com/NOAA-EMC/global-workflow.git"
+  workflow_branch="develop"
+else
+  echo "Testing stand-alone GDASApp"
+
+  CI_LABEL="${GDAS_CI_HOST}-RT"
+  OPEN_PR_LIST_DIR=$GDAS_CI_ROOT/open_pr_list
+  PR_TEST_DIR=$GDAS_CI_ROOT/PR
+  BASE_REPO=GDASApp
+fi
+
 # ==============================================================================
 # pull on the repo and get list of open PRs
+
 cd $GDAS_CI_ROOT/repo
-CI_LABEL="${GDAS_CI_HOST}-RT"
-gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $GDAS_CI_ROOT/open_pr_list
 
-open_pr=`cat $GDAS_CI_ROOT/open_pr_list | wc -l`
+gh pr list --label "$CI_LABEL" --state "open" | awk '{print $1;}' > $OPEN_PR_LIST_DIR
+
+open_pr=`cat $OPEN_PR_LIST_DIR | wc -l`
 if (( $open_pr == 0 )); then
   echo "No open PRs with ${CI_LABEL}, exit."
-  echo "Finish at $(date)"    
+  echo "Finished automated testing at $(date)"    
   exit
 fi
 
-open_pr_list=$(cat $GDAS_CI_ROOT/open_pr_list)
+open_pr_list=$(cat $OPEN_PR_LIST_DIR)
 
 # ==============================================================================
 # clone, checkout, build, test, etc.
-repo_url="https://github.com/NOAA-EMC/GDASApp.git"
 # loop through all open PRs
 for pr in $open_pr_list; do
   echo " "
-  echo "Start processing Pull Request #${pr} at $(date)"
+  echo "Starting processing of pull request #${pr} at $(date)"
 
   # get the branch name used for the PR
   gdasapp_branch=$(gh pr view $pr --json headRefName -q ".headRefName")
 
-  # get additional branch information
-  branch_owner=$(gh pr view $pr --repo ${repo_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login')
-  branch_name=$(gh pr view $pr --repo ${repo_url} --json headRepository --jq '.headRepository.name')
-  pr_assignees=$(gh pr view $pr --repo ${repo_url} --json assignees --jq '.assignees[].login')
-
+  # get additional branch informatio
+  branch_owner=$(gh pr view $pr --repo ${gdasapp_url} --json headRepositoryOwner --jq '.headRepositoryOwner.login')
+  branch_name=$(gh pr view $pr --repo ${gdasapp_url} --json headRepository --jq '.headRepository.name')
+  pr_assignees=$(gh pr view $pr --repo ${gdasapp_url} --json assignees --jq '.assignees[].login')
+  
   # check if any assignee is authorized to run CI
-  authorized_by=""
+  rc=1
   for str in ${pr_assignees[@]}; do
-    grep $str /scratch1/NCEPDEV/da/role.jedipara/CI/GDASApp/authorized_users
-    rc=$?
+    grep $str $AUTHORIZED_USERS_FILE > /dev/null
+    if (( rc != 0 )); then
+	rc=$?
+    fi	
     if (( rc == 0 )); then
-      authorized_by=${str}
-      echo "FOUND MATCH $str, rc $rc"
-      break
+      echo "Authorized user $str assigned to this PR"
     fi
   done
 
   # Authorized to run CI
   if (( rc == 0 )); then
-    echo "Run CI"
+    echo "CI authorized. Running CI..."
 
     # update PR label
     gh pr edit $pr --remove-label $CI_LABEL --add-label ${CI_LABEL}-Running
-
-    # construct the fork URL
-    gdasapp_url="https://github.com/$branch_owner/${branch_name}.git"
-
+
     echo "GDASApp URL: $gdasapp_url"
     echo "GDASApp branch Name: $gdasapp_branch"
-    echo "CI authorized by $authorized_by at $(date)"
+
+    if [[ $TEST_WORKFLOW == 1 ]]; then
+      # check for a companion PR in the global-workflow
+      companion_pr_exists=$(gh pr list --repo ${workflow_url} --head ${gdasapp_branch} --state open)
+
+      if [ -n "$companion_pr_exists" ]; then
+        # get the PR number
+        companion_pr=$(echo "$companion_pr_exists" | awk '{print $1;}')
+
+        # extract the necessary info
+        branch_owner=$(gh pr view $companion_pr --repo $workflow_url --json headRepositoryOwner --jq '.headRepositoryOwner.login')
+        branch_name=$(gh pr view $companion_pr --repo $workflow_url --json headRepository --jq '.headRepository.name')
+
+        # Construct fork URL. Update workflow branch name
+        workflow_url="https://github.com/$branch_owner/$branch_name.git"
+        workflow_branch=$gdasapp_branch
+      fi
+
+      echo "Found companion Global Workflow PR #${companion_pr}!"
+      echo "Global Workflow URL: $workflow_url"
+      echo "Global Workflow branch name: $workflow_branch"      
+    fi
 
-    # create PR specific directory
-    if [ -d $GDAS_CI_ROOT/PR/$pr ]; then
-        rm -rf $GDAS_CI_ROOT/PR/$pr
+    # create PR specific directory    
+    if [ -d $PR_TEST_DIR/$pr ]; then
+        rm -rf $PR_TEST_DIR/$pr
     fi
-    mkdir -p $GDAS_CI_ROOT/PR/$pr
-    cd $GDAS_CI_ROOT/PR/$pr
+    mkdir -p $PR_TEST_DIR/$pr
+    cd $PR_TEST_DIR/$pr        
     pwd
 
     # clone copy of repo
-    git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url
-    cd GDASApp
+    if [[ $TEST_WORKFLOW == 1 ]]; then
+      echo "Cloning Global Workflow branch $workflow_branch from $workflow_url at $(date)"
+      git clone --recursive --jobs 8 --branch $workflow_branch $workflow_url
+      cd global-workflow/sorc/gdas.cd
+    else
+      echo "Cloning GDASApp branch $workflow_branch at $(date)"	
+      git clone --recursive --jobs 8 --branch $gdasapp_branch $gdasapp_url
+      cd GDASApp
+    fi
     pwd
 
     # checkout GDASApp pull request
-    git pull
     gh pr checkout $pr
     git submodule update --init --recursive
 
     # get commit hash
     commit=$(git log --pretty=format:'%h' -n 1)
-    echo "$commit" > $GDAS_CI_ROOT/PR/$pr/commit
+    echo "$commit" > $PR_TEST_DIR/$pr/commit
 
     # run build and testing command
-    echo "Execute $my_dir/run_ci.sh for $GDAS_CI_ROOT/PR/$pr/GDASApp at $(date)"
-    $my_dir/run_ci.sh -d $GDAS_CI_ROOT/PR/$pr/GDASApp -o $GDAS_CI_ROOT/PR/$pr/output_${commit}
+    echo "Running run_ci.sh for $PR_TEST_DIR/$pr/$BASE_REPO at $(date)"
+    run_ci_cmd="$my_dir/run_ci.sh -d $PR_TEST_DIR/$pr/$BASE_REPO -o $PR_TEST_DIR/$pr/output_${commit}"
+    if [[ $TEST_WORKFLOW == 1 ]]; then
+      # get ci tests from PR description and convert into a regular expressions to be excluded
+      branch_body=$(gh pr view $pr --repo ${gdasapp_url} --json body --jq '.body')
+      ci_checklist=$(echo "$branch_body" | grep '\[x\]')
+      ctest_regex_exclude=""  
+      for ci_test in ${CI_TESTS[@]}; do
+        if ! echo "$ci_checklist" | grep -q "$ci_test"; then
+	  ctest_regex_exclude+="${ctest_regex_exclude:+|}$ci_test"
+	fi      
+      done
+
+      # setup run_ci.sh arguments to test in the Global Workflow and exclude chosen CI tests
+      run_ci_cmd+=" -w"
+      if [ -n "$ctest_regex_exclude" ]; then
+        run_ci_cmd+=" -E $ctest_regex_exclude"
+      fi
+    fi
+    $run_ci_cmd
     ci_status=$?
-    echo "After run_ci.sh with ci_status ${ci_status} at $(date)"
-    gh pr comment $pr --repo ${repo_url} --body-file $GDAS_CI_ROOT/PR/$pr/output_${commit}
+    echo "Finished running run_ci.sh with ci_status ${ci_status} at $(date)"
+
+    gh pr comment $pr --repo ${gdasapp_url} --body-file $PR_TEST_DIR/$pr/output_${commit}
     if [ $ci_status -eq 0 ]; then
-      gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed
+      gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Passed
     else
-      gh pr edit $pr --repo ${repo_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed
+      gh pr edit $pr --repo ${gdasapp_url} --remove-label ${CI_LABEL}-Running --add-label ${CI_LABEL}-Failed
     fi
 
   # Not authorized to run CI
   else
-    echo "Do NOT run CI"
+    echo "No authorized users assigned to this PR. Aborting CI..."
   fi
 
-  echo "Finish processing Pull Request #{pr} at $(date)"
+  echo "Finished processing Pull Request #${pr} at $(date)"
 done
 
 # ==============================================================================
 # scrub working directory for older files
-find $GDAS_CI_ROOT/PR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \;
-echo "Finish at $(date)"
+find $PR_TEST_DIR/* -maxdepth 1 -mtime +3 -exec rm -rf {} \;
+echo "Finished automated testing at $(date)"