Merge branch 'master' of https://github.com/instructure/canvas-lms

Signed-off-by: Pratap Chand <[email protected]>

.nvmrc

@@ -1 +1 @@
-v10.15.3
+v10.19.0

Dockerfile

@@ -78,7 +78,6 @@ RUN mkdir -p .yardoc \
              client_apps/canvas_quizzes/tmp \
              config/locales/generated \
              gems/canvas_i18nliner/node_modules \
-             gems/selinimum/node_modules \
              log \
              node_modules \
              packages/canvas-planner/lib \

Gemfile.d/app.rb

@@ -74,12 +74,12 @@
   gem 'statsd-ruby', '1.4.0', require: false
   gem 'aroi', '0.0.7', require: false
   gem 'dogstatsd-ruby', '4.7.0'
-gem 'inst-jobs', '0.15.16'
+gem 'inst-jobs', '0.15.20'
   gem 'fugit', '1.3.3', require: false
     gem 'et-orbi', '1.2.2', require: false
 gem 'inst-jobs-autoscaling', '1.0.5'
   gem 'aws-sdk-autoscaling', '1.32.0', require: false
-gem 'inst-jobs-statsd', '1.3.2'
+gem 'inst-jobs-statsd', '1.4.0'
 gem 'json', '2.3.0'
 gem 'json_schemer', '0.2.10'
 gem 'json-jwt', '1.11.0', require: false
@@ -118,9 +118,9 @@
 gem 'sentry-raven', '2.13.0', require: false
 gem 'shackles', '1.4.2'
 gem 'simple_oauth', '0.3.1', require: false
-gem 'switchman', '1.14.9'
+gem 'switchman', '1.14.10'
   gem 'open4', '1.3.4', require: false
-gem 'switchman-inst-jobs', '1.3.7'
+gem 'switchman-inst-jobs', '1.4.1'
 gem 'twilio-ruby', '5.31.1', require: false
 gem 'tzinfo', '1.2.5'
 gem 'vault', '0.13.0', require: false

Gemfile.d/test.rb

@@ -48,7 +48,6 @@
   gem 'selenium-webdriver', '3.142.7'
     gem 'childprocess', '3.0.0', require: false
   gem 'webdrivers', '4.2.0', require: false
-  gem 'selinimum', '0.0.1', require: false, path: 'gems/selinimum'
   gem 'test-queue', github: 'instructure/test-queue', ref: 'd35166408df3a5396cd809e85dcba175136a69ba', require: false
   gem 'testrailtagging', '0.3.8.7', require: false
 

Jenkinsfile

@@ -17,6 +17,9 @@
  * You should have received a copy of the GNU Affero General Public License along
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+import org.jenkinsci.plugins.workflow.support.steps.build.DownstreamFailureCause
+import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException
+
 def buildParameters = [
   string(name: 'GERRIT_REFSPEC', value: "${env.GERRIT_REFSPEC}"),
   string(name: 'GERRIT_EVENT_TYPE', value: "${env.GERRIT_EVENT_TYPE}"),
@@ -49,13 +52,55 @@ def skipIfPreviouslySuccessful(name, block) {
   }
 }
 
+def wrapBuildExecution(jobName, parameters, propagate, urlExtra) {
+  try {
+    build(job: jobName, parameters: parameters, propagate: propagate)
+  }
+  catch(FlowInterruptedException ex) {
+    // if its this type, then that means its a build failure.
+    // other reasons can be user cancelling or jenkins aborting, etc...
+    def failure = ex.causes.find { it instanceof DownstreamFailureCause }
+    if (failure != null) {
+      def downstream = failure.getDownstreamBuild()
+      def url = downstream.getAbsoluteUrl() + urlExtra
+      load('build/new-jenkins/groovy/reports.groovy').appendFailMessageReport(jobName, url)
+    }
+    throw ex
+  }
+}
+
+// if the build never starts or gets into a node block, then we
+// can never load a file. and a very noisy/confusing error is thrown.
+def ignoreBuildNeverStartedError(block) {
+  try {
+    block()
+  }
+  catch (org.jenkinsci.plugins.workflow.steps.MissingContextVariableException ex) {
+    if (!ex.message.startsWith('Required context class hudson.FilePath is missing')) {
+      throw ex
+    }
+    else {
+      echo "ignored MissingContextVariableException: \n${ex.message}"
+    }
+    // we can ignore this very noisy error
+  }
+}
+
+def buildRegistryFQDN() {
+  load('build/new-jenkins/groovy/configuration.groovy').buildRegistryFQDN()
+}
+
 // ignore builds where the current patchset tag doesn't match the
 // mainline publishable tag. i.e. ignore ruby-passenger-2.6/pg-12
 // upgrade builds
 def isPatchsetPublishable() {
   env.PATCHSET_TAG == env.PUBLISHABLE_TAG
 }
 
+def isPatchsetSlackableOnFailure() {
+  env.SLACK_MESSAGE_ON_FAILURE == 'true' && env.GERRIT_EVENT_TYPE == 'change-merged'
+}
+
 // WARNING! total hack, being removed after covid...
 def isCovid() {
   env.GERRIT_BRANCH == 'covid'
@@ -71,26 +116,31 @@ pipeline {
     GERRIT_URL = "$GERRIT_HOST:$GERRIT_PORT"
     NAME = getImageTagVersion()
     CANVAS_LMS_IMAGE = "$DOCKER_REGISTRY_FQDN/jenkins/canvas-lms"
+    BUILD_REGISTRY_FQDN = buildRegistryFQDN()
+    BUILD_IMAGE = "$BUILD_REGISTRY_FQDN/jenkins/canvas-lms"
 
     // e.g. postgres-9.5-ruby-passenger-2.4-xenial
     TAG_SUFFIX = "postgres-$POSTGRES-ruby-passenger-$RUBY_PASSENGER"
 
     // e.g. canvas-lms:01.123456.78-postgres-12-ruby-passenger-2.6
-    PATCHSET_TAG = "$CANVAS_LMS_IMAGE:$NAME-$TAG_SUFFIX"
+    PATCHSET_TAG = "$BUILD_IMAGE:$NAME-$TAG_SUFFIX"
 
     // e.g. canvas-lms:01.123456.78-postgres-9.5-ruby-passenger-2.4-xenial
-    PUBLISHABLE_TAG = "$CANVAS_LMS_IMAGE:$NAME-postgres-9.5-ruby-passenger-2.4-xenial"
+    PUBLISHABLE_TAG = "$BUILD_IMAGE:$NAME-postgres-9.5-ruby-passenger-2.4-xenial"
 
     // e.g. canvas-lms:master when not on another branch
     MERGE_TAG = "$CANVAS_LMS_IMAGE:$GERRIT_BRANCH"
+
+    // e.g. canvas-lms:01.123456.78; this is for consumers like Portal 2 who want to build a patchset
+    EXTERNAL_TAG = "$CANVAS_LMS_IMAGE:$NAME"
   }
 
   stages {
     stage('Setup') {
       steps {
         timeout(time: 5) {
           script {
-            runDatadogMetric("Setup"){
+            runDatadogMetric("Setup") {
               sh 'build/new-jenkins/print-env-excluding-secrets.sh'
               sh 'build/new-jenkins/docker-cleanup.sh'
 
@@ -112,8 +162,9 @@ pipeline {
               if (isCovid() && env.GERRIT_PROJECT != 'canvas-lms') {
                 echo 'checking out canvas-lms covid branch'
                 credentials.withGerritCredentials {
-                  sh '''
-                    set -ex
+                  sh '''#!/bin/bash
+                    set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
+
                     git branch -D covid || true
                     GIT_SSH_COMMAND='ssh -i \"$SSH_KEY_PATH\" -l \"$SSH_USERNAME\"' \
                       git fetch origin $GERRIT_BRANCH:origin/$GERRIT_BRANCH
@@ -144,27 +195,27 @@ pipeline {
                   // end of hack (covid)
                 }
               }
-            credentials.fetchFromGerrit('gergich_user_config', '.')
-            credentials.fetchFromGerrit('qti_migration_tool', 'vendor', 'QTIMigrationTool')
-
-            sh 'mv -v gerrit_builder/canvas-lms/config/* config/'
-            sh 'rm -v config/cache_store.yml'
-            sh 'rmdir -p gerrit_builder/canvas-lms/config'
-            sh 'rm -v config/database.yml'
-            sh 'rm -v config/security.yml'
-            sh 'rm -v config/selenium.yml'
-            sh 'cp -v docker-compose/config/selenium.yml config/'
-            sh 'cp -vR docker-compose/config/new-jenkins/* config/'
-            sh 'cp -v config/delayed_jobs.yml.example config/delayed_jobs.yml'
-            sh 'cp -v config/domain.yml.example config/domain.yml'
-            sh 'cp -v config/external_migration.yml.example config/external_migration.yml'
-            sh 'cp -v config/outgoing_mail.yml.example config/outgoing_mail.yml'
-            sh 'cp -v ./gergich_user_config/gergich_user_config.yml ./gems/dr_diff/config/gergich_user_config.yml'
+              credentials.fetchFromGerrit('gergich_user_config', '.')
+              credentials.fetchFromGerrit('qti_migration_tool', 'vendor', 'QTIMigrationTool')
+
+              sh 'mv -v gerrit_builder/canvas-lms/config/* config/'
+              sh 'rm -v config/cache_store.yml'
+              sh 'rmdir -p gerrit_builder/canvas-lms/config'
+              sh 'rm -v config/database.yml'
+              sh 'rm -v config/security.yml'
+              sh 'rm -v config/selenium.yml'
+              sh 'cp -v docker-compose/config/selenium.yml config/'
+              sh 'cp -vR docker-compose/config/new-jenkins/* config/'
+              sh 'cp -v config/delayed_jobs.yml.example config/delayed_jobs.yml'
+              sh 'cp -v config/domain.yml.example config/domain.yml'
+              sh 'cp -v config/external_migration.yml.example config/external_migration.yml'
+              sh 'cp -v config/outgoing_mail.yml.example config/outgoing_mail.yml'
+              sh 'cp -v ./gergich_user_config/gergich_user_config.yml ./gems/dr_diff/config/gergich_user_config.yml'
+            }
           }
         }
       }
     }
-  }
 
     stage('Rebase') {
       when { expression { env.GERRIT_EVENT_TYPE == 'patchset-created' && env.GERRIT_PROJECT == 'canvas-lms' } }
@@ -174,7 +225,9 @@ pipeline {
             runDatadogMetric("Rebase") {
               def credentials = load('build/new-jenkins/groovy/credentials.groovy')
               credentials.withGerritCredentials({ ->
-                sh '''
+                sh '''#!/bin/bash
+                  set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
+
                   GIT_SSH_COMMAND='ssh -i \"$SSH_KEY_PATH\" -l \"$SSH_USERNAME\"' \
                     git fetch origin $GERRIT_BRANCH:origin/$GERRIT_BRANCH
 
@@ -227,8 +280,12 @@ pipeline {
                     .
                 """
               }
+              sh "docker push $PATCHSET_TAG"
+              if (isPatchsetPublishable()) {
+                sh 'docker tag $PATCHSET_TAG $EXTERNAL_TAG'
+                sh 'docker push $EXTERNAL_TAG'
+              }
             }
-            sh "docker push $PATCHSET_TAG"
           }
         }
       }
@@ -256,32 +313,31 @@ pipeline {
           echo 'adding Vendored Gems'
           stages['Vendored Gems'] = {
             skipIfPreviouslySuccessful("vendored-gems") {
-              build(job: 'test-suites/vendored-gems', parameters: buildParameters)
+              wrapBuildExecution('test-suites/vendored-gems', buildParameters, true, "")
             }
           }
 
           echo 'adding Javascript'
           stages['Javascript'] = {
             skipIfPreviouslySuccessful("javascript") {
-              build(job: 'test-suites/JS', parameters: buildParameters)
+              wrapBuildExecution('test-suites/JS', buildParameters, true, "testReport")
             }
           }
 
           echo 'adding Contract Tests'
           stages['Contract Tests'] = {
             skipIfPreviouslySuccessful("contract-tests") {
-              build(job: 'test-suites/contract-tests', parameters: buildParameters)
+              wrapBuildExecution('test-suites/contract-tests', buildParameters, true, "")
             }
           }
 
           if (env.GERRIT_EVENT_TYPE != 'change-merged' && !isCovid()) {
             echo 'adding Flakey Spec Catcher'
             stages['Flakey Spec Catcher'] = {
               skipIfPreviouslySuccessful("flakey-spec-catcher") {
-                build(
-                  job: 'test-suites/flakey-spec-catcher',
-                  parameters: buildParameters
-                )
+                def propagate = load('build/new-jenkins/groovy/configuration.groovy').fscPropagate()
+                echo "fsc propagation: $propagate"
+                wrapBuildExecution('test-suites/flakey-spec-catcher', buildParameters, propagate, "")
               }
             }
           }
@@ -368,7 +424,7 @@ pipeline {
   post {
     failure {
       script {
-        if (isPatchsetPublishable() && env.GERRIT_EVENT_TYPE == 'change-merged') {
+        if (isPatchsetSlackableOnFailure()) {
           def branchSegment = env.GERRIT_BRANCH ? "[$env.GERRIT_BRANCH]" : ''
           def authorSegment = env.GERRIT_EVENT_ACCOUNT_NAME ? "Patchset by ${env.GERRIT_EVENT_ACCOUNT_NAME}. " : ''
           slackSend(
@@ -381,13 +437,18 @@ pipeline {
     }
     always {
       script {
-        def rspec = load 'build/new-jenkins/groovy/rspec.groovy'
-        rspec.uploadSeleniumFailures()
-        rspec.uploadRSpecFailures()
+        ignoreBuildNeverStartedError {
+          def rspec = load 'build/new-jenkins/groovy/rspec.groovy'
+          rspec.uploadSeleniumFailures()
+          rspec.uploadRSpecFailures()
+          load('build/new-jenkins/groovy/reports.groovy').sendFailureMessageIfPresent()
+        }
       }
     }
     cleanup {
-      sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
+      ignoreBuildNeverStartedError {
+        sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
+      }
     }
   }
 }

Jenkinsfile.contract-tests

@@ -25,6 +25,7 @@ pipeline {
   environment {
     COMPOSE_FILE = 'docker-compose.new-jenkins.yml'
     PACT_BROKER = credentials('PACT_BROKER')
+    POSTGRES_PASSWORD = 'sekret'
   }
 
   stages {
@@ -41,7 +42,7 @@ pipeline {
         timeout(time: 10) {
           sh 'build/new-jenkins/docker-compose-pull.sh'
           sh 'build/new-jenkins/docker-compose-build-up.sh'
-          sh 'build/new-jenkins/docker-compose-create-migrate-database.sh'
+          sh 'build/new-jenkins/docker-compose-setup-databases.sh'
         }
       }
     }