Merge branch 'master' into feature/APPEALS-21351-merged

app/controllers/idt/api/v2/distributions_controller.rb

@@ -40,14 +40,18 @@ def format_response(response)
     response_body = response.raw_body
 
     begin
-      parsed_response = JSON.parse(response_body)
+      parsed_response = if [ActiveSupport::HashWithIndifferentAccess, Hash].include?(response_body.class)
+                          response_body
+                        else
+                          JSON.parse(response_body)
+                        end
 
       # Convert keys from camelCase to snake_case
       parsed_response.deep_transform_keys do |key|
         key.to_s.underscore.gsub(/e(\d)/, 'e_\1')
       end
-    rescue JSON::ParseError => error
-      log_error(error + " Distribution ID: #{params[:distribution_id]}")
+    rescue StandardError => error
+      log_error(error)
 
       response_body
     end

app/jobs/ama_notification_efolder_sync_job.rb

@@ -15,9 +15,15 @@ class AmaNotificationEfolderSyncJob < CaseflowJob
   def perform
     RequestStore[:current_user] = User.system_user
 
-    all_active_ama_appeals = appeals_recently_outcoded + appeals_never_synced + ready_for_resync
-
-    sync_notification_reports(all_active_ama_appeals.first(BATCH_LIMIT.to_i))
+    all_active_ama_appeals = if FeatureToggle.enabled?(:phase_1_notification_sync_job_rollout)
+                               appeals_never_synced
+                             elsif FeatureToggle.enabled?(:phase_2_notification_sync_job_rollout)
+                               appeals_never_synced + ready_for_resync
+                             else
+                               appeals_recently_outcoded + appeals_never_synced + ready_for_resync
+                             end
+
+    sync_notification_reports(all_active_ama_appeals.uniq(&:id).first(BATCH_LIMIT.to_i))
   end
 
   private
@@ -98,18 +104,21 @@ def ready_for_resync
   # Return: Array of active appeals
   def get_appeals_from_prev_synced_ids(appeal_ids)
     appeal_ids.in_groups_of(1000, false).flat_map do |ids|
-      Appeal.active.find_by_sql(
+      Appeal.find_by_sql(
         <<-SQL
-          SELECT appeals.*
-          FROM appeals
+          SELECT appeals.* FROM appeals
+          JOIN tasks t ON appeals.id = t.appeal_id
+          AND t.appeal_type = 'Appeal'
           JOIN (#{appeals_on_latest_notifications(ids)}) AS notifs ON
             notifs.appeals_id = appeals."uuid"::text AND notifs.appeals_type = 'Appeal'
           JOIN (#{appeals_on_latest_doc_uploads(ids)}) AS vbms_uploads ON
             vbms_uploads.appeal_id = appeals.id AND vbms_uploads.appeal_type = 'Appeal'
-          WHERE
+          WHERE (
             notifs.notified_at > vbms_uploads.attempted_at
           OR
             notifs.created_at > vbms_uploads.attempted_at
+          )
+          AND t.TYPE = 'RootTask' AND t.status NOT IN ('completed', 'cancelled')
           GROUP BY appeals.id
         SQL
       )
@@ -120,8 +129,16 @@ def appeals_on_latest_notifications(appeal_ids)
     <<-SQL
       SELECT n1.* FROM appeals a
       JOIN notifications n1 on n1.appeals_id = a."uuid"::text AND n1.appeals_type = 'Appeal'
-      LEFT OUTER JOIN notifications n2 ON (n2.appeals_id = a."uuid"::text AND n1.appeals_type = 'Appeal' AND
-          (n1.notified_at < n2.notified_at OR (n1.notified_at = n2.notified_at AND n1.id < n2.id)))
+      AND (n1.email_notification_status IS NULL OR
+        n1.email_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+      AND (n1.sms_notification_status IS NULL OR
+          n1.sms_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+      LEFT OUTER JOIN notifications n2 ON (n2.appeals_id = a."uuid"::text AND n1.appeals_type = 'Appeal'
+        AND (n2.email_notification_status IS NULL OR
+          n2.email_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+        AND (n2.sms_notification_status IS NULL OR
+            n2.sms_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+        AND (n1.notified_at < n2.notified_at OR (n1.notified_at = n2.notified_at AND n1.id < n2.id)))
       WHERE n2.id IS NULL
         AND n1.id IS NOT NULL
         AND (n1.email_notification_status <> 'Failure Due to Deceased'

app/jobs/legacy_notification_efolder_sync_job.rb

@@ -15,9 +15,15 @@ class LegacyNotificationEfolderSyncJob < CaseflowJob
   def perform
     RequestStore[:current_user] = User.system_user
 
-    all_active_legacy_appeals = appeals_recently_outcoded + appeals_never_synced + ready_for_resync
-
-    sync_notification_reports(all_active_legacy_appeals.first(BATCH_LIMIT.to_i))
+    all_active_legacy_appeals = if FeatureToggle.enabled?(:phase_1_notification_sync_job_rollout)
+                                  appeals_never_synced
+                                elsif FeatureToggle.enabled?(:phase_2_notification_sync_job_rollout)
+                                  appeals_never_synced + ready_for_resync
+                                else
+                                  appeals_recently_outcoded + appeals_never_synced + ready_for_resync
+                                end
+
+    sync_notification_reports(all_active_legacy_appeals.uniq(&:id).first(BATCH_LIMIT.to_i))
   end
 
   private
@@ -71,7 +77,7 @@ def previous_case_notifications_document_join_clause
   end
 
   def open_root_task_join_clause
-    "JOIN tasks t ON t.appeal_type = 'LegacyAppeal' AND t.id = legacy_appeals.id \
+    "JOIN tasks t ON t.appeal_type = 'LegacyAppeal' AND t.appeal_id = legacy_appeals.id \
       AND t.type = 'RootTask' AND t.status NOT IN ('completed', 'cancelled')"
   end
 
@@ -99,31 +105,41 @@ def ready_for_resync
   # Return: Array of active appeals
   def get_appeals_from_prev_synced_ids(appeal_ids)
     appeal_ids.in_groups_of(1000, false).flat_map do |ids|
-      LegacyAppeal.where(id: RootTask.open.where(appeal_type: "LegacyAppeal").pluck(:appeal_id))
-        .find_by_sql(
-          <<-SQL
-              SELECT la.*
-              FROM legacy_appeals la
+      LegacyAppeal.find_by_sql(
+        <<-SQL
+              SELECT la.* FROM legacy_appeals la
+              JOIN tasks t ON la.id = t.appeal_id
+              AND t.appeal_type = 'LegacyAppeal'
               JOIN (#{appeals_on_latest_notifications(ids)}) AS notifs ON
                 notifs.appeals_id = la.vacols_id AND notifs.appeals_type = 'LegacyAppeal'
               JOIN (#{appeals_on_latest_doc_uploads(ids)}) AS vbms_uploads ON
                 vbms_uploads.appeal_id = la.id AND vbms_uploads.appeal_type = 'LegacyAppeal'
-              WHERE
+              WHERE (
                 notifs.notified_at > vbms_uploads.attempted_at
               OR
                 notifs.created_at > vbms_uploads.attempted_at
+              )
+              AND t.type = 'RootTask' AND t.status NOT IN ('completed', 'cancelled')
               GROUP BY la.id
-          SQL
-        )
+        SQL
+      )
     end
   end
 
   def appeals_on_latest_notifications(appeal_ids)
     <<-SQL
       SELECT n1.* FROM legacy_appeals a
       JOIN notifications n1 on n1.appeals_id = a.vacols_id AND n1.appeals_type = 'LegacyAppeal'
-      LEFT OUTER JOIN notifications n2 ON (n2.appeals_id = a.vacols_id AND n1.appeals_type = 'LegacyAppeal' AND
-          (n1.notified_at < n2.notified_at OR (n1.notified_at = n2.notified_at AND n1.id < n2.id)))
+      AND (n1.email_notification_status IS NULL OR
+        n1.email_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+      AND (n1.sms_notification_status IS NULL OR
+          n1.sms_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+      LEFT OUTER JOIN notifications n2 ON (n2.appeals_id = a.vacols_id AND n1.appeals_type = 'LegacyAppeal'
+        AND (n2.email_notification_status IS NULL OR
+          n2.email_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+        AND (n2.sms_notification_status IS NULL OR
+            n2.sms_notification_status NOT IN ('No Participant Id Found', 'No Claimant Found', 'No External Id'))
+        AND (n1.notified_at < n2.notified_at OR (n1.notified_at = n2.notified_at AND n1.id < n2.id)))
       WHERE n2.id IS NULL
         AND n1.id IS NOT NULL
         AND (n1.email_notification_status <> 'Failure Due to Deceased'

app/models/end_product_establishment.rb

@@ -50,7 +50,7 @@ class ContentionNotFound < StandardError; end
 
   class << self
     def order_by_sync_priority
-      active.order("last_synced_at IS NOT NULL, last_synced_at ASC")
+      active.order(Arel.sql("last_synced_at IS NOT NULL, last_synced_at ASC"))
     end
 
     def established

app/queries/batch_users_for_reader_query.rb

@@ -7,7 +7,7 @@ def self.process
     User.where("(efolder_documents_fetched_at <= ? " \
                "OR efolder_documents_fetched_at IS NULL) " \
                "AND last_login_at >= ?", 24.hours.ago, 1.week.ago)
-      .order("efolder_documents_fetched_at IS NULL DESC, efolder_documents_fetched_at ASC")
+      .order(Arel.sql("efolder_documents_fetched_at IS NULL DESC, efolder_documents_fetched_at ASC"))
       .limit(DEFAULT_USERS_LIMIT)
   end
 end

app/services/deprecation_warnings/base_handler.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# @abstract Subclass and override {.call} to implement a custom DeprecationWarnings handler class.
+# @note For use with `ActiveSupport::Deprecation.behavior=`.
+module DeprecationWarnings
+  class BaseHandler
+    class << self
+      # Subclasses must respond to `.call` to play nice with `ActiveSupport::Deprecation.behavior=`.
+      #   https://github.com/rails/rails/blob/a4581b53aae93a8dd3205abae0630398cbce9204/activesupport/lib/active_support/deprecation/behaviors.rb#L70-L71
+      # :reek:LongParameterList
+      def call(_message, _callstack, _deprecation_horizon, _gem_name)
+        fail NotImplementedError
+      end
+
+      # Subclasses must respond to `.arity` to play nice with `ActiveSupport::Deprecation.behavior=`.
+      #   Must return number of arguments accepted by `.call`.
+      #   https://github.com/rails/rails/blob/a4581b53aae93a8dd3205abae0630398cbce9204/activesupport/lib/active_support/deprecation/behaviors.rb#L101
+      def arity
+        method(:call).arity
+      end
+    end
+  end
+end

app/services/deprecation_warnings/development_handler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "disallowed_deprecations"
+
+# @note For use with `ActiveSupport::Deprecation.behavior=`.
+module DeprecationWarnings
+  class DevelopmentHandler < BaseHandler
+    extend DisallowedDeprecations
+
+    class << self
+      # :reek:LongParameterList
+      def call(message, _callstack, _deprecation_horizon, _gem_name)
+        raise_if_disallowed_deprecation!(message)
+        emit_warning_to_application_logs(message)
+      end
+
+      private
+
+      def emit_warning_to_application_logs(message)
+        Rails.logger.warn(message)
+      end
+    end
+  end
+end

app/services/deprecation_warnings/disallowed_deprecations.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# @note Temporary solution for disallowed deprecation warnings.
+#   To be replaced by ActiveSupport Disallowed Deprecations after upgrading to Rails 6.1:
+#   https://rubyonrails.org/2020/12/9/Rails-6-1-0-release#disallowed-deprecation-support
+module DisallowedDeprecations
+  class ::DisallowedDeprecationError < StandardError; end
+
+  # Regular expressions for Rails 5.2 deprecation warnings that we have addressed in the codebase
+  RAILS_5_2_FIXED_DEPRECATION_WARNING_REGEXES = [
+    /Dangerous query method \(method whose arguments are used as raw SQL\) called with non\-attribute argument\(s\)/
+  ].freeze
+
+  # Regular expressions for deprecation warnings that should raise an exception on detection
+  DISALLOWED_DEPRECATION_WARNING_REGEXES = [
+    *RAILS_5_2_FIXED_DEPRECATION_WARNING_REGEXES
+  ].freeze
+
+  # @param message [String] deprecation warning message to be checked against disallow list
+  def raise_if_disallowed_deprecation!(message)
+    if DISALLOWED_DEPRECATION_WARNING_REGEXES.any? { |re| re.match?(message) }
+      fail DisallowedDeprecationError, message
+    end
+  end
+end

app/services/deprecation_warnings/production_handler.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# @note For use with `ActiveSupport::Deprecation.behavior=`.
+module DeprecationWarnings
+  class ProductionHandler < BaseHandler
+    APP_NAME = "caseflow"
+    SLACK_ALERT_CHANNEL = "#appeals-deprecation-alerts"
+
+    class << self
+      # :reek:LongParameterList
+      def call(message, callstack, deprecation_horizon, gem_name)
+        emit_warning_to_application_logs(message)
+        emit_warning_to_sentry(message, callstack, deprecation_horizon, gem_name)
+        emit_warning_to_slack_alerts_channel(message)
+      rescue StandardError => error
+        Raven.capture_exception(error)
+      end
+
+      private
+
+      def emit_warning_to_application_logs(message)
+        Rails.logger.warn(message)
+      end
+
+      # :reek:LongParameterList
+      def emit_warning_to_sentry(message, callstack, deprecation_horizon, gem_name)
+        # Pre-emptive bugfix for future versions of the `sentry-raven` gem:
+        #   Need to convert callstack elements from `Thread::Backtrace::Location` objects to `Strings`
+        #   to avoid a `TypeError` on `options.deep_dup` in `Raven.capture_message`:
+        #   https://github.com/getsentry/sentry-ruby/blob/2e07e0295ba83df4c76c7bf3315d199c7050a7f9/lib/raven/instance.rb#L114
+        callstack_strings = callstack.map(&:to_s)
+
+        Raven.capture_message(
+          message,
+          level: "warning",
+          extra: {
+            message: message,
+            gem_name: gem_name,
+            deprecation_horizon: deprecation_horizon,
+            callstack: callstack_strings,
+            environment: Rails.env
+          }
+        )
+      end
+
+      def emit_warning_to_slack_alerts_channel(message)
+        slack_alert_title = "Deprecation Warning - #{APP_NAME} (#{ENV['DEPLOY_ENV']})"
+
+        SlackService
+          .new(url: ENV["SLACK_DISPATCH_ALERT_URL"])
+          .send_notification(message, slack_alert_title, SLACK_ALERT_CHANNEL)
+      end
+    end
+  end
+end

app/services/deprecation_warnings/test_handler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "disallowed_deprecations"
+
+# @note For use with `ActiveSupport::Deprecation.behavior=`.
+module DeprecationWarnings
+  class TestHandler < BaseHandler
+    extend DisallowedDeprecations
+
+    class << self
+      # :reek:LongParameterList
+      def call(message, _callstack, _deprecation_horizon, _gem_name)
+        raise_if_disallowed_deprecation!(message)
+        emit_error_to_stderr(message)
+      end
+
+      private
+
+      def emit_error_to_stderr(message)
+        ActiveSupport::Logger.new($stderr).error(message)
+      end
+    end
+  end
+end