From 2e476e76c181fa96d65e2819757bc703115ad2c3 Mon Sep 17 00:00:00 2001 From: AnandEdara <131183324+AnandEdara@users.noreply.github.com> Date: Fri, 28 Jul 2023 00:19:09 -0500 Subject: [PATCH 01/15] Adding a rails initializer for monitoring deprecation warnings --- config/initializers/deprecation_alerts.rb | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 config/initializers/deprecation_alerts.rb diff --git a/config/initializers/deprecation_alerts.rb b/config/initializers/deprecation_alerts.rb new file mode 100644 index 00000000000..dadc9229281 --- /dev/null +++ b/config/initializers/deprecation_alerts.rb @@ -0,0 +1,25 @@ +require "#{Rails.root}/app/services/slack_service" + +if Rails.env.production? + ActiveSupport::Deprecation.behavior = ->(message, callstack) do + # Log the deprecation warning to Rails logger + Rails.logger.warn(message) + + # Log the deprecation warning to Sentry + Raven.capture_message( + 'Rails Deprecation Warning in Production', + level: 'warning', + extra: { + message: message, + callstack: callstack, + environment: Rails.env + } + ) + + # Send the deprecation warning to the Slack channel + slack_msg = "Deprecation Warning: #{message}" + SlackService.new(url: ENV['SLACK_DISPATCH_ALERT_URL']).send_notification(slack_msg, "Deprecation Warning", "#appeals-deprecation-alerts") + + ActiveSupport::Deprecation::DEFAULT_BEHAVIORS[:stderr].call(message, callstack) + end +end From 31c33f4eadfdf4c496bb223cc0773bdbacdef1ab Mon Sep 17 00:00:00 2001 From: AnandEdara <131183324+AnandEdara@users.noreply.github.com> Date: Thu, 3 Aug 2023 03:28:59 -0500 Subject: [PATCH 02/15] Adding Subscriber to monitor deprecation warnings --- .../deprecation_warning_subscriber.rb | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 config/initializers/deprecation_warning_subscriber.rb diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb new file mode 100644 index 00000000000..ae6a8bc4a64 --- /dev/null +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# @note For use in conjuction with setting `Rails.application.config.active_support.deprecation = :notify`. +# Whenever a “deprecation.rails” notification is published, it will dispatch the event +# (ActiveSupport::Notifications::Event) to method #deprecation. +class DeprecationWarningSubscriber < ActiveSupport::Subscriber + SLACK_ALERT_TITLE = "Deprecation Warning" + SLACK_ALERT_CHANNEL = "#appeals-deprecation-alerts" + + attach_to :rails + + def deprecation(event) + emit_warning_to_application_logs(event) + emit_warning_to_sentry(event) + emit_warning_to_slack_alerts_channel(event) + end + + private + + def emit_warning_to_application_logs(event) + Rails.logger.warn(event.payload[:message]) + end + + def emit_warning_to_sentry(event) + Raven.capture_message( + event.payload[:message], + level: "warning", + extra: { + message: event.payload[:message], + callstack: event.payload[:callstack], + environment: Rails.env + } + ) + end + + def emit_warning_to_slack_alerts_channel(event) + SlackService + .new(url: ENV["SLACK_DISPATCH_ALERT_URL"]) + .send_notification(event.payload[:message], SLACK_ALERT_TITLE, SLACK_ALERT_CHANNEL) + end +end From 9428cb240b5b2523ee0c06a9453d78cfad7c9281 Mon Sep 17 00:00:00 2001 From: AnandEdara <131183324+AnandEdara@users.noreply.github.com> Date: Thu, 3 Aug 2023 03:30:40 -0500 Subject: [PATCH 03/15] Add spec file for deprecation warnings --- .../deprecation_warning_subscriber_spec.rb | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 spec/initializers/deprecation_warning_subscriber_spec.rb diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb new file mode 100644 index 00000000000..bc6edf5d00c --- /dev/null +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +describe "DeprecationWarningSubscriber" do + let(:rails_logger) { Rails.logger } + let(:slack_service) { SlackService.new(url: "dummy-url") } + + before do + allow(Rails).to receive(:logger).and_return(rails_logger) + allow(rails_logger).to receive(:warn) + + allow(Raven).to receive(:capture_message) + + allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) + allow(slack_service).to receive(:send_notification) + end + + context "when a 'deprecation.rails' event is instrumented" do + let(:payload) { { message: "test message", callstack: "test callstack" } } + + before { ActiveSupport::Notifications.instrument("deprecation.rails", payload) } + + it "emits a warning to the application logs" do + expect(rails_logger).to have_received(:warn).with(payload[:message]) + end + + it "emits a warning to Sentry" do + expect(Raven).to have_received(:capture_message).with( + payload[:message], + level: "warning", + extra: { + message: payload[:message], + callstack: payload[:callstack], + environment: Rails.env + } + ) + end + + it "emits a warning to Slack channel" do + expect(slack_service).to have_received(:send_notification).with( + payload[:message], + "Deprecation Warning", + "#appeals-deprecation-alerts" + ) + end + end +end From b4b078fecbe670b8e4eddfddd88c97e066a2517a Mon Sep 17 00:00:00 2001 From: AnandEdara <131183324+AnandEdara@users.noreply.github.com> Date: Thu, 3 Aug 2023 12:43:03 -0500 Subject: [PATCH 04/15] Delete deprecation_alerts.rb --- config/initializers/deprecation_alerts.rb | 25 ----------------------- 1 file changed, 25 deletions(-) delete mode 100644 config/initializers/deprecation_alerts.rb diff --git a/config/initializers/deprecation_alerts.rb b/config/initializers/deprecation_alerts.rb deleted file mode 100644 index dadc9229281..00000000000 --- a/config/initializers/deprecation_alerts.rb +++ /dev/null @@ -1,25 +0,0 @@ -require "#{Rails.root}/app/services/slack_service" - -if Rails.env.production? - ActiveSupport::Deprecation.behavior = ->(message, callstack) do - # Log the deprecation warning to Rails logger - Rails.logger.warn(message) - - # Log the deprecation warning to Sentry - Raven.capture_message( - 'Rails Deprecation Warning in Production', - level: 'warning', - extra: { - message: message, - callstack: callstack, - environment: Rails.env - } - ) - - # Send the deprecation warning to the Slack channel - slack_msg = "Deprecation Warning: #{message}" - SlackService.new(url: ENV['SLACK_DISPATCH_ALERT_URL']).send_notification(slack_msg, "Deprecation Warning", "#appeals-deprecation-alerts") - - ActiveSupport::Deprecation::DEFAULT_BEHAVIORS[:stderr].call(message, callstack) - end -end From b0e21efe6c5002bccdae20c0bb35ad63a9358afc Mon Sep 17 00:00:00 2001 From: AnandEdara <131183324+AnandEdara@users.noreply.github.com> Date: Mon, 7 Aug 2023 11:04:34 -0500 Subject: [PATCH 05/15] Apply suggestions from code review Co-authored-by: Jeremy Croteau --- config/initializers/deprecation_warning_subscriber.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb index ae6a8bc4a64..6f73783e406 100644 --- a/config/initializers/deprecation_warning_subscriber.rb +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -4,7 +4,7 @@ # Whenever a “deprecation.rails” notification is published, it will dispatch the event # (ActiveSupport::Notifications::Event) to method #deprecation. class DeprecationWarningSubscriber < ActiveSupport::Subscriber - SLACK_ALERT_TITLE = "Deprecation Warning" + APP_NAME = "caseflow" SLACK_ALERT_CHANNEL = "#appeals-deprecation-alerts" attach_to :rails @@ -34,8 +34,10 @@ def emit_warning_to_sentry(event) end def emit_warning_to_slack_alerts_channel(event) + slack_alert_title = "Deprecation Warning - #{APP_NAME} (#{ENV['DEPLOY_ENV']})" + SlackService .new(url: ENV["SLACK_DISPATCH_ALERT_URL"]) - .send_notification(event.payload[:message], SLACK_ALERT_TITLE, SLACK_ALERT_CHANNEL) + .send_notification(event.payload[:message], slack_alert_title, SLACK_ALERT_CHANNEL) end end From 710649adc833a99de042fbf5d366fc632732c327 Mon Sep 17 00:00:00 2001 From: AnandEdara <131183324+AnandEdara@users.noreply.github.com> Date: Mon, 7 Aug 2023 11:33:08 -0500 Subject: [PATCH 06/15] Update deprecation_warning_subscriber_spec.rb --- spec/initializers/deprecation_warning_subscriber_spec.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb index bc6edf5d00c..61deb8d44fa 100644 --- a/spec/initializers/deprecation_warning_subscriber_spec.rb +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -15,6 +15,8 @@ end context "when a 'deprecation.rails' event is instrumented" do + let(:app_name") { "caseflow" } + let(:deploy_env) { "test" } let(:payload) { { message: "test message", callstack: "test callstack" } } before { ActiveSupport::Notifications.instrument("deprecation.rails", payload) } @@ -36,9 +38,10 @@ end it "emits a warning to Slack channel" do + slack_alert_title = "Deprecation Warning - #{app_name} (#{deploy_env})" expect(slack_service).to have_received(:send_notification).with( payload[:message], - "Deprecation Warning", + slack_alert_title, "#appeals-deprecation-alerts" ) end From 7efc64e0cdbf62d77727f838f98adbb190ac9dff Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Mon, 7 Aug 2023 13:20:55 -0400 Subject: [PATCH 07/15] Fix typo --- spec/initializers/deprecation_warning_subscriber_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb index 61deb8d44fa..39b4e9553a4 100644 --- a/spec/initializers/deprecation_warning_subscriber_spec.rb +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -15,7 +15,7 @@ end context "when a 'deprecation.rails' event is instrumented" do - let(:app_name") { "caseflow" } + let(:app_name) { "caseflow" } let(:deploy_env) { "test" } let(:payload) { { message: "test message", callstack: "test callstack" } } From 7559dab4170141c0ca5c9eae12757114eaa68359 Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Mon, 7 Aug 2023 21:12:12 -0400 Subject: [PATCH 08/15] :bug: Pre-emptive bugfix - TypeError Exception: allocator undefined for Thread::Backtrace::Location --- .../initializers/deprecation_warning_subscriber.rb | 8 +++++++- .../deprecation_warning_subscriber_spec.rb | 13 +++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb index 6f73783e406..7cb3e615c75 100644 --- a/config/initializers/deprecation_warning_subscriber.rb +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -22,12 +22,18 @@ def emit_warning_to_application_logs(event) end def emit_warning_to_sentry(event) + # Pre-emptive bugfix for future versions of the `sentry-raven` gem: + # Need to convert callstack elements from `Thread::Backtrace::Location` objects to `Strings` + # to avoid a `TypeError` on `options.deep_dup` in `Raven.capture_message`: + # https://github.com/getsentry/sentry-ruby/blob/2e07e0295ba83df4c76c7bf3315d199c7050a7f9/lib/raven/instance.rb#L114 + callstack_strings = event.payload[:callstack].map(&:to_s) + Raven.capture_message( event.payload[:message], level: "warning", extra: { message: event.payload[:message], - callstack: event.payload[:callstack], + callstack: callstack_strings, environment: Rails.env } ) diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb index 39b4e9553a4..fffc8fe9224 100644 --- a/spec/initializers/deprecation_warning_subscriber_spec.rb +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -17,7 +17,16 @@ context "when a 'deprecation.rails' event is instrumented" do let(:app_name) { "caseflow" } let(:deploy_env) { "test" } - let(:payload) { { message: "test message", callstack: "test callstack" } } + let(:payload) do + { + message: "test message", + gem_name: "Rails", + deprecation_horizon: "6.0", + callstack: [location_1, location_2] + } + end + let(:location_1) { instance_double("Thread::Backtrace::Location", to_s: "location 1") } + let(:location_2) { instance_double("Thread::Backtrace::Location", to_s: "location 2") } before { ActiveSupport::Notifications.instrument("deprecation.rails", payload) } @@ -31,7 +40,7 @@ level: "warning", extra: { message: payload[:message], - callstack: payload[:callstack], + callstack: ["location 1", "location 2"], environment: Rails.env } ) From 49ce118307d37b87b452e25a049ced510ce89ab5 Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Mon, 7 Aug 2023 21:13:45 -0400 Subject: [PATCH 09/15] :loud_sound: Log additional metadata to Sentry on deprecation warnings --- config/initializers/deprecation_warning_subscriber.rb | 2 ++ spec/initializers/deprecation_warning_subscriber_spec.rb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb index 7cb3e615c75..a867b59128b 100644 --- a/config/initializers/deprecation_warning_subscriber.rb +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -33,6 +33,8 @@ def emit_warning_to_sentry(event) level: "warning", extra: { message: event.payload[:message], + gem_name: event.payload[:gem_name], + deprecation_horizon: event.payload[:deprecation_horizon], callstack: callstack_strings, environment: Rails.env } diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb index fffc8fe9224..40cb812af3b 100644 --- a/spec/initializers/deprecation_warning_subscriber_spec.rb +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -40,6 +40,8 @@ level: "warning", extra: { message: payload[:message], + gem_name: "Rails", + deprecation_horizon: "6.0", callstack: ["location 1", "location 2"], environment: Rails.env } From f912429e197d6844f975b6b5fa259f90330fcd6f Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Thu, 10 Aug 2023 02:30:01 -0400 Subject: [PATCH 10/15] :rotating_light: Lint: Remove trailing whites space c --- config/initializers/deprecation_warning_subscriber.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb index a867b59128b..4e05554c161 100644 --- a/config/initializers/deprecation_warning_subscriber.rb +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -43,7 +43,7 @@ def emit_warning_to_sentry(event) def emit_warning_to_slack_alerts_channel(event) slack_alert_title = "Deprecation Warning - #{APP_NAME} (#{ENV['DEPLOY_ENV']})" - + SlackService .new(url: ENV["SLACK_DISPATCH_ALERT_URL"]) .send_notification(event.payload[:message], slack_alert_title, SLACK_ALERT_CHANNEL) From c6339669f5e9b5154ac490012f3cf15fc0502bdd Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Thu, 10 Aug 2023 02:33:27 -0400 Subject: [PATCH 11/15] :goal_net: Add blanket rescue to DeprecationWarningSubscriber (Puts on tin foil hat) Prevent potential exceptions from SlackService from impacting user experience --- .../deprecation_warning_subscriber.rb | 2 ++ .../deprecation_warning_subscriber_spec.rb | 26 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb index 4e05554c161..89ee7dafced 100644 --- a/config/initializers/deprecation_warning_subscriber.rb +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -13,6 +13,8 @@ def deprecation(event) emit_warning_to_application_logs(event) emit_warning_to_sentry(event) emit_warning_to_slack_alerts_channel(event) + rescue StandardError => error + Raven.capture_exception(error) end private diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb index 40cb812af3b..9602b97179b 100644 --- a/spec/initializers/deprecation_warning_subscriber_spec.rb +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -9,6 +9,7 @@ allow(rails_logger).to receive(:warn) allow(Raven).to receive(:capture_message) + allow(Raven).to receive(:capture_exception) allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) allow(slack_service).to receive(:send_notification) @@ -28,13 +29,19 @@ let(:location_1) { instance_double("Thread::Backtrace::Location", to_s: "location 1") } let(:location_2) { instance_double("Thread::Backtrace::Location", to_s: "location 2") } - before { ActiveSupport::Notifications.instrument("deprecation.rails", payload) } + def instrument_deprecation_warning + ActiveSupport::Notifications.instrument("deprecation.rails", payload) + end it "emits a warning to the application logs" do + instrument_deprecation_warning + expect(rails_logger).to have_received(:warn).with(payload[:message]) end it "emits a warning to Sentry" do + instrument_deprecation_warning + expect(Raven).to have_received(:capture_message).with( payload[:message], level: "warning", @@ -50,11 +57,28 @@ it "emits a warning to Slack channel" do slack_alert_title = "Deprecation Warning - #{app_name} (#{deploy_env})" + + instrument_deprecation_warning + expect(slack_service).to have_received(:send_notification).with( payload[:message], slack_alert_title, "#appeals-deprecation-alerts" ) end + + context "when an exeption ocurrs" do + before { allow(slack_service).to receive(:send_notification).and_raise(StandardError) } + + it "logs error to Sentry" do + instrument_deprecation_warning + + expect(Raven).to have_received(:capture_exception).with(StandardError) + end + + it "does not raise error" do + expect { instrument_deprecation_warning }.not_to raise_error + end + end end end From 2c78c59d2052fe497d45d5ca02da79185dece674 Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Thu, 10 Aug 2023 02:52:00 -0400 Subject: [PATCH 12/15] :pencil2: Fix typo in test description --- spec/initializers/deprecation_warning_subscriber_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb index 9602b97179b..25eacc02fd8 100644 --- a/spec/initializers/deprecation_warning_subscriber_spec.rb +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -67,7 +67,7 @@ def instrument_deprecation_warning ) end - context "when an exeption ocurrs" do + context "when an exception occurs" do before { allow(slack_service).to receive(:send_notification).and_raise(StandardError) } it "logs error to Sentry" do From 93eab403d86a48746d6c830d77dfdfdf81ba7f90 Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Thu, 10 Aug 2023 10:27:12 -0400 Subject: [PATCH 13/15] :test_tube: Fix test so that it will pass in non-'test' environments as well --- spec/initializers/deprecation_warning_subscriber_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/initializers/deprecation_warning_subscriber_spec.rb b/spec/initializers/deprecation_warning_subscriber_spec.rb index 25eacc02fd8..57e88778983 100644 --- a/spec/initializers/deprecation_warning_subscriber_spec.rb +++ b/spec/initializers/deprecation_warning_subscriber_spec.rb @@ -17,7 +17,7 @@ context "when a 'deprecation.rails' event is instrumented" do let(:app_name) { "caseflow" } - let(:deploy_env) { "test" } + let(:deploy_env) { ENV["DEPLOY_ENV"] } let(:payload) do { message: "test message", From 50bf09f3cb28e9ffda5ef8b1405edb53ab73ee39 Mon Sep 17 00:00:00 2001 From: AnandEdara <131183324+AnandEdara@users.noreply.github.com> Date: Mon, 14 Aug 2023 06:59:58 -0500 Subject: [PATCH 14/15] Update deprecation_warning_subscriber.rb --- config/initializers/deprecation_warning_subscriber.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb index 89ee7dafced..ef099127aed 100644 --- a/config/initializers/deprecation_warning_subscriber.rb +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -50,4 +50,14 @@ def emit_warning_to_slack_alerts_channel(event) .new(url: ENV["SLACK_DISPATCH_ALERT_URL"]) .send_notification(event.payload[:message], slack_alert_title, SLACK_ALERT_CHANNEL) end + + def raise_if_fixed_deprecation_triggered(event) + # Checking for deprecated message in development and test environments + if Rails.env.development? || Rails.env.test? + if event.payload[:message].include?("The success? predicate is deprecated") + message = "Fixed deprecation warning triggered: #{event.payload[:message]}" + raise DeprecationWarning, message + end + end + end end From a645a409718fa4c81fd5605aae4990992736f050 Mon Sep 17 00:00:00 2001 From: Jeremy Croteau Date: Mon, 14 Aug 2023 10:39:52 -0400 Subject: [PATCH 15/15] Revert "Update deprecation_warning_subscriber.rb" This reverts commit 50bf09f3cb28e9ffda5ef8b1405edb53ab73ee39. --- config/initializers/deprecation_warning_subscriber.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/config/initializers/deprecation_warning_subscriber.rb b/config/initializers/deprecation_warning_subscriber.rb index ef099127aed..89ee7dafced 100644 --- a/config/initializers/deprecation_warning_subscriber.rb +++ b/config/initializers/deprecation_warning_subscriber.rb @@ -50,14 +50,4 @@ def emit_warning_to_slack_alerts_channel(event) .new(url: ENV["SLACK_DISPATCH_ALERT_URL"]) .send_notification(event.payload[:message], slack_alert_title, SLACK_ALERT_CHANNEL) end - - def raise_if_fixed_deprecation_triggered(event) - # Checking for deprecated message in development and test environments - if Rails.env.development? || Rails.env.test? - if event.payload[:message].include?("The success? predicate is deprecated") - message = "Fixed deprecation warning triggered: #{event.payload[:message]}" - raise DeprecationWarning, message - end - end - end end