From 6f9aaab573c09eada96d29e433130d55adb5d3fc Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 10:27:54 -0800 Subject: [PATCH 01/20] Add VIP NotOptions Mitigation Detect invalid values in the notoptions cache. --- notoptions-mitigation.php | 81 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 notoptions-mitigation.php diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php new file mode 100644 index 0000000000..12c3a37933 --- /dev/null +++ b/notoptions-mitigation.php @@ -0,0 +1,81 @@ +blogid ) ? ", blog ID {$wpdb->blogid}" : ''), + $total_invalid, + ( $flushed ? '' : 'un' ) + ); + + $to_irc = $subject . ' #vipnotoptions'; + + // Send to IRC, if we have a host configured + if ( defined( 'ALERT_SERVICE_ADDRESS' ) && ALERT_SERVICE_ADDRESS ) { + if ( 'production' === $environment ) { + wpcom_vip_irc( '#vip-deploy-on-call', $to_irc , $irc_alert_level, 'a8c-notoptions' ); + + // Send to OpsGenie + $alerts = Alerts::instance(); + $alerts->opsgenie( + $subject, + array( + 'alias' => 'notoptions/' . $site_id, + 'description' => 'Invalid values found in notoptions cache', + 'entity' => (string) $site_id, + 'priority' => $opsgenie_alert_level, + 'source' => 'sites/notoptions-value', + ), + 'notoptions-value-alert', + '10' + ); + + } + } + + } + + } + +} From 629ea916d01d04efd5bfbb7991ed681e7693a02c Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 10:47:41 -0800 Subject: [PATCH 02/20] lint fixes --- notoptions-mitigation.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 12c3a37933..2baa4b3d56 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -7,6 +7,7 @@ */ namespace Automattic\VIP; + use Automattic\VIP\Utils\Alerts; if ( is_admin() || ( defined( 'WP_CLI' ) && WP_CLI ) ) { @@ -17,7 +18,9 @@ function notoptions_mitigation() { $notoptions = wp_cache_get( 'notoptions', 'options' ); // filter for any values not equal to (bool)true. - $not_trues = array_filter( $notoptions, function( $v ) { return $v !== true; } ); + $not_trues = array_filter( $notoptions, function( $v ) { + return $v !== true; + } ); // if they exist, something's borked if ( $not_trues ) { @@ -41,7 +44,7 @@ function notoptions_mitigation() { $subject = sprintf( $subject, - esc_url( home_url() ), + esc_url( home_url( ) ), esc_html( $environment ), (int) $site_id, ( ( 0 !== $wpdb->blogid ) ? ", blog ID {$wpdb->blogid}" : ''), @@ -54,7 +57,7 @@ function notoptions_mitigation() { // Send to IRC, if we have a host configured if ( defined( 'ALERT_SERVICE_ADDRESS' ) && ALERT_SERVICE_ADDRESS ) { if ( 'production' === $environment ) { - wpcom_vip_irc( '#vip-deploy-on-call', $to_irc , $irc_alert_level, 'a8c-notoptions' ); + wpcom_vip_irc( '#vip-deploy-on-call', $to_irc, $irc_alert_level, 'a8c-notoptions' ); // Send to OpsGenie $alerts = Alerts::instance(); @@ -73,9 +76,6 @@ function notoptions_mitigation() { } } - } - } - } From 77cb8a29ad752bad91d187c0228a747d1b738a3b Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 10:56:16 -0800 Subject: [PATCH 03/20] lint fix again --- notoptions-mitigation.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 2baa4b3d56..9f21bd00cf 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -44,10 +44,10 @@ function notoptions_mitigation() { $subject = sprintf( $subject, - esc_url( home_url( ) ), + esc_url( home_url() ), esc_html( $environment ), (int) $site_id, - ( ( 0 !== $wpdb->blogid ) ? ", blog ID {$wpdb->blogid}" : ''), + ( ( 0 !== $wpdb->blogid ) ? ", blog ID {$wpdb->blogid}" : '' ), $total_invalid, ( $flushed ? '' : 'un' ) ); From b56c2c8b72bf3783c4bcf6e84f9c1be8d99dec74 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 10:57:15 -0800 Subject: [PATCH 04/20] yodify --- notoptions-mitigation.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 9f21bd00cf..915c8ed2c1 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -19,7 +19,7 @@ function notoptions_mitigation() { // filter for any values not equal to (bool)true. $not_trues = array_filter( $notoptions, function( $v ) { - return $v !== true; + return true !== $v; } ); // if they exist, something's borked From 8be0fb90d8f9ca3a37dadbe8540c6f4d93a34cb5 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 11:19:34 -0800 Subject: [PATCH 05/20] Bail if notoptions isn't an array --- notoptions-mitigation.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 915c8ed2c1..c56783c0ab 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -17,6 +17,10 @@ function notoptions_mitigation() { $notoptions = wp_cache_get( 'notoptions', 'options' ); + if ( ! is_array( $notoptions ) ) { + return; + } + // filter for any values not equal to (bool)true. $not_trues = array_filter( $notoptions, function( $v ) { return true !== $v; From 272af44e76c60f53c1872f1a9ccc3ca516342225 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 11:20:40 -0800 Subject: [PATCH 06/20] More explicit check for invalid values --- notoptions-mitigation.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index c56783c0ab..807c4fdef2 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -27,13 +27,12 @@ function notoptions_mitigation() { } ); // if they exist, something's borked - if ( $not_trues ) { + if ( 1 <= ( $total_invalid = count( $not_trues ) ) ) { // attempt repair $flushed = wp_cache_delete( 'notoptions', 'options' ); // begin prepping notification - $total_invalid = count( $not_trues ); $is_vip_env = ( defined( 'WPCOM_IS_VIP_ENV' ) && true === WPCOM_IS_VIP_ENV ); $environment = ( ( defined( 'VIP_GO_APP_ENVIRONMENT' ) && VIP_GO_APP_ENVIRONMENT ) ? VIP_GO_APP_ENVIRONMENT : 'unknown' ); $site_id = defined( 'VIP_GO_APP_ID' ) ? VIP_GO_APP_ID : false; From 3bcb6e0d3d407341829732d82cf8a302177efd9d Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 11:21:20 -0800 Subject: [PATCH 07/20] Use the Alerts class to send a message to internal VIP channels, and do it for all environments --- notoptions-mitigation.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 807c4fdef2..01a7b1d673 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -59,8 +59,10 @@ function notoptions_mitigation() { // Send to IRC, if we have a host configured if ( defined( 'ALERT_SERVICE_ADDRESS' ) && ALERT_SERVICE_ADDRESS ) { + + Alerts::send_to_chat( '#vip-deploy-on-call', $to_irc, $irc_alert_level, 'a8c-notoptions' ); + if ( 'production' === $environment ) { - wpcom_vip_irc( '#vip-deploy-on-call', $to_irc, $irc_alert_level, 'a8c-notoptions' ); // Send to OpsGenie $alerts = Alerts::instance(); From 24504428ba0fd0e2fb92778bf7168ef1e7f0c461 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 11:45:44 -0800 Subject: [PATCH 08/20] Fix send_to_chat call --- notoptions-mitigation.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 01a7b1d673..2142bf1ca5 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -60,12 +60,12 @@ function notoptions_mitigation() { // Send to IRC, if we have a host configured if ( defined( 'ALERT_SERVICE_ADDRESS' ) && ALERT_SERVICE_ADDRESS ) { - Alerts::send_to_chat( '#vip-deploy-on-call', $to_irc, $irc_alert_level, 'a8c-notoptions' ); + $alerts = Alerts::instance(); + $alerts->send_to_chat( '#vip-deploy-on-call', $to_irc, $irc_alert_level, 'a8c-notoptions' ); if ( 'production' === $environment ) { // Send to OpsGenie - $alerts = Alerts::instance(); $alerts->opsgenie( $subject, array( From bd7536b9ef40d91775b8ae0fea7d5eb49e2dead2 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 11:49:18 -0800 Subject: [PATCH 09/20] unsquash code to please the linter --- notoptions-mitigation.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 2142bf1ca5..eb36fdf4a0 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -26,8 +26,10 @@ function notoptions_mitigation() { return true !== $v; } ); + $total_invalid = count( $not_trues ); + // if they exist, something's borked - if ( 1 <= ( $total_invalid = count( $not_trues ) ) ) { + if ( 1 <= $total_invalid ) { // attempt repair $flushed = wp_cache_delete( 'notoptions', 'options' ); From f3b3fcf47118f2ff6a10cb3cc2f2df44792668b0 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Wed, 3 Feb 2021 18:29:11 -0800 Subject: [PATCH 10/20] make sure wpdb global is available --- notoptions-mitigation.php | 1 + 1 file changed, 1 insertion(+) diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index eb36fdf4a0..260e5a0d30 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -15,6 +15,7 @@ } function notoptions_mitigation() { + global $wpdb; $notoptions = wp_cache_get( 'notoptions', 'options' ); if ( ! is_array( $notoptions ) ) { From 261aa80b9e88eaed4f917fd2a1524b1eb8e80686 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Thu, 4 Feb 2021 10:02:33 -0800 Subject: [PATCH 11/20] Use foreach and break early, far more efficent --- .vscode/settings.json | 7 +++++++ notoptions-mitigation.php | 18 +++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..e1e6e3e2d6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "workbench.colorCustomizations": { + "activityBar.background": "#4A2127", + "titleBar.activeBackground": "#682E36", + "titleBar.activeForeground": "#FEFCFC" + } +} \ No newline at end of file diff --git a/notoptions-mitigation.php b/notoptions-mitigation.php index 260e5a0d30..082388bf78 100644 --- a/notoptions-mitigation.php +++ b/notoptions-mitigation.php @@ -22,15 +22,19 @@ function notoptions_mitigation() { return; } - // filter for any values not equal to (bool)true. - $not_trues = array_filter( $notoptions, function( $v ) { - return true !== $v; - } ); - - $total_invalid = count( $not_trues ); + $invalid = false; + + foreach ( $notoptions as $v ) { + // notoptions should never have !true values, see get_option() + // break on any values not equal to (bool)true. + if ( true !== $v ) { + $invalid = true; + break; + } + } // if they exist, something's borked - if ( 1 <= $total_invalid ) { + if ( $invalid ) { // attempt repair $flushed = wp_cache_delete( 'notoptions', 'options' ); From 90f2a453db586561f9e2c968ee7e935aa9012b49 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Thu, 4 Feb 2021 10:48:14 -0800 Subject: [PATCH 12/20] move notoptions-mitig ahead in load order --- 000-vip-init.php | 3 ++- .../vip-notoptions-mitigation.php | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename notoptions-mitigation.php => vip-helpers/vip-notoptions-mitigation.php (100%) diff --git a/000-vip-init.php b/000-vip-init.php index 9bf6e61e7f..06dcda9cee 100644 --- a/000-vip-init.php +++ b/000-vip-init.php @@ -163,6 +163,7 @@ require_once( __DIR__ . '/lib/utils/class-alerts.php' ); // Load our development and environment helpers +require_once( __DIR__ . '/vip-helpers/vip-notoptions-mitigation.php' ); require_once( __DIR__ . '/vip-helpers/vip-utils.php' ); require_once( __DIR__ . '/vip-helpers/vip-newrelic.php' ); require_once( __DIR__ . '/vip-helpers/vip-caching.php' ); @@ -223,7 +224,7 @@ // VIP sites should not be set as staging in Jetpack // since it breaks SSO and prevents data from being passed to - // WordPress.com + // WordPress.com add_filter( 'jetpack_is_staging_site', '__return_false' ); } diff --git a/notoptions-mitigation.php b/vip-helpers/vip-notoptions-mitigation.php similarity index 100% rename from notoptions-mitigation.php rename to vip-helpers/vip-notoptions-mitigation.php From 38d284a8f90f3afcb959b1ce25b728aefb71db8f Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Thu, 4 Feb 2021 10:48:32 -0800 Subject: [PATCH 13/20] Include documentation link in alert messaging --- vip-helpers/vip-notoptions-mitigation.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vip-helpers/vip-notoptions-mitigation.php b/vip-helpers/vip-notoptions-mitigation.php index 082388bf78..9da79a37ce 100644 --- a/vip-helpers/vip-notoptions-mitigation.php +++ b/vip-helpers/vip-notoptions-mitigation.php @@ -62,7 +62,7 @@ function notoptions_mitigation() { ( $flushed ? '' : 'un' ) ); - $to_irc = $subject . ' #vipnotoptions'; + $to_irc = $subject . ' [https://wp.me/PCYsg-vqN] #vipnotoptions'; // Send to IRC, if we have a host configured if ( defined( 'ALERT_SERVICE_ADDRESS' ) && ALERT_SERVICE_ADDRESS ) { From f752c33ed37202b9da30c1af4343db57fcf93c32 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Thu, 4 Feb 2021 10:50:29 -0800 Subject: [PATCH 14/20] remove stray .vscode :( --- .vscode/settings.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index e1e6e3e2d6..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "workbench.colorCustomizations": { - "activityBar.background": "#4A2127", - "titleBar.activeBackground": "#682E36", - "titleBar.activeForeground": "#FEFCFC" - } -} \ No newline at end of file From a6cba968cdc8cfad495b3b825cc5d10fb5fee5ac Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Thu, 4 Feb 2021 10:55:40 -0800 Subject: [PATCH 15/20] Remove old reference to total_invalid --- vip-helpers/vip-notoptions-mitigation.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vip-helpers/vip-notoptions-mitigation.php b/vip-helpers/vip-notoptions-mitigation.php index 9da79a37ce..d0da5ae68f 100644 --- a/vip-helpers/vip-notoptions-mitigation.php +++ b/vip-helpers/vip-notoptions-mitigation.php @@ -50,7 +50,7 @@ function notoptions_mitigation() { $irc_alert_level = 2; // ALERT $opsgenie_alert_level = 'P4'; - $subject = 'NOTOPTIONS: %s (%s VIP Go site ID: %s%s) %s invalid values found in notoptions. Cache flush was %ssuccessful.'; + $subject = 'NOTOPTIONS: %s (%s VIP Go site ID: %s%s) - Invalid values found in notoptions. Cache flush was %ssuccessful.'; $subject = sprintf( $subject, @@ -58,7 +58,6 @@ function notoptions_mitigation() { esc_html( $environment ), (int) $site_id, ( ( 0 !== $wpdb->blogid ) ? ", blog ID {$wpdb->blogid}" : '' ), - $total_invalid, ( $flushed ? '' : 'un' ) ); From 3fa88a513a9da9586836bc9cb12e9125fb1ecf77 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Mon, 8 Feb 2021 09:19:46 -0800 Subject: [PATCH 16/20] increase alert level --- vip-helpers/vip-notoptions-mitigation.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vip-helpers/vip-notoptions-mitigation.php b/vip-helpers/vip-notoptions-mitigation.php index d0da5ae68f..937257a3f2 100644 --- a/vip-helpers/vip-notoptions-mitigation.php +++ b/vip-helpers/vip-notoptions-mitigation.php @@ -47,8 +47,8 @@ function notoptions_mitigation() { // Send notices to VIP staff if this is happening on VIP-hosted sites if ( $is_vip_env && $site_id ) { - $irc_alert_level = 2; // ALERT - $opsgenie_alert_level = 'P4'; + $irc_alert_level = 3; // CRITICAL + $opsgenie_alert_level = 'P3'; $subject = 'NOTOPTIONS: %s (%s VIP Go site ID: %s%s) - Invalid values found in notoptions. Cache flush was %ssuccessful.'; From 8536a52d8c5a93b0194ebdee22d867f41ed4d149 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Mon, 8 Feb 2021 09:21:27 -0800 Subject: [PATCH 17/20] improve opsgenie alert data --- vip-helpers/vip-notoptions-mitigation.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vip-helpers/vip-notoptions-mitigation.php b/vip-helpers/vip-notoptions-mitigation.php index 937257a3f2..9e7220c755 100644 --- a/vip-helpers/vip-notoptions-mitigation.php +++ b/vip-helpers/vip-notoptions-mitigation.php @@ -75,11 +75,11 @@ function notoptions_mitigation() { $alerts->opsgenie( $subject, array( - 'alias' => 'notoptions/' . $site_id, + 'alias' => 'notoptions/cache-corruption/' . $site_id, 'description' => 'Invalid values found in notoptions cache', 'entity' => (string) $site_id, 'priority' => $opsgenie_alert_level, - 'source' => 'sites/notoptions-value', + 'source' => 'sites/notoptions-value-monitor', ), 'notoptions-value-alert', '10' From b8857a7fb8d1af3d82d72b6cc8257b0a57ff614e Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Mon, 8 Feb 2021 09:22:05 -0800 Subject: [PATCH 18/20] Do not require file in same commit the file is introduced in --- 000-vip-init.php | 1 - 1 file changed, 1 deletion(-) diff --git a/000-vip-init.php b/000-vip-init.php index 06dcda9cee..666d2ff24b 100644 --- a/000-vip-init.php +++ b/000-vip-init.php @@ -163,7 +163,6 @@ require_once( __DIR__ . '/lib/utils/class-alerts.php' ); // Load our development and environment helpers -require_once( __DIR__ . '/vip-helpers/vip-notoptions-mitigation.php' ); require_once( __DIR__ . '/vip-helpers/vip-utils.php' ); require_once( __DIR__ . '/vip-helpers/vip-newrelic.php' ); require_once( __DIR__ . '/vip-helpers/vip-caching.php' ); From 4f662a8e0fd63f9770f548c6fc5e86e221fc5739 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Mon, 8 Feb 2021 09:34:04 -0800 Subject: [PATCH 19/20] add back trailing whitespace for reasons --- 000-vip-init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/000-vip-init.php b/000-vip-init.php index 666d2ff24b..9bf6e61e7f 100644 --- a/000-vip-init.php +++ b/000-vip-init.php @@ -223,7 +223,7 @@ // VIP sites should not be set as staging in Jetpack // since it breaks SSO and prevents data from being passed to - // WordPress.com + // WordPress.com add_filter( 'jetpack_is_staging_site', '__return_false' ); } From ce1b58295542714ddf013d669aa0850ea0706c83 Mon Sep 17 00:00:00 2001 From: Kailey Lampert Date: Mon, 8 Feb 2021 09:47:52 -0800 Subject: [PATCH 20/20] sneaky whitespace --- vip-helpers/vip-notoptions-mitigation.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vip-helpers/vip-notoptions-mitigation.php b/vip-helpers/vip-notoptions-mitigation.php index 9e7220c755..fe7fb72ce4 100644 --- a/vip-helpers/vip-notoptions-mitigation.php +++ b/vip-helpers/vip-notoptions-mitigation.php @@ -26,7 +26,7 @@ function notoptions_mitigation() { foreach ( $notoptions as $v ) { // notoptions should never have !true values, see get_option() - // break on any values not equal to (bool)true. + // break on any values not equal to (bool)true. if ( true !== $v ) { $invalid = true; break;