From 2824da20446643e73a8d63a042d861a877104dad Mon Sep 17 00:00:00 2001 From: JP Rodrigues <70jprodrigues@gmail.com> Date: Tue, 12 May 2020 19:28:51 -0300 Subject: [PATCH 1/2] Fix logs and authorization of real bots --- src/Jobs/CheckIfBotIsReal.php | 3 ++- src/Jobs/ProcessLogWithIpInfo.php | 21 ++++++++++++++------- src/Middleware/BlockBots.php | 13 ++++++------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/Jobs/CheckIfBotIsReal.php b/src/Jobs/CheckIfBotIsReal.php index 9efda6c..8ca605e 100755 --- a/src/Jobs/CheckIfBotIsReal.php +++ b/src/Jobs/CheckIfBotIsReal.php @@ -16,6 +16,7 @@ class CheckIfBotIsReal implements ShouldQueue protected $client; protected $options; + protected $allowedBots; /** @@ -52,7 +53,7 @@ public function handle() } // Lets remove from the pending list - Redis::srem($this->options->pending_bots_key, $this->client->ip); + Redis::srem($this->options->pending_bot_list_key, $this->client->ip); if ($this->isValid($found_bot_key)) { Redis::sadd($this->options->whitelist_key, $this->client->ip); diff --git a/src/Jobs/ProcessLogWithIpInfo.php b/src/Jobs/ProcessLogWithIpInfo.php index 418be66..0bbe4c2 100755 --- a/src/Jobs/ProcessLogWithIpInfo.php +++ b/src/Jobs/ProcessLogWithIpInfo.php @@ -18,7 +18,7 @@ class ProcessLogWithIpInfo implements ShouldQueue protected $action; protected $client; protected $options; - + protected $accessLimit; /** * Checks whether the given IP address really belongs to a valid host or not @@ -26,11 +26,14 @@ class ProcessLogWithIpInfo implements ShouldQueue * @param $ip the IP address to check * @return bool true if the given IP address belongs to any of the valid hosts, otherwise false */ - public function __construct($client, $action, $options = null) + public function __construct($client, $action, $options = null, $accessLimit = null) { $this->action = $action; $this->client = $client; $this->options = $options; + if (!is_null($accessLimit)) { + $this->accessLimit = $accessLimit; + } } /** @@ -43,7 +46,11 @@ public function handle() $hits = Redis::get($this->client->key); $host = strtolower(gethostbyaddr($this->client->ip)); - $messsage = "[Block-Bots] IP: {$this->client->ip}; After {$hits} requests, Host: {$host} \n with User agent: {$this->client->userAgent}; was {$this->action}"; + if (!empty($this->accessLimit)) { + $message = "[Block-Bots] IP: {$this->client->ip}; After {$hits}/{$this->accessLimit} requests, Host: {$host} \n with User agent: {$this->client->userAgent}; was {$this->action}"; + } else { + $message = "[Block-Bots] IP: {$this->client->ip}; After {$hits} requests, Host: {$host} \n with User agent: {$this->client->userAgent}; was {$this->action}"; + } if ($this->options->ip_info_key) { $http = new HTTP(); @@ -67,18 +74,18 @@ public function handle() $region = $json_response["region"]; $country = $json_response["country"]; - $messsage .= "Org: {$org} | city: {$city} | region: {$region} | country: {$country} "; + $message .= "Org: {$org} | city: {$city} | region: {$region} | country: {$country} "; } } if ($this->client->url) { - $messsage .= " when accessing the URL: {$this->client->url} "; + $message .= " when accessing the URL: {$this->client->url} "; } if (($this->action === 'WHITELISTED') || ($this->action === 'GOOD_CRAWLER')) { - Log::stack($this->options->channels_info)->info($messsage); + Log::stack($this->options->channels_info)->info($message); } else { - Log::stack($this->options->channels_info)->error($messsage); + Log::stack($this->options->channels_info)->error($message); } } } diff --git a/src/Middleware/BlockBots.php b/src/Middleware/BlockBots.php index eaca74b..dadf08d 100644 --- a/src/Middleware/BlockBots.php +++ b/src/Middleware/BlockBots.php @@ -38,7 +38,7 @@ public function handle($request, Closure $next, $limit = 100, $frequency = 'dail $this->setUp($request, $limit, $frequency); $this->countHits(); - return $this->isAllowed() ? $next($this->request) : $this->notAllowed(); + return $this->isAllowed() ? $next($request) : $this->notAllowed(); } /** @@ -77,8 +77,8 @@ protected function isAllowed() return false; } elseif (Auth::check()) { return $this->passesAuthRules() && !$this->isLimitExceeded(); - } elseif (Auth::guest()) { - return $this->passesGuestRules() && !$this->isLimitExceeded(); + } elseif (Auth::guest() && $this->passesGuestRules() && !$this->isLimitExceeded()) { + return true; } return $this->passesBotRules(); @@ -94,6 +94,7 @@ protected function countHits() if (!Redis::exists($this->client->key)) { Redis::set($this->client->key, 1); Redis::expireat($this->client->key, $this->timeOutAt); + return $this->hits = 1; } return $this->hits = Redis::incr($this->client->key); @@ -104,8 +105,7 @@ private function logDisallowance() if (!Redis::exists($this->client->logKey)) { Redis::set($this->client->logKey, 1); Redis::expireat($this->client->logKey, $this->timeOutAt); - - ProcessLogWithIpInfo::dispatch($this->client, "BLOCKED", $this->options); + ProcessLogWithIpInfo::dispatch($this->client, "BLOCKED", $this->options, $this->limit); } } @@ -193,7 +193,6 @@ public function passesBotRules() if ($this->isWhitelisted()) { return true; } - //Lets block fake bots if (Redis::sismember($this->options->fake_bot_list_key, $this->client->ip)) { return false; @@ -203,7 +202,7 @@ public function passesBotRules() // While the bot is on pending_list, it's unchecked, so we allow this bot to pass-thru if (!Redis::sismember($this->options->pending_bot_list_key, $this->client->ip)) { // If we got here, it is an unknown bot. Let's create a job to test it - CheckIfBotIsReal::dispatch($this->client, $this->getAllowedBots()); + CheckIfBotIsReal::dispatch($this->client, $this->getAllowedBots(), $this->options); Redis::sadd($this->options->pending_bot_list_key, $this->client->ip); } From 8eb66c0e9686a7165be489ab342d9fdf0caba532 Mon Sep 17 00:00:00 2001 From: JP Rodrigues <70jprodrigues@gmail.com> Date: Tue, 12 May 2020 19:37:14 -0300 Subject: [PATCH 2/2] Disable log of auth blocks by default --- src/Middleware/BlockBots.php | 4 +++- src/config/block-bots.php | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Middleware/BlockBots.php b/src/Middleware/BlockBots.php index dadf08d..c91bf4a 100644 --- a/src/Middleware/BlockBots.php +++ b/src/Middleware/BlockBots.php @@ -49,7 +49,9 @@ public function handle($request, Closure $next, $limit = 100, $frequency = 'dail protected function notAllowed() { if ($this->options->log) { - $this->logDisallowance(); + if (!$this->options->log_only_guest || Auth::guest()) { + $this->logDisallowance(); + } } if (Auth::check() && $this->isTheFirstOverflow()) { diff --git a/src/config/block-bots.php b/src/config/block-bots.php index 489081b..d03878c 100755 --- a/src/config/block-bots.php +++ b/src/config/block-bots.php @@ -54,6 +54,7 @@ */ 'log' => env('BLOCK_BOTS_LOG_ENABLED', env('BLOCK_BOTS_LOG_BLOCKED_REQUESTS', true)), + 'log_only_guest' => env('BLOCK_BOTS_LOG_ONLY_GUEST', true), /* * The list of allowed user-agents. The value of the key should be a keyword in hostname or * for enable to everyone