Merge branch '__rultor'

lib/fbe/middleware.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# MIT License
+#
+# Copyright (c) 2024 Zerocracy
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# The module.
+module Fbe::Middleware
+  # empty
+end

lib/fbe/middleware/quota.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# MIT License
+#
+# Copyright (c) 2024 Zerocracy
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'faraday'
+
+# Faraday Middleware that monitors GitHub API rate limits.
+class Fbe::Middleware::Quota < Faraday::Middleware
+  def initialize(app, logger: Loog::NULL, pause: 60, limit: 100, rate: 5)
+    super(app)
+    @limit = limit
+    @requests = 0
+    @app = app
+    @logger = logger
+    @pause = pause
+    @rate = rate
+  end
+
+  def call(env)
+    @requests += 1
+    response = @app.call(env)
+    if out_of_limit?(env)
+      @logger.info(
+        "Too much GitHub API quota consumed, pausing for #{@pause} seconds"
+      )
+      sleep(@pause)
+      @requests = 0
+    end
+    response
+  end
+
+  private
+
+  def out_of_limit?(env)
+    remaining = env.response_headers['x-ratelimit-remaining'].to_i
+    (@requests % @limit).zero? && remaining < @rate
+  end
+end

lib/fbe/octo.rb

@@ -30,6 +30,7 @@
 require 'faraday/http_cache'
 require 'faraday/retry'
 require_relative '../fbe'
+require_relative 'middleware/quota'
 
 def Fbe.octo(options: $options, global: $global, loog: $loog)
   raise 'The $global is not set' if global.nil?
@@ -75,6 +76,11 @@ def Fbe.octo(options: $options, global: $global, loog: $loog)
           methods: [:get],
           backoff_factor: 2
         )
+        builder.use(
+          Fbe::Middleware::Quota,
+          logger: loog,
+          pause: options.github_api_pause
+        )
         builder.use(Faraday::HttpCache, serializer: Marshal, shared_cache: false, logger: Loog::NULL)
         builder.use(Octokit::Response::RaiseError)
         builder.use(Faraday::Response::Logger, Loog::NULL)

test/fbe/middleware/test_quota.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# MIT License
+#
+# Copyright (c) 2024 Zerocracy
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'minitest/autorun'
+require 'faraday'
+require 'logger'
+require_relative '../../../lib/fbe/middleware'
+
+class QuotaTest < Minitest::Test
+  class FakeApp
+    def initialize
+      @calls = 0
+    end
+
+    def call(env)
+      @calls += 1
+      response_headers = {
+        'x-ratelimit-remaining' => (100 - @calls).to_s
+      }
+      env[:response_headers] = response_headers
+      env
+    end
+  end
+
+  def test_quota_middleware_pauses_when_quota_low
+    loog = Loog::NULL
+    pause = 0
+    app = FakeApp.new
+    middleware = Fbe::Middleware::Quota.new(app, logger: loog, pause:)
+    start_time = Time.now
+    105.times do
+      env = Judges::Options.new(
+        'method' => :get,
+        'url' => 'http://example.com',
+        'request_headers' => {},
+        'response_headers' => {}
+      )
+      middleware.call(env)
+    end
+    assert_in_delta pause, Time.now - start_time, 0.4
+  end
+
+  def test_quota_middleware_logs_when_quota_low
+    pause = 1
+    log_output = StringIO.new
+    loog = Logger.new(log_output)
+    app = FakeApp.new
+    middleware = Fbe::Middleware::Quota.new(app, logger: loog, pause:)
+    105.times do
+      env = Judges::Options.new(
+        'method' => :get,
+        'url' => 'http://example.com',
+        'request_headers' => {},
+        'response_headers' => {}
+      )
+      middleware.call(env)
+    end
+    assert_match(/Too much GitHub API quota/, log_output.string)
+  end
+end

test/fbe/test_octo.rb

@@ -117,4 +117,31 @@ def test_commit_pulls
     assert_equal(1, o.commit_pulls('zerocracy/fbe', '0b7d0699bd744b62c0731064c2adaad0c58e1416').size)
     assert_equal(0, o.commit_pulls('zerocracy/fbe', '16b3ea6b71c6e932ba7666c40ca846ecaa6d6f0d').size)
   end
+
+  def test_pauses_when_quota_is_exceeded
+    WebMock.disable_net_connect!
+    global = {}
+    pause = 1
+    o = Fbe.octo(loog: Loog::NULL, global:, options: Judges::Options.new({ 'github_api_pause' => pause }))
+    limit = 100
+    start_time = Time.now
+    105.times do |i|
+      n = i + 1
+      user = "test#{n}"
+      limit = 100 if n > 100
+      stub_request(:get, "https://api.github.com/users/#{user}")
+        .then
+        .to_return(
+          status: 200, body: '{}',
+          headers: {
+            'x-ratelimit-remaining' => limit.to_s
+          }
+        )
+        .times(1)
+      o.user(user)
+      assert(!o.off_quota) if n > 100
+      limit -= 1
+    end
+    assert_in_delta(pause, Time.now - start_time, 0.4)
+  end
 end