diff --git a/spec/lib/appsignal/environment_spec.rb b/spec/lib/appsignal/environment_spec.rb
index 4befdc2e..9bff9615 100644
--- a/spec/lib/appsignal/environment_spec.rb
+++ b/spec/lib/appsignal/environment_spec.rb
@@ -115,12 +115,26 @@ def inspect
   end
 
   describe ".report_supported_gems" do
-    it "reports about all AppSignal supported gems in the bundle" do
+    it "reports about all AppSignal supported gems in the bundle using Bundler all_specs" do
       logs = capture_logs { described_class.report_supported_gems }
 
       expect(logs).to be_empty
 
-      bundle_gem_specs = ::Bundler.rubygems.all_specs
+      bundle_gem_specs = silence { ::Bundler.rubygems.all_specs }
+      rack_spec = bundle_gem_specs.find { |s| s.name == "rack" }
+      rake_spec = bundle_gem_specs.find { |s| s.name == "rake" }
+      expect_environment_metadata("ruby_rack_version", rack_spec.version.to_s)
+      expect_environment_metadata("ruby_rake_version", rake_spec.version.to_s)
+      expect(rack_spec.version.to_s).to_not be_empty
+      expect(rake_spec.version.to_s).to_not be_empty
+    end
+
+    it "reports about all AppSignal supported gems in the bundle using bundler installed_specs" do
+      logs = capture_logs { described_class.report_supported_gems }
+
+      expect(logs).to be_empty
+
+      bundle_gem_specs = ::Bundler.rubygems.installed_specs
       rack_spec = bundle_gem_specs.find { |s| s.name == "rack" }
       rake_spec = bundle_gem_specs.find { |s| s.name == "rake" }
       expect_environment_metadata("ruby_rack_version", rack_spec.version.to_s)