diff --git a/Gemfile b/Gemfile index 83b7b2811fbd..8f8e4795cb36 100644 --- a/Gemfile +++ b/Gemfile @@ -35,7 +35,7 @@ group :development, :test do gem 'rake' # Define `rake spec`. Must be in development AND test so that its available by default as a rake test when the # environment is development - gem 'rspec-rails' + gem 'rspec-rails', '~> 7.0' gem 'rspec-rerun' # Required during CI as well local development gem 'rubocop' diff --git a/Gemfile.lock b/Gemfile.lock index 81bf73fcf103..704e3652bb56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,9 +4,9 @@ PATH metasploit-framework (6.4.37) aarch64 abbrev - actionpack (~> 7.0.0) - activerecord (~> 7.0.0) - activesupport (~> 7.0.0) + actionpack (~> 7.1.0) + activerecord (~> 7.1.0) + activesupport (~> 7.1.0) aws-sdk-ec2 aws-sdk-ec2instanceconnect aws-sdk-iam @@ -95,7 +95,7 @@ PATH ruby_smb (~> 3.3.3) rubyntlm rubyzip - sinatra + sinatra (~> 3) sqlite3 (= 1.7.3) sshkey swagger-blocks @@ -118,28 +118,40 @@ GEM aarch64 (2.1.0) racc (~> 1.6) abbrev (0.1.2) - actionpack (7.0.8.6) - actionview (= 7.0.8.6) - activesupport (= 7.0.8.6) - rack (~> 2.0, >= 2.2.4) + actionpack (7.1.5) + actionview (= 7.1.5) + activesupport (= 7.1.5) + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4) + rack-session (>= 1.0.1) rack-test (>= 0.6.3) - rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.2.0) - actionview (7.0.8.6) - activesupport (= 7.0.8.6) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + actionview (7.1.5) + activesupport (= 7.1.5) builder (~> 3.1) - erubi (~> 1.4) - rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.1, >= 1.2.0) - activemodel (7.0.8.6) - activesupport (= 7.0.8.6) - activerecord (7.0.8.6) - activemodel (= 7.0.8.6) - activesupport (= 7.0.8.6) - activesupport (7.0.8.6) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activemodel (7.1.5) + activesupport (= 7.1.5) + activerecord (7.1.5) + activemodel (= 7.1.5) + activesupport (= 7.1.5) + timeout (>= 0.4.0) + activesupport (7.1.5) + base64 + benchmark (>= 0.3) + bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) + mutex_m + securerandom (>= 0.3) tzinfo (~> 2.0) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) @@ -186,6 +198,7 @@ GEM base64 (0.2.0) bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) + benchmark (0.4.0) bigdecimal (3.1.8) bindata (2.4.15) bootsnap (1.18.4) @@ -196,6 +209,7 @@ GEM chunky_png (1.4.0) coderay (1.1.3) concurrent-ruby (1.3.4) + connection_pool (2.4.1) cookiejar (0.3.4) crass (1.0.6) csv (3.3.0) @@ -380,8 +394,13 @@ GEM rack-protection (3.2.0) base64 (>= 0.1.0) rack (~> 2.2, >= 2.2.4) + rack-session (1.0.2) + rack (< 3) rack-test (2.1.0) rack (>= 1.3) + rackup (1.0.1) + rack (< 3) + webrick rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -389,13 +408,14 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.0.8.6) - actionpack (= 7.0.8.6) - activesupport (= 7.0.8.6) - method_source + railties (7.1.5) + actionpack (= 7.1.5) + activesupport (= 7.1.5) + irb + rackup (>= 1.0.0) rake (>= 12.2) - thor (~> 1.0) - zeitwerk (~> 2.5) + thor (~> 1.0, >= 1.2.2) + zeitwerk (~> 2.6) rainbow (3.1.1) rake (13.2.1) rasn1 (0.13.0) @@ -405,7 +425,7 @@ GEM nokogiri redcarpet (3.6.0) regexp_parser (2.9.2) - reline (0.5.10) + reline (0.5.11) io-console (~> 0.5) require_all (3.0.0) rex-arch (0.1.16) @@ -470,7 +490,7 @@ GEM rspec-mocks (3.13.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-rails (7.0.1) + rspec-rails (7.1.0) actionpack (>= 7.0) activesupport (>= 7.0) railties (>= 7.0) @@ -511,6 +531,7 @@ GEM sawyer (0.9.2) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) + securerandom (0.3.2) simplecov (0.18.2) docile (~> 1.1) simplecov-html (~> 0.11) @@ -548,7 +569,7 @@ GEM macaddr (~> 1.0) warden (1.2.9) rack (>= 2.0.9) - webrick (1.8.2) + webrick (1.9.0) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -586,7 +607,7 @@ DEPENDENCIES pry-byebug rake redcarpet - rspec-rails + rspec-rails (~> 7.0) rspec-rerun rubocop ruby-prof (= 1.4.2) diff --git a/config/README.md b/config/README.md index 431188811c3c..5eb397d0fd43 100644 --- a/config/README.md +++ b/config/README.md @@ -1,3 +1,12 @@ +# Metasploit Framework Config Folder + Contains various files that help configure Metasploit. Most files here you'll never have to deal with, though `database.yml.example` might be useful for those looking to configure their database, and `openssl.conf` -might be helpful for those trying to troubleshoot OpenSSL issues in Metasploit. \ No newline at end of file +might be helpful for those trying to troubleshoot OpenSSL issues in Metasploit. + +> [!IMPORTANT] +> Because the behavior of Ruby on Rails changes between versions, +> and code needs to be considered thread-safe when dealing with Ruby on Rails, +> we ensure that the `reconnect: true` property is configured for our database +> connection. This allows the console/framework to reconnect when a thread messes +> up the connection pool. diff --git a/config/application.rb b/config/application.rb index bda8166b912e..f9cce0942646 100644 --- a/config/application.rb +++ b/config/application.rb @@ -37,22 +37,53 @@ module Framework class Application < Rails::Application include Metasploit::Framework::CommonEngine - config.paths['log'] = "#{Msf::Config.log_directory}/#{Rails.env}.log" + config.paths['log'] = "#{Msf::Config.log_directory}/#{Rails.env}.log" config.paths['config/database'] = [Metasploit::Framework::Database.configurations_pathname.try(:to_path)] + config.autoloader = :zeitwerk - case Rails.env - when "development" - config.eager_load = false - when "test" - config.eager_load = false - when "production" - config.eager_load = false - end + # Load the Rails 7.1 defaults. + config.load_defaults 7.1 + + # The cache behavior changed with Rails 7.1, and requires the desired version to be set. + config.active_support.cache_format_version = 7.1 + + # Timezone shenanigans + config.time_zone = 'UTC' + + if config.respond_to?(:active_record) + # The default column serializer was YAML prior to Rails 7.1 + config.active_record.default_column_serializer = ::YAML + + # Timezone settings + config.active_record.default_timezone = :utc - if ActiveRecord.respond_to?(:legacy_connection_handling=) - ActiveRecord.legacy_connection_handling = false + # Partials inserts are disabled by default in Rails 7 + # This only writes attributes that changed. + config.active_record.partial_inserts = true + + # Foreign Key Validation - Belongs-to + # Was not enabled by default + config.active_record.belongs_to_required_validates_foreign_key = true + + # This behavior changed in 7.1 + config.active_record.commit_transaction_on_non_local_return = false + + # Originally allowed but silently ignored, raises in 7.1 + config.active_record.raise_on_assign_to_attr_readonly = false + + # Rails originally ran the callbacks on the first commit change. + # In Rails 7.1 this is done on all models, so we need to retain the behavior for now. + config.active_record.run_commit_callbacks_on_first_saved_instances_in_transaction = true + + # Rails 7.1 will execute after commit callbacks in order they are defined. + # Originally it was in reverse order. + config.active_record.run_after_transaction_callbacks_in_order_defined = false end + + # We never eager load files. + config.eager_load = false + config.enable_reloading = ::Rails.env.test? end end end diff --git a/config/database.yml.example b/config/database.yml.example index b04aede6b087..f90b3ac9a8d0 100644 --- a/config/database.yml.example +++ b/config/database.yml.example @@ -5,14 +5,16 @@ # managing your database, which may be more convenient than rolling your own. development: &pgsql + allow_concurrency: true adapter: postgresql database: metasploit_framework_development username: metasploit_framework_development password: __________________________________ host: localhost port: 5432 - pool: 200 + pool: 10 timeout: 5 + reconnect: true # You will often want to seperate your databases between dev # mode and prod mode. Absent a production db, though, defaulting diff --git a/config/database.yml.github_actions b/config/database.yml.github_actions index bd2fef7cb7df..0b8eae2e3999 100644 --- a/config/database.yml.github_actions +++ b/config/database.yml.github_actions @@ -7,13 +7,16 @@ # # update password fields for each environment's user development: &pgsql + allow_concurrency: true adapter: postgresql database: metasploit_framework_development + port: 5432 host: localhost username: postgres password: postgres - pool: 25 + pool: 10 timeout: 5 + reconnect: true # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". diff --git a/config/database.yml.vagrant b/config/database.yml.vagrant index d5a94c96b238..dd03c6c178b7 100644 --- a/config/database.yml.vagrant +++ b/config/database.yml.vagrant @@ -1,12 +1,14 @@ development: &pgsql + allow_concurrency: true adapter: postgresql database: msf_dev_db username: vagrant password: vagrant host: localhost port: 5432 - pool: 200 + pool: 10 timeout: 5 + reconnect: true production: &production <<: *pgsql diff --git a/db/schema.rb b/db/schema.rb index 90d6436444c1..74f5b65d3a44 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_12_09_005658) do +ActiveRecord::Schema[7.1].define(version: 2022_12_09_005658) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 11f46b4dc280..c30ac70d83d9 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -7,6 +7,6 @@ services: BUNDLER_ARGS: --jobs=8 image: metasploit:dev environment: - DATABASE_URL: postgres://postgres@db:5432/msf_dev?pool=200&timeout=5 + DATABASE_URL: postgres://postgres@db:5432/msf_dev?pool=100&timeout=5&reconnect=true&allow_concurrency=true volumes: - .:/usr/src/metasploit-framework diff --git a/docker-compose.yml b/docker-compose.yml index e4b1d12d7d5e..7e648015d16f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: ms: image: metasploitframework/metasploit-framework:latest environment: - DATABASE_URL: postgres://postgres@db:5432/msf?pool=200&timeout=5 + DATABASE_URL: postgres://postgres@db:5432/msf?pool=10&timeout=5&reconnect=true&allow_concurrency=true links: - db ports: @@ -11,7 +11,7 @@ services: - $HOME/.msf4:/home/msf/.msf4 db: - image: postgres:10-alpine + image: postgres:14-alpine volumes: - pg_data:/var/lib/postgresql/data environment: diff --git a/docs/metasploit-framework.wiki/Work-needed-to-allow-msfdb-to-use-postgresql-common.md b/docs/metasploit-framework.wiki/Work-needed-to-allow-msfdb-to-use-postgresql-common.md index c8e934695188..d4d1b3c12212 100644 --- a/docs/metasploit-framework.wiki/Work-needed-to-allow-msfdb-to-use-postgresql-common.md +++ b/docs/metasploit-framework.wiki/Work-needed-to-allow-msfdb-to-use-postgresql-common.md @@ -104,7 +104,7 @@ development: &pgsql password: Password123 host: 127.0.0.1 port: 5433 - pool: 200 + pool: 10 production: &production <<: *pgsql diff --git a/lib/metasploit/framework/common_engine.rb b/lib/metasploit/framework/common_engine.rb index 6198dbc176bb..41d56f69f75c 100644 --- a/lib/metasploit/framework/common_engine.rb +++ b/lib/metasploit/framework/common_engine.rb @@ -40,10 +40,6 @@ module Metasploit::Framework::CommonEngine config.active_support.deprecation = :stderr - if ActiveRecord.respond_to?(:legacy_connection_handling=) - ActiveRecord.legacy_connection_handling = false - end - # @see https://github.com/rapid7/metasploit_data_models/blob/54a17149d5ccd0830db742d14c4987b48399ceb7/lib/metasploit_data_models/yaml.rb#L10 # @see https://github.com/rapid7/metasploit_data_models/blob/54a17149d5ccd0830db742d14c4987b48399ceb7/lib/metasploit_data_models/base64_serializer.rb#L28-L31 ActiveRecord.yaml_column_permitted_classes = (ActiveRecord.yaml_column_permitted_classes + MetasploitDataModels::YAML::PERMITTED_CLASSES).uniq diff --git a/lib/metasploit/framework/rails_version_constraint.rb b/lib/metasploit/framework/rails_version_constraint.rb index 474a5b494f6c..042643891782 100644 --- a/lib/metasploit/framework/rails_version_constraint.rb +++ b/lib/metasploit/framework/rails_version_constraint.rb @@ -3,7 +3,7 @@ module Metasploit module Framework module RailsVersionConstraint - RAILS_VERSION = '~> 7.0.0' + RAILS_VERSION = '~> 7.1.0' end end end diff --git a/lib/msf/core/db_manager/connection.rb b/lib/msf/core/db_manager/connection.rb index 97c878b8867d..e8004f68d0a9 100644 --- a/lib/msf/core/db_manager/connection.rb +++ b/lib/msf/core/db_manager/connection.rb @@ -82,31 +82,14 @@ def create_db(opts) begin case opts["adapter"] when 'postgresql' - # Try to force a connection to be made to the database, if it succeeds - # then we know we don't need to create it :) - ApplicationRecord.establish_connection(opts) - # Do the checkout, checkin dance here to make sure this thread doesn't - # hold on to a connection we don't need - conn = ApplicationRecord.connection_pool.checkout - ApplicationRecord.connection_pool.checkin(conn) - end - rescue ::Exception => e - errstr = e.to_s - if errstr =~ /does not exist/i or errstr =~ /Unknown database/ - ilog("Database doesn't exist \"#{opts['database']}\", attempting to create it.") - ApplicationRecord.establish_connection( - opts.merge( - 'database' => 'postgres', - 'schema_search_path' => 'public' - ) - ) - - ApplicationRecord.connection.create_database(opts['database']) + existing_db = ::ApplicationRecord.connection_pool.with_connection(&:active) rescue false + ::ApplicationRecord.connection.create_database(opts['database']) unless existing_db else - ilog("Trying to continue despite failed database creation: #{e}") + ilog("Unknown database adapter: #{opts['adapter']}") end + rescue ::Exception => e + ilog("Trying to continue despite failed database creation: #{e}") end - ApplicationRecord.remove_connection end # Checks if the spec passed to `ApplicationRecord.establish_connection` can connect to the database. diff --git a/lib/msf/core/thread_manager.rb b/lib/msf/core/thread_manager.rb index 7fe59f501805..9bc89ae37ca6 100644 --- a/lib/msf/core/thread_manager.rb +++ b/lib/msf/core/thread_manager.rb @@ -111,16 +111,6 @@ def spawn(name, crit, *args, &block) error: e ) raise e - ensure - if framework.db && framework.db.active && framework.db.is_local? - # NOTE: despite the Deprecation Warning's advice, this should *NOT* - # be ApplicationRecord.connection.close which causes unrelated - # threads to raise ActiveRecord::StatementInvalid exceptions at - # some point in the future, presumably due to the pool manager - # believing that the connection is still usable and handing it out - # to another thread. - ::ApplicationRecord.connection_pool.release_connection - end end end else diff --git a/metasploit-framework.gemspec b/metasploit-framework.gemspec index ddec5cf4b716..0e181c15dfaf 100644 --- a/metasploit-framework.gemspec +++ b/metasploit-framework.gemspec @@ -109,7 +109,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'puma' spec.add_runtime_dependency 'ruby-mysql' spec.add_runtime_dependency 'thin' - spec.add_runtime_dependency 'sinatra' + spec.add_runtime_dependency 'sinatra', '~> 3' spec.add_runtime_dependency 'warden' spec.add_runtime_dependency 'swagger-blocks' # Required for JSON-RPC client diff --git a/msfdb b/msfdb index 6649fd250c2e..7037bac2c647 100755 --- a/msfdb +++ b/msfdb @@ -66,7 +66,7 @@ require 'msfenv' msftest_db_user: 'msftest', db_host: '127.0.0.1', db_port: 5433, - db_pool: 200, + db_pool: 10, address: 'localhost', port: 5443, daemon: true, diff --git a/spec/allure_config.rb b/spec/allure_config.rb index 97b9a34df4d5..84b0e33a9f7a 100644 --- a/spec/allure_config.rb +++ b/spec/allure_config.rb @@ -1,4 +1,6 @@ require "allure-rspec" +require "active_support" +require "active_support/core_ext/object" AllureRspec.configure do |config| config.results_directory = "tmp/allure-raw-data" diff --git a/spec/modules/auxiliary/admin/kerberos/forge_ticket_spec.rb b/spec/modules/auxiliary/admin/kerberos/forge_ticket_spec.rb index b64200d39776..957da6156fe0 100644 --- a/spec/modules/auxiliary/admin/kerberos/forge_ticket_spec.rb +++ b/spec/modules/auxiliary/admin/kerberos/forge_ticket_spec.rb @@ -56,8 +56,8 @@ Addresses: 0 Authdatas: 0 Times: - Auth time: #{Time.parse('2022-07-15 13:33:40 +0100').to_time} - Start time: #{Time.parse('2022-07-15 13:33:40 +0100').to_time} + Auth time: #{Time.parse('2022-07-15 12:33:40 +0000').to_time} + Start time: #{Time.parse('2022-07-15 12:33:40 +0000').to_time} End time: #{Time.parse('2032-07-12 13:33:40 +0100').to_time} Renew Till: #{Time.parse('2032-07-12 13:33:40 +0100').to_time} Ticket: @@ -82,7 +82,7 @@ Flags: 0x50e00000 (FORWARDABLE, PROXIABLE, RENEWABLE, INITIAL, PRE_AUTHENT) PAC: Validation Info: - Logon Time: #{Time.parse('2022-07-15 13:33:40 +0100').to_time} + Logon Time: #{Time.parse('2022-07-15 12:33:40 +0000').to_time} Logoff Time: Never Expires (inf) Kick Off Time: Never Expires (inf) Password Last Set: No Time Set (0) @@ -185,7 +185,7 @@ Logon Domain Name: 'DEMO.LOCAL' Client Info: Name: 'Administrator' - Client ID: #{Time.parse('2022-07-15 13:33:40 +0100').to_time} + Client ID: #{Time.parse('2022-07-15 12:33:40 +0000').to_time} Pac Requestor: SID: S-1-5-21-1266190811-2419310613-1856291569-500 Pac Attributes: diff --git a/spec/modules/auxiliary/admin/kerberos/keytab_spec.rb b/spec/modules/auxiliary/admin/kerberos/keytab_spec.rb index cc3e2bd9496f..4884f1678933 100644 --- a/spec/modules/auxiliary/admin/kerberos/keytab_spec.rb +++ b/spec/modules/auxiliary/admin/kerberos/keytab_spec.rb @@ -295,9 +295,9 @@ def report_creds( kvno type principal hash date ---- ---- --------- ---- ---- - 1 23 (RC4_HMAC) user_without_realm@ e02bc503339d51f71d913c245d35b50b #{Time.parse('1970-01-01 01:00:00 +0100').to_time} - 1 23 (RC4_HMAC) user_with_realm@example.local 32ede47af254546a82b1743953cc4950 #{Time.parse('1970-01-01 01:00:00 +0100').to_time} - 1 18 (AES256) user_with_krbkey@demo.local 63346133663331643634616661363438613664303864303737363536336531323338623937366430623930663739656130373231393433363832393465393239 #{Time.parse('1970-01-01 01:00:00 +0100').to_time} + 1 23 (RC4_HMAC) user_without_realm@ e02bc503339d51f71d913c245d35b50b #{Time.parse('1970-01-01 00:00:00 +0000').to_time} + 1 23 (RC4_HMAC) user_with_realm@example.local 32ede47af254546a82b1743953cc4950 #{Time.parse('1970-01-01 00:00:00 +0000').to_time} + 1 18 (AES256) user_with_krbkey@demo.local 63346133663331643634616661363438613664303864303737363536336531323338623937366430623930663739656130373231393433363832393465393239 #{Time.parse('1970-01-01 00:00:00 +0000').to_time} TABLE end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6e0463ca4672..f009fca22100 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -99,7 +99,9 @@ config.order = :random if load_metasploit + # Run fixtures and examples in transactions to keep the database clean. config.use_transactional_fixtures = true + config.use_transactional_examples = true # rspec-rails 3 will no longer automatically infer an example group's spec type # from the file location. You can explicitly opt-in to the feature using this