From ae01d554c54863ca0ca6e9727d8234be544ac876 Mon Sep 17 00:00:00 2001 From: adfoster-r7 Date: Tue, 30 Apr 2024 21:36:30 +0100 Subject: [PATCH] Ensure omnibus on windows can be rebuilt multiple times --- .github/workflows/verify.yml | 147 +++++++++++++++++- .../metasploit-framework/bundler.patch | 10 ++ config/patches/rubygems/rubygems-3.5.10.patch | 24 +++ config/projects/metasploit-framework.rb | 1 + config/software/bundler.rb | 2 +- config/software/metasploit-framework.rb | 30 +++- config/software/ruby-windows.rb | 6 +- config/software/ruby.rb | 3 +- config/software/rubygems.rb | 13 +- local/cache | 2 +- 10 files changed, 229 insertions(+), 9 deletions(-) create mode 100644 config/patches/metasploit-framework/bundler.patch create mode 100644 config/patches/rubygems/rubygems-3.5.10.patch diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 6149bc97..75e191bb 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -19,7 +19,7 @@ permissions: on: push: branches: - - '*' + - 'master' pull_request: branches: - '*' @@ -97,6 +97,8 @@ jobs: with: repository: rapid7/metasploit-framework path: metasploit-framework + # If testing a custom branch is required + # ref: 'update-bundler-version' - name: Run omnibus run: | @@ -126,6 +128,17 @@ jobs: docker rm -v $id docker rmi ${TEMP_DOCKER_IMAGE} + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: metasploit-${{ matrix.docker.name }}-installers + path: | + metasploit-omnibus/pkg/*.pkg + metasploit-omnibus/pkg/*.rpm + metasploit-omnibus/pkg/*.msi + metasploit-omnibus/pkg/*.deb + retention-days: 1 + docker_intel: runs-on: ${{ matrix.os }} timeout-minutes: 180 @@ -139,20 +152,26 @@ jobs: docker: - name: 'centos6-x64' previousDockerhubImage: 'rapid7/msf-centos6-x64-omnibus:2020_03' + installer: 'sudo rpm -i metasploit-omnibus/pkg/metasploit-framework*.rpm' # Currently fails as it uses an older Ruby version: - name: 'fedora30-x64' # XXX: Previous dockerhub image fails as using Ruby 2.5.3 still previousDockerhubImage: 'rapid7/msf-fedora30-x64-omnibus:2019_09' + installer: 'sudo rpm -i metasploit-omnibus/pkg/metasploit-framework*.rpm' # Currently fails on rate limiting on Kali's side: # - name: 'kali109-x64' # previousDockerhubImage: 'rapid7/msf-kali109-x64-omnibus:2020_03' + # installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_amd64.deb' - name: 'ubuntu1204-x64' previousDockerhubImage: 'rapid7/msf-ubuntu1204-x86-omnibus:2021_11' + installer: 'sudo dpkg -i metasploit-omnibus/pkg/*.deb' - name: 'ubuntu1204-x86' previousDockerhubImage: 'rapid7/msf-ubuntu1204-x64-omnibus:2019_01' linux32: true + installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_i386.deb' - name: 'ubuntu1804-x64' previousDockerhubImage: 'rapid7/msf-ubuntu1804-x64-omnibus:2019_09' + installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_amd64.deb' name: ${{ matrix.os }} - ${{ matrix.docker.name }} steps: @@ -215,6 +234,55 @@ jobs: env: LINUX32: ${{ matrix.docker.linux32 }} + - name: Test artifact + run: | + echo "Testing artifact" + + cat > test_script.sh < debug.log + $artifact = (Get-ChildItem -Path "metasploit-omnibus/pkg/*.msi")[0].Name + $install_process = Start-Process msiexec.exe -ArgumentList "/i metasploit-omnibus\pkg\$artifact /quiet /qn /l*v debug.log" -NoNewWindow -PassThru + $log_process = Start-Process "powershell" "Get-Content -Path debug.log -Wait" -NoNewWindow -PassThru + $install_process.WaitForExit() + $log_process.Kill() + echo "finished install" + + c:\metasploit-framework\bin\msfconsole -qx 'setg variable test; version; exit' + c:\metasploit-framework\bin\msfvenom -p windows/meterpreter/reverse_tcp -f exe -o test.exe + c:\metasploit-framework\bin\msfd -h + c:\metasploit-framework\bin\msfrpc -h + c:\metasploit-framework\bin\msfrpcd -h + c:\metasploit-framework\bin\msfdb -h + c:\metasploit-framework\bin\msfbinscan -h + c:\metasploit-framework\bin\msfrop -h + c:\metasploit-framework\bin\msfelfscan -h + c:\metasploit-framework\bin\msfmachscan -h + c:\metasploit-framework\bin\msfpescan -h + c:\metasploit-framework\bin\msfupdate + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: metasploit-windows-installers + path: | + metasploit-omnibus/pkg/*.pkg + metasploit-omnibus/pkg/*.rpm + metasploit-omnibus/pkg/*.msi + metasploit-omnibus/pkg/*.deb + retention-days: 1 diff --git a/config/patches/metasploit-framework/bundler.patch b/config/patches/metasploit-framework/bundler.patch new file mode 100644 index 00000000..d6aa414b --- /dev/null +++ b/config/patches/metasploit-framework/bundler.patch @@ -0,0 +1,10 @@ +diff --git a/Gemfile.lock b/Gemfile.lock +index 6802316759..89aaddd322 100644 +--- Gemfile.lock 2024-05-17 12:32:16 ++++ Gemfile.lock 2024-05-17 12:32:59 +@@ -574,4 +574,4 @@ + yard + + BUNDLED WITH +- 2.1.4 ++ 2.5.10 diff --git a/config/patches/rubygems/rubygems-3.5.10.patch b/config/patches/rubygems/rubygems-3.5.10.patch new file mode 100644 index 00000000..9900ce03 --- /dev/null +++ b/config/patches/rubygems/rubygems-3.5.10.patch @@ -0,0 +1,24 @@ +diff --git a/lib/rubygems/platform.rb b/lib/rubygems/platform.rb +index 48b7344aee..08728c209c 100644 +--- a/lib/rubygems/platform.rb ++++ b/lib/rubygems/platform.rb +@@ -1,7 +1,5 @@ + # frozen_string_literal: true + +-require_relative "deprecate" +- + ## + # Available list of platforms for targeting Gem installations. + # +@@ -24,11 +22,6 @@ def self.match(platform) + match_platforms?(platform, Gem.platforms) + end + +- class << self +- extend Gem::Deprecate +- rubygems_deprecate :match, "Gem::Platform.match_spec? or match_gem?" +- end +- + def self.match_platforms?(platform, platforms) + platform = Gem::Platform.new(platform) unless platform.is_a?(Gem::Platform) + platforms.any? do |local_platform| diff --git a/config/projects/metasploit-framework.rb b/config/projects/metasploit-framework.rb index 545cf576..b9396440 100644 --- a/config/projects/metasploit-framework.rb +++ b/config/projects/metasploit-framework.rb @@ -4,6 +4,7 @@ install_dir "#{default_root}/metasploit-framework" +# Version is extracted from the latest Git tag found in the local Git repository build_version Omnibus::BuildVersion.semver + "-1rapid7" build_iteration 1 diff --git a/config/software/bundler.rb b/config/software/bundler.rb index 0c712006..ca443e29 100644 --- a/config/software/bundler.rb +++ b/config/software/bundler.rb @@ -28,7 +28,7 @@ dependency "rubygems" end -default_version "2.1.4" +default_version "2.5.10" build do env = with_standard_compiler_flags(with_embedded_path) diff --git a/config/software/metasploit-framework.rb b/config/software/metasploit-framework.rb index 438c2f55..cf9e1456 100644 --- a/config/software/metasploit-framework.rb +++ b/config/software/metasploit-framework.rb @@ -1,8 +1,17 @@ name "metasploit-framework" + +# Detect a local checkout of metasploit-framework at '../metasploit-framework' - i.e. for the scenario of: +# - c:/temp/metasploit-omnibus +# - c:/temp/metasploit-framework (A local checkout of framework to use during the build process) +# but try and use 'C:/metasploit-framework' - as that's the metasploit-omnibus artifacts output directory +def has_windows_metasploit_framework_repo? + windows? && File.exist?('../metasploit-framework') && File.expand_path(File.join(Dir.pwd, "..", "metasploit-framework")) != "c:/metasploit-framework" +end + if linux? && File.exist?("/metasploit-framework") # supply current version of metasploit-framework at root of filesystem source path: "/metasploit-framework" -elsif windows? && File.exist?('../metasploit-framework') +elsif has_windows_metasploit_framework_repo? # supply current version of metasploit-framework relative to the current directory source path: "../metasploit-framework" else @@ -43,6 +52,8 @@ whitelist_file "#{install_dir}//embedded/lib/ruby/gems/#{ruby_abi_version}/.*/sqlite3.*" build do + patch_env = with_standard_compiler_flags(with_embedded_path) + patch source: "bundler.patch", plevel: 0, env: patch_env copy "#{project_dir}", "#{install_dir}/embedded/framework" major, minor, patch = Omnibus::BuildVersion.semver.split('.') @@ -79,12 +90,21 @@ vars: { install_dir: install_dir } end + bundle "version", env: env bundle "config set force_ruby_platform true", env: env bundle_env = with_standard_compiler_flags(with_embedded_path) bundle_env['MAKE'] = 'make -j4' + bundle_env['BUNDLE_FORCE_RUBY_PLATFORM'] = 'true' bundle "install --jobs=4", env: bundle_env if windows? + # Ensure we additionally copy out 'libssp-0.dll', which is required for multiple gems: + # > dumpbin /dependents C:/metasploit-framework/embedded/lib/ruby/gems/3.1.0/gems/msgpack-1.6.1/lib/msgpack/msgpack.so + # ... + # libssp-0.dll + # ... + copy "#{install_dir}/embedded/msys64/ucrt64/bin/libssp-0.dll", "#{install_dir}/embedded/bin/libssp-0.dll" + delete "#{install_dir}/embedded/msys64" end copy "#{project_dir}/Gemfile.lock", "#{install_dir}/embedded/framework/Gemfile.lock" @@ -154,4 +174,12 @@ end end end + + # Workaround for a Windows bug with chef r7_9.0.23_custom that allows the `.git` folders through + # into the final build result, leading to the .exe being an extra 1gb in size + block do + self.project.exclusions.each do |exclusion| + Pathname(install_dir).glob(exclusion).each(&:rmtree) + end + end end diff --git a/config/software/ruby-windows.rb b/config/software/ruby-windows.rb index d5b60b21..b403ff20 100644 --- a/config/software/ruby-windows.rb +++ b/config/software/ruby-windows.rb @@ -15,7 +15,7 @@ # name "ruby-windows" -default_version "3.1.4-1" +default_version "3.1.5-1" if windows_arch_i386? relative_path "rubyinstaller-#{version}-x86" @@ -58,6 +58,10 @@ source sha256: "6701088607ea4b587a31af76d75cb3fe9f7bcd75fc175cffcca22369ebb6331d" end + version "3.1.5-1" do + source sha256: "f31eece5b5c64563829117d41c09ed87572c595e5d64a525d49c1dd9c50dccca" + end + source url: "https://github.com/oneclick/rubyinstaller2/releases/download/rubyinstaller-#{version}/rubyinstaller-#{version}-x64.7z" end diff --git a/config/software/ruby.rb b/config/software/ruby.rb index ca66aeac..1b50af6f 100644 --- a/config/software/ruby.rb +++ b/config/software/ruby.rb @@ -26,7 +26,7 @@ # the default versions should always be the latest release of ruby # if you consume this definition it is your responsibility to pin # to the desired version of ruby. don't count on this not changing. -default_version "3.1.4" +default_version "3.1.5" dependency "zlib" dependency "openssl" @@ -45,6 +45,7 @@ version("3.2.2") { source sha256: "96c57558871a6748de5bc9f274e93f4b5aad06cd8f37befa0e8d94e7b8a423bc" } version("3.2.0") { source sha256: "daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" } version("3.1.4") { source sha256: "a3d55879a0dfab1d7141fdf10d22a07dbf8e5cdc4415da1bde06127d5cc3c7b6" } +version("3.1.5") { source sha256: "3685c51eeee1352c31ea039706d71976f53d00ab6d77312de6aa1abaf5cda2c5" } version("3.1.3") { source sha256: "5ea498a35f4cd15875200a52dde42b6eb179e1264e17d78732c3a57cd1c6ab9e" } version("3.1.2") { source sha256: "61843112389f02b735428b53bb64cf988ad9fb81858b8248e22e57336f24a83e" } version("3.1.1") { source sha256: "fe6e4782de97443978ddba8ba4be38d222aa24dc3e3f02a6a8e7701c0eeb619d" } diff --git a/config/software/rubygems.rb b/config/software/rubygems.rb index 68f5acf6..1ae9fd95 100644 --- a/config/software/rubygems.rb +++ b/config/software/rubygems.rb @@ -28,7 +28,7 @@ dependency "ruby" end -default_version "3.2.22" +default_version "3.5.10" if version && !source # NOTE: 2.1.11 is the last version of rubygems before the 2.2.x change to native gem install location @@ -40,7 +40,9 @@ # we pin the previously known tarballs. known_tarballs = { "3.1.4" => "d117187a8f016cbe8f52011ae02e858b", - "3.2.22"=> "b128d5493da2ec7a1da49a7189c04b35", + "3.2.22" => "b128d5493da2ec7a1da49a7189c04b35", + "3.3.26" => "ba171c52fd9beda6dac7194413601795", + "3.5.10" => "70f46c096b4e11c42b0190cc3e3375e2" } known_tarballs.each do |vsn, md5| version vsn do @@ -69,6 +71,13 @@ build do env = with_standard_compiler_flags(with_embedded_path) + # Fix warning on msfconsle bootup that will be fixed when framework's bundler version is updated + # NOTE: Gem::Platform.match is deprecated; use Gem::Platform.match_spec? or match_gem? instead. It will be removed in Rubygems 4 + # Gem::Platform.match called from /Users/adfoster/.rvm/gems/ruby-3.3.0/gems/bundler-2.1.4/lib/bundler/index.rb:198. + if version.satisfies?('>= 3.5.10') + patch source: "rubygems-3.5.10.patch", plevel: 1, env: env + end + if source # Building from source: ruby "setup.rb --no-document", env: env diff --git a/local/cache b/local/cache index 984fcfb6..0ad37117 160000 --- a/local/cache +++ b/local/cache @@ -1 +1 @@ -Subproject commit 984fcfb697151a942583ba831102c584d13773ad +Subproject commit 0ad37117f09e0fd78f109c36a17a48421b2ac7c6