Ensure omnibus on windows can be rebuilt multiple times

rapid7 · May 17, 2024 · ae01d55 · ae01d55
1 parent 0008a23
commit ae01d55
Show file tree

Hide file tree

Showing 10 changed files with 229 additions and 9 deletions.
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
@@ -19,7 +19,7 @@ permissions:
 on:
   push:
     branches:
-      - '*'
+      - 'master'
   pull_request:
     branches:
       - '*'
@@ -97,6 +97,8 @@ jobs:
         with:
           repository: rapid7/metasploit-framework
           path: metasploit-framework
+          # If testing a custom branch is required
+          # ref: 'update-bundler-version'
 
       - name: Run omnibus
         run: |
@@ -126,6 +128,17 @@ jobs:
           docker rm -v $id
           docker rmi ${TEMP_DOCKER_IMAGE}
 
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: metasploit-${{ matrix.docker.name }}-installers
+          path: |
+            metasploit-omnibus/pkg/*.pkg
+            metasploit-omnibus/pkg/*.rpm
+            metasploit-omnibus/pkg/*.msi
+            metasploit-omnibus/pkg/*.deb
+          retention-days: 1
+
   docker_intel:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
@@ -139,20 +152,26 @@ jobs:
         docker:
           - name: 'centos6-x64'
             previousDockerhubImage: 'rapid7/msf-centos6-x64-omnibus:2020_03'
+            installer: 'sudo rpm -i metasploit-omnibus/pkg/metasploit-framework*.rpm'
           # Currently fails as it uses an older Ruby version:
           - name: 'fedora30-x64'
             # XXX: Previous dockerhub image fails as using Ruby 2.5.3 still
             previousDockerhubImage: 'rapid7/msf-fedora30-x64-omnibus:2019_09'
+            installer: 'sudo rpm -i metasploit-omnibus/pkg/metasploit-framework*.rpm'
           # Currently fails on rate limiting on Kali's side:
           # - name: 'kali109-x64'
           #   previousDockerhubImage: 'rapid7/msf-kali109-x64-omnibus:2020_03'
+          #   installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_amd64.deb'
           - name: 'ubuntu1204-x64'
             previousDockerhubImage: 'rapid7/msf-ubuntu1204-x86-omnibus:2021_11'
+            installer: 'sudo dpkg -i metasploit-omnibus/pkg/*.deb'
           - name: 'ubuntu1204-x86'
             previousDockerhubImage: 'rapid7/msf-ubuntu1204-x64-omnibus:2019_01'
             linux32: true
+            installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_i386.deb'
           - name: 'ubuntu1804-x64'
             previousDockerhubImage: 'rapid7/msf-ubuntu1804-x64-omnibus:2019_09'
+            installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_amd64.deb'
 
     name: ${{ matrix.os }} - ${{ matrix.docker.name }}
     steps:
@@ -215,6 +234,55 @@ jobs:
         env:
           LINUX32: ${{ matrix.docker.linux32 }}
 
+      - name: Test artifact
+        run: |
+          echo "Testing artifact"
+
+          cat > test_script.sh <<EOF
+          #!/bin/bash -ex
+          find metasploit-omnibus/pkg
+          $INSTALL_ARTIFACT
+
+          mkdir ~/.msf4; touch ~/.msf4/initial_setup_complete
+
+          msfconsole -qx 'setg variable test; version; exit'
+          msfvenom -p windows/meterpreter/reverse_tcp -f exe -o test.exe
+          msfd -h
+          msfrpc -h
+          msfrpcd -h
+          msfdb -h
+          msfbinscan -h
+          msfrop -h
+          msfelfscan -h
+          msfmachscan -h
+          msfpescan -h
+          # msfupdate
+          EOF
+
+          chmod +x ./test_script.sh
+
+          # If required, change reported architecture in new program environment and set personality flags
+          if [ ! -z "${LINUX32}" ] ; then
+            echo 'setting linux32'
+            /bin/bash -x -c "docker run --rm --volume $(pwd):$(pwd) --workdir $(pwd) --user jenkins ${DOCKER_IMAGE} linux32 /bin/bash -l -c './test_script.sh'"
+          else
+            /bin/bash -x -c "docker run --rm --volume $(pwd):$(pwd) --workdir $(pwd) --user jenkins ${DOCKER_IMAGE} /bin/bash -l -c './test_script.sh'"
+          fi
+        env:
+          LINUX32: ${{ matrix.docker.linux32 }}
+          INSTALL_ARTIFACT: ${{ matrix.docker.installer }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: metasploit-${{ matrix.docker.name }}-installers
+          path: |
+            metasploit-omnibus/pkg/*.pkg
+            metasploit-omnibus/pkg/*.rpm
+            metasploit-omnibus/pkg/*.msi
+            metasploit-omnibus/pkg/*.deb
+          retention-days: 1
+
   osx:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
@@ -255,6 +323,39 @@ jobs:
           cd metasploit-omnibus
           make
 
+      - name: Test artifact
+        run: |
+          echo "Testing artifact"
+
+          sudo rm -rf /opt/metasploit-framework
+          /usr/bin/find metasploit-omnibus/pkg
+          PACKAGE=$(pwd)/$(/usr/bin/find metasploit-omnibus/pkg/metasploit*.pkg | head -n 1)
+          sudo installer -pkg ${PACKAGE} -target /
+
+          /opt/metasploit-framework/bin/msfconsole -qx 'setg variable test; version; exit'
+          /opt/metasploit-framework/bin/msfvenom -p windows/meterpreter/reverse_tcp -f exe -o test.exe
+          /opt/metasploit-framework/bin/msfd -h
+          /opt/metasploit-framework/bin/msfrpc -h
+          /opt/metasploit-framework/bin/msfrpcd -h
+          /opt/metasploit-framework/bin/msfdb -h
+          /opt/metasploit-framework/bin/msfbinscan -h
+          /opt/metasploit-framework/bin/msfrop -h
+          /opt/metasploit-framework/bin/msfelfscan -h
+          /opt/metasploit-framework/bin/msfmachscan -h
+          /opt/metasploit-framework/bin/msfpescan -h
+          # /opt/metasploit-framework/bin/msfupdate
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: metasploit-osx-installers
+          path: |
+            metasploit-omnibus/pkg/*.pkg
+            metasploit-omnibus/pkg/*.rpm
+            metasploit-omnibus/pkg/*.msi
+            metasploit-omnibus/pkg/*.deb
+          retention-days: 1
+
   windows:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
@@ -338,6 +439,48 @@ jobs:
           cd metasploit-omnibus
           make dependencies
 
-          rem Don't run the main build itslef under `make`, as the process will be spawned under msys2
+          rem Don't run the main build itself under `make`, as the process will be spawned under msys2
           rem and the ridk.cmd Ruby installer will forcibly kill the msys2 process before attempting to install ruby
           ruby bin/omnibus build metasploit-framework
+
+      - name: Test artifact
+        shell: pwsh
+        run: |
+          Set-PSDebug -Trace 1
+          echo "Testing artifact"
+
+          Remove-Item c:\metasploit-framework -Recurse -ErrorAction Ignore
+
+          dir metasploit-omnibus\pkg
+
+          echo '' > debug.log
+          $artifact = (Get-ChildItem -Path "metasploit-omnibus/pkg/*.msi")[0].Name
+          $install_process = Start-Process msiexec.exe -ArgumentList "/i metasploit-omnibus\pkg\$artifact /quiet /qn /l*v debug.log" -NoNewWindow -PassThru
+          $log_process = Start-Process "powershell" "Get-Content -Path debug.log -Wait" -NoNewWindow -PassThru
+          $install_process.WaitForExit()
+          $log_process.Kill()
+          echo "finished install"
+
+          c:\metasploit-framework\bin\msfconsole -qx 'setg variable test; version; exit'
+          c:\metasploit-framework\bin\msfvenom -p windows/meterpreter/reverse_tcp -f exe -o test.exe
+          c:\metasploit-framework\bin\msfd -h
+          c:\metasploit-framework\bin\msfrpc -h
+          c:\metasploit-framework\bin\msfrpcd -h
+          c:\metasploit-framework\bin\msfdb -h
+          c:\metasploit-framework\bin\msfbinscan -h
+          c:\metasploit-framework\bin\msfrop -h
+          c:\metasploit-framework\bin\msfelfscan -h
+          c:\metasploit-framework\bin\msfmachscan -h
+          c:\metasploit-framework\bin\msfpescan -h
+          c:\metasploit-framework\bin\msfupdate
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: metasploit-windows-installers
+          path: |
+            metasploit-omnibus/pkg/*.pkg
+            metasploit-omnibus/pkg/*.rpm
+            metasploit-omnibus/pkg/*.msi
+            metasploit-omnibus/pkg/*.deb
+          retention-days: 1
diff --git a/config/patches/metasploit-framework/bundler.patch b/config/patches/metasploit-framework/bundler.patch
@@ -0,0 +1,10 @@
+diff --git a/Gemfile.lock b/Gemfile.lock
+index 6802316759..89aaddd322 100644
+--- Gemfile.lock	2024-05-17 12:32:16
++++ Gemfile.lock	2024-05-17 12:32:59
+@@ -574,4 +574,4 @@
+   yard
+
+ BUNDLED WITH
+-   2.1.4
++   2.5.10
diff --git a/config/patches/rubygems/rubygems-3.5.10.patch b/config/patches/rubygems/rubygems-3.5.10.patch
@@ -0,0 +1,24 @@
+diff --git a/lib/rubygems/platform.rb b/lib/rubygems/platform.rb
+index 48b7344aee..08728c209c 100644
+--- a/lib/rubygems/platform.rb
++++ b/lib/rubygems/platform.rb
+@@ -1,7 +1,5 @@
+ # frozen_string_literal: true
+
+-require_relative "deprecate"
+-
+ ##
+ # Available list of platforms for targeting Gem installations.
+ #
+@@ -24,11 +22,6 @@ def self.match(platform)
+     match_platforms?(platform, Gem.platforms)
+   end
+
+-  class << self
+-    extend Gem::Deprecate
+-    rubygems_deprecate :match, "Gem::Platform.match_spec? or match_gem?"
+-  end
+-
+   def self.match_platforms?(platform, platforms)
+     platform = Gem::Platform.new(platform) unless platform.is_a?(Gem::Platform)
+     platforms.any? do |local_platform|
diff --git a/config/projects/metasploit-framework.rb b/config/projects/metasploit-framework.rb
@@ -4,6 +4,7 @@
 
 install_dir "#{default_root}/metasploit-framework"
 
+# Version is extracted from the latest Git tag found in the local Git repository
 build_version Omnibus::BuildVersion.semver + "-1rapid7"
 build_iteration 1
 

diff --git a/config/software/bundler.rb b/config/software/bundler.rb
@@ -28,7 +28,7 @@
   dependency "rubygems"
 end
 
-default_version "2.1.4"
+default_version "2.5.10"
 
 build do
   env = with_standard_compiler_flags(with_embedded_path)

diff --git a/config/software/metasploit-framework.rb b/config/software/metasploit-framework.rb
@@ -1,8 +1,17 @@
 name "metasploit-framework"
+
+# Detect a local checkout of metasploit-framework at '../metasploit-framework' - i.e. for the scenario of:
+# - c:/temp/metasploit-omnibus
+# - c:/temp/metasploit-framework (A local checkout of framework to use during the build process)
+# but try and use 'C:/metasploit-framework' - as that's the metasploit-omnibus artifacts output directory
+def has_windows_metasploit_framework_repo?
+  windows? && File.exist?('../metasploit-framework') && File.expand_path(File.join(Dir.pwd, "..", "metasploit-framework")) != "c:/metasploit-framework"
+end
+
 if linux? && File.exist?("/metasploit-framework")
   # supply current version of metasploit-framework at root of filesystem
   source path: "/metasploit-framework"
-elsif windows? && File.exist?('../metasploit-framework')
+elsif has_windows_metasploit_framework_repo?
   # supply current version of metasploit-framework relative to the current directory
   source path: "../metasploit-framework"
 else
@@ -43,6 +52,8 @@
 whitelist_file "#{install_dir}//embedded/lib/ruby/gems/#{ruby_abi_version}/.*/sqlite3.*"
 
 build do
+  patch_env = with_standard_compiler_flags(with_embedded_path)
+  patch source: "bundler.patch", plevel: 0, env: patch_env
   copy "#{project_dir}", "#{install_dir}/embedded/framework"
 
   major, minor, patch = Omnibus::BuildVersion.semver.split('.')
@@ -79,12 +90,21 @@
         vars: { install_dir: install_dir }
   end
 
+  bundle "version", env: env
   bundle "config set force_ruby_platform true", env: env
   bundle_env = with_standard_compiler_flags(with_embedded_path)
   bundle_env['MAKE'] = 'make -j4'
+  bundle_env['BUNDLE_FORCE_RUBY_PLATFORM'] = 'true'
   bundle "install --jobs=4", env: bundle_env
 
   if windows?
+    # Ensure we additionally copy out 'libssp-0.dll', which is required for multiple gems:
+    #   > dumpbin /dependents  C:/metasploit-framework/embedded/lib/ruby/gems/3.1.0/gems/msgpack-1.6.1/lib/msgpack/msgpack.so
+    #       ...
+    #       libssp-0.dll
+    #       ...
+    copy "#{install_dir}/embedded/msys64/ucrt64/bin/libssp-0.dll", "#{install_dir}/embedded/bin/libssp-0.dll"
+
     delete "#{install_dir}/embedded/msys64"
   end
   copy "#{project_dir}/Gemfile.lock", "#{install_dir}/embedded/framework/Gemfile.lock"
@@ -154,4 +174,12 @@
       end
     end
   end
+
+  # Workaround for a Windows bug with chef r7_9.0.23_custom that allows the `.git` folders through
+  # into the final build result, leading to the .exe being an extra 1gb in size
+  block do
+    self.project.exclusions.each do |exclusion|
+      Pathname(install_dir).glob(exclusion).each(&:rmtree)
+    end
+  end
 end
diff --git a/config/software/ruby-windows.rb b/config/software/ruby-windows.rb
@@ -15,7 +15,7 @@
 #
 
 name "ruby-windows"
-default_version "3.1.4-1"
+default_version "3.1.5-1"
 
 if windows_arch_i386?
   relative_path "rubyinstaller-#{version}-x86"
@@ -58,6 +58,10 @@
     source sha256: "6701088607ea4b587a31af76d75cb3fe9f7bcd75fc175cffcca22369ebb6331d"
   end
 
+  version "3.1.5-1" do
+    source sha256: "f31eece5b5c64563829117d41c09ed87572c595e5d64a525d49c1dd9c50dccca"
+  end
+
   source url: "https://github.com/oneclick/rubyinstaller2/releases/download/rubyinstaller-#{version}/rubyinstaller-#{version}-x64.7z"
 end
 

diff --git a/config/software/ruby.rb b/config/software/ruby.rb
@@ -26,7 +26,7 @@
 # the default versions should always be the latest release of ruby
 # if you consume this definition it is your responsibility to pin
 # to the desired version of ruby. don't count on this not changing.
-default_version "3.1.4"
+default_version "3.1.5"
 
 dependency "zlib"
 dependency "openssl"
@@ -45,6 +45,7 @@
 version("3.2.2")      { source sha256: "96c57558871a6748de5bc9f274e93f4b5aad06cd8f37befa0e8d94e7b8a423bc" }
 version("3.2.0")      { source sha256: "daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" }
 version("3.1.4")      { source sha256: "a3d55879a0dfab1d7141fdf10d22a07dbf8e5cdc4415da1bde06127d5cc3c7b6" }
+version("3.1.5")      { source sha256: "3685c51eeee1352c31ea039706d71976f53d00ab6d77312de6aa1abaf5cda2c5" }
 version("3.1.3")      { source sha256: "5ea498a35f4cd15875200a52dde42b6eb179e1264e17d78732c3a57cd1c6ab9e" }
 version("3.1.2")      { source sha256: "61843112389f02b735428b53bb64cf988ad9fb81858b8248e22e57336f24a83e" }
 version("3.1.1")      { source sha256: "fe6e4782de97443978ddba8ba4be38d222aa24dc3e3f02a6a8e7701c0eeb619d" }

diff --git a/config/software/rubygems.rb b/config/software/rubygems.rb
@@ -28,7 +28,7 @@
   dependency "ruby"
 end
 
-default_version "3.2.22"
+default_version "3.5.10"
 
 if version && !source
   # NOTE: 2.1.11 is the last version of rubygems before the 2.2.x change to native gem install location
@@ -40,7 +40,9 @@
   # we pin the previously known tarballs.
   known_tarballs = {
     "3.1.4" => "d117187a8f016cbe8f52011ae02e858b",
-    "3.2.22"=> "b128d5493da2ec7a1da49a7189c04b35",
+    "3.2.22" => "b128d5493da2ec7a1da49a7189c04b35",
+    "3.3.26" => "ba171c52fd9beda6dac7194413601795",
+    "3.5.10" => "70f46c096b4e11c42b0190cc3e3375e2"
   }
   known_tarballs.each do |vsn, md5|
     version vsn do
@@ -69,6 +71,13 @@
 build do
   env = with_standard_compiler_flags(with_embedded_path)
 
+  # Fix warning on msfconsle bootup that will be fixed when framework's bundler version is updated
+  #   NOTE: Gem::Platform.match is deprecated; use Gem::Platform.match_spec? or match_gem? instead. It will be removed in Rubygems 4
+  #   Gem::Platform.match called from /Users/adfoster/.rvm/gems/ruby-3.3.0/gems/bundler-2.1.4/lib/bundler/index.rb:198.
+  if version.satisfies?('>= 3.5.10')
+    patch source: "rubygems-3.5.10.patch", plevel: 1, env: env
+  end
+
   if source
     # Building from source:
     ruby "setup.rb --no-document", env: env

diff --git a/local/cache b/local/cache
+ −		ruby-3.1.5.tar.gz
+ −		rubygems-3.3.26.tgz
+ −		rubygems-3.5.10.tgz
+ −		rubyinstaller-3.1.5-1-x64.7z