Skip to content

Commit

Permalink
Ensure omnibus on windows can be rebuilt multiple times
Browse files Browse the repository at this point in the history
  • Loading branch information
adfoster-r7 committed May 17, 2024
1 parent 0008a23 commit c04a257
Show file tree
Hide file tree
Showing 10 changed files with 220 additions and 9 deletions.
140 changes: 138 additions & 2 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ permissions:
on:
push:
branches:
- '*'
- 'master'
pull_request:
branches:
- '*'
Expand Down Expand Up @@ -97,6 +97,8 @@ jobs:
with:
repository: rapid7/metasploit-framework
path: metasploit-framework
# If testing a custom branch is required
# ref: 'update-bundler-version'

- name: Run omnibus
run: |
Expand Down Expand Up @@ -126,6 +128,17 @@ jobs:
docker rm -v $id
docker rmi ${TEMP_DOCKER_IMAGE}
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: metasploit-${{ matrix.docker.name }}-installers
path: |
metasploit-omnibus/pkg/*.pkg
metasploit-omnibus/pkg/*.rpm
metasploit-omnibus/pkg/*.msi
metasploit-omnibus/pkg/*.deb
retention-days: 1

docker_intel:
runs-on: ${{ matrix.os }}
timeout-minutes: 180
Expand All @@ -139,20 +152,26 @@ jobs:
docker:
- name: 'centos6-x64'
previousDockerhubImage: 'rapid7/msf-centos6-x64-omnibus:2020_03'
installer: 'sudo rpm -i metasploit-omnibus/pkg/metasploit-framework*.rpm'
# Currently fails as it uses an older Ruby version:
- name: 'fedora30-x64'
# XXX: Previous dockerhub image fails as using Ruby 2.5.3 still
previousDockerhubImage: 'rapid7/msf-fedora30-x64-omnibus:2019_09'
installer: 'sudo rpm -i metasploit-omnibus/pkg/metasploit-framework*.rpm'
# Currently fails on rate limiting on Kali's side:
# - name: 'kali109-x64'
# previousDockerhubImage: 'rapid7/msf-kali109-x64-omnibus:2020_03'
# installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_amd64.deb'
- name: 'ubuntu1204-x64'
previousDockerhubImage: 'rapid7/msf-ubuntu1204-x86-omnibus:2021_11'
installer: 'sudo dpkg -i metasploit-omnibus/pkg/*.deb'
- name: 'ubuntu1204-x86'
previousDockerhubImage: 'rapid7/msf-ubuntu1204-x64-omnibus:2019_01'
linux32: true
installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_i386.deb'
- name: 'ubuntu1804-x64'
previousDockerhubImage: 'rapid7/msf-ubuntu1804-x64-omnibus:2019_09'
installer: 'sudo dpkg -i metasploit-omnibus/pkg/metasploit-framework_*_amd64.deb'

name: ${{ matrix.os }} - ${{ matrix.docker.name }}
steps:
Expand Down Expand Up @@ -215,6 +234,55 @@ jobs:
env:
LINUX32: ${{ matrix.docker.linux32 }}

- name: Test artifact
run: |
echo "Testing artifact"
cat > test_script.sh <<EOF
#!/bin/bash -ex
find metasploit-omnibus/pkg
$INSTALL_ARTIFACT
mkdir ~/.msf4; touch ~/.msf4/initial_setup_complete
msfconsole -qx 'setg variable test; version; exit'
msfvenom -p windows/meterpreter/reverse_tcp -f exe -o test.exe
msfd -h
msfrpc -h
msfrpcd -h
msfdb -h
msfbinscan -h
msfrop -h
msfelfscan -h
msfmachscan -h
msfpescan -h
# msfupdate
EOF
chmod +x ./test_script.sh
# If required, change reported architecture in new program environment and set personality flags
if [ ! -z "${LINUX32}" ] ; then
echo 'setting linux32'
/bin/bash -x -c "docker run --rm --volume $(pwd):$(pwd) --workdir $(pwd) --user jenkins ${DOCKER_IMAGE} linux32 /bin/bash -l -c './test_script.sh'"
else
/bin/bash -x -c "docker run --rm --volume $(pwd):$(pwd) --workdir $(pwd) --user jenkins ${DOCKER_IMAGE} /bin/bash -l -c './test_script.sh'"
fi
env:
LINUX32: ${{ matrix.docker.linux32 }}
INSTALL_ARTIFACT: ${{ matrix.docker.installer }}

- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: metasploit-${{ matrix.docker.name }}-installers
path: |
metasploit-omnibus/pkg/*.pkg
metasploit-omnibus/pkg/*.rpm
metasploit-omnibus/pkg/*.msi
metasploit-omnibus/pkg/*.deb
retention-days: 1

osx:
runs-on: ${{ matrix.os }}
timeout-minutes: 180
Expand Down Expand Up @@ -255,6 +323,39 @@ jobs:
cd metasploit-omnibus
make
- name: Test artifact
run: |
echo "Testing artifact"
sudo rm -rf /opt/metasploit-framework
/usr/bin/find metasploit-omnibus/pkg
PACKAGE=$(pwd)/$(/usr/bin/find metasploit-omnibus/pkg/metasploit*.pkg | head -n 1)
sudo installer -pkg ${PACKAGE} -target /
/opt/metasploit-framework/bin/msfconsole -qx 'setg variable test; version; exit'
/opt/metasploit-framework/bin/msfvenom -p windows/meterpreter/reverse_tcp -f exe -o test.exe
/opt/metasploit-framework/bin/msfd -h
/opt/metasploit-framework/bin/msfrpc -h
/opt/metasploit-framework/bin/msfrpcd -h
/opt/metasploit-framework/bin/msfdb -h
/opt/metasploit-framework/bin/msfbinscan -h
/opt/metasploit-framework/bin/msfrop -h
/opt/metasploit-framework/bin/msfelfscan -h
/opt/metasploit-framework/bin/msfmachscan -h
/opt/metasploit-framework/bin/msfpescan -h
# /opt/metasploit-framework/bin/msfupdate
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: metasploit-osx-installers
path: |
metasploit-omnibus/pkg/*.pkg
metasploit-omnibus/pkg/*.rpm
metasploit-omnibus/pkg/*.msi
metasploit-omnibus/pkg/*.deb
retention-days: 1

windows:
runs-on: ${{ matrix.os }}
timeout-minutes: 180
Expand Down Expand Up @@ -338,6 +439,41 @@ jobs:
cd metasploit-omnibus
make dependencies
rem Don't run the main build itslef under `make`, as the process will be spawned under msys2
rem Don't run the main build itself under `make`, as the process will be spawned under msys2
rem and the ridk.cmd Ruby installer will forcibly kill the msys2 process before attempting to install ruby
ruby bin/omnibus build metasploit-framework
- name: Test artifact
shell: pwsh
run: |
echo "Testing artifact"
Remove-Item c:\metasploit-framework -Recurse -ErrorAction Ignore
dir metasploit-omnibus\pkg
$artifact = (Get-ChildItem -Path "metasploit-omnibus/pkg/*.msi")[0].Name
Start-Process msiexec.exe -ArgumentList "/i metasploit-omnibus\pkg\$artifact /quiet /qn /l*v debug.log" -Wait
c:\metasploit-framework\bin\msfconsole -qx 'setg variable test; version; exit'
c:\metasploit-framework\bin\msfvenom -p windows/meterpreter/reverse_tcp -f exe -o test.exe
c:\metasploit-framework\bin\msfd -h
c:\metasploit-framework\bin\msfrpc -h
c:\metasploit-framework\bin\msfrpcd -h
c:\metasploit-framework\bin\msfdb -h
c:\metasploit-framework\bin\msfbinscan -h
c:\metasploit-framework\bin\msfrop -h
c:\metasploit-framework\bin\msfelfscan -h
c:\metasploit-framework\bin\msfmachscan -h
c:\metasploit-framework\bin\msfpescan -h
c:\metasploit-framework\bin\msfupdate
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: metasploit-windows-installers
path: |
metasploit-omnibus/pkg/*.pkg
metasploit-omnibus/pkg/*.rpm
metasploit-omnibus/pkg/*.msi
metasploit-omnibus/pkg/*.deb
retention-days: 1
8 changes: 8 additions & 0 deletions config/patches/metasploit-framework/bundler.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
--- Gemfile.lock 2024-05-17 12:32:16
+++ Gemfile.lock.new 2024-05-17 12:32:59
@@ -574,4 +574,4 @@
yard

BUNDLED WITH
- 2.1.4
+ 2.5.10
24 changes: 24 additions & 0 deletions config/patches/rubygems/rubygems-3.5.10.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
diff --git a/lib/rubygems/platform.rb b/lib/rubygems/platform.rb
index 48b7344aee..08728c209c 100644
--- a/lib/rubygems/platform.rb
+++ b/lib/rubygems/platform.rb
@@ -1,7 +1,5 @@
# frozen_string_literal: true

-require_relative "deprecate"
-
##
# Available list of platforms for targeting Gem installations.
#
@@ -24,11 +22,6 @@ def self.match(platform)
match_platforms?(platform, Gem.platforms)
end

- class << self
- extend Gem::Deprecate
- rubygems_deprecate :match, "Gem::Platform.match_spec? or match_gem?"
- end
-
def self.match_platforms?(platform, platforms)
platform = Gem::Platform.new(platform) unless platform.is_a?(Gem::Platform)
platforms.any? do |local_platform|
1 change: 1 addition & 0 deletions config/projects/metasploit-framework.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

install_dir "#{default_root}/metasploit-framework"

# Version is extracted from the latest Git tag found in the local Git repository
build_version Omnibus::BuildVersion.semver + "-1rapid7"
build_iteration 1

Expand Down
2 changes: 1 addition & 1 deletion config/software/bundler.rb
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
dependency "rubygems"
end

default_version "2.1.4"
default_version "2.5.10"

build do
env = with_standard_compiler_flags(with_embedded_path)
Expand Down
30 changes: 29 additions & 1 deletion config/software/metasploit-framework.rb
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
name "metasploit-framework"

# Detect a local checkout of metasploit-framework at '../metasploit-framework' - i.e. for the scenario of:
# - c:/temp/metasploit-omnibus
# - c:/temp/metasploit-framework (A local checkout of framework to use during the build process)
# but try and use 'C:/metasploit-framework' - as that's the metasploit-omnibus artifacts output directory
def has_windows_metasploit_framework_repo?
windows? && File.exist?('../metasploit-framework') && File.expand_path(File.join(Dir.pwd, "..", "metasploit-framework")) != "c:/metasploit-framework"
end

if linux? && File.exist?("/metasploit-framework")
# supply current version of metasploit-framework at root of filesystem
source path: "/metasploit-framework"
elsif windows? && File.exist?('../metasploit-framework')
elsif has_windows_metasploit_framework_repo?
# supply current version of metasploit-framework relative to the current directory
source path: "../metasploit-framework"
else
Expand Down Expand Up @@ -43,6 +52,8 @@
whitelist_file "#{install_dir}//embedded/lib/ruby/gems/#{ruby_abi_version}/.*/sqlite3.*"

build do
patch_env = with_standard_compiler_flags(with_embedded_path)
patch source: "bundler.patch", plevel: 1, env: patch_env
copy "#{project_dir}", "#{install_dir}/embedded/framework"

major, minor, patch = Omnibus::BuildVersion.semver.split('.')
Expand Down Expand Up @@ -79,12 +90,21 @@
vars: { install_dir: install_dir }
end

bundle "version", env: env
bundle "config set force_ruby_platform true", env: env
bundle_env = with_standard_compiler_flags(with_embedded_path)
bundle_env['MAKE'] = 'make -j4'
bundle_env['BUNDLE_FORCE_RUBY_PLATFORM'] = 'true'
bundle "install --jobs=4", env: bundle_env

if windows?
# Ensure we additionally copy out 'libssp-0.dll', which is required for multiple gems:
# > dumpbin /dependents C:/metasploit-framework/embedded/lib/ruby/gems/3.1.0/gems/msgpack-1.6.1/lib/msgpack/msgpack.so
# ...
# libssp-0.dll
# ...
copy "#{install_dir}/embedded/msys64/ucrt64/bin/libssp-0.dll", "#{install_dir}/embedded/bin/libssp-0.dll"

delete "#{install_dir}/embedded/msys64"
end
copy "#{project_dir}/Gemfile.lock", "#{install_dir}/embedded/framework/Gemfile.lock"
Expand Down Expand Up @@ -154,4 +174,12 @@
end
end
end

# Workaround for a Windows bug with chef r7_9.0.23_custom that allows the `.git` folders through
# into the final build result, leading to the .exe being an extra 1gb in size
block do
self.project.exclusions.each do |exclusion|
Pathname(install_dir).glob(exclusion).each(&:rmtree)
end
end
end
6 changes: 5 additions & 1 deletion config/software/ruby-windows.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#

name "ruby-windows"
default_version "3.1.4-1"
default_version "3.1.5-1"

if windows_arch_i386?
relative_path "rubyinstaller-#{version}-x86"
Expand Down Expand Up @@ -58,6 +58,10 @@
source sha256: "6701088607ea4b587a31af76d75cb3fe9f7bcd75fc175cffcca22369ebb6331d"
end

version "3.1.5-1" do
source sha256: "f31eece5b5c64563829117d41c09ed87572c595e5d64a525d49c1dd9c50dccca"
end

source url: "https://github.com/oneclick/rubyinstaller2/releases/download/rubyinstaller-#{version}/rubyinstaller-#{version}-x64.7z"
end

Expand Down
3 changes: 2 additions & 1 deletion config/software/ruby.rb
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
# the default versions should always be the latest release of ruby
# if you consume this definition it is your responsibility to pin
# to the desired version of ruby. don't count on this not changing.
default_version "3.1.4"
default_version "3.1.5"

dependency "zlib"
dependency "openssl"
Expand All @@ -45,6 +45,7 @@
version("3.2.2") { source sha256: "96c57558871a6748de5bc9f274e93f4b5aad06cd8f37befa0e8d94e7b8a423bc" }
version("3.2.0") { source sha256: "daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" }
version("3.1.4") { source sha256: "a3d55879a0dfab1d7141fdf10d22a07dbf8e5cdc4415da1bde06127d5cc3c7b6" }
version("3.1.5") { source sha256: "3685c51eeee1352c31ea039706d71976f53d00ab6d77312de6aa1abaf5cda2c5" }
version("3.1.3") { source sha256: "5ea498a35f4cd15875200a52dde42b6eb179e1264e17d78732c3a57cd1c6ab9e" }
version("3.1.2") { source sha256: "61843112389f02b735428b53bb64cf988ad9fb81858b8248e22e57336f24a83e" }
version("3.1.1") { source sha256: "fe6e4782de97443978ddba8ba4be38d222aa24dc3e3f02a6a8e7701c0eeb619d" }
Expand Down
13 changes: 11 additions & 2 deletions config/software/rubygems.rb
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
dependency "ruby"
end

default_version "3.2.22"
default_version "3.5.10"

if version && !source
# NOTE: 2.1.11 is the last version of rubygems before the 2.2.x change to native gem install location
Expand All @@ -40,7 +40,9 @@
# we pin the previously known tarballs.
known_tarballs = {
"3.1.4" => "d117187a8f016cbe8f52011ae02e858b",
"3.2.22"=> "b128d5493da2ec7a1da49a7189c04b35",
"3.2.22" => "b128d5493da2ec7a1da49a7189c04b35",
"3.3.26" => "ba171c52fd9beda6dac7194413601795",
"3.5.10" => "70f46c096b4e11c42b0190cc3e3375e2"
}
known_tarballs.each do |vsn, md5|
version vsn do
Expand Down Expand Up @@ -69,6 +71,13 @@
build do
env = with_standard_compiler_flags(with_embedded_path)

# Fix warning on msfconsle bootup that will be fixed when framework's bundler version is updated
# NOTE: Gem::Platform.match is deprecated; use Gem::Platform.match_spec? or match_gem? instead. It will be removed in Rubygems 4
# Gem::Platform.match called from /Users/adfoster/.rvm/gems/ruby-3.3.0/gems/bundler-2.1.4/lib/bundler/index.rb:198.
if version.satisfies?('>= 3.5.10')
patch source: "rubygems-3.5.10.patch", plevel: 1, env: env
end

if source
# Building from source:
ruby "setup.rb --no-document", env: env
Expand Down
2 changes: 1 addition & 1 deletion local/cache

0 comments on commit c04a257

Please sign in to comment.