Revive aarch64 image

.github/workflows/verify.yml

@@ -27,7 +27,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 90
+    timeout-minutes: 180
 
     strategy:
       fail-fast: false
@@ -37,9 +37,11 @@ jobs:
         # From: 'ls ./docker/'
         image_name:
           - centos6-x64
+          - debian-aarch64
           - debian-armv7
           - fedora30-x64
-          - kali109-x64
+          # Currently fails on rate limiting on Kali's side:
+          # - kali109-x64
           - ubuntu1204-x64
           - ubuntu1204-x86
           - ubuntu1804-x64
@@ -49,9 +51,38 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set the docker build metadata
+        run: |
+          export BUILD_DATE=$(date "+%Y_%m")
+          echo "BUILD_DATE=$BUILD_DATE" >> "$GITHUB_ENV"
+          echo "BUILD_TAG=rapid7/${IMAGE_NAME}-omnibus:${BUILD_DATE}" >> "$GITHUB_ENV"
+        env:
+          IMAGE_NAME: ${{ matrix.image_name }}
+
       - name: Build Docker image
         run: |
-          export BUILD_DATE=$(date "+%y_%m")
-          /bin/bash -x -c "docker build --tag rapid7/${IMAGE_NAME}-omnibus:${BUILD_DATE} -f ./docker/${IMAGE_NAME}/Dockerfile ./docker/${IMAGE_NAME}"
+          /bin/bash -x -c "docker build --tag ${BUILD_TAG} -f ./docker/${IMAGE_NAME}/Dockerfile ./docker/${IMAGE_NAME}"
         env:
           IMAGE_NAME: ${{ matrix.image_name }}
+
+      # Checkout again - but with the submodules enabled to start a real build
+      - name: Checkout submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Run omnibus
+        run: |
+          cat <<EOF >>build_docker.sh
+          #!/bin/bash --login
+
+          # install omnibus' dependencies
+          bundle install
+          bundle binstubs --all
+
+          # build the metasploit-framework package
+          bin/omnibus build metasploit-framework
+          EOF
+
+          chmod +x ./build_docker.sh
+          /bin/bash -x -c "docker run --rm --volume $(pwd):$(pwd) --workdir $(pwd) --user jenkins --entrypoint '' ${BUILD_TAG} /bin/bash -l -c './build_docker.sh'"

README.md

@@ -32,7 +32,7 @@ docker build --tag metasploit-omnibus-builder - < ./docker/ubuntu1204-x86/Docker
 Or on OSX you can use the following script to build all images following the latest Docker image naming convention:
 
 ```shell
-export BUILD_DATE=$(date "+%y_%m"); ls ./docker | xargs -I IMAGE_NAME /bin/bash -x -c "docker build --tag rapid7/IMAGE_NAME-omnibus:$BUILD_DATE -f ./docker/IMAGE_NAME/Dockerfile ./docker/IMAGE_NAME"
+export BUILD_DATE=$(date "+%Y_%m"); ls ./docker | xargs -I IMAGE_NAME /bin/bash -x -c "docker build --tag rapid7/IMAGE_NAME-omnibus:$BUILD_DATE -f ./docker/IMAGE_NAME/Dockerfile ./docker/IMAGE_NAME"
 ```
 
 You can then run a new container using the above tagged image, whilst mounting the current directory as a volume:

config/software/openssl.rb

@@ -23,29 +23,50 @@
 dependency "cacerts"
 dependency "openssl-fips" if fips_mode?
 
-default_version "1.1.1m"
+default_version "1.1.1t" # # do not remove - Rapid7 custom - do not remove
 
 # Openssl builds engines as libraries into a special directory. We need to include
 # that directory in lib_dirs so omnibus can sign them during macOS deep signing.
 lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/engines"])
 lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/engines-1.1"]) if version.start_with?("1.1")
+if version.start_with?("3.")
+  lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/engines-3"])
+  lib_dirs lib_dirs.concat(["#{install_dir}/embedded/lib/ossl-modules"])
+end
 
 # 1.0.2u was the last public release of 1.0.2. Subsequent releases come from a support contract with OpenSSL Software Services
 if version.satisfies?("< 1.1.0")
   source url: "https://s3.amazonaws.com/chef-releng/openssl/openssl-#{version}.tar.gz", extract: :lax_tar
+  internal_source url: "#{ENV["ARTIFACTORY_REPO_URL"]}/#{name}/#{name}-#{version}.tar.gz", extract: :lax_tar,
+                  authorization: "X-JFrog-Art-Api:#{ENV["ARTIFACTORY_TOKEN"]}"
 else
   # As of 2020-09-09 even openssl-1.0.0.tar.gz can be downloaded from /source/openssl-VERSION.tar.gz
   # However, the latest releases are not in /source/old/VERSION/openssl-VERSION.tar.gz.
   # Let's stick with the simpler one for now.
   source url: "https://www.openssl.org/source/openssl-#{version}.tar.gz", extract: :lax_tar
+  internal_source url: "#{ENV["ARTIFACTORY_REPO_URL"]}/#{name}/#{name}-#{version}.tar.gz", extract: :lax_tar,
+                  authorization: "X-JFrog-Art-Api:#{ENV["ARTIFACTORY_TOKEN"]}"
 end
 
-version("1.1.1m") { source sha256: "f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96" }
-version("1.1.1l") { source sha256: "0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1" }
-version("1.1.1f") { source sha256: "186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35" }
+version("3.0.5")   { source sha256: "aa7d8d9bef71ad6525c55ba11e5f4397889ce49c2c9349dcea6d3e4f0b024a7a" }
+version("3.0.4")   { source sha256: "2831843e9a668a0ab478e7020ad63d2d65e51f72977472dc73efcefbafc0c00f" }
+version("3.0.3")   { source sha256: "ee0078adcef1de5f003c62c80cc96527721609c6f3bb42b7795df31f8b558c0b" }
+version("3.0.1")   { source sha256: "c311ad853353bce796edad01a862c50a8a587f62e7e2100ef465ab53ec9b06d1" } # only ruby 3.1 supports openssl-3.0.1
+
+version("1.1.1t")  { source sha256: "8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b" }
+version("1.1.1q")  { source sha256: "d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca" }
+version("1.1.1p")  { source sha256: "bf61b62aaa66c7c7639942a94de4c9ae8280c08f17d4eac2e44644d9fc8ace6f" }
+version("1.1.1o")  { source sha256: "9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f" }
+version("1.1.1m")  { source sha256: "f89199be8b23ca45fc7cb9f1d8d3ee67312318286ad030f5316aca6462db6c96" }
+version("1.1.1l")  { source sha256: "0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1" }
+version("1.1.1w")  { source sha256: "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8" }
 
+version("1.0.2zg") { source sha256: "09f8372eaede77ec8e6945e2d2d8eeb1b91662980cf23fe95f627b377162296c" }
 version("1.0.2zb") { source sha256: "b7d8f8c895279caa651e7f3de9a7b87b8dd01a452ca3d9327f45a9ef31d0c518" }
 version("1.0.2za") { source sha256: "86ec5d2ecb53839e9ec999db7f8715d0eb7e534d8a1d8688ef25280fbeee2ff8" }
+version("1.0.2ze") { source sha256: "796624c593c361c695bd16314bc6f944184f5d2ff87efcf0bfa0545aa84c4d88" }
+version("1.0.2zf") { source sha256: "85d2242b7d11a33d5f239f1f34a1ff7eb37431a554b7df99c52c646b70b14b2e" }
+version("1.0.2zi") { source sha256: "80b6c07995fc92456e31c61cf1b2a18f75e314063189bb183af6ae66d0261d84" }
 
 relative_path "openssl-#{version}"
 
@@ -55,9 +76,6 @@
     env["M4"] = "/opt/freeware/bin/m4"
   elsif mac_os_x? && arm?
     env["CFLAGS"] << " -Qunused-arguments"
-  elsif freebsd?
-    # Should this just be in standard_compiler_flags?
-    env["LDFLAGS"] += " -Wl,-rpath,#{install_dir}/embedded/lib"
   elsif windows?
     # XXX: OpenSSL explicitly sets -march=i486 and expects that to be honored.
     # It has OPENSSL_IA32_SSE2 controlling whether it emits optimized SSE2 code
@@ -77,15 +95,21 @@
     "no-mdc2",
     "no-rc5",
     "no-ssl2",
-    "enable-ssl3",
+    "no-ssl3",
     "no-zlib",
     "shared",
   ]
 
+  configure_args += ["--libdir=#{install_dir}/embedded/lib"] if version.satisfies?(">=3.0.1")
+
   # https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/
   configure_args += [ "-DOPENSSL_TRUSTED_FIRST_DEFAULT" ] if version.satisfies?(">= 1.0.2zb") && version.satisfies?("< 1.1.0")
 
-  configure_args += ["--with-fipsdir=#{install_dir}/embedded", "fips"] if fips_mode?
+  if version.satisfies?("< 3.0.0")
+    configure_args += ["--with-fipsdir=#{install_dir}/embedded", "fips"] if fips_mode?
+  else
+    configure_args += ["-enable-fips"] if fips_mode?
+  end
 
   configure_cmd =
     if aix?
@@ -135,6 +159,8 @@
     patch source: "openssl-1.0.1f-do-not-build-docs.patch", env: patch_env
   elsif version.start_with? "1.1"
     patch source: "openssl-1.1.0f-do-not-install-docs.patch", env: patch_env
+  elsif version.start_with? "3.0"
+    patch source: "openssl-3.0.1-do-not-install-docs.patch", env: patch_env
   end
 
   if version.start_with?("1.0.2") && mac_os_x? && arm?
@@ -148,7 +174,7 @@
 
   # Out of abundance of caution, we put the feature flags first and then
   # the crazy platform specific compiler flags at the end.
-  configure_args << env["CFLAGS"] << env["LDFLAGS"]
+  configure_args << env["CFLAGS"]
 
   configure_command = configure_args.unshift(configure_cmd).join(" ")
 

config/software/ruby.rb

@@ -84,7 +84,7 @@
 
 env = with_standard_compiler_flags(with_embedded_path)
 
-# do not remote - Rapid7 custom - do not remove:
+# do not remove - Rapid7 custom - do not remove:
 jemalloc_required = linux? || mac_os_x?
 if jemalloc_required
   dependency "jemalloc"

docker/centos6-x64/Dockerfile

@@ -5,7 +5,7 @@ MAINTAINER Rapid7 Release Engineering <[email protected]>
 
 VOLUME /pkg
 
-# https://stackoverflow.com/questions/21396508/yumrepo-error-all-mirror-urls-are-not-using-ftp-https-or-file
+# # Copy newer apt sources that are valid/use https
 COPY yum.repos.d /etc/yum.repos.d
 
 RUN rpm -ivh https://dl.fedoraproject.org/pub/archive/epel/6/x86_64/epel-release-6-8.noarch.rpm

docker/debian-aarch64/.dockerignore

@@ -0,0 +1,3 @@
+*
+
+!apt

docker/debian-aarch64/Dockerfile

@@ -0,0 +1,69 @@
+# docker build -t rapid7/msf-debian-aarch64-omnibus .
+FROM rapid7/msf-debian-aarch64-omnibus:2020_03
+MAINTAINER Rapid7 Release Engineering <[email protected]>
+
+RUN ["docker-build-start"]
+
+# Copy newer apt sources that are valid/use https
+COPY apt /etc/apt
+
+# Update certs as the current ones have expired
+RUN mkdir -p /tmp/updated_certs && \
+    cd /tmp/updated_certs && \
+    curl --location --insecure --output cacert-2023-08-22.pem https://curl.se/ca/cacert-2023-08-22.pem && \
+    echo '23c2469e2a568362a62eecf1b49ed90a15621e6fa30e29947ded3436422de9b9  cacert-2023-08-22.pem' > cacert-2023-08-22.pem.sha256 && \
+    sha256sum -c cacert-2023-08-22.pem.sha256 && \
+    mv cacert-2023-08-22.pem /etc/ssl/certs/ca-certificates.crt && \
+    rm -rf /tmp/updated_certs
+
+# Import newer debian signing keys, as the current version available in the docker image is outdated and apt-get fails
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 0E98404D386FA1D9 6ED0E7B82643E131 F8D2585B8783D481
+
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    autoconf \
+    binutils-doc \
+    bison \
+    flex \
+    gettext \
+    build-essential \
+    ruby \
+    rsync \
+    ccache \
+    devscripts \
+    fakeroot \
+    unzip \
+    procps \
+    gnupg \
+	build-essential \
+    m4 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN git config --global user.email "[email protected]" && \
+    git config --global user.name "Omnibus Packager"
+
+RUN command curl -sSL https://rvm.io/mpapis.asc | gpg --import - && \
+    command curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - && \
+    curl -L -sSL https://get.rvm.io | bash -s stable
+RUN /bin/bash -l -c "rvm requirements"
+RUN /bin/bash -l -c "rvm install 3.0.6"
+RUN /bin/bash -l -c "gem install bundler -v 2.2.3 --no-document"
+
+# pre-load the omnibus dependencies
+RUN /bin/bash -l -c 'cd ~/ && git clone https://github.com/rapid7/metasploit-omnibus.git && \
+        cd ~/metasploit-omnibus && bundle install && bundle binstubs --all && cd ~/ && rm -fr metasploit-omnibus'
+
+ENV JENKINS_HOME /home/jenkins
+RUN useradd -d "$JENKINS_HOME" -u 1001 -m -s /bin/sh jenkins
+RUN cp ~/.gitconfig "$JENKINS_HOME"
+RUN chown -R jenkins "$JENKINS_HOME"
+
+RUN mkdir -p /var/cache/omnibus
+RUN mkdir -p /opt/metasploit-framework
+RUN chown jenkins /var/cache/omnibus
+RUN chown jenkins /opt/metasploit-framework
+RUN chown -R jenkins /var/lib/gems/
+
+RUN ["docker-build-end"]

docker/debian-aarch64/apt/sources.list

@@ -0,0 +1,8 @@
+deb http://deb.debian.org/debian bullseye main contrib non-free
+deb-src http://deb.debian.org/debian bullseye main contrib non-free
+
+deb http://deb.debian.org/debian bullseye-updates main contrib non-free
+deb-src http://deb.debian.org/debian bullseye-updates main contrib non-free
+
+deb http://deb.debian.org/debian bullseye-backports main contrib non-free
+deb-src http://deb.debian.org/debian bullseye-backports main contrib non-free

docker/debian-armv7/Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Rapid7 Release Engineering <[email protected]>
 
 RUN ["cross-build-start"]
 
-# https://unix.stackexchange.com/questions/508724/failed-to-fetch-jessie-backports-repository/508728
+# Copy newer apt sources that are valid
 COPY apt /etc/apt
 
 RUN apt-get update && apt-get install -y \

docker/kali109-x64/Dockerfile

@@ -53,10 +53,7 @@ RUN curl -O ftp://ftp.tcl.tk/pub/tcl/tcl8_6/tcl8.6.9-src.tar.gz && \
   ./configure && make && make install && \
   cd ../../ && rm -rf tcl8.6.9*
 
-# Update certs to fix later git clone issues:
-#   error: server certificate verification failed. CAfile: /etc/ssl/certs/ca-certificates.crt CRLfile: none while accessing https://github.com/rapid7/metasploit-omnibus.git/info/refs
-# We extract a new https://curl.se/docs/caextract.html bundle using the sha256summ
-# from https://curl.se/ca/cacert-2023-08-22.pem.sha256
+# Update certs as the current ones have expired
 RUN mkdir -p /tmp/updated_certs && \
     cd /tmp/updated_certs && \
     curl --location --insecure --output cacert-2023-08-22.pem https://curl.se/ca/cacert-2023-08-22.pem && \

docker/ubuntu1204-x86/Dockerfile

@@ -19,6 +19,7 @@ RUN rm -fr /var/lib/apt/lists && \
         libcurl4-openssl-dev \
         libexpat1-dev \
         libicu-dev \
+        libc6-dev-amd64 \
         reprepro \
         sudo && \
     apt-get clean && \