[secure-boot] ease of use update and bug fixes (#90)

* [secure-boot] ease of use update and bug fixes * using variable values instead of hard-coding paths * incorrect path to db.der/mok.pub corrected * If the named secret exists in secret manager and tls/ directory does not exist, fetch contents of tls/ from gcp APIs * Otherwise create a new MOK key pair and publish in secret manager * make expected md5sum immutable * removed definition of unused project number variable ; added docs link regarding instructions for creating service account * image name may not include dots * suggested another role to grant to the dataproc service account * tested with this documentation * test of install_gpu_driver.sh as a customization script * more selective about portion of the script which has access to private key material * first pass at using custom script instance to install GPU driver using init action ; to do: install the trust db before running the installer ; turn on secure boot * removed redundant gpu argument ; launching 2.2 customization instances with secure boot enabled * returning version to 42 * enable dkms certificate use anywhere in main function * moved example code from README.md to cuda.sh update python version in Dockerfile reduce noise from serial console read config values from env.json * included license text with example * added example script to run generate-custom_image.py with secure boot and cuda
GoogleCloudDataproc · Oct 6, 2024 · 112b5f5 · 112b5f5
1 parent a17548b
commit 112b5f5
Show file tree

Hide file tree

Showing 9 changed files with 1,443 additions and 110 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,4 @@
-# since this script depends on python 2.7, we need a stable base from which to run it
-FROM python:2.7-slim
+FROM python:3.11-slim
 
 # To build: docker build -t dataproc-custom-images:latest .
 # To run: docker run -it dataproc-custom-images:latest /bin/bash
@@ -20,16 +19,18 @@ FROM python:2.7-slim
 
 WORKDIR /custom-images
 
-RUN apt-get update && apt-get -y install apt-transport-https ca-certificates gnupg curl
+RUN apt-get -qq update \
+  && apt-get -y -qq install \
+     apt-transport-https ca-certificates gnupg curl jq less
 RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \
     | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
 RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
     | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
 
 # This will only work so long as google-cloud-cli installs to Buster
-RUN apt-get -y update && apt-get -y install google-cloud-cli && apt-get clean
+RUN apt-get -y -qq update && apt-get -y -qq install google-cloud-cli && apt-get clean
 
-RUN apt-get -y install emacs-nox vim && apt-get clean
+RUN apt-get -y -qq install emacs-nox vim && apt-get clean
 
 COPY . ${WORKDIR}
 

diff --git a/custom_image_utils/args_parser.py b/custom_image_utils/args_parser.py
@@ -227,6 +227,7 @@ def parse_args(args):
       "--trusted-cert",
       type=str,
       required=False,
+      default="tls/db.der",
       help="""(Optional) Inserts the specified DER-format certificate into
       the custom image's EFI boot sector for use with secure boot.""")
 

diff --git a/custom_image_utils/shell_script_generator.py b/custom_image_utils/shell_script_generator.py
@@ -63,34 +63,61 @@
 
   echo 'Creating disk.'
   if [[ '{base_image_family}' = '' ||  '{base_image_family}' = 'None' ]]; then
-     IMAGE_SOURCE="--image={dataproc_base_image}"
+     src_image="--source-image={dataproc_base_image}"
   else
-     IMAGE_SOURCE="--image-family={base_image_family}"
+     src_image="--source-image-family={base_image_family}"
   fi
 
-  gcloud compute disks create {image_name}-install \
-      --project={project_id} \
-      --zone={zone} \
-      ${{IMAGE_SOURCE}} \
-      --type=pd-ssd \
-      --size={disk_size}GB
+  # build tls/ directory from variables defined near the header of
+  # the examples/secure-boot/create-key-pair.sh file
+
+  # by default, a gcloud secret with the name of efi-db-pub-key-042 is
+  # created in the current project to store the certificate installed
+  # as the signature database file for this disk image
+
+  eval "$(bash examples/secure-boot/create-key-pair.sh)"
+
+  # The MS UEFI CA is a reasonable base from which to build trust.  We
+  # will trust code signed by this CA as well as code signed by
+  # trusted_cert (tls/db.der)
+
+  # The Microsoft Corporation UEFI CA 2011
+  local -r MS_UEFI_CA="tls/MicCorUEFCA2011_2011-06-27.crt"
+  test -f "${{MS_UEFI_CA}}" || \
+    curl -L -o ${{MS_UEFI_CA}} 'https://go.microsoft.com/fwlink/p/?linkid=321194'
+
+  local cert_args=""
+  if [[ -n '{trusted_cert}' ]] && [[ -f '{trusted_cert}' ]]; then
+    cert_args="--signature-database-file={trusted_cert},${{MS_UEFI_CA}} --guest-os-features=UEFI_COMPATIBLE"
+  fi
+
+  gcloud compute images create {image_name}-install \
+       --project={project_id} \
+       ${{src_image}} \
+       ${{cert_args}} \
+       {storage_location_flag} \
+       --family={family}
 
   touch "/tmp/{run_id}/disk_created"
 
   echo 'Creating VM instance to run customization script.'
-  gcloud compute instances create {image_name}-install \
+  time gcloud compute instances create {image_name}-install \
       --project={project_id} \
       --zone={zone} \
       {network_flag} \
       {subnetwork_flag} \
       {no_external_ip_flag} \
       --machine-type={machine_type} \
-      --disk=auto-delete=yes,boot=yes,mode=rw,name={image_name}-install \
+      --image-project {project_id} \
+      --image="{image_name}-install" \
+      --boot-disk-size={disk_size}G \
+      --boot-disk-type=pd-ssd \
       {accelerator_flag} \
       {service_account_flag} \
       --scopes=cloud-platform \
       {metadata_flag} \
       --metadata-from-file startup-script=startup_script/run.sh
+
   touch /tmp/{run_id}/vm_created
 
   echo 'Waiting for customization script to finish and VM shutdown.'
@@ -99,6 +126,7 @@
       --zone={zone} \
       --port=1 2>&1 \
       | grep 'startup-script' \
+      | sed -e 's/ {image_name}-install.*startup-script://g' \
       | tee {log_dir}/startup-script.log \
       || true
 
@@ -114,29 +142,12 @@
   fi
 
   echo 'Creating custom image.'
-  if [[ -n '{trusted_cert}' ]] && [[ -f '{trusted_cert}' ]]; then
-     # The Microsoft Corporation UEFI CA 2011
-     mkdir -p tls
-     MS_UEFI_CA="tls/MicCorUEFCA2011_2011-06-27.crt"
-     test -f "${{MS_UEFI_CA}}" || \
-         curl -L -o ${{MS_UEFI_CA}} 'https://go.microsoft.com/fwlink/p/?linkid=321194'
-
-     gcloud compute images create {image_name} \
-        --project={project_id} \
-        --source-disk-zone={zone} \
-        --source-disk={image_name}-install \
-        --signature-database-file="{trusted_cert},${{MS_UEFI_CA}}" \
-        --guest-os-features="UEFI_COMPATIBLE" \
-        {storage_location_flag} \
-        --family={family}
-  else
-     gcloud compute images create {image_name} \
-        --project={project_id} \
-        --source-disk-zone={zone} \
-        --source-disk={image_name}-install \
-        {storage_location_flag} \
-        --family={family}
-  fi
+  gcloud compute images create {image_name} \
+    --project={project_id} \
+    --source-disk-zone={zone} \
+    --source-disk={image_name}-install \
+    {storage_location_flag} \
+    --family={family}
   touch /tmp/{run_id}/image_created
 }}
 

diff --git a/examples/secure-boot/README.md b/examples/secure-boot/README.md
@@ -1,62 +1,15 @@
-To generate a key pair for use with the custom image, run the
-create-key-pair.sh script.  You can then specify the full path to
-**tls/db.der** with the argument **--trusted-cert=.../tls/db.der**
-
-Kernel drivers signed with the private side of this key pair can then
-be loaded into kernels on systems with secure boot enabled.
-
 To create a custom image with a self-signed, trusted certificate
-inserted into the boot sector, and then run a script to install nvidia
-kernel drivers on a Dataproc image, the following commands can be
-run from the root of the custom-images git repository:
-
-```bash
-PROJECT_ID=your-project-here
-PROJECT_NUMBER=your-project-nnnn-here
-CLUSTER_NAME=your-cluster-name-here
-my_bucket=your-bucket-here
-custom_image_zone=your-zone-here
-
-export SA_NAME=sa-${CLUSTER_NAME}
-export GSA=${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com
-
-gcloud projects add-iam-policy-binding ${PROJECT_ID} \
-	--member=serviceAccount:${GSA} \
-	--role=roles/secretmanager.secretAccessor
-gcloud config set project ${PROJECT_ID}
-
-gcloud auth login
+inserted into the boot sector, and then run a script to install cuda
+on a Dataproc image, the commands from cuda.sh can be run from the
+root of the custom-images git repository or from a docker container.
 
-# variables *_secret_name_, secret_project, secret_version defined here:
-eval $(bash examples/secure-boot/create-key-pair.sh)
-metadata="public_secret_name=${public_secret_name}"
-metadata="${metadata},private_secret_name=${private_secret_name}"
-metadata="${metadata},secret_project=${secret_project}"
-metadata="${metadata},secret_version=${secret_version}"
+First, write an env.json to the directory from which you will run the
+customization script.  There is a sample which you can copy and edit
+in the file examples/secure-boot/env.json.sample.
 
-#dataproc_version=2.2-rocky9
-#dataproc_version=2.2-ubuntu22
-dataproc_version=2.2-debian12
-#customization_script=examples/secure-boot/install-nvidia-driver-debian11.sh
-customization_script=examples/secure-boot/install-nvidia-driver-debian12.sh
-#image_name="nvidia-open-kernel-2.2-ubuntu22-$(date +%F)"
-#image_name="nvidia-open-kernel-2.2-rocky9-$(date +%F)"
-#image_name="nvidia-open-kernel-2.2-debian12-$(date +%F)"
-image_name="nvidia-open-kernel-${dataproc_version}-$(date +%F)"
-disk_size_gb="50"
-
-python generate_custom_image.py \
-    --image-name ${image_name} \
-    --dataproc-version ${dataproc_version} \
-    --trusted-cert "tls/db.der" \
-    --customization-script ${customization_script} \
-    --metadata "${metadata}" \
-    --zone "${custom_image_zone}" \
-    --disk-size "${disk_size_gb}" \
-    --no-smoke-test \
-    --gcs-bucket "${my_bucket}"
+```bash
+cp examples/secure-boot/env.json.sample env.json
+vi env.json
+docker build -t dataproc-custom-images:latest .
+docker run -it dataproc-custom-images:latest /bin/bash examples/secure-boot/cuda.sh
 ```
-
-
-
-
diff --git a/examples/secure-boot/create-key-pair.sh b/examples/secure-boot/create-key-pair.sh
@@ -1,6 +1,21 @@
 #!/bin/bash
-
-set -e
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script creates a key pair and publishes to cloud secrets or
+# fetches an already published key pair from cloud secrets
+
+set -ex
 
 # https://github.com/glevand/secure-boot-utils
 
@@ -22,20 +37,49 @@ PROJECT_ID="${CURRENT_PROJECT_ID}"
 function create_key () {
     local EFI_VAR_NAME="$1"
     local CN_VAL="$2"
+    local PRIVATE_KEY="tls/${EFI_VAR_NAME}.rsa"
+    local CACERT="tls/${EFI_VAR_NAME}.pem"
+    local CACERT_DER="tls/${EFI_VAR_NAME}.der"
+    CA_KEY_SECRET_NAME="efi-${EFI_VAR_NAME}-priv-key-${ITERATION}"
+    CA_CERT_SECRET_NAME="efi-${EFI_VAR_NAME}-pub-key-${ITERATION}"
+    # If the secrets exist in secret manager, populate the tls/ directory
+    if [[ ! -f "${PRIVATE_KEY}" ]] && gcloud secrets describe "${CA_CERT_SECRET_NAME}" > /dev/null ; then
+      mkdir -p tls
+
+      gcloud secrets versions access "1" \
+        --project="${PROJECT_ID}" \
+        --secret="${CA_KEY_SECRET_NAME}" \
+        | dd of="${PRIVATE_KEY}"
+
+      gcloud secrets versions access "1" \
+        --project="${PROJECT_ID}" \
+        --secret="${CA_CERT_SECRET_NAME}" \
+        | base64 --decode \
+        | dd of="${CACERT_DER}"
+
+      # Create a PEM-format version of the cert
+      openssl x509 \
+        -inform DER \
+        -in "${CACERT_DER}" \
+        -outform PEM \
+        -out "${CACERT}"
+
+      MS_UEFI_CA="tls/MicCorUEFCA2011_2011-06-27.crt"
+      curl -L -o "${MS_UEFI_CA}" 'https://go.microsoft.com/fwlink/p/?linkid=321194'
+
+      echo "${CA_KEY_SECRET_NAME}" > tls/private-key-secret-name.txt
+      echo "${CA_CERT_SECRET_NAME}" > tls/public-key-secret-name.txt
+      modulus_md5sum="$(openssl rsa -noout -modulus -in ${PRIVATE_KEY} | openssl md5 | awk '{print $2}' | tee tls/modulus-md5sum.txt)"
+      return
+    fi
 
-    if [[ -f "tls/${EFI_VAR_NAME}.rsa" ]]; then
+    if [[ -f "${PRIVATE_KEY}" ]]; then
         echo "key already exists.  Skipping generation." >&2
-        CA_KEY_SECRET_NAME="$(cat tls/private-key-secret-name.txt)"
-        CA_CERT_SECRET_NAME="$(cat tls/public-key-secret-name.txt)"
         modulus_md5sum="$(cat tls/modulus-md5sum.txt)"
         return
     fi
     mkdir -p tls
 
-    local PRIVATE_KEY="tls/${EFI_VAR_NAME}.rsa"
-    local CACERT="tls/${EFI_VAR_NAME}.pem"
-    local CACERT_DER="tls/${EFI_VAR_NAME}.der"
-
     echo "generating '${CN_VAL}' '${CACERT}', '${CACERT_DER}' and '${PRIVATE_KEY}'" >&2
     # Generate new x.509 key and cert
     openssl req \
@@ -58,7 +102,6 @@ function create_key () {
             -out "${CACERT_DER}"
 
     # Create a new secret containing private key
-    CA_KEY_SECRET_NAME="efi-${EFI_VAR_NAME}-priv-key-${ITERATION}"
     gcloud secrets create "${CA_KEY_SECRET_NAME}" \
            --project="${PROJECT_ID}" \
            --replication-policy="automatic" \
@@ -68,14 +111,13 @@ function create_key () {
     echo "${CA_KEY_SECRET_NAME}" > tls/private-key-secret-name.txt
 
     # Create a new secret containing public key
-    CA_CERT_SECRET_NAME="efi-${EFI_VAR_NAME}-pub-key-${ITERATION}"
     cat "${CACERT_DER}" | base64 > "${CACERT_DER}.base64"
     gcloud secrets create "${CA_CERT_SECRET_NAME}" \
            --project="${PROJECT_ID}" \
            --replication-policy="automatic" \
            --data-file="${CACERT_DER}.base64"
 
-    modulus_md5sum="$(openssl x509 -noout -modulus -in /var/lib/dkms/mok.pub | openssl md5 | awk '{print $2}')"
+    modulus_md5sum="$(openssl x509 -noout -modulus -in ${CACERT} | openssl md5 | awk '{print $2}')"
     echo "modulus-md5sum: ${modulus_md5sum}" >&2
     echo "${modulus_md5sum}" > tls/modulus-md5sum.txt
     echo "Public key secret name: '${CA_CERT_SECRET_NAME}'" >&2
@@ -92,3 +134,5 @@ echo "private_secret_name=${CA_KEY_SECRET_NAME}"
 echo "public_secret_name=${CA_CERT_SECRET_NAME}"
 echo "secret_project=${PROJECT_ID}"
 echo "secret_version=1"
+
+set +x