Add optional embedding of container images (#239)

## Description

<!--- Please describe what this PR is going to change -->
This adds the optional capability to embed container images into
hook-docker. This helps use cases where images already existing in the
DinD cache is needed. Air gap envs, latency constrained/concerned envs,
etc. Just FYI, with this change but without embedding any images the
final 6.6 kernel initramfs is 171 MB. The v0.9.0 initramfs is 178.9 MB

## Why is this needed

<!--- Link to issue you have raised -->

Fixes: #

## How Has This Been Tested?
<!--- Please describe in detail how you tested your changes. -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->


## How are existing users impacted? What migration steps/scripts do we
need?

<!--- Fixes a bug, unblocks installation, removes a component of the
stack etc -->
<!--- Requires a DB migration script, etc. -->


## Checklist:

I have:

- [ ] updated the documentation and/or roadmap (if required)
- [ ] added unit or e2e tests
- [ ] provided instructions on how to upgrade

.gitignore

@@ -12,3 +12,10 @@ cache/
 *.swp
 .idea
 kernel/Dockerfile.autogen.*
+images/hook-embedded/images/*
+!images/hook-embedded/images/.keep
+images/hook-embedded/images.txt
+images/hook-embedded/docker/*
+!images/hook-embedded/docker/.keep
+images/hook-embedded/images_tar/*
+!images/hook-embedded/images_tar/.keep

README.md

@@ -160,9 +160,21 @@ The `gha-matrix` CLI command prepares a set of JSON outputs for GitHub Actions m
 - `DOCKER_ARCH` is used by the `linuxkit-containers` command to build the containers for the specified architecture.
 - `DO_PUSH`: `yes` or `no`, will push the built containers to the OCI registry; defaults to `no`.
 
+### Embedding container images into the DinD (docker-in-docker), also known as [hook-docker](images/hook-docker/), container
+
+For use cases where having container images already available in Docker is needed, the following steps can be taken to embed container images into hook-docker (DinD):
+
+> Note: This is optional and no container images will be embedded by default.
+
+> Note: This will increase the overall size of HookOS. As HookOS is an in memory OS, make sure that the size increase works for the machines you are provisioning.
+
+1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory.
+1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format.
+1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run [`pull-images.sh`](images/hook-embedded/pull-images.sh) script when building amd64 images and run [`pull-images.sh arm64`](images/hook-embedded/pull-images.sh) when building arm64 images. Read the comments at the top of the script for more details.
+1. Change directories to the root of the HookOS repository and run `sudo ./build.sh build ...` to build the HookOS kernel and ramdisk. FYI, `sudo` is needed as DIND changes file ownerships to root.
+
 ### Build system TO-DO list
 
-- [ ] Update to Linuxkit 1.2.0 and new linuxkit pkgs; this might lead into the containerd vs dind;
 - [ ] `make debug` functionality (sshd enabled) was lost in the Makefile -> bash transition;
 
 [formats]: https://github.com/linuxkit/linuxkit/blob/master/README.md#booting-and-testing

bash/hook-lk-containers.sh

@@ -13,6 +13,7 @@ function build_all_hook_linuxkit_containers() {
 	build_hook_linuxkit_container hook-mdev HOOK_CONTAINER_MDEV_IMAGE
 	build_hook_linuxkit_container hook-containerd HOOK_CONTAINER_CONTAINERD_IMAGE
 	build_hook_linuxkit_container hook-runc HOOK_CONTAINER_RUNC_IMAGE
+	build_hook_linuxkit_container hook-embedded HOOK_CONTAINER_EMBEDDED_IMAGE
 }
 
 function build_hook_linuxkit_container() {

bash/linuxkit.sh

@@ -70,7 +70,8 @@ function linuxkit_build() {
 			HOOK_CONTAINER_MDEV_IMAGE="${HOOK_CONTAINER_MDEV_IMAGE}" \
 			HOOK_CONTAINER_CONTAINERD_IMAGE="${HOOK_CONTAINER_CONTAINERD_IMAGE}" \
 			HOOK_CONTAINER_RUNC_IMAGE="${HOOK_CONTAINER_RUNC_IMAGE}" \
-			envsubst '$HOOK_VERSION $HOOK_KERNEL_IMAGE $HOOK_KERNEL_ID $HOOK_KERNEL_VERSION $HOOK_CONTAINER_IP_IMAGE $HOOK_CONTAINER_BOOTKIT_IMAGE $HOOK_CONTAINER_DOCKER_IMAGE $HOOK_CONTAINER_MDEV_IMAGE $HOOK_CONTAINER_CONTAINERD_IMAGE $HOOK_CONTAINER_RUNC_IMAGE' \
+			HOOK_CONTAINER_EMBEDDED_IMAGE="${HOOK_CONTAINER_EMBEDDED_IMAGE}" \
+			envsubst '$HOOK_VERSION $HOOK_KERNEL_IMAGE $HOOK_KERNEL_ID $HOOK_KERNEL_VERSION $HOOK_CONTAINER_IP_IMAGE $HOOK_CONTAINER_BOOTKIT_IMAGE $HOOK_CONTAINER_DOCKER_IMAGE $HOOK_CONTAINER_MDEV_IMAGE $HOOK_CONTAINER_CONTAINERD_IMAGE $HOOK_CONTAINER_RUNC_IMAGE $HOOK_CONTAINER_EMBEDDED_IMAGE' \
 			> "hook.${inventory_id}.yaml"
 
 	declare -g linuxkit_bin=""

build.sh

@@ -29,7 +29,7 @@ declare -g HOOK_LK_CONTAINERS_OCI_BASE="${HOOK_LK_CONTAINERS_OCI_BASE:-"quay.io/
 declare -g SKOPEO_IMAGE="${SKOPEO_IMAGE:-"quay.io/skopeo/stable:latest"}"
 
 # See https://github.com/linuxkit/linuxkit/releases
-declare -g -r LINUXKIT_VERSION_DEFAULT="1.2.0" # LinuxKit version to use by default; each flavor can set its own too
+declare -g -r LINUXKIT_VERSION_DEFAULT="1.5.0" # LinuxKit version to use by default; each flavor can set its own too
 
 # Directory to use for storing downloaded artifacts: LinuxKit binary, shellcheck binary, etc.
 declare -g -r CACHE_DIR="${CACHE_DIR:-"cache"}"

images/hook-bootkit/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine as dev
+FROM golang:1.22.6-alpine AS dev
 COPY . /src/
 WORKDIR /src
 RUN go mod download

images/hook-bootkit/go.mod

@@ -1,6 +1,8 @@
 module github.com/tinkerbell/hook/hook-bootkit
 
-go 1.17
+go 1.22
+
+toolchain go1.22.6
 
 require (
 	github.com/cenkalti/backoff/v4 v4.3.0

images/hook-bootkit/main.go

@@ -15,8 +15,8 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
-	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/client"
@@ -127,13 +127,22 @@ func run(ctx context.Context, log logr.Logger) error {
 
 	authStr := base64.URLEncoding.EncodeToString(encodedJSON)
 
-	pullOpts := types.ImagePullOptions{
+	pullOpts := image.PullOptions{
 		RegistryAuth: authStr,
 	}
 	var out io.ReadCloser
 	imagePullOperation := func() error {
+		// with embedded images, the tink worker could potentially already exist
+		// in the local Docker image cache. And the image name could be something
+		// unreachable via the network (for example: 127.0.0.1/embedded/tink-worker).
+		// Because of this we check if the image already exists and don't return an
+		// error if the image does not exist and the pull fails.
+		var imageExists bool
+		if _, _, err := cli.ImageInspectWithRaw(ctx, imageName); err == nil {
+			imageExists = true
+		}
 		out, err = cli.ImagePull(ctx, imageName, pullOpts)
-		if err != nil {
+		if err != nil && !imageExists {
 			log.Error(err, "image pull failure", "imageName", imageName)
 			return err
 		}
@@ -143,18 +152,20 @@ func run(ctx context.Context, log logr.Logger) error {
 		return err
 	}
 
-	buf := bufio.NewScanner(out)
-	for buf.Scan() {
-		structured := make(map[string]interface{})
-		if err := json.Unmarshal(buf.Bytes(), &structured); err != nil {
-			log.Info("image pull logs", "output", buf.Text())
-		} else {
-			log.Info("image pull logs", "logs", structured)
-		}
+	if out != nil {
+		buf := bufio.NewScanner(out)
+		for buf.Scan() {
+			structured := make(map[string]interface{})
+			if err := json.Unmarshal(buf.Bytes(), &structured); err != nil {
+				log.Info("image pull logs", "output", buf.Text())
+			} else {
+				log.Info("image pull logs", "logs", structured)
+			}
 
-	}
-	if err := out.Close(); err != nil {
-		log.Error(err, "closing image pull logs failed")
+		}
+		if err := out.Close(); err != nil {
+			log.Error(err, "closing image pull logs failed")
+		}
 	}
 
 	log.Info("Removing any existing tink-worker container")

images/hook-docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as dev
+FROM golang:1.20-alpine AS dev
 COPY . /src/
 WORKDIR /src
 RUN CGO_ENABLED=0 go build -a -ldflags '-s -w -extldflags "-static"' -o /hook-docker
@@ -13,4 +13,5 @@ RUN strip /usr/local/bin/docker /usr/local/bin/dockerd /usr/local/bin/docker-pro
 # Purge binutils package after stripping
 RUN apk del binutils
 COPY --from=dev /hook-docker .
+
 ENTRYPOINT ["/hook-docker"]

images/hook-embedded/Dockerfile

@@ -0,0 +1,9 @@
+FROM scratch
+ENTRYPOINT []
+WORKDIR /
+COPY ./images/ /etc/embedded-images/
+# the name 001 is important as that is the order in which the scripts are executed
+# we need this mounting to happen before the other init.d scripts run so that
+# the mount points are available to them.
+COPY ./images-mount.sh /etc/init.d/001-images-mount.sh
+CMD []

images/hook-embedded/images-mount.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+exec 3>&1 4>&2
+trap 'exec 2>&4 1>&3' 0 1 2 3
+exec 1>/var/log/embedded-images.log 2>&1
+
+set -xeuo pipefail
+
+# We can't have a Linuxkit "init" container that dumps its file contents to /var and be writable
+# because the init process overwrites it and the contents are lost.
+# Instead, we have the init container, with all the Docker images, dump its contents to /etc/embedded-images.
+# Then we bind mount /etc/embedded-images to /run/images (/var/run is symlinked to /run) and make sure it's
+# read/write. This allows the DinD container to bind mount /var/run/images to /var/lib/docker and the Docker
+# images are available right away and /var/lib/docker is writable.
+mkdir -p /run/images
+mount -o bind,rw /etc/embedded-images/ /run/images
+mount -o remount,rw /run/images

images/hook-embedded/images.txt.example

@@ -0,0 +1,9 @@
+# This is an example file. It explains the required format.
+# For the actual file, you must remove all the comments.
+# The format is source image, a single space, optional additional tag of the source image, a single space, true or false to remove the original tag.
+#<source image> <optional additional tag of the source image> <remove original tag>
+# for example:
+quay.io/tinkerbell/tink-worker:v0.10.0
+quay.io/tinkerbell/tink-worker:v0.10.0 tink-worker:v0.10.0 true
+quay.io/tinkerbell/actions/image2disk embedded/actions/image2disk
+quay.io/tinkerbell/actions/cexec 127.0.0.1/embedded/actions/cexec true

images/hook-embedded/pull-images.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+# This script is used to build container images that are embedded in HookOS.
+# When HookOS boots up, the DinD container will have all the images in its cache.
+
+set -euo pipefail
+
+function docker_save_image() {
+    local image="$1"
+    local output_dir="$2"
+    local output_file="${output_dir}/$(echo "${image}" | tr '/' '-')"
+
+    docker save -o "${output_file}".tar "${image}"
+}
+
+function docker_pull_image() {
+    local image="$1"
+    local arch="${2-amd64}"
+
+    docker pull --platform=linux/"${arch}" "${image}"
+}
+
+function docker_remove_image() {
+    local image="$1"
+
+    docker rmi "${image}" || true
+}
+
+function trap_handler() {
+    local dind_container="$1"
+
+    if [[ "${remove_dind_container}" == "true" ]]; then
+        docker rm -f "${dind_container}" &> /dev/null
+    else
+        echo "DinD container NOT removed, please remove it manually"
+    fi
+}
+
+function main() {
+    local dind_container="$1"
+    local images_file="$2"
+    local arch="$3"
+    local dind_container_image="$4"
+
+    # Pull the images
+    while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
+        echo -e "----------------------- $first_image -----------------------"
+        # Remove the image if it exists so that the image pulls the correct architecture
+        docker_remove_image "${first_image}"
+        docker_pull_image "${first_image}" "${arch}"
+    done < "${images_file}"
+
+    # Save the images
+    local output_dir="${PWD}/images_tar"
+    mkdir -p "${output_dir}"
+    while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do
+        docker_save_image "${first_image}" "${output_dir}"
+    done < "${images_file}"
+
+    export remove_dind_container="true"
+    # as this function maybe called multiple times, we need to ensure the container is removed
+    trap "trap_handler ${dind_container}" RETURN
+    # we're using set -e so the trap on RETURN will not be executed when a command fails
+    trap "trap_handler ${dind_container}" EXIT
+
+    # start DinD container
+    # In order to avoid the src bind mount directory (./images/) ownership from changing to root
+    # we don't bind mount to /var/lib/docker in the container because the DinD container is running as root and
+    # will change the permissions of the bind mount directory (images/) to root.
+    echo -e "Starting DinD container"
+    echo -e "-----------------------"
+    docker run -d --privileged --name "${dind_container}" -v "${PWD}/images_tar":/images_tar -v "${PWD}"/images/:/var/lib/docker-embedded/ -d "${dind_container_image}"
+
+    # wait until the docker daemon is ready
+    until docker exec "${dind_container}" docker info &> /dev/null; do
+        sleep 1
+        if [[ $(docker inspect -f '{{.State.Status}}' "${dind_container}") == "exited" ]]; then
+            echo "DinD container exited unexpectedly"
+            docker logs "${dind_container}"
+            exit 1
+        fi
+    done
+
+    # As hook-docker uses the overlay2 storage driver the DinD must use the overlay2 storage driver too.
+    # make sure the overlay2 storage driver is used by the DinD container.
+    # The VFS storage driver might get used if /var/lib/docker in the DinD container cannot be used by overlay2.
+    storage_driver=$(docker exec "${dind_container}" docker info --format '{{.Driver}}')
+    if [[ "${storage_driver}" != "overlay2" ]]; then
+        export remove_dind_container="false"
+        echo "DinD container is not using overlay2 storage driver, storage driver detected: ${storage_driver}"
+        exit 1
+    fi
+
+    # remove the contents of /var/lib/docker-embedded so that any previous images are removed. Without this it seems to cause boot issues.
+    docker exec "${dind_container}" sh -c "rm -rf /var/lib/docker-embedded/*"
+
+    # Load the images
+    for image_file in "${output_dir}"/*; do
+        echo -e "Loading image: ${image_file}"
+        docker exec "${dind_container}" docker load -i "/images_tar/$(basename ${image_file})"
+    done
+
+    # clean up tar files
+    rm -rf "${output_dir}"/*
+
+    # Create any tags for the images and remove any original tags
+    while IFS=" " read -r first_image image_tag remove_original || [ -n "${first_image}" ] ; do
+        if [[ "${image_tag}" != "" ]]; then
+            docker exec "${dind_container}" docker tag "${first_image}" "${image_tag}"
+            if [[ "${remove_original}" == "true" ]]; then
+                docker exec "${dind_container}" docker rmi "${first_image}"
+            fi
+        fi
+    done < "${images_file}"
+
+    # We need to copy /var/lib/docker to /var/lib/docker-embedded in order for HookOS to use the Docker images in its build.
+    docker exec "${dind_container}" sh -c "cp -a /var/lib/docker/* /var/lib/docker-embedded/"    
+}
+
+arch="${1-amd64}"
+dind_container_name="hookos-dind"
+images_file="images.txt"
+dind_container_image="${2-docker:dind}"
+main "${dind_container_name}" "${images_file}" "${arch}" "${dind_container_image}"

linuxkit-templates/hook.template.yaml

@@ -9,19 +9,21 @@
 # - HOOK_CONTAINER_MDEV_IMAGE: ${HOOK_CONTAINER_MDEV_IMAGE}
 # - HOOK_CONTAINER_CONTAINERD_IMAGE: ${HOOK_CONTAINER_CONTAINERD_IMAGE}
 # - HOOK_CONTAINER_RUNC_IMAGE: ${HOOK_CONTAINER_RUNC_IMAGE}
+# - HOOK_CONTAINER_EMBEDDED_IMAGE: ${HOOK_CONTAINER_EMBEDDED_IMAGE}
 # - Other variables are not replaced: for example this is a literal dollarsign-SOMETHING: $SOMETHING and with braces: ${SOMETHING}
 
 kernel:
   image: "${HOOK_KERNEL_IMAGE}"
-  cmdline: "this_is_not_used=at_at_all_in_hook command_line_is_determined_by=ipxe"
+  cmdline: "this_is_not_used=at_all_in_hook command_line_is_determined_by=ipxe"
 
 init:
-  # this sha is the first with cgroups v2 as the default
-  - linuxkit/init:8a7b6cdb89197dc94eb6db69ef9dc90b750db598
+  # this init container sha has support for volumes
+  - linuxkit/init:872d2e1be745f1acb948762562cf31c367303a3b
   - "${HOOK_CONTAINER_RUNC_IMAGE}"
   - "${HOOK_CONTAINER_CONTAINERD_IMAGE}"
   - linuxkit/ca-certificates:v1.0.0
   - linuxkit/firmware:24402a25359c7bc290f7fc3cd23b6b5f0feb32a5 # "Some" firmware from Linuxkit pkg; see https://github.com/linuxkit/linuxkit/blob/master/pkg/firmware/Dockerfile
+  - "${HOOK_CONTAINER_EMBEDDED_IMAGE}"
 
 onboot:
   - name: rngd1
@@ -100,6 +102,8 @@ services:
     devices:
     - path: all
       type: b
+    - path: all
+      type: c
     - path: "/dev/console"
       type: c
       major: 5
@@ -187,6 +191,8 @@ services:
     devices:
     - path: all
       type: b
+    - path: all
+      type: c
 
   - name: hook-bootkit
     image: "${HOOK_CONTAINER_BOOTKIT_IMAGE}"
@@ -219,8 +225,15 @@ services:
       mkdir:
         - /var/lib/dhcpcd
 
-#dbg  - name: sshd
-#dbg    image: linuxkit/sshd:v1.0.0
+#SSH_SERVER  - name: sshd
+#SSH_SERVER    image: linuxkit/sshd:v1.0.0
+#SSH_SERVER    binds.add:
+#SSH_SERVER      - /etc/profile.d/local.sh:/etc/profile.d/local.sh
+#SSH_SERVER      - /root/.ssh/authorized_keys:/root/.ssh/authorized_keys
+#SSH_SERVER      - /usr/bin/nerdctl:/usr/bin/nerdctl
+#SSH_SERVER      - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt
+#SSH_SERVER      - /:/host_root
+
 
 files:
   - path: etc/profile.d/local.sh
@@ -299,10 +312,10 @@ files:
       ttyUSB1
       ttyUSB2
 
-#dbg  - path: root/.ssh/authorized_keys
-#dbg    source: ~/.ssh/id_rsa.pub
-#dbg    mode: "0600"
-#dbg    optional: true
+#SSH_SERVER  - path: root/.ssh/authorized_keys
+#SSH_SERVER    source: ~/.ssh/id_rsa.pub
+#SSH_SERVER    mode: "0600"
+#SSH_SERVER    optional: true
 
 trust:
   org: