diff --git a/.gitignore b/.gitignore index 4bba5900..42c2ec5c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,10 @@ cache/ *.swp .idea kernel/Dockerfile.autogen.* +images/hook-embedded/images/* +!images/hook-embedded/images/.keep +images/hook-embedded/images.txt +images/hook-embedded/docker/* +!images/hook-embedded/docker/.keep +images/hook-embedded/images_tar/* +!images/hook-embedded/images_tar/.keep diff --git a/README.md b/README.md index c2f88c78..b5a14185 100644 --- a/README.md +++ b/README.md @@ -160,9 +160,21 @@ The `gha-matrix` CLI command prepares a set of JSON outputs for GitHub Actions m - `DOCKER_ARCH` is used by the `linuxkit-containers` command to build the containers for the specified architecture. - `DO_PUSH`: `yes` or `no`, will push the built containers to the OCI registry; defaults to `no`. +### Embedding container images into the DinD (docker-in-docker), also known as [hook-docker](images/hook-docker/), container + +For use cases where having container images already available in Docker is needed, the following steps can be taken to embed container images into hook-docker (DinD): + +> Note: This is optional and no container images will be embedded by default. + +> Note: This will increase the overall size of HookOS. As HookOS is an in memory OS, make sure that the size increase works for the machines you are provisioning. + +1. Create a file named `images.txt` in the [images/hook-embedded/](images/hook-embedded/) directory. +1. Populate this `images.txt` file with the list of images to be embedded. See [images/hook-embedded/images.txt.example](images/hook-embedded/images.txt.example) for details on the required file format. +1. Change directories to [images/hook-embedded/](images/hook-embedded/) and run [`pull-images.sh`](images/hook-embedded/pull-images.sh) script when building amd64 images and run [`pull-images.sh arm64`](images/hook-embedded/pull-images.sh) when building arm64 images. Read the comments at the top of the script for more details. +1. Change directories to the root of the HookOS repository and run `sudo ./build.sh build ...` to build the HookOS kernel and ramdisk. FYI, `sudo` is needed as DIND changes file ownerships to root. + ### Build system TO-DO list -- [ ] Update to Linuxkit 1.2.0 and new linuxkit pkgs; this might lead into the containerd vs dind; - [ ] `make debug` functionality (sshd enabled) was lost in the Makefile -> bash transition; [formats]: https://github.com/linuxkit/linuxkit/blob/master/README.md#booting-and-testing diff --git a/bash/hook-lk-containers.sh b/bash/hook-lk-containers.sh index 1c9a8a87..9c5c8b15 100644 --- a/bash/hook-lk-containers.sh +++ b/bash/hook-lk-containers.sh @@ -13,6 +13,7 @@ function build_all_hook_linuxkit_containers() { build_hook_linuxkit_container hook-mdev HOOK_CONTAINER_MDEV_IMAGE build_hook_linuxkit_container hook-containerd HOOK_CONTAINER_CONTAINERD_IMAGE build_hook_linuxkit_container hook-runc HOOK_CONTAINER_RUNC_IMAGE + build_hook_linuxkit_container hook-embedded HOOK_CONTAINER_EMBEDDED_IMAGE } function build_hook_linuxkit_container() { diff --git a/bash/linuxkit.sh b/bash/linuxkit.sh index bf4dea0d..5f053256 100644 --- a/bash/linuxkit.sh +++ b/bash/linuxkit.sh @@ -70,7 +70,8 @@ function linuxkit_build() { HOOK_CONTAINER_MDEV_IMAGE="${HOOK_CONTAINER_MDEV_IMAGE}" \ HOOK_CONTAINER_CONTAINERD_IMAGE="${HOOK_CONTAINER_CONTAINERD_IMAGE}" \ HOOK_CONTAINER_RUNC_IMAGE="${HOOK_CONTAINER_RUNC_IMAGE}" \ - envsubst '$HOOK_VERSION $HOOK_KERNEL_IMAGE $HOOK_KERNEL_ID $HOOK_KERNEL_VERSION $HOOK_CONTAINER_IP_IMAGE $HOOK_CONTAINER_BOOTKIT_IMAGE $HOOK_CONTAINER_DOCKER_IMAGE $HOOK_CONTAINER_MDEV_IMAGE $HOOK_CONTAINER_CONTAINERD_IMAGE $HOOK_CONTAINER_RUNC_IMAGE' \ + HOOK_CONTAINER_EMBEDDED_IMAGE="${HOOK_CONTAINER_EMBEDDED_IMAGE}" \ + envsubst '$HOOK_VERSION $HOOK_KERNEL_IMAGE $HOOK_KERNEL_ID $HOOK_KERNEL_VERSION $HOOK_CONTAINER_IP_IMAGE $HOOK_CONTAINER_BOOTKIT_IMAGE $HOOK_CONTAINER_DOCKER_IMAGE $HOOK_CONTAINER_MDEV_IMAGE $HOOK_CONTAINER_CONTAINERD_IMAGE $HOOK_CONTAINER_RUNC_IMAGE $HOOK_CONTAINER_EMBEDDED_IMAGE' \ > "hook.${inventory_id}.yaml" declare -g linuxkit_bin="" diff --git a/build.sh b/build.sh index 6a670848..985bea88 100755 --- a/build.sh +++ b/build.sh @@ -29,7 +29,7 @@ declare -g HOOK_LK_CONTAINERS_OCI_BASE="${HOOK_LK_CONTAINERS_OCI_BASE:-"quay.io/ declare -g SKOPEO_IMAGE="${SKOPEO_IMAGE:-"quay.io/skopeo/stable:latest"}" # See https://github.com/linuxkit/linuxkit/releases -declare -g -r LINUXKIT_VERSION_DEFAULT="1.2.0" # LinuxKit version to use by default; each flavor can set its own too +declare -g -r LINUXKIT_VERSION_DEFAULT="1.5.0" # LinuxKit version to use by default; each flavor can set its own too # Directory to use for storing downloaded artifacts: LinuxKit binary, shellcheck binary, etc. declare -g -r CACHE_DIR="${CACHE_DIR:-"cache"}" diff --git a/images/hook-bootkit/Dockerfile b/images/hook-bootkit/Dockerfile index 6dc81241..125e57a4 100644 --- a/images/hook-bootkit/Dockerfile +++ b/images/hook-bootkit/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21-alpine as dev +FROM golang:1.22.6-alpine AS dev COPY . /src/ WORKDIR /src RUN go mod download diff --git a/images/hook-bootkit/go.mod b/images/hook-bootkit/go.mod index 76a259a7..bec5c940 100644 --- a/images/hook-bootkit/go.mod +++ b/images/hook-bootkit/go.mod @@ -1,6 +1,8 @@ module github.com/tinkerbell/hook/hook-bootkit -go 1.17 +go 1.22 + +toolchain go1.22.6 require ( github.com/cenkalti/backoff/v4 v4.3.0 diff --git a/images/hook-bootkit/main.go b/images/hook-bootkit/main.go index 8a8d6d97..a752ed0f 100644 --- a/images/hook-bootkit/main.go +++ b/images/hook-bootkit/main.go @@ -15,8 +15,8 @@ import ( "time" "github.com/cenkalti/backoff/v4" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/registry" "github.com/docker/docker/client" @@ -127,13 +127,22 @@ func run(ctx context.Context, log logr.Logger) error { authStr := base64.URLEncoding.EncodeToString(encodedJSON) - pullOpts := types.ImagePullOptions{ + pullOpts := image.PullOptions{ RegistryAuth: authStr, } var out io.ReadCloser imagePullOperation := func() error { + // with embedded images, the tink worker could potentially already exist + // in the local Docker image cache. And the image name could be something + // unreachable via the network (for example: 127.0.0.1/embedded/tink-worker). + // Because of this we check if the image already exists and don't return an + // error if the image does not exist and the pull fails. + var imageExists bool + if _, _, err := cli.ImageInspectWithRaw(ctx, imageName); err == nil { + imageExists = true + } out, err = cli.ImagePull(ctx, imageName, pullOpts) - if err != nil { + if err != nil && !imageExists { log.Error(err, "image pull failure", "imageName", imageName) return err } @@ -143,18 +152,20 @@ func run(ctx context.Context, log logr.Logger) error { return err } - buf := bufio.NewScanner(out) - for buf.Scan() { - structured := make(map[string]interface{}) - if err := json.Unmarshal(buf.Bytes(), &structured); err != nil { - log.Info("image pull logs", "output", buf.Text()) - } else { - log.Info("image pull logs", "logs", structured) - } + if out != nil { + buf := bufio.NewScanner(out) + for buf.Scan() { + structured := make(map[string]interface{}) + if err := json.Unmarshal(buf.Bytes(), &structured); err != nil { + log.Info("image pull logs", "output", buf.Text()) + } else { + log.Info("image pull logs", "logs", structured) + } - } - if err := out.Close(); err != nil { - log.Error(err, "closing image pull logs failed") + } + if err := out.Close(); err != nil { + log.Error(err, "closing image pull logs failed") + } } log.Info("Removing any existing tink-worker container") diff --git a/images/hook-docker/Dockerfile b/images/hook-docker/Dockerfile index 3aad7bd1..737a9670 100644 --- a/images/hook-docker/Dockerfile +++ b/images/hook-docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-alpine as dev +FROM golang:1.20-alpine AS dev COPY . /src/ WORKDIR /src RUN CGO_ENABLED=0 go build -a -ldflags '-s -w -extldflags "-static"' -o /hook-docker @@ -13,4 +13,5 @@ RUN strip /usr/local/bin/docker /usr/local/bin/dockerd /usr/local/bin/docker-pro # Purge binutils package after stripping RUN apk del binutils COPY --from=dev /hook-docker . + ENTRYPOINT ["/hook-docker"] diff --git a/images/hook-embedded/Dockerfile b/images/hook-embedded/Dockerfile new file mode 100644 index 00000000..b716d552 --- /dev/null +++ b/images/hook-embedded/Dockerfile @@ -0,0 +1,9 @@ +FROM scratch +ENTRYPOINT [] +WORKDIR / +COPY ./images/ /etc/embedded-images/ +# the name 001 is important as that is the order in which the scripts are executed +# we need this mounting to happen before the other init.d scripts run so that +# the mount points are available to them. +COPY ./images-mount.sh /etc/init.d/001-images-mount.sh +CMD [] diff --git a/images/hook-embedded/docker/.keep b/images/hook-embedded/docker/.keep new file mode 100644 index 00000000..e69de29b diff --git a/images/hook-embedded/images-mount.sh b/images/hook-embedded/images-mount.sh new file mode 100755 index 00000000..ff939c5c --- /dev/null +++ b/images/hook-embedded/images-mount.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +exec 3>&1 4>&2 +trap 'exec 2>&4 1>&3' 0 1 2 3 +exec 1>/var/log/embedded-images.log 2>&1 + +set -xeuo pipefail + +# We can't have a Linuxkit "init" container that dumps its file contents to /var and be writable +# because the init process overwrites it and the contents are lost. +# Instead, we have the init container, with all the Docker images, dump its contents to /etc/embedded-images. +# Then we bind mount /etc/embedded-images to /run/images (/var/run is symlinked to /run) and make sure it's +# read/write. This allows the DinD container to bind mount /var/run/images to /var/lib/docker and the Docker +# images are available right away and /var/lib/docker is writable. +mkdir -p /run/images +mount -o bind,rw /etc/embedded-images/ /run/images +mount -o remount,rw /run/images diff --git a/images/hook-embedded/images.txt.example b/images/hook-embedded/images.txt.example new file mode 100644 index 00000000..c21c8267 --- /dev/null +++ b/images/hook-embedded/images.txt.example @@ -0,0 +1,9 @@ +# This is an example file. It explains the required format. +# For the actual file, you must remove all the comments. +# The format is source image, a single space, optional additional tag of the source image, a single space, true or false to remove the original tag. +# +# for example: +quay.io/tinkerbell/tink-worker:v0.10.0 +quay.io/tinkerbell/tink-worker:v0.10.0 tink-worker:v0.10.0 true +quay.io/tinkerbell/actions/image2disk embedded/actions/image2disk +quay.io/tinkerbell/actions/cexec 127.0.0.1/embedded/actions/cexec true diff --git a/images/hook-embedded/images/.keep b/images/hook-embedded/images/.keep new file mode 100644 index 00000000..e69de29b diff --git a/images/hook-embedded/images_tar/.keep b/images/hook-embedded/images_tar/.keep new file mode 100644 index 00000000..e69de29b diff --git a/images/hook-embedded/pull-images.sh b/images/hook-embedded/pull-images.sh new file mode 100755 index 00000000..45bf18da --- /dev/null +++ b/images/hook-embedded/pull-images.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# This script is used to build container images that are embedded in HookOS. +# When HookOS boots up, the DinD container will have all the images in its cache. + +set -euo pipefail + +function docker_save_image() { + local image="$1" + local output_dir="$2" + local output_file="${output_dir}/$(echo "${image}" | tr '/' '-')" + + docker save -o "${output_file}".tar "${image}" +} + +function docker_pull_image() { + local image="$1" + local arch="${2-amd64}" + + docker pull --platform=linux/"${arch}" "${image}" +} + +function docker_remove_image() { + local image="$1" + + docker rmi "${image}" || true +} + +function trap_handler() { + local dind_container="$1" + + if [[ "${remove_dind_container}" == "true" ]]; then + docker rm -f "${dind_container}" &> /dev/null + else + echo "DinD container NOT removed, please remove it manually" + fi +} + +function main() { + local dind_container="$1" + local images_file="$2" + local arch="$3" + local dind_container_image="$4" + + # Pull the images + while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do + echo -e "----------------------- $first_image -----------------------" + # Remove the image if it exists so that the image pulls the correct architecture + docker_remove_image "${first_image}" + docker_pull_image "${first_image}" "${arch}" + done < "${images_file}" + + # Save the images + local output_dir="${PWD}/images_tar" + mkdir -p "${output_dir}" + while IFS=" " read -r first_image image_tag || [ -n "${first_image}" ] ; do + docker_save_image "${first_image}" "${output_dir}" + done < "${images_file}" + + export remove_dind_container="true" + # as this function maybe called multiple times, we need to ensure the container is removed + trap "trap_handler ${dind_container}" RETURN + # we're using set -e so the trap on RETURN will not be executed when a command fails + trap "trap_handler ${dind_container}" EXIT + + # start DinD container + # In order to avoid the src bind mount directory (./images/) ownership from changing to root + # we don't bind mount to /var/lib/docker in the container because the DinD container is running as root and + # will change the permissions of the bind mount directory (images/) to root. + echo -e "Starting DinD container" + echo -e "-----------------------" + docker run -d --privileged --name "${dind_container}" -v "${PWD}/images_tar":/images_tar -v "${PWD}"/images/:/var/lib/docker-embedded/ -d "${dind_container_image}" + + # wait until the docker daemon is ready + until docker exec "${dind_container}" docker info &> /dev/null; do + sleep 1 + if [[ $(docker inspect -f '{{.State.Status}}' "${dind_container}") == "exited" ]]; then + echo "DinD container exited unexpectedly" + docker logs "${dind_container}" + exit 1 + fi + done + + # As hook-docker uses the overlay2 storage driver the DinD must use the overlay2 storage driver too. + # make sure the overlay2 storage driver is used by the DinD container. + # The VFS storage driver might get used if /var/lib/docker in the DinD container cannot be used by overlay2. + storage_driver=$(docker exec "${dind_container}" docker info --format '{{.Driver}}') + if [[ "${storage_driver}" != "overlay2" ]]; then + export remove_dind_container="false" + echo "DinD container is not using overlay2 storage driver, storage driver detected: ${storage_driver}" + exit 1 + fi + + # remove the contents of /var/lib/docker-embedded so that any previous images are removed. Without this it seems to cause boot issues. + docker exec "${dind_container}" sh -c "rm -rf /var/lib/docker-embedded/*" + + # Load the images + for image_file in "${output_dir}"/*; do + echo -e "Loading image: ${image_file}" + docker exec "${dind_container}" docker load -i "/images_tar/$(basename ${image_file})" + done + + # clean up tar files + rm -rf "${output_dir}"/* + + # Create any tags for the images and remove any original tags + while IFS=" " read -r first_image image_tag remove_original || [ -n "${first_image}" ] ; do + if [[ "${image_tag}" != "" ]]; then + docker exec "${dind_container}" docker tag "${first_image}" "${image_tag}" + if [[ "${remove_original}" == "true" ]]; then + docker exec "${dind_container}" docker rmi "${first_image}" + fi + fi + done < "${images_file}" + + # We need to copy /var/lib/docker to /var/lib/docker-embedded in order for HookOS to use the Docker images in its build. + docker exec "${dind_container}" sh -c "cp -a /var/lib/docker/* /var/lib/docker-embedded/" +} + +arch="${1-amd64}" +dind_container_name="hookos-dind" +images_file="images.txt" +dind_container_image="${2-docker:dind}" +main "${dind_container_name}" "${images_file}" "${arch}" "${dind_container_image}" diff --git a/linuxkit-templates/hook.template.yaml b/linuxkit-templates/hook.template.yaml index 26db724d..56187f09 100644 --- a/linuxkit-templates/hook.template.yaml +++ b/linuxkit-templates/hook.template.yaml @@ -9,19 +9,21 @@ # - HOOK_CONTAINER_MDEV_IMAGE: ${HOOK_CONTAINER_MDEV_IMAGE} # - HOOK_CONTAINER_CONTAINERD_IMAGE: ${HOOK_CONTAINER_CONTAINERD_IMAGE} # - HOOK_CONTAINER_RUNC_IMAGE: ${HOOK_CONTAINER_RUNC_IMAGE} +# - HOOK_CONTAINER_EMBEDDED_IMAGE: ${HOOK_CONTAINER_EMBEDDED_IMAGE} # - Other variables are not replaced: for example this is a literal dollarsign-SOMETHING: $SOMETHING and with braces: ${SOMETHING} kernel: image: "${HOOK_KERNEL_IMAGE}" - cmdline: "this_is_not_used=at_at_all_in_hook command_line_is_determined_by=ipxe" + cmdline: "this_is_not_used=at_all_in_hook command_line_is_determined_by=ipxe" init: - # this sha is the first with cgroups v2 as the default - - linuxkit/init:8a7b6cdb89197dc94eb6db69ef9dc90b750db598 + # this init container sha has support for volumes + - linuxkit/init:872d2e1be745f1acb948762562cf31c367303a3b - "${HOOK_CONTAINER_RUNC_IMAGE}" - "${HOOK_CONTAINER_CONTAINERD_IMAGE}" - linuxkit/ca-certificates:v1.0.0 - linuxkit/firmware:24402a25359c7bc290f7fc3cd23b6b5f0feb32a5 # "Some" firmware from Linuxkit pkg; see https://github.com/linuxkit/linuxkit/blob/master/pkg/firmware/Dockerfile + - "${HOOK_CONTAINER_EMBEDDED_IMAGE}" onboot: - name: rngd1 @@ -100,6 +102,8 @@ services: devices: - path: all type: b + - path: all + type: c - path: "/dev/console" type: c major: 5 @@ -187,6 +191,8 @@ services: devices: - path: all type: b + - path: all + type: c - name: hook-bootkit image: "${HOOK_CONTAINER_BOOTKIT_IMAGE}" @@ -219,8 +225,15 @@ services: mkdir: - /var/lib/dhcpcd -#dbg - name: sshd -#dbg image: linuxkit/sshd:v1.0.0 +#SSH_SERVER - name: sshd +#SSH_SERVER image: linuxkit/sshd:v1.0.0 +#SSH_SERVER binds.add: +#SSH_SERVER - /etc/profile.d/local.sh:/etc/profile.d/local.sh +#SSH_SERVER - /root/.ssh/authorized_keys:/root/.ssh/authorized_keys +#SSH_SERVER - /usr/bin/nerdctl:/usr/bin/nerdctl +#SSH_SERVER - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt +#SSH_SERVER - /:/host_root + files: - path: etc/profile.d/local.sh @@ -299,10 +312,10 @@ files: ttyUSB1 ttyUSB2 -#dbg - path: root/.ssh/authorized_keys -#dbg source: ~/.ssh/id_rsa.pub -#dbg mode: "0600" -#dbg optional: true +#SSH_SERVER - path: root/.ssh/authorized_keys +#SSH_SERVER source: ~/.ssh/id_rsa.pub +#SSH_SERVER mode: "0600" +#SSH_SERVER optional: true trust: org: