diff --git a/Makefile b/Makefile index 4d8d4174..16b48043 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,8 @@ shellcheck: shellcheck --exclude=SC1090,SC1091 \ $$(find scripts -type f $(FINDEXEC)) \ $$(find live-build/misc/live-build-hooks -type f $(FINDEXEC)) \ - $$(find live-build/misc/upgrade-scripts -type f) + $$(find live-build/misc/upgrade-scripts -type f) \ + $$(find live-build/misc/migration-scripts -type f) # # There doesn't appear to be a way to have "shfmt" return non-zero when @@ -82,7 +83,8 @@ shellcheck: shfmtcheck: ! shfmt -d $$(find scripts -type f $(FINDEXEC)) \ $$(find live-build/misc/live-build-hooks -type f $(FINDEXEC)) \ - $$(find live-build/misc/upgrade-scripts -type f) | grep . + $$(find live-build/misc/upgrade-scripts -type f) \ + $$(find live-build/misc/migration-scripts -type f) | grep . ansiblecheck: ansible-lint $$(find bootstrap live-build/variants -name playbook.yml) diff --git a/docker/Dockerfile b/docker/Dockerfile index dda2db6c..d990489c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,6 +44,7 @@ RUN \ livecd-rootfs \ make \ man \ + pigz \ rename \ shellcheck \ vim \ diff --git a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml index 0ac54936..e5555d5c 100644 --- a/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml @@ -44,6 +44,21 @@ password: "{{ lookup('env', 'APPLIANCE_PASSWORD') | password_hash('sha512') }}" +# +# In a similar fashion with other critical data that are required +# we specify "no_log" for any information related to signing images. +# Currently: DLPX_KEY_URL, DLPX_SIGNING_LOGIN. +# +- fail: + msg: "Required environment variable 'DLPX_KEY_URL' is empty." + when: lookup('env', 'DLPX_KEY_URL') == '' + no_log: true + +- fail: + msg: "Required environment variable 'DLPX_SIGNING_LOGIN' is empty." + when: lookup('env', 'DLPX_SIGNING_LOGIN') == '' + no_log: true + # # The virtualization package uses the /etc/issue file to store a # customer-supplied banner that is displayed prior to login. By default, diff --git a/live-build/misc/live-build-hooks/90-linux-migration-artifact.binary b/live-build/misc/live-build-hooks/90-linux-migration-artifact.binary new file mode 100755 index 00000000..f0c70a0b --- /dev/null +++ b/live-build/misc/live-build-hooks/90-linux-migration-artifact.binary @@ -0,0 +1,165 @@ +#!/bin/bash -eux +# +# Copyright 2018 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +die() { + echo "$*" 1>&2 + exit 1 +} + +# +# This script is intended to be used as part of Delphix's build process. +# It's role is to convert the "binary" directory generated by live-build, +# into a cpio archive that will be wrapper in a tarball. That tarball is +# the migration artifact that illumos-based engines use to migrate to +# Linux. +# +MIGRATION_ARTIFACT="$APPLIANCE_VARIANT.migration.tar.gz" + +# +# At the time of this writing dx_unpack.sh expects a tarball of the +# following structure: +# +# depot/ the top level container +# depot/dx_prepare top-level script to prepare the archive +# for upgrade +# depot/version.info version and date information about the +# upgrade image +# depot/hashes the sha256sum of any upgrade artifacts +# (see below) +# depot/hashes.sig[.] one cryptographic signature of depot/hashes +# per Delphix release +# depot/ generally cpio archives of OS and appliance +# +# As dx_upack.sh is the only piece of code that runs from the old +# world, we treat it as a contract that lists the bare minimum +# requirements for our artifact. +# +DEPOT_DIRECTORY=depot + +# +# Delete any leftover files from previous runs. +# +rm -rf "$MIGRATION_ARTIFACT" "$DEPOT_DIRECTORY" + +if [ ! -d binary/opt/delphix ]; then + echo "This artifact does not contain the appliance." \ + " Skipping migration image..." + exit 0 +fi + +mkdir $DEPOT_DIRECTORY + +# +# Generate archive for os root directory. +# +# Note: We temporarily change directory in the subshell (we move to the +# "binary" directory) because we find prints evenrything as a relative +# path to the current directory and we don't want the generated archive +# to have "binary" as its top-level directory. +# +( + cd binary + find ./* -depth -print | cpio -ocB +) >$DEPOT_DIRECTORY/os-root.cpio + +# +# Copy all migration scripts including dx_prepare which is needed by +# dx_unpack. +# +cp migration-scripts/* $DEPOT_DIRECTORY + +# +# There may be a version.info file in the current directory already. +# That version.info is Linux-specific and does not necessarily have +# all the fields that the illumos upgrade process requires. Thus, we +# create our own special version info directly in the depot directory +# with all the expected fields plus a special variable that indicates +# that this is a migration image, so dx_prepare knows how to tell the +# difference between normal illumos upgrade and illumos-to-Linux +# migrations. +# +KERNEL_VERSION=$(basename "$(find binary/boot/vmlinuz-* | sort -r | head -n 1)") +{ + echo "DLPX_DATE=$(date '+%Y.%m.%d.%H.%M.%S')" + echo "DLPX_OS_VERSION=$KERNEL_VERSION" + echo "DLPX_VERSION=$(date '+%Y.%m.%d.%H')" + echo "DLPX_MIN_VERSION=5.3.0.0" + echo "MIGRATION_IMAGE=y" +} >$DEPOT_DIRECTORY/version.info + +# +# Generate hashes file. +# +( + cd binary + find ./* -type f -print0 | xargs -0 sha256sum +) >$DEPOT_DIRECTORY/os-root.hashes || die "generating os-root.hashes failed" +( + cd $DEPOT_DIRECTORY + sha256sum ./* >hashes +) + +# +# DLPX_SIGNING_LOGIN is used for signing our migration image. +# The signature is later unpacked and verified on the VM +# performing the migration. If DLPX_SIGNING_LOGIN has not been +# specified by the user when running this hook we skip generating +# the signature for this image. +# +# Note that DLPX_KEY_URL should also be specified for this to +# work, as the default is really a placeholder. +# +if [ "$DLPX_SIGNING_LOGIN" != "delphix" ]; then + # + # Generate depot/hashes.sig[.] + # + # Assumption: we always migrate from version 5.3 + # + UPGRADE_VERSION="5.3" + DLPX_SIGNING_URL="$DLPX_KEY_URL/upgrade/keyVersion/$UPGRADE_VERSION/sign" + + hashes_data=$(base64 -w 0 $DEPOT_DIRECTORY/hashes) + json_payload="{\"data\": \"${hashes_data}\"}" + request_url="$DLPX_SIGNING_URL" + + # + # Compute signature + # + result=$( + set -o pipefail + echo "$json_payload" | curl -s -f -H \ + "Content-Type: application/json" \ + -u "$DLPX_SIGNING_LOGIN" "$request_url" -d @- + ) || die "Failure signing hashes with $request_url" + + # + # Extract signature from JSON response, ensure it is not empty + # and save it. + # + signature=$(echo "$result" | grep signature | + sed 's/.*"signature": "\([^"]*\)".*/\1/') + [ -n "$signature" ] || die "Empty signature obtained with $request_url" + echo "$signature" | base64 -d >$DEPOT_DIRECTORY/hashes.sig.$UPGRADE_VERSION +fi + +# +# Tape-ARchive & Compress Zee File! +# +# We enable verbose output in the hope that all the files that are part of +# the artifact are logged somewhere which could aid debugging in the future. +# +tar -cvf - $DEPOT_DIRECTORY | pigz >"$MIGRATION_ARTIFACT" diff --git a/live-build/misc/migration-scripts/dx_prepare b/live-build/misc/migration-scripts/dx_prepare new file mode 100755 index 00000000..6430016e --- /dev/null +++ b/live-build/misc/migration-scripts/dx_prepare @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2018 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script is a no-op and only exist to satisfy dx_unpack.sh +# from illumos. +# + +exit 0 diff --git a/live-build/variants/external-standard/.gitignore b/live-build/variants/external-standard/.gitignore index 80c4defa..629981d9 100644 --- a/live-build/variants/external-standard/.gitignore +++ b/live-build/variants/external-standard/.gitignore @@ -19,3 +19,4 @@ !ansible/** !config/hooks !upgrade-scripts +!migration-scripts diff --git a/live-build/variants/external-standard/migration-scripts b/live-build/variants/external-standard/migration-scripts new file mode 120000 index 00000000..9aae020d --- /dev/null +++ b/live-build/variants/external-standard/migration-scripts @@ -0,0 +1 @@ +../../misc/migration-scripts \ No newline at end of file diff --git a/live-build/variants/internal-dcenter/migration-scripts b/live-build/variants/internal-dcenter/migration-scripts new file mode 120000 index 00000000..9aae020d --- /dev/null +++ b/live-build/variants/internal-dcenter/migration-scripts @@ -0,0 +1 @@ +../../misc/migration-scripts \ No newline at end of file diff --git a/live-build/variants/internal-dev/.gitignore b/live-build/variants/internal-dev/.gitignore index 80c4defa..629981d9 100644 --- a/live-build/variants/internal-dev/.gitignore +++ b/live-build/variants/internal-dev/.gitignore @@ -19,3 +19,4 @@ !ansible/** !config/hooks !upgrade-scripts +!migration-scripts diff --git a/live-build/variants/internal-dev/migration-scripts b/live-build/variants/internal-dev/migration-scripts new file mode 120000 index 00000000..9aae020d --- /dev/null +++ b/live-build/variants/internal-dev/migration-scripts @@ -0,0 +1 @@ +../../misc/migration-scripts \ No newline at end of file diff --git a/live-build/variants/internal-minimal/.gitignore b/live-build/variants/internal-minimal/.gitignore index 80c4defa..629981d9 100644 --- a/live-build/variants/internal-minimal/.gitignore +++ b/live-build/variants/internal-minimal/.gitignore @@ -19,3 +19,4 @@ !ansible/** !config/hooks !upgrade-scripts +!migration-scripts diff --git a/live-build/variants/internal-minimal/migration-scripts b/live-build/variants/internal-minimal/migration-scripts new file mode 120000 index 00000000..9aae020d --- /dev/null +++ b/live-build/variants/internal-minimal/migration-scripts @@ -0,0 +1 @@ +../../misc/migration-scripts \ No newline at end of file diff --git a/live-build/variants/internal-qa/.gitignore b/live-build/variants/internal-qa/.gitignore index 80c4defa..629981d9 100644 --- a/live-build/variants/internal-qa/.gitignore +++ b/live-build/variants/internal-qa/.gitignore @@ -19,3 +19,4 @@ !ansible/** !config/hooks !upgrade-scripts +!migration-scripts diff --git a/live-build/variants/internal-qa/migration-scripts b/live-build/variants/internal-qa/migration-scripts new file mode 120000 index 00000000..9aae020d --- /dev/null +++ b/live-build/variants/internal-qa/migration-scripts @@ -0,0 +1 @@ +../../misc/migration-scripts \ No newline at end of file diff --git a/scripts/create-variant.sh b/scripts/create-variant.sh index 02de216b..0157e4e2 100755 --- a/scripts/create-variant.sh +++ b/scripts/create-variant.sh @@ -63,3 +63,6 @@ ln -s ../../../misc/live-build-hooks \ ln -s ../../misc/upgrade-scripts \ "$TOP/live-build/variants/$1/upgrade-scripts" + +ln -s ../../misc/migration-scripts \ + "$TOP/live-build/variants/$1/migration-scripts" diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh index 572b9dd2..5c15a745 100755 --- a/scripts/docker-run.sh +++ b/scripts/docker-run.sh @@ -54,6 +54,8 @@ $DOCKER_RUN --rm \ --env APPLIANCE_PASSWORD \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ + --env DLPX_KEY_URL \ + --env DLPX_SIGNING_LOGIN \ --volume "$TOP:/opt/appliance-build" \ --workdir "/opt/appliance-build" \ appliance-build "$@" diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index bb52673f..26acbb26 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -46,6 +46,13 @@ set -o pipefail # export APPLIANCE_PASSWORD="${APPLIANCE_PASSWORD:-delphix}" +# +# Similarly to the appliance credentials above do the same for info +# needed to sign artifacts. +# +export DLPX_KEY_URL="${DLPX_KEY_URL:-https://example.com}" +export DLPX_SIGNING_LOGIN="${DLPX_SIGNING_LOGIN:-delphix}" + # # We need to be careful to set xtrace after we set the USERNAME and # PASSWORD variables above; otherwise, we could leak the values of the @@ -122,7 +129,7 @@ fi # user (e.g. other software); this is most useful when multiple variants # are built via a single call to "make" (e.g. using the "all" target). # -for ext in ova qcow2 upgrade.tar.gz gcp.tar.gz vhdx vmdk; do +for ext in ova qcow2 upgrade.tar.gz migration.tar.gz gcp.tar.gz vhdx vmdk; do if [[ -f "$APPLIANCE_VARIANT.$ext" ]]; then mv "$APPLIANCE_VARIANT.$ext" "$TOP/live-build/artifacts" fi