LX-1550 create migration image

Makefile

@@ -22,6 +22,11 @@ FINDEXEC.Darwin := -perm +111
 FINDEXEC.Linux := -executable
 FINDEXEC := $(FINDEXEC.$(shell uname -s))
 
+SHELL_CHECKSTYLE_FILES = $(shell find scripts -type f $(FINDEXEC)) \
+                $(shell find live-build/misc/live-build-hooks -type f $(FINDEXEC)) \
+                $(shell find live-build/misc/upgrade-scripts -type f) \
+                $(shell find live-build/misc/migration-scripts -type f)
+
 .PHONY: \
 	all-external \
 	all-internal \
@@ -54,10 +59,7 @@ ancillary-repository:
 	./scripts/build-ancillary-repository.sh
 
 shellcheck:
-	shellcheck --exclude=SC1090,SC1091 \
-		$$(find scripts -type f $(FINDEXEC)) \
-		$$(find live-build/misc/live-build-hooks -type f $(FINDEXEC)) \
-		$$(find live-build/misc/upgrade-scripts -type f)
+	shellcheck --exclude=SC1090,SC1091 $(SHELL_CHECKSTYLE_FILES)
 
 #
 # There doesn't appear to be a way to have "shfmt" return non-zero when
@@ -80,9 +82,7 @@ shellcheck:
 # problematic lines are conveyed to the user so they can be fixed.
 #
 shfmtcheck:
-	! shfmt -d $$(find scripts -type f $(FINDEXEC)) \
-		$$(find live-build/misc/live-build-hooks -type f $(FINDEXEC)) \
-		$$(find live-build/misc/upgrade-scripts -type f) | grep .
+	! shfmt -d $(SHELL_CHECKSTYLE_FILES) | grep .
 
 ansiblecheck:
 	ansible-lint $$(find bootstrap live-build/variants -name playbook.yml)

docker/Dockerfile

@@ -44,6 +44,7 @@ RUN \
       livecd-rootfs \
       make \
       man \
+      pigz \
       rename \
       shellcheck \
       vim \

live-build/misc/live-build-hooks/90-linux-migration-artifact.binary

@@ -0,0 +1,164 @@
+#!/bin/bash -eux
+#
+# Copyright 2018 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This script is intended to be used as part of Delphix's build process.
+# Its role is to convert the "binary" directory generated by live-build,
+# into a cpio archive that will be wrapper in a tarball. That tarball is
+# the migration artifact that illumos-based engines use to migrate to
+# Linux.
+#
+MIGRATION_ARTIFACT="$APPLIANCE_VARIANT.migration.tar.gz"
+
+#
+# At the time of this writing dx_unpack.sh expects a tarball of the
+# following structure:
+#
+# depot/                       the top level container
+# depot/dx_prepare             top-level script to prepare the archive
+#                              for upgrade
+# depot/version.info           version and date information about the
+#                              upgrade image
+# depot/hashes                 the sha256sum of any upgrade artifacts
+#                              (see below)
+# depot/hashes.sig[.<release>] one cryptographic signature of depot/hashes
+#                              per Delphix release
+# depot/<upgrade artifacts>    generally cpio archives of OS and appliance
+#
+# As dx_upack.sh is the only piece of code that runs from the old
+# world, we treat it as a contract that lists the bare minimum
+# requirements for our artifact.
+#
+DEPOT_DIRECTORY=depot
+
+#
+# Delete any leftover files from previous runs.
+#
+rm -rf "$MIGRATION_ARTIFACT" "$DEPOT_DIRECTORY"
+
+#
+# Not all variants include the virtualization application (e.g.
+# internal-minimal) so don't bother generating a migration image
+# for those. /opt/delphix is always created as part of the
+# appliance. Thus, checking if it exists is a quick way for us to
+# distinguish between variants that have the appliance from
+# the ones that don't.
+#
+if [[ ! -d binary/opt/delphix ]]; then
+	echo "This artifact does not contain the appliance." \
+		" Skipping migration image..."
+	exit 0
+fi
+
+mkdir $DEPOT_DIRECTORY
+
+#
+# Generate archive for os root directory.
+#
+# Note: We temporarily change directory in the subshell (we move to the
+# "binary" directory) because find prints everything as a relative
+# path to the current directory and we don't want the generated archive
+# to have "binary" as its top-level directory.
+#
+(
+	cd binary
+	find . -print | cpio -oc
+) >$DEPOT_DIRECTORY/os-root.cpio
+
+#
+# Copy all migration scripts including dx_prepare which is needed by
+# dx_unpack.
+#
+cp migration-scripts/* $DEPOT_DIRECTORY
+
+#
+# There may be a version.info file in the current directory already.
+# That version.info is Linux-specific and does not necessarily have
+# all the fields that the illumos upgrade process requires. Thus, we
+# create our own special version info directly in the depot directory
+# with all the fields expected by dx_unpack.
+#
+{
+	#
+	# Note that both the following fields are bogus and supplied
+	# solely for making dx_unpack happy.
+	#
+	echo "DLPX_DATE=bogus"
+	echo "DLPX_OS_VERSION=Linux"
+
+	#
+	# DLPX_MIN_VERSION is specified for ensuring that we are
+	# migrating from the right version.
+	#
+	echo "DLPX_MIN_VERSION=5.3.0.0"
+
+	#
+	# DLPX_VERSION is set explicitly to match the version of the
+	# virtualization appliance, which is what upgrades in Linux
+	# do and therefore we try to stay consistent as this field
+	# really can be anything (e.g. a date, a point-version, a
+	# nickname, etc.)
+	#
+	# shellcheck disable=SC2016
+	echo "DLPX_VERSION=$(chroot binary dpkg-query \
+		-Wf '${Version}' delphix-virtualization)"
+} >$DEPOT_DIRECTORY/version.info
+
+#
+# Generate hashes file.
+#
+(
+	cd binary
+	find . -type f -print0 | xargs -0 sha256sum
+) >$DEPOT_DIRECTORY/os-root.hashes
+(
+	cd $DEPOT_DIRECTORY
+	sha256sum ./* >hashes
+)
+
+#
+# DLPX_SIGNING_LOGIN is used for signing our migration image.
+# The signature is later unpacked and verified on the VM
+# performing the migration. If DLPX_SIGNING_LOGIN has not been
+# specified by the user when running this hook we skip generating
+# the signature for this image.
+#
+if [[ -n "${DLPX_SIGNING_LOGIN:-}" ]] && [[ -n "${DLPX_KEY_URL:-}" ]]; then
+	#
+	# Generate depot/hashes.sig[.<release>]
+	#
+	# Assumption: we always migrate from version 5.3
+	#
+	UPGRADE_VERSION="5.3"
+	DLPX_SIGNING_URL="$DLPX_KEY_URL/upgrade/keyVersion/$UPGRADE_VERSION/sign"
+
+	set -o pipefail
+	curl -s -f -H "Content-Type: application/json" \
+		-u "$DLPX_SIGNING_LOGIN" "$DLPX_SIGNING_URL" -d @- <<-EOF |
+			{"data": "$(base64 -w 0 $DEPOT_DIRECTORY/hashes)"}
+		EOF
+		jq -r .signature |
+		base64 -d >$DEPOT_DIRECTORY/hashes.sig.$UPGRADE_VERSION
+fi
+
+#
+# Tape-ARchive & Compress Zee File!
+#
+# We enable verbose output in the hope that all the files that are part of
+# the artifact are logged somewhere which could aid debugging in the future.
+#
+tar -cvf - $DEPOT_DIRECTORY | pigz >"$MIGRATION_ARTIFACT"

live-build/misc/migration-scripts/dx_apply

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Copyright 2018 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# [WIP]
+# The existence of this file is checked by dx_unpack, thus we provide
+# it as an empty file until the needed functionality is added.
+#
+
+exit 0

live-build/misc/migration-scripts/dx_execute

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Copyright 2018 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# [WIP]
+# The existence of this file is checked by dx_unpack, thus we provide
+# it as an empty file until the needed functionality is added.
+#
+
+exit 0

live-build/misc/migration-scripts/dx_prepare

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2018 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This script is a no-op and only exist to satisfy dx_unpack.sh
+# from illumos.
+#
+
+exit 0

live-build/misc/migration-scripts/dx_verify

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Copyright 2018 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# [WIP]
+# The existence of this file is checked by dx_unpack, thus we provide
+# it as an empty file until the needed functionality is added.
+#
+
+exit 0

live-build/variants/external-standard/.gitignore

@@ -19,3 +19,4 @@
 !ansible/**
 !config/hooks
 !upgrade-scripts
+!migration-scripts

live-build/variants/external-standard/migration-scripts

@@ -0,0 +1 @@
+../../misc/migration-scripts

live-build/variants/internal-dcenter/migration-scripts

@@ -0,0 +1 @@
+../../misc/migration-scripts

live-build/variants/internal-dev/.gitignore

@@ -19,3 +19,4 @@
 !ansible/**
 !config/hooks
 !upgrade-scripts
+!migration-scripts

live-build/variants/internal-dev/migration-scripts

@@ -0,0 +1 @@
+../../misc/migration-scripts

live-build/variants/internal-minimal/.gitignore

@@ -19,3 +19,4 @@
 !ansible/**
 !config/hooks
 !upgrade-scripts
+!migration-scripts

live-build/variants/internal-minimal/migration-scripts

@@ -0,0 +1 @@
+../../misc/migration-scripts

live-build/variants/internal-qa/.gitignore

@@ -19,3 +19,4 @@
 !ansible/**
 !config/hooks
 !upgrade-scripts
+!migration-scripts

live-build/variants/internal-qa/migration-scripts

@@ -0,0 +1 @@
+../../misc/migration-scripts

scripts/create-variant.sh

@@ -63,3 +63,6 @@ ln -s ../../../misc/live-build-hooks \
 
 ln -s ../../misc/upgrade-scripts \
 	"$TOP/live-build/variants/$1/upgrade-scripts"
+
+ln -s ../../misc/migration-scripts \
+	"$TOP/live-build/variants/$1/migration-scripts"

scripts/docker-run.sh

@@ -54,6 +54,8 @@ $DOCKER_RUN --rm \
 	--env APPLIANCE_PASSWORD \
 	--env AWS_ACCESS_KEY_ID \
 	--env AWS_SECRET_ACCESS_KEY \
+	--env DLPX_KEY_URL \
+	--env DLPX_SIGNING_LOGIN \
 	--volume "$TOP:/opt/appliance-build" \
 	--workdir "/opt/appliance-build" \
 	appliance-build "$@"

scripts/run-live-build.sh

@@ -122,7 +122,7 @@ fi
 # user (e.g. other software); this is most useful when multiple variants
 # are built via a single call to "make" (e.g. using the "all" target).
 #
-for ext in ova qcow2 upgrade.tar.gz gcp.tar.gz vhdx vmdk; do
+for ext in ova qcow2 upgrade.tar.gz migration.tar.gz gcp.tar.gz vhdx vmdk; do
 	if [[ -f "$APPLIANCE_VARIANT.$ext" ]]; then
 		mv "$APPLIANCE_VARIANT.$ext" "$TOP/live-build/artifacts"
 	fi