Upgrade OpenStack support from AMI compatibility to qcow2 native

bin/generate-buildenv

@@ -45,7 +45,7 @@ _git_state() {
 }
 
 _common() {
-    git_state $BT
+    _git_state $BT
     echo "turnkey_version $(cat /etc/turnkey_version)"
     echo "ami-id $(which ec2metadata && ec2metadata --ami-id || echo none)"
     echo "awscli $(python -c 'import awscli; print awscli.__version__')"

bin/openstack-bundle

@@ -33,6 +33,8 @@ rootfs=$1
 name=$(echo $rootfs | sed 's/.rootfs//')
 appname=$(echo $name |sed 's/turnkey-\(.*\)-[0-9].*/\1/')
 
+
+
 case "$appname" in
     canvas)           loopsize_padding=524288 ;;
     ejabberd)         loopsize_padding=524288 ;;
@@ -46,29 +48,79 @@ loopsize=$[$rootsize + $loopsize_padding]
 
 info "creating sparse loopback"
 dd if=/dev/null of=$rootfs.img bs=1 seek=${loopsize}K
-mkfs.ext4 -F -j $rootfs.img
 
-mkdir $rootfs.img.mount
-mount -o loop $rootfs.img $rootfs.img.mount
+info "creating partition"
+#inspired by package openstack-debian-images
+PARTED=/sbin/parted
+AMI_NAME=$rootfs.img
+${PARTED} -s ${AMI_NAME} mktable msdos
+${PARTED} -s -a optimal ${AMI_NAME} mkpart primary ext3 1Mi 100%
+${PARTED} -s ${AMI_NAME} set 1 boot on
+install-mbr ${AMI_NAME}
+RESULT_KPARTX=`kpartx -asv ${AMI_NAME} 2>&1`
+
+if echo "${RESULT_KPARTX}" | grep "^add map" ; then
+    LOOP_DEVICE=`echo ${RESULT_KPARTX} | cut -d" " -f3`
+    info "kpartx mounted using: ${LOOP_DEVICE}"
+else
+    fatal "It seems kpartx didn't mount the image correctly: exiting."
+fi
+
+cleanup(){
+    error=$?
+    [ ! -d "${MOUNT_DIR}" ] && return
+    echo
+    echo "error $error, umounting $MOUNT_DIR"
+    chroot ${MOUNT_DIR} umount /proc || true
+    chroot ${MOUNT_DIR} umount /sys || true
+    umount ${MOUNT_DIR}
+    rmdir ${MOUNT_DIR}
+    kpartx -d ${AMI_NAME}
+    exit $error
+}
+trap "cleanup" EXIT TERM INT
+
+mkfs.ext4 -F -j /dev/mapper/${LOOP_DEVICE}
+# No fsck because of X days without checks
+tune2fs -i 0 /dev/mapper/${LOOP_DEVICE}
+
+MOUNT_DIR=`mktemp -d -t build-debimg.XXXXXX`
+mount -o loop /dev/mapper/${LOOP_DEVICE} ${MOUNT_DIR}
 
 info "syncing rootfs to loopback"
-rsync -a -t -r -S -I -H $rootfs/ $rootfs.img.mount
+rsync -a -t -r -S -I -H $rootfs/ ${MOUNT_DIR}
 
-info "umount loopback"
-umount -d $rootfs.img.mount
-rmdir $rootfs.img.mount
+info "install extlinux"
+mkdir -p ${MOUNT_DIR}/boot/extlinux 
+echo "default linux
+timeout 1
+label linux
+kernel /vmlinuz
+append initrd=/initrd.img root=/dev/vda1 biosdevname=0 net.ifnames=0 console=tty0 console=ttyS0,115200 ro" > ${MOUNT_DIR}/boot/extlinux/extlinux.conf
+extlinux --install ${MOUNT_DIR}/boot/extlinux
 
-info "setting up image directory"
-mkdir $name
-mv $rootfs.img $name/$name.img
-cp $rootfs/boot/vmlinuz-* $name/$name-kernel
-cp $rootfs/boot/initrd.img-* $name/$name-initrd
+info "umount loopback"
+umount -d ${MOUNT_DIR}
+rmdir ${MOUNT_DIR}
+
+fsck.ext3 -f /dev/mapper/${LOOP_DEVICE} || true
+
+#the next command failed once, so
+sync
+kpartx -d ${AMI_NAME}
+
+info "creating qcow2 image"
+QCOW2_NAME=$name-openstack.qcow2
+QEMU_VERSION=`qemu-img --help | head -n 1 | cut -d" " -f3 | cut -d"," -f1`
+if dpkg --compare-versions ${QEMU_VERSION} gt 1.0 ; then
+    OTHER_QEMU_IMG_OPTIONS=" -o compat=0.10"
+else
+    OTHER_QEMU_IMG_OPTIONS=""
+fi
 
-info "creating $name-openstack.tar.gz"
-tar --sparse -zcvf $name-openstack.tar.gz $name
+qemu-img convert -c -f raw ${AMI_NAME}${OTHER_QEMU_IMG_OPTIONS} -O qcow2 ${QCOW2_NAME}
 
 if [ -z "$BT_DEBUG" ]; then
     info "removing directory"
     rm -rf $name
 fi
-

bin/openstack-bundle-ami

@@ -0,0 +1,74 @@
+#!/bin/bash -e
+# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org
+# 
+# This file is part of buildtasks.
+# 
+# Buildtasks is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+
+
+fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; }
+info() { echo "INFO [$(basename $0)]: $@"; }
+
+usage() {
+cat<<EOF
+Syntax: $0 rootfs
+Bundles rootfs into an openstack tarball
+
+Arguments::
+
+    rootfs          - root filesystem path
+
+EOF
+exit 1
+}
+
+if [[ "$#" != "1" ]]; then
+    usage
+fi
+
+rootfs=$1
+name=$(echo $rootfs | sed 's/.rootfs//')
+appname=$(echo $name |sed 's/turnkey-\(.*\)-[0-9].*/\1/')
+
+case "$appname" in
+    canvas)           loopsize_padding=524288 ;;
+    ejabberd)         loopsize_padding=524288 ;;
+    appengine-python) loopsize_padding=524288 ;;
+    *)                loopsize_padding=262144 ;;
+esac
+
+info "getting size for loopback"
+rootsize=$(du -s $rootfs | awk '{print $1}')
+loopsize=$[$rootsize + $loopsize_padding]
+
+info "creating sparse loopback"
+dd if=/dev/null of=$rootfs.img bs=1 seek=${loopsize}K
+mkfs.ext4 -F -j $rootfs.img
+
+mkdir $rootfs.img.mount
+mount -o loop $rootfs.img $rootfs.img.mount
+
+info "syncing rootfs to loopback"
+rsync -a -t -r -S -I -H $rootfs/ $rootfs.img.mount
+
+info "umount loopback"
+umount -d $rootfs.img.mount
+rmdir $rootfs.img.mount
+
+info "setting up image directory"
+mkdir $name
+mv $rootfs.img $name/$name.img
+cp $rootfs/boot/vmlinuz-* $name/$name-kernel
+cp $rootfs/boot/initrd.img-* $name/$name-initrd
+
+info "creating $name-openstack.tar.gz"
+tar --sparse -zcvf $name-openstack.tar.gz $name
+
+if [ -z "$BT_DEBUG" ]; then
+    info "removing directory"
+    rm -rf $name
+fi
+

bt-openstack

@@ -73,17 +73,17 @@ $BT/bin/aptconf-tag $rootfs openstack
 
 $BT/bin/openstack-bundle $rootfs
 
-$BT/bin/generate-signature $O/$name-openstack.tar.gz
+$BT/bin/generate-signature $O/$name-openstack.qcow2
 
-$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.tar.gz.buildenv
+$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.qcow2.buildenv
 
 # publish if specified
 if [ "$publish" == "yes" ]; then
     export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/
-    $BT/bin/publish-files $O/$name-openstack.tar.gz
+    $BT/bin/publish-files $O/$name-openstack.qcow2
 
     export PUBLISH_DEST=${BT_PUBLISH_META}/
-    $BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv}
+    $BT/bin/publish-files $O/$name-openstack.{qcow2.sig,qcow2.buildenv}
 fi
 
 if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then

bt-openstack-ami

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org
+# 
+# This file is part of buildtasks.
+# 
+# Buildtasks is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+
+
+fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; }
+warning() { echo "WARNING [$(basename $0)]: $@"; }
+info() { echo "INFO [$(basename $0)]: $@"; }
+
+usage() {
+cat<<EOF
+Syntax: $(basename $0) [--publish] appname
+Converts appliance appname (e.g., core) to openstack image
+
+Options::
+
+    --publish           - if set, image will be devpay'ed and made public
+
+Environment::
+
+    BT_DEBUG            - turn on debugging
+EOF
+exit 1
+}
+
+while [ "$1" != "" ]; do
+    case $1 in
+        --help|-h )  usage;;
+        --publish)   publish="yes";;
+        *)           if [ -n "$appname" ]; then usage; else appname=$1; fi ;;
+    esac
+    shift
+done
+
+[ -n "$appname" ] || usage
+[ -n "$publish" ] || warning "--publish was not specified"
+
+[ -n "$BT_DEBUG" ] && set -x
+
+export BT=$(dirname $(readlink -f $0))
+export BT_CONFIG=$BT/config
+. $BT_CONFIG/common.cfg
+. $BT_CONFIG/build.cfg
+
+O=$BT_BUILDS/openstack
+mkdir -p $O
+
+[ -n "$BT_VERSION" ] || fatal "BT_VERSION not set"
+
+isofile=turnkey-$appname-$BT_VERSION.iso
+name=turnkey-$appname-$BT_VERSION
+rootfs=$name.rootfs
+cdroot=$name.cdroot
+
+$BT/bin/iso-download $BT_ISOS $BT_VERSION $appname
+$BT/bin/iso-verify $BT_ISOS $BT_VERSION $appname
+
+cd $O
+tklpatch-extract-iso $BT_ISOS/$isofile
+
+tklpatch-apply $rootfs $BT/patches/headless
+tklpatch-apply $rootfs $BT/patches/cloud
+tklpatch-apply $rootfs $BT/patches/openstack-ami
+$BT/bin/rootfs-cleanup $rootfs
+
+$BT/bin/aptconf-tag $rootfs openstack
+
+$BT/bin/openstack-bundle-ami $rootfs
+
+$BT/bin/generate-signature $O/$name-openstack.tar.gz
+
+$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.tar.gz.buildenv
+
+# publish if specified
+if [ "$publish" == "yes" ]; then
+    export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/
+    $BT/bin/publish-files $O/$name-openstack.tar.gz
+
+    export PUBLISH_DEST=${BT_PUBLISH_META}/
+    $BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv}
+fi
+
+if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then
+    rm -rf $rootfs
+    rm -rf $cdroot
+fi
+

docs/setup

@@ -22,6 +22,11 @@ bt-ec2 ::
     apt-get install parted ec2metadata
     pip install boto
 
+bt-openstack ::
+
+    apt-get install parted ec2metadata mbr qemu kpartx extlinux
+    pip install boto
+
 bt-vm ::
 
     # vmware ovftool

patches/openstack-ami/conf

@@ -0,0 +1,36 @@
+#!/bin/bash -ex
+
+install() {
+    apt-get update
+    DEBIAN_FRONTEND=noninteractive apt-get -y \
+        -o DPkg::Options::=--force-confdef \
+        -o DPkg::Options::=--force-confold \
+        install $@
+}
+
+# install useful packages
+install ebsmount sysvinit-core systemd-shim
+
+# remove systemd (sysvinit used in container)
+dpkg --purge systemd-sysv systemd || true
+
+# support hot-plugging of attached volumes
+echo "acpiphp" >> /etc/modules
+
+# hold kernel (not used in image, pro-longs sec-updates)
+ARCH=$(dpkg --print-architecture)
+case "$ARCH" in
+    "i386")
+        META_KERNEL="linux-image-686";
+        ;;
+    "amd64")
+        META_KERNEL="linux-image-amd64";
+        ;;
+    *)
+        fatal "non-supported architecture: $ARCH";
+        ;;
+esac
+KERNEL=$(echo /boot/vmlinuz-* | sed 's|/boot/vmlinuz-|linux-image-|')
+echo "$KERNEL hold" | dpkg --set-selections
+echo "$META_KERNEL hold" | dpkg --set-selections
+

patches/openstack-ami/overlay/etc/fstab

@@ -0,0 +1,6 @@
+# /etc/fstab: static file system information.
+# <file system>        <mount point>   <type>  <options>       <dump>  <pass>
+proc                    /proc           proc    nodev,noexec,nosuid 0       0
+/dev/vda                /               ext4    defaults            0       0
+/dev/vdb                /mnt            auto    defaults            0       0
+

patches/openstack/conf

@@ -8,29 +8,16 @@ install() {
         install $@
 }
 
+#conflicts with cloud-utils, which is a dependency of cloud-initramfs-growroot
+dpkg --purge ec2metadata
+
 # install useful packages
-install ebsmount sysvinit-core systemd-shim
+install ebsmount cloud-initramfs-growroot extlinux
 
-# remove systemd (sysvinit used in container)
-dpkg --purge systemd-sysv systemd || true
+#on the other hand, inithooks needs it and the other implementation is written by Alon Swartz anyway
+DEBIAN_FRONTEND=noninteractive apt-get -y -o DPkg::Options::=--force-confdef \
+  -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-overwrite \
+  install ec2metadata
 
 # support hot-plugging of attached volumes
 echo "acpiphp" >> /etc/modules
-
-# hold kernel (not used in image, pro-longs sec-updates)
-ARCH=$(dpkg --print-architecture)
-case "$ARCH" in
-    "i386")
-        META_KERNEL="linux-image-686";
-        ;;
-    "amd64")
-        META_KERNEL="linux-image-amd64";
-        ;;
-    *)
-        fatal "non-supported architecture: $ARCH";
-        ;;
-esac
-KERNEL=$(echo /boot/vmlinuz-* | sed 's|/boot/vmlinuz-|linux-image-|')
-echo "$KERNEL hold" | dpkg --set-selections
-echo "$META_KERNEL hold" | dpkg --set-selections
-