diff --git a/bin/generate-buildenv b/bin/generate-buildenv index 24addda..62c9aa9 100755 --- a/bin/generate-buildenv +++ b/bin/generate-buildenv @@ -45,7 +45,7 @@ _git_state() { } _common() { - git_state $BT + _git_state $BT echo "turnkey_version $(cat /etc/turnkey_version)" echo "ami-id $(which ec2metadata && ec2metadata --ami-id || echo none)" echo "awscli $(python -c 'import awscli; print awscli.__version__')" diff --git a/bin/openstack-bundle b/bin/openstack-bundle index 27141cc..df85a98 100755 --- a/bin/openstack-bundle +++ b/bin/openstack-bundle @@ -33,6 +33,8 @@ rootfs=$1 name=$(echo $rootfs | sed 's/.rootfs//') appname=$(echo $name |sed 's/turnkey-\(.*\)-[0-9].*/\1/') + + case "$appname" in canvas) loopsize_padding=524288 ;; ejabberd) loopsize_padding=524288 ;; @@ -46,29 +48,82 @@ loopsize=$[$rootsize + $loopsize_padding] info "creating sparse loopback" dd if=/dev/null of=$rootfs.img bs=1 seek=${loopsize}K -mkfs.ext4 -F -j $rootfs.img -mkdir $rootfs.img.mount -mount -o loop $rootfs.img $rootfs.img.mount +info "creating partition" +#inspired by package openstack-debian-images +PARTED=/sbin/parted +AMI_NAME=$rootfs.img +${PARTED} -s ${AMI_NAME} mktable msdos +${PARTED} -s -a optimal ${AMI_NAME} mkpart primary ext3 1Mi 100% +${PARTED} -s ${AMI_NAME} set 1 boot on +install-mbr ${AMI_NAME} +RESULT_KPARTX=`kpartx -asv ${AMI_NAME} 2>&1` + +if echo "${RESULT_KPARTX}" | grep "^add map" ; then + LOOP_DEVICE=`echo ${RESULT_KPARTX} | cut -d" " -f3` + info "kpartx mounted using: ${LOOP_DEVICE}" +else + fatal "It seems kpartx didn't mount the image correctly: exiting." +fi + +cleanup(){ + error=$? + [ ! -d "${MOUNT_DIR}" ] && return + echo + echo "error $error, umounting $MOUNT_DIR" + chroot ${MOUNT_DIR} umount /proc || true + chroot ${MOUNT_DIR} umount /sys || true + umount ${MOUNT_DIR} + rmdir ${MOUNT_DIR} + kpartx -d ${AMI_NAME} + exit $error +} +trap "cleanup" EXIT TERM INT + +mkfs.ext4 -F -j -L root /dev/mapper/${LOOP_DEVICE} +# No fsck because of X days without checks +tune2fs -i 0 /dev/mapper/${LOOP_DEVICE} + +MOUNT_DIR=`mktemp -d -t build-debimg.XXXXXX` +mount -o loop /dev/mapper/${LOOP_DEVICE} ${MOUNT_DIR} info "syncing rootfs to loopback" -rsync -a -t -r -S -I -H $rootfs/ $rootfs.img.mount +rsync -a -t -r -S -I -H $rootfs/ ${MOUNT_DIR} + +info "install extlinux" +mkdir -p ${MOUNT_DIR}/boot/extlinux +echo "default linux +timeout 1 +label linux +kernel /vmlinuz +append initrd=/initrd.img root=LABEL=root biosdevname=0 net.ifnames=0 console=tty0 console=ttyS0,115200 ro" > ${MOUNT_DIR}/boot/extlinux/extlinux.conf +rm ${MOUNT_DIR}/extlinux.conf || true +ln ${MOUNT_DIR}/boot/extlinux/extlinux.conf ${MOUNT_DIR}/extlinux.conf +extlinux --install ${MOUNT_DIR}/boot/extlinux + info "umount loopback" -umount -d $rootfs.img.mount -rmdir $rootfs.img.mount +umount -d ${MOUNT_DIR} +rmdir ${MOUNT_DIR} -info "setting up image directory" -mkdir $name -mv $rootfs.img $name/$name.img -cp $rootfs/boot/vmlinuz-* $name/$name-kernel -cp $rootfs/boot/initrd.img-* $name/$name-initrd +fsck.ext3 -f /dev/mapper/${LOOP_DEVICE} || true -info "creating $name-openstack.tar.gz" -tar --sparse -zcvf $name-openstack.tar.gz $name +#the next command failed once, so +sync +kpartx -d ${AMI_NAME} if [ -z "$BT_DEBUG" ]; then info "removing directory" rm -rf $name fi +info "creating qcow2 image" +QCOW2_NAME=$name-openstack.qcow2 +QEMU_VERSION=`qemu-img --help | head -n 1 | cut -d" " -f3 | cut -d"," -f1` +if dpkg --compare-versions ${QEMU_VERSION} gt 1.0 ; then + OTHER_QEMU_IMG_OPTIONS=" -o compat=0.10" +else + OTHER_QEMU_IMG_OPTIONS="" +fi + +qemu-img convert -f raw ${AMI_NAME}${OTHER_QEMU_IMG_OPTIONS} -O qcow2 ${QCOW2_NAME} diff --git a/bin/openstack-bundle-ami b/bin/openstack-bundle-ami new file mode 100755 index 0000000..27141cc --- /dev/null +++ b/bin/openstack-bundle-ami @@ -0,0 +1,74 @@ +#!/bin/bash -e +# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org +# +# This file is part of buildtasks. +# +# Buildtasks is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. + + +fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; } +info() { echo "INFO [$(basename $0)]: $@"; } + +usage() { +cat< $O/$name-openstack.tar.gz.buildenv +$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.qcow2.buildenv # publish if specified if [ "$publish" == "yes" ]; then export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/ - $BT/bin/publish-files $O/$name-openstack.tar.gz + $BT/bin/publish-files $O/$name-openstack.qcow2 export PUBLISH_DEST=${BT_PUBLISH_META}/ - $BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv} + $BT/bin/publish-files $O/$name-openstack.{qcow2.sig,qcow2.buildenv} fi if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then diff --git a/bt-openstack-ami b/bt-openstack-ami new file mode 100755 index 0000000..57d925f --- /dev/null +++ b/bt-openstack-ami @@ -0,0 +1,93 @@ +#!/bin/bash -e +# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org +# +# This file is part of buildtasks. +# +# Buildtasks is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. + + +fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; } +warning() { echo "WARNING [$(basename $0)]: $@"; } +info() { echo "INFO [$(basename $0)]: $@"; } + +usage() { +cat< $O/$name-openstack.tar.gz.buildenv + +# publish if specified +if [ "$publish" == "yes" ]; then + export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/ + $BT/bin/publish-files $O/$name-openstack.tar.gz + + export PUBLISH_DEST=${BT_PUBLISH_META}/ + $BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv} +fi + +if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then + rm -rf $rootfs + rm -rf $cdroot +fi + diff --git a/docs/setup b/docs/setup index 45789fc..a4afb67 100644 --- a/docs/setup +++ b/docs/setup @@ -22,6 +22,11 @@ bt-ec2 :: apt-get install parted ec2metadata pip install boto +bt-openstack :: + + apt-get install parted ec2metadata mbr qemu kpartx extlinux + pip install boto + bt-vm :: # vmware ovftool diff --git a/patches/cloud/conf b/patches/cloud/conf index 4174ef4..1c10e96 100755 --- a/patches/cloud/conf +++ b/patches/cloud/conf @@ -20,4 +20,3 @@ update-rc.d -f confconsole disable # redirect inithook output (preseeded headless deployment) sed -i '/REDIRECT_OUTPUT/ s/=.*/=true/g' /etc/default/inithooks - diff --git a/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/25ec2-userdata b/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/25ec2-userdata deleted file mode 100755 index a7fbc24..0000000 --- a/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/25ec2-userdata +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python -# Author: Alon Swartz - -import os -import sys - -if '_TURNKEY_INIT' in os.environ: - sys.exit(0) - -import tempfile - -import executil -import ec2metadata - -class TempFile(file): - def __init__(self, prefix='tmp', suffix=''): - fd, path = tempfile.mkstemp(suffix, prefix) - os.close(fd) - self.path = path - self.pid = os.getpid() - file.__init__(self, path, "w") - - def __del__(self): - if self.pid == os.getpid(): - os.remove(self.path) - -def main(): - userdata = ec2metadata.get('user-data') - - if userdata and userdata.startswith("#!"): - fh = TempFile(prefix="ec2userdata") - fh.writelines(userdata) - fh.close() - - os.chmod(fh.path, 0750) - executil.system(fh.path) - print "# executed ec2 user-data script" - -if __name__ == "__main__": - main() - diff --git a/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys b/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys deleted file mode 100755 index 91fa009..0000000 --- a/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python -# Author: Alon Swartz - -import os -import sys - -if '_TURNKEY_INIT' in os.environ: - sys.exit(0) - -import pwd -import ec2metadata - -USERNAME = 'root' - -def authorize_sshkeys(keys, username): - pw = pwd.getpwnam(username) - sshdir = os.path.join(pw.pw_dir, '.ssh') - authorized_keys = os.path.join(sshdir, 'authorized_keys') - - if not os.path.exists(sshdir): - os.makedirs(sshdir) - os.chmod(sshdir, 0700) - os.chown(sshdir, pw.pw_uid, pw.pw_gid) - - f = open(authorized_keys, 'a') - f.write(''.join(['%s\n' % key for key in keys])) - f.close() - os.chown(authorized_keys, pw.pw_uid, pw.pw_gid) - -def main(): - keys = ec2metadata.get('public-keys') - if keys: - authorize_sshkeys(keys, USERNAME) - - -if __name__ == "__main__": - main() - diff --git a/patches/openstack-ami/conf b/patches/openstack-ami/conf new file mode 100755 index 0000000..84cffd5 --- /dev/null +++ b/patches/openstack-ami/conf @@ -0,0 +1,36 @@ +#!/bin/bash -ex + +install() { + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get -y \ + -o DPkg::Options::=--force-confdef \ + -o DPkg::Options::=--force-confold \ + install $@ +} + +# install useful packages +install ebsmount sysvinit-core systemd-shim + +# remove systemd (sysvinit used in container) +dpkg --purge systemd-sysv systemd || true + +# support hot-plugging of attached volumes +echo "acpiphp" >> /etc/modules + +# hold kernel (not used in image, pro-longs sec-updates) +ARCH=$(dpkg --print-architecture) +case "$ARCH" in + "i386") + META_KERNEL="linux-image-686"; + ;; + "amd64") + META_KERNEL="linux-image-amd64"; + ;; + *) + fatal "non-supported architecture: $ARCH"; + ;; +esac +KERNEL=$(echo /boot/vmlinuz-* | sed 's|/boot/vmlinuz-|linux-image-|') +echo "$KERNEL hold" | dpkg --set-selections +echo "$META_KERNEL hold" | dpkg --set-selections + diff --git a/patches/openstack-ami/overlay/etc/fstab b/patches/openstack-ami/overlay/etc/fstab new file mode 100644 index 0000000..cfbe00d --- /dev/null +++ b/patches/openstack-ami/overlay/etc/fstab @@ -0,0 +1,6 @@ +# /etc/fstab: static file system information. +# +proc /proc proc nodev,noexec,nosuid 0 0 +/dev/vda / ext4 defaults 0 0 +/dev/vdb /mnt auto defaults 0 0 + diff --git a/patches/openstack/conf b/patches/openstack/conf index 84cffd5..4087852 100755 --- a/patches/openstack/conf +++ b/patches/openstack/conf @@ -8,29 +8,33 @@ install() { install $@ } +#conflicts with cloud-utils, which is a dependency of cloud-initramfs-growroot +dpkg --purge ec2metadata + # install useful packages -install ebsmount sysvinit-core systemd-shim +install ebsmount cloud-initramfs-growroot extlinux cloud-init sudo -# remove systemd (sysvinit used in container) -dpkg --purge systemd-sysv systemd || true +#on the other hand, inithooks needs it and the other implementation is written by Alon Swartz anyway +#on the third hand, we've got cloud-init, these inithooks can go away +#DEBIAN_FRONTEND=noninteractive apt-get -y -o DPkg::Options::=--force-confdef \ +# -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-overwrite \ +# install ec2metadata # support hot-plugging of attached volumes echo "acpiphp" >> /etc/modules -# hold kernel (not used in image, pro-longs sec-updates) -ARCH=$(dpkg --print-architecture) -case "$ARCH" in - "i386") - META_KERNEL="linux-image-686"; - ;; - "amd64") - META_KERNEL="linux-image-amd64"; - ;; - *) - fatal "non-supported architecture: $ARCH"; - ;; -esac -KERNEL=$(echo /boot/vmlinuz-* | sed 's|/boot/vmlinuz-|linux-image-|') -echo "$KERNEL hold" | dpkg --set-selections -echo "$META_KERNEL hold" | dpkg --set-selections +#duplication with cloud-init +rm /usr/lib/inithooks/firstboot.d/10regen-sshkeys + +#we're setting this another way in 29sudoadmin-fencethemall +rm /usr/lib/inithooks/firstboot.d/29sudoadmin + +#For OpenStack, we would like to use Ec2 and no other API +echo "# to update this file, run dpkg-reconfigure cloud-init +datasource_list: [ConfigDrive, Openstack, Ec2]" >/etc/cloud/cloud.cfg.d/90_dpkg.cfg +#fix the ordering of the init scripts, so that cloud-init runs before inithooks and friends +sed -i '/^# Required-Start/ s/$/ cloud-config/' /etc/init.d/inithooks +sed -i '/^# Required-Start/ s/$/ cloud-config/' /etc/init.d/hubdns +sed -i '/^# Required-Start/ s/$/ cloud-config/' /etc/init.d/turnkey-init-fence +insserv diff --git a/patches/openstack/overlay/etc/cloud/cloud.cfg b/patches/openstack/overlay/etc/cloud/cloud.cfg new file mode 100644 index 0000000..20f5c31 --- /dev/null +++ b/patches/openstack/overlay/etc/cloud/cloud.cfg @@ -0,0 +1,101 @@ +# The top level settings are used as module +# and system configuration. + +# Needed to have automatic mounts of /dev/vdb +mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] + +# Add hostname to /etc/hosts +manage_etc_hosts: true + +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below +users: + - default + +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the above $user (ubuntu) +disable_root: true + +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +# The modules that run in the 'init' stage +cloud_init_modules: + - migrator + - bootcmd + - write-files + - resizefs + - set_hostname + - update_hostname + - update_etc_hosts + - ca-certs + - rsyslog + - users-groups + - ssh + +# The modules that run in the 'config' stage +cloud_config_modules: +# Emit the cloud config ready event +# this can be used by upstart jobs for 'start on cloud-config'. + - emit_upstart + - mounts + - ssh-import-id + - locale + - set-passwords + - grub-dpkg + - apt-pipelining + - apt-configure + - package-update-upgrade-install + - landscape + - timezone + - puppet + - chef + - salt-minion + - mcollective + - disable-ec2-metadata + - runcmd + - byobu + - rightscale_userdata + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - scripts-user + +# The modules that run in the 'final' stage +cloud_final_modules: +# - ssh-authkey-fingerprints + - keys-to-console + - phone-home + - final-message + - power-state-change + +# System and/or distro specific settings +# (not accessible to handlers/transforms) +system_info: + # This will affect which distro class gets used + distro: debian + # Default user name + that default users groups (if added/used) + default_user: + name: debian + lock_passwd: True + gecos: Debian + groups: [adm, audio, cdrom, dialout, floppy, video, plugdev, dip] + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + shell: /bin/bash + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + upstart_dir: /etc/init/ + package_mirrors: + - arches: [default] + failsafe: + primary: http://ftp.debian.org/debian diff --git a/patches/openstack/overlay/etc/fstab b/patches/openstack/overlay/etc/fstab index cfbe00d..0f85c2c 100644 --- a/patches/openstack/overlay/etc/fstab +++ b/patches/openstack/overlay/etc/fstab @@ -1,6 +1,4 @@ # /etc/fstab: static file system information. # proc /proc proc nodev,noexec,nosuid 0 0 -/dev/vda / ext4 defaults 0 0 -/dev/vdb /mnt auto defaults 0 0 - +LABEL=root / ext4 discard 0 0 diff --git a/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/29sudoadmin-fencethemall b/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/29sudoadmin-fencethemall new file mode 100755 index 0000000..d08689e --- /dev/null +++ b/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/29sudoadmin-fencethemall @@ -0,0 +1,16 @@ +#!/bin/bash + +#because cloud-init may create multiple users +for username in `ls /home`; do + root_profiled=/root/.profile.d/turnkey-init-fence + user_profiled=/home/${username}/.profile.d/turnkey-init-fence + if [ -e $root_profiled ]; then + if [ $username != "root" ]; then + mkdir -p $(dirname $user_profiled) + cp $root_profiled $user_profiled + sed -i "s|^|sudo |" $user_profiled + sed -i "s|/root|/home\/${username}|g" $user_profiled + chown -R $username:$username $(dirname $user_profiled) + fi + fi +done \ No newline at end of file diff --git a/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/30turnkey-init-fence b/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/30turnkey-init-fence new file mode 100755 index 0000000..2731650 --- /dev/null +++ b/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/30turnkey-init-fence @@ -0,0 +1,14 @@ +#!/bin/bash + +[ -n "$_TURNKEY_INIT" ] && exit 0 + +. /etc/default/inithooks + +for USERNAME in root `ls /home`; do + PROFILE_FIRSTLOGIN=$(eval printf ~$USERNAME)/.profile.d/turnkey-init-fence + [ -f $PROFILE_FIRSTLOGIN ] && chmod +x $PROFILE_FIRSTLOGIN +done + +update-rc.d turnkey-init-fence defaults +/etc/init.d/turnkey-init-fence start +echo 'turnkey-init-fence is up' diff --git a/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/97turnkey-init-fence-disable b/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/97turnkey-init-fence-disable new file mode 100755 index 0000000..88b8a12 --- /dev/null +++ b/patches/openstack/overlay/usr/lib/inithooks/firstboot.d/97turnkey-init-fence-disable @@ -0,0 +1,18 @@ +#!/bin/bash + +[ -z "$_TURNKEY_INIT" ] && exit 0; # ONLY run from within turnkey-init + +if [ -x /etc/init.d/turnkey-init-fence ]; then + + update-rc.d turnkey-init-fence remove; + /etc/init.d/turnkey-init-fence stop; + +fi + +for home in /root `find /home/*`; do + [ -e $home/.profile.d/turnkey-init-fence ] && chmod -x $home/.profile.d/turnkey-init-fence; +done + +chmod -x /usr/lib/inithooks/firstboot.d/??turnkey-init-fence* + +echo 'turnkey-init-fence is down'