-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #5 from aws4embeddedlinux/ami-pipeline
Initial Poky based AMI pipeline
- Loading branch information
Showing
9 changed files
with
2,374 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| import { Construct } from 'constructs'; | ||
| import * as s3 from 'aws-cdk-lib/aws-s3'; | ||
| import * as iam from 'aws-cdk-lib/aws-iam'; | ||
|
|
||
| /** | ||
| * ... | ||
| */ | ||
| export class VMImportBucket extends s3.Bucket { | ||
| constructor(scope: Construct, id: string, props: s3.BucketProps) { | ||
| super(scope, id, { | ||
| ...props, | ||
| }); | ||
|
|
||
| // Adapted from meta-aws-ewaol and | ||
| // https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html | ||
| const importPolicy = new iam.PolicyDocument({ | ||
| statements: [ | ||
| new iam.PolicyStatement({ | ||
| actions: ['s3:GetBucketLocation', 's3:GetObject', 's3:ListBucket'], | ||
| resources: [this.bucketArn, `${this.bucketArn}/*`], | ||
| }), | ||
| new iam.PolicyStatement({ | ||
| actions: [ | ||
| 'ec2:ModifySnapshotAttribute', | ||
| 'ec2:CopySnapshot', | ||
| 'ec2:RegisterImage', | ||
| 'ec2:Describe*', | ||
| ], | ||
| resources: ['*'], | ||
| }), | ||
| ], | ||
| }); | ||
|
|
||
| new iam.Role(scope, 'VMImportRole', { | ||
| roleName: 'vmimport', | ||
| assumedBy: new iam.ServicePrincipal('vmie.amazonaws.com'), | ||
| externalIds: ['vmimport'], | ||
| inlinePolicies: { importPolicy }, | ||
| }); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| version: 0.2 | ||
| run-as: yoctouser | ||
|
|
||
| env: | ||
| shell: bash | ||
|
|
||
| phases: | ||
| pre_build: | ||
| run-as: root | ||
| commands: | ||
| - chown -R yoctouser /sstate-cache | ||
| - chown -R yoctouser /downloads | ||
| - chown yoctouser /build-output | ||
| - chmod 755 /sstate-cache | ||
| - chmod 755 /downloads | ||
| - chmod 755 /build-output | ||
| build: | ||
| commands: | ||
| - echo Build started on `date` | ||
| - repo init -u $CODEBUILD_SRC_DIR -b main -m manifest.xml | ||
| - repo sync | ||
| - . poky/oe-init-build-env $TMP_DIR | ||
| - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-oe | ||
| - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-python | ||
| - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-networking | ||
| - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-multimedia | ||
| - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-filesystems | ||
| - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-virtualization | ||
| - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-aws | ||
| # Update our local conf to build the EC2 AMI. | ||
| - echo 'MACHINE = "aws-ec2-arm64"' >> $TMP_DIR/conf/local.conf | ||
| - echo 'INHERIT += "aws-ec2-image"' >> $TMP_DIR/conf/local.conf | ||
| - echo 'DISTRO_FEATURES += "usrmerge"' >> $TMP_DIR/conf/local.conf | ||
| - bitbake core-image-minimal | ||
| - echo Build completed on `date` | ||
| post_build: | ||
| commands: | ||
| - echo AMI Upload started on `date` | ||
| # This is required because the CDK asset uploader does not seem to carry permissions though. | ||
| - chmod +x $CODEBUILD_SRC_DIR/create-ec2-ami.sh | ||
| - $CODEBUILD_SRC_DIR/create-ec2-ami.sh $IMPORT_BUCKET 16 core-image-minimal aws-ec2-arm64 $TMP_DIR/tmp | ||
| artifacts: | ||
| discard-paths: true | ||
| files: | ||
| - $TMP_DIR/tmp/deploy/images/aws-ec2-arm64/core-image-minimal* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| #!/usr/bin/env bash | ||
| # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| # SPDX-License-Identifier: MIT-0 | ||
| set -e | ||
| [ "$DEBUG" == 'true' ] && set -x | ||
|
|
||
| ARGC=$# | ||
| if [ $ARGC -lt 3 ]; then | ||
| echo "ERROR: Please inform import bucket name as first argument and AMI disk size in GB as second, IMAGE_NAME as third and MACHINE_NAME as last." | ||
| exit 1 | ||
| fi | ||
| IMPORT_BUCKET_NAME=$1 | ||
| AMI_DISK_SIZE_GB=$2 | ||
| IMAGE_NAME=$3 | ||
| MACHINE_NAME=$4 | ||
| TMPDIR=${5:-build/tmp} | ||
|
|
||
| IMG_DIR="${TMPDIR}/deploy/images/${MACHINE_NAME}" | ||
|
|
||
| TESTDATA_JSON="${IMG_DIR}/${IMAGE_NAME}-${MACHINE_NAME}.rootfs.testdata.json" | ||
|
|
||
| DISTRO=$(jq -r '.DISTRO' $TESTDATA_JSON) | ||
| DISTRO_CODENAME=$(jq -r '.DISTRO_CODENAME' $TESTDATA_JSON) | ||
| DISTRO_NAME=$(jq -r '.DISTRO_NAME' $TESTDATA_JSON) | ||
| DISTRO_VERSION=$(jq -r '.DISTRO_VERSION' $TESTDATA_JSON) | ||
| BUILDNAME=$(jq -r '.BUILDNAME' $TESTDATA_JSON) | ||
| TARGET_ARCH=$(jq -r '.TARGET_ARCH' $TESTDATA_JSON) | ||
| IMAGE_NAME=$(jq -r '.IMAGE_NAME' $TESTDATA_JSON) | ||
| IMAGE_ROOTFS_SIZE=$(jq -r '.IMAGE_ROOTFS_SIZE' $TESTDATA_JSON) | ||
|
|
||
|
|
||
| echo DISTRO=$DISTRO | ||
| echo DISTRO_CODENAME=$DISTRO_CODENAME | ||
| echo DISTRO_NAME=$DISTRO_NAME | ||
| echo DISTRO_VERSION=$DISTRO_VERSION | ||
| echo BUILDNAME=$BUILDNAME | ||
| echo TARGET_ARCH=$TARGET_ARCH | ||
| echo IMAGE_ROOTFS_SIZE=$IMAGE_ROOTFS_SIZE | ||
| echo AMI_DISK_SIZE_GB=$AMI_DISK_SIZE_GB | ||
|
|
||
| echo "Pushing image ${IMAGE_NAME}.wic.vhdx to s3://${IMPORT_BUCKET_NAME}" | ||
| aws s3 cp ${IMG_DIR}/${IMAGE_NAME}.wic.vhdx s3://${IMPORT_BUCKET_NAME} | ||
|
|
||
| cat <<EOF > image-import.json | ||
| { | ||
| "Description": "ewaol docker image", | ||
| "Format": "vhd", | ||
| "UserBucket": { | ||
| "S3Bucket": "${IMPORT_BUCKET_NAME}", | ||
| "S3Key": "${IMAGE_NAME}.wic.vhdx" | ||
| } | ||
| } | ||
| EOF | ||
| echo "Importing image file into snapshot " | ||
| IMPORT_TASK_ID=$(aws ec2 import-snapshot --disk-container "file://image-import.json" | jq -r '.ImportTaskId') | ||
|
|
||
| IMPORT_STATUS=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status') | ||
| x=0 | ||
| rm image-import.json | ||
| while [ "$IMPORT_STATUS" = "active" ] && [ $x -lt 120 ] | ||
| do | ||
| IMPORT_STATUS=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status') | ||
| IMPORT_STATUS_MSG=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.StatusMessage') | ||
| echo "Import Status: ${IMPORT_STATUS} / ${IMPORT_STATUS_MSG}" | ||
| x=$(( $x + 1 )) | ||
| sleep 15 | ||
| done | ||
| if [ $x -eq 120 ]; then | ||
| echo "ERROR: Import task taking too long, exiting..."; exit 1; | ||
| elif [ "$IMPORT_STATUS" = "completed" ]; then | ||
| echo "Import completed Successfully" | ||
| else | ||
| echo "Import Failed, exiting"; exit 2; | ||
| fi | ||
|
|
||
| SNAPSHOT_ID=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId') | ||
|
|
||
| aws ec2 wait snapshot-completed --snapshot-ids $SNAPSHOT_ID | ||
|
|
||
| if [[ "$TARGET_ARCH" == "x86_64" ]]; then | ||
| ARCHITECTURE="x86_64" | ||
| elif [[ "$TARGET_ARCH" == "aarch64" ]]; then | ||
| ARCHITECTURE="arm64" | ||
| else | ||
| echo "Architecture not supported" | ||
| exit 1 | ||
| fi | ||
| DESCRIPTION=$(echo "DISTRO=$DISTRO;DISTRO_CODENAME=$DISTRO_CODENAME;DISTRO_NAME=$DISTRO_NAME;DISTRO_VERSION=$DISTRO_VERSION;BUILDNAME=$BUILDNAME;TARGET_ARCH=$ARCHITECTURE;IMAGE_NAME=$IMAGE_NAME" | cut -c -255) | ||
|
|
||
| cat <<EOF > register-ami.json | ||
| { | ||
| "Architecture": "$ARCHITECTURE", | ||
| "BlockDeviceMappings": [ | ||
| { | ||
| "DeviceName": "/dev/sda1", | ||
| "Ebs": { | ||
| "DeleteOnTermination": true, | ||
| "SnapshotId": "$SNAPSHOT_ID", | ||
| "VolumeSize": ${AMI_DISK_SIZE_GB}, | ||
| "VolumeType": "gp2" | ||
| } | ||
| } | ||
| ], | ||
| "Description": "$DESCRIPTION", | ||
| "RootDeviceName": "/dev/sda1", | ||
| "BootMode": "uefi", | ||
| "VirtualizationType": "hvm", | ||
| "EnaSupport": true | ||
| } | ||
| EOF | ||
|
|
||
| AMI_NAME=$(echo "${IMAGE_NAME}-${DISTRO}-${DISTRO_CODENAME}-${DISTRO_VERSION}-${BUILDNAME}-${ARCHITECTURE}" | cut -c -128 | sed -e s/+/-/g) | ||
| IMAGE_ID=$(aws ec2 describe-images --filters Name=name,Values=${AMI_NAME} | jq -r '.Images[].ImageId') | ||
| if [ "$IMAGE_ID" != "" ]; then | ||
| echo "Deregistering existing image $IMAGE_ID" | ||
| aws ec2 deregister-image --image-id ${IMAGE_ID} 2>&1 > /dev/null | ||
| fi | ||
| echo "Registering AMI with Snapshot $SNAPSHOT_ID" | ||
| AMI_ID=$(aws ec2 register-image --name ${AMI_NAME} --cli-input-json="file://register-ami.json" | jq -r '.ImageId') | ||
| echo "AMI name: $AMI_NAME" | ||
| echo "AMI ID: $AMI_ID" | ||
| rm register-ami.json | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <manifest> | ||
| <remote fetch="https://git.yoctoproject.org/" name="yocto"/> | ||
| <remote fetch="https://github.com/openembedded/" name="oe"/> | ||
| <remote fetch="https://github.com/aws4embeddedlinux/" name="aws"/> | ||
|
|
||
| <project name="poky" remote="yocto" path="poky" revision="master" /> | ||
| <project name="meta-openembedded" remote="oe" path="meta-openembedded" revision="master" /> | ||
| <project name="meta-virtualization" remote="yocto" path="meta-virtualization" revision="4c21ae49dbe05923309892d2fac63d166fd7d508" upstream="master" /> | ||
| <project name="meta-aws" remote="aws" path="meta-aws" revision="master" /> | ||
| </manifest> |
Oops, something went wrong.