Initial Poky based AMI pipeline

assets/build-image/ubuntu_22_04/Dockerfile

@@ -10,12 +10,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev \
     python3-subunit mesa-common-dev zstd liblz4-tool file locales \
     uuid-runtime sysstat iproute2 openssh-client curl python-is-python3 \
-    parted mtools dosfstools
+    parted mtools dosfstools jq
 
 # install repo tool
 RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /bin/repo
 RUN chmod a+rx /bin/repo
 
+# Install aws cli v2
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
+RUN unzip /tmp/awscliv2.zip -d /tmp
+RUN /tmp/aws/install
+
 # install kas tool
 RUN pip3 install PyYAML==5.3.1 && pip3 install kas==3.0.2
 

lib/constructs/source-repo.ts

@@ -12,6 +12,8 @@ export enum ProjectKind {
   Poky = 'poky',
   /** Build the Qemu meta-aws Demonstration Distribution. */
   MetaAwsDemo = 'meta-aws-demo',
+  /** Build an EC2 AMI */
+  PokyAmi = 'poky-ami',
 }
 
 export interface SourceRepoProps extends cdk.StackProps {

lib/demo-pipeline.ts

@@ -9,6 +9,7 @@ import * as iam from 'aws-cdk-lib/aws-iam';
 import * as efs from 'aws-cdk-lib/aws-efs';
 
 import {
+  BuildEnvironmentVariableType,
   BuildSpec,
   ComputeType,
   FileSystemLocation,
@@ -24,8 +25,9 @@ import {
   Port,
   SecurityGroup,
 } from 'aws-cdk-lib/aws-ec2';
-import { Bucket } from 'aws-cdk-lib/aws-s3';
+import { Bucket, IBucket } from 'aws-cdk-lib/aws-s3';
 import { SourceRepo, ProjectKind } from './constructs/source-repo';
+import { VMImportBucket } from './vm-import-bucket';
 
 /**
  * Properties to allow customizing the build.
@@ -37,8 +39,8 @@ export interface DemoPipelineProps extends cdk.StackProps {
   readonly imageTag?: string;
   /** VPC where the networking setup resides. */
   readonly vpc: IVpc;
-  /** The type of Layer  */
-  readonly distroKind?: ProjectKind;
+  /** The type of project being built.  */
+  readonly projectKind?: ProjectKind;
   /** A name for the layer-repo that is created. Default is 'layer-repo' */
   readonly layerRepoName?: string;
 }
@@ -66,12 +68,27 @@ export class DemoPipelineStack extends cdk.Stack {
     const dlFS = this.addFileSystem('Downloads', props.vpc, projectSg);
     const tmpFS = this.addFileSystem('Temp', props.vpc, projectSg);
 
+    let artifactBucket: IBucket;
+
+    if (props.projectKind && props.projectKind == ProjectKind.PokyAmi) {
+      /** The bucket our images are sent to. */
+      artifactBucket = new VMImportBucket(this, 'DemoArtifact', {
+        versioned: true,
+        enforceSSL: true,
+      });
+    } else {
+      artifactBucket = new Bucket(this, 'DemoArtifact', {
+        versioned: true,
+        enforceSSL: true,
+      });
+    }
+
     /** Create our CodePipeline Actions. */
 
     const sourceRepo = new SourceRepo(this, 'SourceRepo', {
       ...props,
       repoName: props.layerRepoName ?? `layer-repo-${this.stackName}`,
-      kind: props.distroKind ?? ProjectKind.Poky,
+      kind: props.projectKind ?? ProjectKind.Poky,
     });
 
     const sourceOutput = new codepipeline.Artifact();
@@ -93,6 +110,12 @@ export class DemoPipelineStack extends cdk.Stack {
           props.imageTag
         ),
         privileged: true,
+        environmentVariables: {
+          IMPORT_BUCKET: {
+            type: BuildEnvironmentVariableType.PLAINTEXT,
+            value: artifactBucket.bucketName,
+          },
+        },
       },
       timeout: cdk.Duration.hours(4),
       vpc: props.vpc,
@@ -116,6 +139,11 @@ export class DemoPipelineStack extends cdk.Stack {
       ],
     });
 
+    if (props.projectKind && props.projectKind == ProjectKind.PokyAmi) {
+      artifactBucket.grantReadWrite(project);
+      project.addToRolePolicy(this.addVMExportPolicy());
+    }
+
     const buildOutput = new codepipeline.Artifact();
     const buildAction = new codepipeline_actions.CodeBuildAction({
       input: sourceOutput,
@@ -124,10 +152,6 @@ export class DemoPipelineStack extends cdk.Stack {
       project,
     });
 
-    const artifactBucket = new Bucket(this, 'DemoArtifact', {
-      versioned: true,
-      enforceSSL: true,
-    });
     const artifactAction = new codepipeline_actions.S3DeployAction({
       actionName: 'Demo-Artifact',
       input: buildOutput,
@@ -243,4 +267,37 @@ def handler(event, context):
 
     return `${fsId}.efs.${region}.amazonaws.com:/`;
   }
+
+  private addVMExportPolicy(): iam.PolicyStatement {
+    return new iam.PolicyStatement({
+      actions: [
+        'ec2:CancelConversionTask',
+        'ec2:CancelExportTask',
+        'ec2:CreateImage',
+        'ec2:CreateInstanceExportTask',
+        'ec2:CreateTags',
+        'ec2:DescribeConversionTasks',
+        'ec2:DescribeExportTasks',
+        'ec2:DescribeExportImageTasks',
+        'ec2:DescribeImages',
+        'ec2:DescribeInstanceStatus',
+        'ec2:DescribeInstances',
+        'ec2:DescribeSnapshots',
+        'ec2:DescribeTags',
+        'ec2:ExportImage',
+        'ec2:ImportInstance',
+        'ec2:ImportVolume',
+        'ec2:StartInstances',
+        'ec2:StopInstances',
+        'ec2:TerminateInstances',
+        'ec2:ImportImage',
+        'ec2:ImportSnapshot',
+        'ec2:DescribeImportImageTasks',
+        'ec2:DescribeImportSnapshotTasks',
+        'ec2:CancelImportTask',
+        'ec2:RegisterImage',
+      ],
+      resources: ['*'],
+    });
+  }
 }

lib/vm-import-bucket.ts

@@ -0,0 +1,42 @@
+import * as cdk from 'aws-cdk-lib';
+import { Construct } from 'constructs';
+import * as s3 from 'aws-cdk-lib/aws-s3';
+import * as iam from 'aws-cdk-lib/aws-iam';
+
+/**
+ * ...
+ */
+export class VMImportBucket extends s3.Bucket {
+  constructor(scope: Construct, id: string, props: s3.BucketProps) {
+    super(scope, id, {
+      ...props,
+    });
+
+    // Adapted from meta-aws-ewaol and
+    // https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html
+    const importPolicy = new iam.PolicyDocument({
+      statements: [
+        new iam.PolicyStatement({
+          actions: ['s3:GetBucketLocation', 's3:GetObject', 's3:ListBucket'],
+          resources: [this.bucketArn, `${this.bucketArn}/*`],
+        }),
+        new iam.PolicyStatement({
+          actions: [
+            'ec2:ModifySnapshotAttribute',
+            'ec2:CopySnapshot',
+            'ec2:RegisterImage',
+            'ec2:Describe*',
+          ],
+          resources: ['*'],
+        }),
+      ],
+    });
+
+    new iam.Role(scope, 'VMImportRole', {
+      roleName: 'vmimport',
+      assumedBy: new iam.ServicePrincipal('vmie.amazonaws.com'),
+      externalIds: ['vmimport'],
+      inlinePolicies: { importPolicy },
+    });
+  }
+}

source-repo/poky-ami/build.buildspec.yml

@@ -0,0 +1,45 @@
+version: 0.2
+run-as: yoctouser
+
+env:
+  shell: bash
+
+phases:
+  pre_build:
+    run-as: root
+    commands:
+      - chown -R yoctouser /sstate-cache
+      - chown -R yoctouser /downloads
+      - chown yoctouser /build-output
+      - chmod 755 /sstate-cache
+      - chmod 755 /downloads
+      - chmod 755 /build-output
+  build:
+    commands:
+      - echo Build started on `date`
+      - repo init -u $CODEBUILD_SRC_DIR -b main -m manifest.xml
+      - repo sync
+      - . poky/oe-init-build-env $TMP_DIR
+      - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-oe
+      - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-python
+      - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-networking
+      - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-multimedia
+      - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-openembedded/meta-filesystems
+      - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-virtualization
+      - bitbake-layers add-layer $CODEBUILD_SRC_DIR/meta-aws
+      # Update our local conf to build the EC2 AMI.
+      - echo 'MACHINE = "aws-ec2-arm64"' >> $TMP_DIR/conf/local.conf
+      - echo 'INHERIT += "aws-ec2-image"' >> $TMP_DIR/conf/local.conf
+      - echo 'DISTRO_FEATURES += "usrmerge"' >> $TMP_DIR/conf/local.conf
+      - bitbake core-image-minimal
+      - echo Build completed on `date`
+  post_build:
+    commands:
+      - echo AMI Upload started on `date`
+      # This is required because the CDK asset uploader does not seem to carry permissions though.
+      - chmod +x $CODEBUILD_SRC_DIR/create-ec2-ami.sh
+      - $CODEBUILD_SRC_DIR/create-ec2-ami.sh $IMPORT_BUCKET 16 core-image-minimal aws-ec2-arm64 $TMP_DIR/tmp
+artifacts:
+  discard-paths: true
+  files:
+    - $TMP_DIR/tmp/deploy/images/aws-ec2-arm64/core-image-minimal*

source-repo/poky-ami/create-ec2-ami.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT-0
+set -e
+[ "$DEBUG" == 'true' ] && set -x
+
+ARGC=$#
+if [ $ARGC -lt 3 ]; then
+    echo "ERROR: Please inform import bucket name as first argument and AMI disk size in GB as second, IMAGE_NAME as third and MACHINE_NAME as last."
+    exit 1
+fi
+IMPORT_BUCKET_NAME=$1
+AMI_DISK_SIZE_GB=$2
+IMAGE_NAME=$3
+MACHINE_NAME=$4
+TMPDIR=${5:-build/tmp}
+
+IMG_DIR="${TMPDIR}/deploy/images/${MACHINE_NAME}"
+
+TESTDATA_JSON="${IMG_DIR}/${IMAGE_NAME}-${MACHINE_NAME}.rootfs.testdata.json"
+
+DISTRO=$(jq -r '.DISTRO' $TESTDATA_JSON)
+DISTRO_CODENAME=$(jq -r '.DISTRO_CODENAME' $TESTDATA_JSON)
+DISTRO_NAME=$(jq -r '.DISTRO_NAME' $TESTDATA_JSON)
+DISTRO_VERSION=$(jq -r '.DISTRO_VERSION' $TESTDATA_JSON)
+BUILDNAME=$(jq -r '.BUILDNAME' $TESTDATA_JSON)
+TARGET_ARCH=$(jq -r '.TARGET_ARCH' $TESTDATA_JSON)
+IMAGE_NAME=$(jq -r '.IMAGE_NAME' $TESTDATA_JSON)
+IMAGE_ROOTFS_SIZE=$(jq -r '.IMAGE_ROOTFS_SIZE' $TESTDATA_JSON)
+
+
+echo DISTRO=$DISTRO
+echo DISTRO_CODENAME=$DISTRO_CODENAME
+echo DISTRO_NAME=$DISTRO_NAME
+echo DISTRO_VERSION=$DISTRO_VERSION
+echo BUILDNAME=$BUILDNAME
+echo TARGET_ARCH=$TARGET_ARCH
+echo IMAGE_ROOTFS_SIZE=$IMAGE_ROOTFS_SIZE
+echo AMI_DISK_SIZE_GB=$AMI_DISK_SIZE_GB
+
+echo "Pushing image ${IMAGE_NAME}.wic.vhdx to s3://${IMPORT_BUCKET_NAME}"
+aws s3 cp ${IMG_DIR}/${IMAGE_NAME}.wic.vhdx s3://${IMPORT_BUCKET_NAME}
+
+cat <<EOF > image-import.json
+{
+    "Description": "ewaol docker image",
+    "Format": "vhd",
+    "UserBucket": {
+        "S3Bucket": "${IMPORT_BUCKET_NAME}",
+        "S3Key": "${IMAGE_NAME}.wic.vhdx"
+    }
+}
+EOF
+echo "Importing image file into snapshot "
+IMPORT_TASK_ID=$(aws ec2 import-snapshot --disk-container "file://image-import.json" | jq -r '.ImportTaskId')
+
+IMPORT_STATUS=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status')
+x=0
+rm image-import.json
+while [ "$IMPORT_STATUS" = "active" ] && [ $x -lt 120 ]
+do
+  IMPORT_STATUS=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.Status')
+  IMPORT_STATUS_MSG=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.StatusMessage')
+  echo "Import Status: ${IMPORT_STATUS} / ${IMPORT_STATUS_MSG}"
+  x=$(( $x + 1 ))
+  sleep 15
+done
+if [ $x -eq 120 ]; then
+    echo "ERROR: Import task taking too long, exiting..."; exit 1;
+elif [ "$IMPORT_STATUS" = "completed" ]; then
+    echo "Import completed Successfully"
+else
+    echo "Import Failed, exiting"; exit 2;
+fi
+
+SNAPSHOT_ID=$(aws ec2 describe-import-snapshot-tasks --import-task-ids $IMPORT_TASK_ID | jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail.SnapshotId')
+
+aws ec2 wait snapshot-completed --snapshot-ids $SNAPSHOT_ID
+
+if [[ "$TARGET_ARCH" == "x86_64" ]]; then
+    ARCHITECTURE="x86_64"
+elif [[ "$TARGET_ARCH" == "aarch64" ]]; then
+    ARCHITECTURE="arm64"
+else
+    echo "Architecture not supported"
+    exit 1
+fi
+DESCRIPTION=$(echo "DISTRO=$DISTRO;DISTRO_CODENAME=$DISTRO_CODENAME;DISTRO_NAME=$DISTRO_NAME;DISTRO_VERSION=$DISTRO_VERSION;BUILDNAME=$BUILDNAME;TARGET_ARCH=$ARCHITECTURE;IMAGE_NAME=$IMAGE_NAME" | cut -c -255)
+
+cat <<EOF > register-ami.json
+{
+    "Architecture": "$ARCHITECTURE",
+    "BlockDeviceMappings": [
+        {
+            "DeviceName": "/dev/sda1",
+            "Ebs": {
+                "DeleteOnTermination": true,
+                "SnapshotId": "$SNAPSHOT_ID",
+                "VolumeSize": ${AMI_DISK_SIZE_GB},
+                "VolumeType": "gp2"
+            }
+        }
+    ],
+    "Description": "$DESCRIPTION",
+    "RootDeviceName": "/dev/sda1",
+    "BootMode": "uefi",
+    "VirtualizationType": "hvm",
+    "EnaSupport": true
+}
+EOF
+
+AMI_NAME=$(echo "${IMAGE_NAME}-${DISTRO}-${DISTRO_CODENAME}-${DISTRO_VERSION}-${BUILDNAME}-${ARCHITECTURE}" | cut -c -128 | sed -e s/+/-/g)
+IMAGE_ID=$(aws ec2 describe-images --filters Name=name,Values=${AMI_NAME} | jq -r '.Images[].ImageId')
+if [ "$IMAGE_ID" != "" ]; then
+    echo "Deregistering existing image $IMAGE_ID"
+    aws ec2 deregister-image --image-id ${IMAGE_ID} 2>&1 > /dev/null
+fi
+echo "Registering AMI with Snapshot $SNAPSHOT_ID"
+AMI_ID=$(aws ec2 register-image --name ${AMI_NAME} --cli-input-json="file://register-ami.json" | jq -r '.ImageId')
+echo "AMI name: $AMI_NAME"
+echo "AMI ID: $AMI_ID"
+rm register-ami.json
+

source-repo/poky-ami/manifest.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <remote fetch="https://git.yoctoproject.org/"         name="yocto"/>
+  <remote fetch="https://github.com/openembedded/"      name="oe"/>
+  <remote fetch="https://github.com/aws4embeddedlinux/" name="aws"/>
+
+  <project name="poky" remote="yocto" path="poky" revision="master" />
+  <project name="meta-openembedded" remote="oe" path="meta-openembedded" revision="master" />
+  <project name="meta-virtualization" remote="yocto" path="meta-virtualization" revision="4c21ae49dbe05923309892d2fac63d166fd7d508" upstream="master" />
+  <project name="meta-aws" remote="aws" path="meta-aws" revision="master" />
+</manifest>