From f8d65053f1b0c8828d6241c260004b1cbbff3d44 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 5 Nov 2024 17:09:25 +0000 Subject: [PATCH] WIP: allow latest build to include branch name --- .github/workflows/nightlybuild.yml | 41 +++++++------- packer/openstack.pkr.hcl | 86 +++++++++--------------------- 2 files changed, 45 insertions(+), 82 deletions(-) diff --git a/.github/workflows/nightlybuild.yml b/.github/workflows/nightlybuild.yml index 5c068635..c97a18e5 100644 --- a/.github/workflows/nightlybuild.yml +++ b/.github/workflows/nightlybuild.yml @@ -22,26 +22,23 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false # allow other matrix jobs to continue even if one fails - matrix: # build RL8, RL9, RL9+CUDA versions - os_version: - - RL8 - - RL9 - build: - - openstack.rocky-latest - # - openstack.rocky-latest-cuda + matrix: + source_image_name: + - Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2 + - Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2 + inventory_groups: + - ["update", "ofed"] + - ["update", "ofed", "cuda"] exclude: - - os_version: RL8 - build: openstack.rocky-latest-cuda - + - source_image_name: Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2 + inventory_groups: ["update", "ofed", "cuda"] env: ANSIBLE_FORCE_COLOR: True OS_CLOUD: openstack CI_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }} - SOURCE_IMAGES_MAP: | - { - "RL8": "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2", - "RL9": "Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2" - } + # set the image suffix to -latest for cron jobs or a branch name if manually-triggered + IMAGE_NAME_SUFFIX: ${{ github.event_name == 'schedule' && 'latest' || github.ref_name }} + steps: - uses: actions/checkout@v2 @@ -49,6 +46,7 @@ jobs: - name: Record settings for CI cloud run: | echo CI_CLOUD: ${{ env.CI_CLOUD }} + echo IMAGE_NAME_SUFFIX: ${{ env.IMAGE_NAME_SUFFIX }} - name: Setup ssh run: | @@ -85,16 +83,17 @@ jobs: cd packer/ packer init . - PACKER_LOG=1 packer build \ + packer build \ -on-error=${{ vars.PACKER_ON_ERROR }} \ - -only=${{ matrix.build }} \ -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \ - -var "source_image_name=${{ env.SOURCE_IMAGE }}" \ + -var source_image_name=${{ matrix.source_image_name }} \ + -var image_name_prefix=rocky-${{ matrix.os_version }} \ + -var image_name_suffix=${{ env.IMAGE_NAME_SUFFIX }} \ + -var 'inventory_groups=${{ toJSON(matrix.inventory_groups) }}' \ openstack.pkr.hcl - env: - PKR_VAR_os_version: ${{ matrix.os_version }} - SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version] }} + PACKER_LOG: '1' + - name: Get created image names from manifest id: manifest diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl index 8deb5df3..6fb04986 100644 --- a/packer/openstack.pkr.hcl +++ b/packer/openstack.pkr.hcl @@ -23,6 +23,7 @@ data "git-commit" "cwd-head" { } locals { git_commit = data.git-commit.cwd-head.hash timestamp = formatdate("YYMMDD-hhmm", timestamp()) + image_name_suffix = var.image_name_suffix == "" ? "${local.timestamp}-${substr(local.git_commit, 0, 8)}" : var.image_name_suffix } # Path pointing to root of repository - automatically set by environment variable PKR_VAR_repo_root @@ -39,12 +40,6 @@ variable "networks" { type = list(string) } -variable "os_version" { - type = string - description = "'RL8' or 'RL9' with default source_image_* mappings" - default = "RL9" -} - # Must supply either source_image_name or source_image_id variable "source_image_name" { type = string @@ -123,19 +118,13 @@ variable "volume_type" { } variable "volume_size" { - type = map(number) - default = { - # fat image builds, GB: - rocky-latest = 15 - rocky-latest-cuda = 30 - openhpc = 15 - openhpc-cuda = 30 - } + type = number + default = 15 # same as default non-CUDA build } -variable "extra_build_volume_size" { +variable "volume_size_cuda" { type = number - default = 15 # same as default non-CUDA build + default = 30 } variable "image_disk_format" { @@ -148,32 +137,32 @@ variable "metadata" { default = {} } -variable "groups" { - type = map(list(string)) - description = "Additional inventory groups (other than 'builder') to add build VM to, keyed by source name" - default = { - # fat image builds: - rocky-latest = ["update", "ofed"] - rocky-latest-cuda = ["update", "ofed", "cuda"] - openhpc = ["control", "compute", "login"] - openhpc-cuda = ["control", "compute", "login"] - } -} - -variable "extra_build_groups" { +variable "inventory_groups" { type = list(string) default = [] } -variable "extra_build_image_name" { +# variable "groups" { +# type = map(list(string)) +# description = "Additional inventory groups (other than 'builder') to add build VM to, keyed by source name" +# default = { + # fat image builds: + # rocky-latest = ["update", "ofed"] + # rocky-latest-cuda = ["update", "ofed", "cuda"] + # openhpc = ["control", "compute", "login"] + # openhpc-cuda = ["control", "compute", "login"] +# } +# } + +variable "image_name_prefix" { type = string - description = "Infix for 'extra' build image name" - default = "extra" + description = "Prefix for built image names" + default = "openhpc" } variable "image_name_suffix" { type = string - description = "Suffix for all build image names" + description = "Suffix for built image names. If not supplied a timestamp+git commit is used" default = "" } @@ -182,7 +171,7 @@ source "openstack" "openhpc" { flavor = var.flavor use_blockstorage_volume = var.use_blockstorage_volume volume_type = var.volume_type - volume_size = lookup(var.volume_size, source.name, var.extra_build_volume_size) + volume_size = contains(var.inventory_groups, "cuda") ? var.volume_size_cuda : var.volume_size metadata = var.metadata instance_metadata = {ansible_init_disable = "true"} networks = var.networks @@ -210,39 +199,14 @@ source "openstack" "openhpc" { build { - # latest nightly image: source "source.openstack.openhpc" { name = "rocky-latest" - image_name = "${source.name}-${var.os_version}${var.image_name_suffix}" - } - - # latest nightly cuda image: - source "source.openstack.openhpc" { - name = "rocky-latest-cuda" - image_name = "${source.name}-${var.os_version}${var.image_name_suffix}" - } - - # OFED fat image: - source "source.openstack.openhpc" { - name = "openhpc" - image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}${var.image_name_suffix}" - } - - # CUDA fat image: - source "source.openstack.openhpc" { - name = "openhpc-cuda" - image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}${var.image_name_suffix}" - } - - # Extended site-specific image, built on fat image: - source "source.openstack.openhpc" { - name = "openhpc-extra" - image_name = "openhpc-${var.extra_build_image_name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}${var.image_name_suffix}" + image_name = "${var.image_name_prefix}-${local.image_name_suffix}" } provisioner "ansible" { playbook_file = "${var.repo_root}/ansible/fatimage.yml" - groups = concat(["builder"], lookup(var.groups, source.name, var.extra_build_groups)) + groups = concat(["builder"], var.inventory_groups) keep_inventory_file = true # for debugging use_proxy = false # see https://www.packer.io/docs/provisioners/ansible#troubleshooting extra_arguments = [