WIP: allow latest build to include branch name

stackhpc · Nov 5, 2024 · f8d6505 · f8d6505
1 parent 1bd2bc4
commit f8d6505
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 82 deletions.
diff --git a/.github/workflows/nightlybuild.yml b/.github/workflows/nightlybuild.yml
@@ -22,33 +22,31 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false # allow other matrix jobs to continue even if one fails
-      matrix: # build RL8, RL9, RL9+CUDA versions
-        os_version:
-          - RL8
-          - RL9
-        build:
-          - openstack.rocky-latest
-          # - openstack.rocky-latest-cuda
+      matrix:
+        source_image_name:
+          - Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2
+          - Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2
+        inventory_groups:
+          - ["update", "ofed"]
+          - ["update", "ofed", "cuda"]
         exclude:
-          - os_version: RL8
-            build: openstack.rocky-latest-cuda
-
+          - source_image_name: Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2
+            inventory_groups: ["update", "ofed", "cuda"]
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }}
-      SOURCE_IMAGES_MAP: |
-        {
-          "RL8": "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2",
-          "RL9": "Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2"
-        }
+      # set the image suffix to -latest for cron jobs or a branch name if manually-triggered
+      IMAGE_NAME_SUFFIX: ${{ github.event_name == 'schedule' && 'latest' || github.ref_name	}}
+
 
     steps:
       - uses: actions/checkout@v2
 
       - name: Record settings for CI cloud
         run: |
           echo CI_CLOUD: ${{ env.CI_CLOUD }}
+          echo IMAGE_NAME_SUFFIX: ${{ env.IMAGE_NAME_SUFFIX }}
 
       - name: Setup ssh
         run: |
@@ -85,16 +83,17 @@ jobs:
           cd packer/
           packer init .
 
-          PACKER_LOG=1 packer build \
+          packer build \
           -on-error=${{ vars.PACKER_ON_ERROR }} \
-          -only=${{ matrix.build }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-          -var "source_image_name=${{ env.SOURCE_IMAGE }}" \
+          -var source_image_name=${{ matrix.source_image_name }} \
+          -var image_name_prefix=rocky-${{ matrix.os_version }} \
+          -var image_name_suffix=${{ env.IMAGE_NAME_SUFFIX }} \
+          -var 'inventory_groups=${{ toJSON(matrix.inventory_groups) }}' \
           openstack.pkr.hcl
-
         env:
-          PKR_VAR_os_version: ${{ matrix.os_version }}
-          SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version] }}
+          PACKER_LOG: '1'
+
 
       - name: Get created image names from manifest
         id: manifest

diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl
@@ -23,6 +23,7 @@ data "git-commit" "cwd-head" { }
 locals {
     git_commit = data.git-commit.cwd-head.hash
     timestamp = formatdate("YYMMDD-hhmm", timestamp())
+    image_name_suffix = var.image_name_suffix == "" ? "${local.timestamp}-${substr(local.git_commit, 0, 8)}" : var.image_name_suffix
 }
 
 # Path pointing to root of repository - automatically set by environment variable PKR_VAR_repo_root
@@ -39,12 +40,6 @@ variable "networks" {
   type = list(string)
 }
 
-variable "os_version" {
-  type = string
-  description = "'RL8' or 'RL9' with default source_image_* mappings"
-  default = "RL9"
-}
-
 # Must supply either source_image_name or source_image_id
 variable "source_image_name" {
   type = string
@@ -123,19 +118,13 @@ variable "volume_type" {
 }
 
 variable "volume_size" {
-  type = map(number)
-  default = {
-    # fat image builds, GB:
-    rocky-latest = 15
-    rocky-latest-cuda = 30
-    openhpc = 15
-    openhpc-cuda = 30
-  }
+  type = number
+  default = 15 # same as default non-CUDA build
 }
 
-variable "extra_build_volume_size" {
+variable "volume_size_cuda" {
   type = number
-  default = 15 # same as default non-CUDA build
+  default = 30
 }
 
 variable "image_disk_format" {
@@ -148,32 +137,32 @@ variable "metadata" {
   default = {}
 }
 
-variable "groups" {
-  type = map(list(string))
-  description = "Additional inventory groups (other than 'builder') to add build VM to, keyed by source name"
-  default = {
-    # fat image builds:
-    rocky-latest = ["update", "ofed"]
-    rocky-latest-cuda = ["update", "ofed", "cuda"]
-    openhpc = ["control", "compute", "login"]
-    openhpc-cuda = ["control", "compute", "login"]
-  }
-}
-
-variable "extra_build_groups" {
+variable "inventory_groups" {
   type = list(string)
   default = []
 }
 
-variable "extra_build_image_name" {
+# variable "groups" {
+#   type = map(list(string))
+#   description = "Additional inventory groups (other than 'builder') to add build VM to, keyed by source name"
+#   default = {
+    # fat image builds:
+    # rocky-latest = ["update", "ofed"]
+    # rocky-latest-cuda = ["update", "ofed", "cuda"]
+    # openhpc = ["control", "compute", "login"]
+    # openhpc-cuda = ["control", "compute", "login"]
+#   }
+# }
+
+variable "image_name_prefix" {
   type = string
-  description = "Infix for 'extra' build image name"
-  default = "extra"
+  description = "Prefix for built image names"
+  default = "openhpc"
 }
 
 variable "image_name_suffix" {
   type = string
-  description = "Suffix for all build image names"
+  description = "Suffix for built image names. If not supplied a timestamp+git commit is used"
   default = ""
 }
 
@@ -182,7 +171,7 @@ source "openstack" "openhpc" {
   flavor = var.flavor
   use_blockstorage_volume = var.use_blockstorage_volume
   volume_type = var.volume_type
-  volume_size = lookup(var.volume_size, source.name, var.extra_build_volume_size)
+  volume_size = contains(var.inventory_groups, "cuda") ? var.volume_size_cuda : var.volume_size
   metadata = var.metadata
   instance_metadata = {ansible_init_disable = "true"}
   networks = var.networks
@@ -210,39 +199,14 @@ source "openstack" "openhpc" {
 
 build {
 
-  # latest nightly image:
   source "source.openstack.openhpc" {
     name = "rocky-latest"
-    image_name = "${source.name}-${var.os_version}${var.image_name_suffix}"
-  }
-
-  # latest nightly cuda image:
-  source "source.openstack.openhpc" {
-    name = "rocky-latest-cuda"
-    image_name = "${source.name}-${var.os_version}${var.image_name_suffix}"
-  }
-
-  # OFED fat image:
-  source "source.openstack.openhpc" {
-    name = "openhpc"
-    image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}${var.image_name_suffix}"
-  }
-
-  # CUDA fat image:
-  source "source.openstack.openhpc" {
-    name = "openhpc-cuda"
-    image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}${var.image_name_suffix}"
-  }
-
-  # Extended site-specific image, built on fat image:
-  source "source.openstack.openhpc" {
-    name = "openhpc-extra"
-    image_name = "openhpc-${var.extra_build_image_name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}${var.image_name_suffix}"
+    image_name = "${var.image_name_prefix}-${local.image_name_suffix}"
   }
 
   provisioner "ansible" {
     playbook_file = "${var.repo_root}/ansible/fatimage.yml"
-    groups = concat(["builder"], lookup(var.groups, source.name, var.extra_build_groups))
+    groups = concat(["builder"], var.inventory_groups)
     keep_inventory_file = true # for debugging
     use_proxy = false # see https://www.packer.io/docs/provisioners/ansible#troubleshooting
     extra_arguments = [