From 4d5a55e6454c8f73fdf101fd24bd5f17266cffa6 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Wed, 20 Dec 2023 09:26:41 +0100 Subject: [PATCH 01/10] wip: use the same machine type as controller ci to fullfill quotas of new subscription --- locals.pkr.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locals.pkr.hcl b/locals.pkr.hcl index e5ff4f781..03b87b0b5 100644 --- a/locals.pkr.hcl +++ b/locals.pkr.hcl @@ -14,8 +14,8 @@ locals { "docker" = "packer" } azure_vm_size = { - "amd64" = "Standard_D4s_v3" # 4 CPU / 16 GB / Huge size required to avoid https:#docs.microsoft.com/en-us/azure/virtual-machines/linux/image-builder-troubleshoot#sysprep-timing and avoid full disk (DS2v2 only have 14 Gb SSD for system) - "arm64" = "Standard_D4ps_v5" # 4 CPU / 16 GB + "amd64" = "Standard_D4ads_v5" # 4 CPU / 16 GB / Huge size required to avoid https:#docs.microsoft.com/en-us/azure/virtual-machines/linux/image-builder-troubleshoot#sysprep-timing and avoid full disk (DS2v2 only have 14 Gb SSD for system) + "arm64" = "Standard_D4pds_v5" # 4 CPU / 16 GB } azure_destination_resource_group = "${var.build_type}-packer-images" azure_galleries = { From 2b5f668340c9d6077a56378e2b76220cbb57d737 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Wed, 20 Dec 2023 09:27:19 +0100 Subject: [PATCH 02/10] wip: remove spot instances as not yet available on the new subscription --- sources.pkr.hcl | 3 --- 1 file changed, 3 deletions(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 4c2ce2cd7..4c90d90e3 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -3,9 +3,6 @@ source "azure-arm" "base" { managed_image_resource_group_name = local.azure_destination_resource_group vm_size = local.azure_vm_size[var.architecture] - spot { - eviction_policy = "Delete" - } # Resource group where to create the VM resources (required to scope permissions into this resource group) build_resource_group_name = "${var.build_type}-packer-builds" From 497ed561d0ca6a1427ccf2e63df0c48f76dbb460 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Wed, 20 Dec 2023 09:27:48 +0100 Subject: [PATCH 03/10] wip: migrate to the new subscription --- Jenkinsfile_k8s | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index e152d6009..e21d6d10e 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -76,7 +76,7 @@ pipeline { } stage('GC on Azure') { environment { - PACKER_AZURE = credentials('packer-azure-serviceprincipal') + PACKER_AZURE = credentials('packer-azure-serviceprincipal-sponsorship') } steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -173,7 +173,7 @@ pipeline { environment { // Defines the following environment variables: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID // Ref. https://plugins.jenkins.io/azure-credentials/#plugin-content-declarative-pipeline - AZURE = credentials('packer-azure-serviceprincipal') + AZURE = credentials('packer-azure-serviceprincipal-sponsorship') // // Split packer plugins/configuration for each matrix cell - ref. https://www.packer.io/docs/configure // PACKER_PLUGIN_PATH = "${env.WORKSPACE}/plugins" // Define Packer Input variables through environment variables prefixed with 'PKR_VAR_' From 0cb8433afe29f13b392ba8bf063ffcc295c8ed33 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Wed, 20 Dec 2023 09:45:05 +0100 Subject: [PATCH 04/10] wip: use the classic subscription for the gallery --- Jenkinsfile_k8s | 2 ++ sources.pkr.hcl | 2 +- variables.pkr.hcl | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index e21d6d10e..655d599c8 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -174,6 +174,7 @@ pipeline { // Defines the following environment variables: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID // Ref. https://plugins.jenkins.io/azure-credentials/#plugin-content-declarative-pipeline AZURE = credentials('packer-azure-serviceprincipal-sponsorship') + AZURE_GALLERY = credentials('packer-azure-serviceprincipal') // // Split packer plugins/configuration for each matrix cell - ref. https://www.packer.io/docs/configure // PACKER_PLUGIN_PATH = "${env.WORKSPACE}/plugins" // Define Packer Input variables through environment variables prefixed with 'PKR_VAR_' @@ -185,6 +186,7 @@ pipeline { PKR_VAR_agent_os_version = "${env.agent_type.split('-')[1]}" PKR_VAR_architecture = "${env.cpu_architecture}" PKR_VAR_image_type = "${env.compute_type}" + PKR_VAR_azure_gallery_subscription_id = "${env.AZURE_GALLERY_SUBSCRIPTION_ID}" PATH = "${WORKSPACE}/.bin:${env.PATH}" // Required if packer needs to be installed } stages { diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 4c90d90e3..f2b654968 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -14,7 +14,7 @@ source "azure-arm" "base" { # Where to export the image shared_image_gallery_destination { - subscription = var.azure_subscription_id + subscription = var.azure_gallery_subscription_id resource_group = local.azure_destination_resource_group gallery_name = "${var.build_type}_packer_images" # Not unique name defined in https://github.com/jenkins-infra/azure/blob/bfe56cb4f843b0c8029413090c383f7ac38dde2a/locals.tf#L4-L41 diff --git a/variables.pkr.hcl b/variables.pkr.hcl index b9c982879..f3116ff3d 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -25,6 +25,10 @@ variable "azure_subscription_id" { type = string default = env("AZURE_SUBSCRIPTION_ID") } +variable "azure_gallery_subscription_id" { + type = string + default = env("AZURE_SUBSCRIPTION_ID") # allow to specify distinct subscription +} variable "image_version" { type = string default = "0.0.1" # Default is a valid version to not fail azure validation From 3d2c2ec06e3dcfd5e91cf9ab538ca85cec7a9933 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Wed, 20 Dec 2023 09:55:23 +0100 Subject: [PATCH 05/10] wip: define virtual network to use new subscription subnet and private ips --- sources.pkr.hcl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index f2b654968..e128f31d3 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -4,6 +4,11 @@ source "azure-arm" "base" { vm_size = local.azure_vm_size[var.architecture] + # network defined in https://github.com/jenkins-infra/azure-net/blob/a396b20f676602703c0dbdf4cee977eae2669cd7/vnets.tf#L535 + virtual_network_name = "infra-ci-jenkins-io-sponsorship-vnet" + virtual_network_subnet_name = "infra-ci-jenkins-io-sponsorship-vnet-packer-builds" + virtual_network_resource_group_name = "infra-ci-jenkins-io-sponsorship" + # Resource group where to create the VM resources (required to scope permissions into this resource group) build_resource_group_name = "${var.build_type}-packer-builds" From 266def3f2e3028654ee0bd59eee8af2166d7383a Mon Sep 17 00:00:00 2001 From: smerle33 Date: Wed, 20 Dec 2023 14:45:45 +0100 Subject: [PATCH 06/10] chore(updatecli): add a manifest for xq --- updatecli/updatecli.d/xq.yml | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 updatecli/updatecli.d/xq.yml diff --git a/updatecli/updatecli.d/xq.yml b/updatecli/updatecli.d/xq.yml new file mode 100644 index 000000000..526940d31 --- /dev/null +++ b/updatecli/updatecli.d/xq.yml @@ -0,0 +1,57 @@ +--- +name: Bump `xq` version + +scms: + default: + kind: github + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + branch: "{{ .github.branch }}" + +sources: + lastReleaseVersion: + kind: githubrelease + name: Get the latest `xq` version + spec: + owner: "sibprogrammer" + repository: "xq" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + versionfilter: + kind: latest + transformers: + - trimprefix: v + +targets: + updateVersion: + name: Update the `xq` version in the tools-versions.yml file + sourceid: lastReleaseVersion + kind: yaml + spec: + file: "provisioning/tools-versions.yml" + key: "$.xq_version" + scmid: default + updateVersionInGoss: + name: "Update the `xq` version in the goss test" + kind: yaml + spec: + files: + - goss/goss-linux.yaml + # - goss/goss-windows.yaml not installed on windows yet + key: $.command.xq.stdout[0] + scmid: default + +actions: + default: + kind: github/pullrequest + scmid: default + title: Bump the `xq` CLI version to {{ source "lastReleaseVersion" }} + spec: + labels: + - enhancement + - xq From 1f3edce0f644afd944fe3b0e651c11e6bf65f7f9 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Thu, 21 Dec 2023 11:10:10 +0100 Subject: [PATCH 07/10] change location to EASTUS-2 --- build-jenkins-agent-ubuntu.pkr.hcl | 6 +++--- build-jenkins-agent-windows.pkr.hcl | 4 ++-- locals.pkr.hcl | 12 ++++++------ sources.pkr.hcl | 11 ++++++----- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/build-jenkins-agent-ubuntu.pkr.hcl b/build-jenkins-agent-ubuntu.pkr.hcl index d7b1dd212..9cbee3260 100644 --- a/build-jenkins-agent-ubuntu.pkr.hcl +++ b/build-jenkins-agent-ubuntu.pkr.hcl @@ -46,7 +46,7 @@ build { } provisioner "shell" { - execute_command = "{{ .Vars }} sudo -E su - jenkins -c \"bash -eu '{{ .Path }}'\"" + execute_command = "{{ .Vars }} sudo -E su - jenkins -c \"bash -eu '{{ .Path }}'\"" inline = [ "source /home/jenkins/.asdf/asdf.sh", # Required as this is a non-interactive and non-login `bash` "goss --version", @@ -55,9 +55,9 @@ build { } post-processor "docker-tag" { - only = ["docker.ubuntu"] + only = ["docker.ubuntu"] # TODO specify architecture in image name with local.image_name - repository = format("%s/jenkins-agent-%s-%s",var.docker_namespace, var.agent_os_type, var.agent_os_version) + repository = format("%s/jenkins-agent-%s-%s", var.docker_namespace, var.agent_os_type, var.agent_os_version) tags = [var.image_version, "latest"] } } diff --git a/build-jenkins-agent-windows.pkr.hcl b/build-jenkins-agent-windows.pkr.hcl index 8589afdd7..cd0482618 100644 --- a/build-jenkins-agent-windows.pkr.hcl +++ b/build-jenkins-agent-windows.pkr.hcl @@ -70,7 +70,7 @@ build { provisioner "file" { source = "./goss/goss-windows.yaml" - destination = "C:/goss-windows.yaml" + destination = "C:/goss-windows.yaml" } provisioner "breakpoint" { @@ -79,7 +79,7 @@ build { } provisioner "powershell" { - pause_before = "2m" # long pause as 1m is not enough + pause_before = "2m" # long pause as 1m is not enough inline = [ "$ErrorActionPreference = 'Stop'", "goss --version", diff --git a/locals.pkr.hcl b/locals.pkr.hcl index 03b87b0b5..9701f5474 100644 --- a/locals.pkr.hcl +++ b/locals.pkr.hcl @@ -10,18 +10,18 @@ locals { "arm64" = "${local.agent_os_version_safe}-lts-arm64" } windows_winrm_user = { - "azure-arm" = "packer" - "docker" = "packer" + "azure-arm" = "packer" + "docker" = "packer" } azure_vm_size = { - "amd64" = "Standard_D4ads_v5" # 4 CPU / 16 GB / Huge size required to avoid https:#docs.microsoft.com/en-us/azure/virtual-machines/linux/image-builder-troubleshoot#sysprep-timing and avoid full disk (DS2v2 only have 14 Gb SSD for system) + "amd64" = "Standard_D4ads_v5" # 4 CPU / 16 GB / Huge size required to avoid https:#docs.microsoft.com/en-us/azure/virtual-machines/linux/image-builder-troubleshoot#sysprep-timing and avoid full disk (DS2v2 only have 14 Gb SSD for system) "arm64" = "Standard_D4pds_v5" # 4 CPU / 16 GB } azure_destination_resource_group = "${var.build_type}-packer-images" azure_galleries = { - "prod_packer_images" = ["East US", "East US 2"] - "staging_packer_images" = ["East US", "East US 2"] # Only the "main" branch, should map the production as much as possible - "dev_packer_images" = ["East US"] # Faster builds for branches, pull requests or local development + "prod_packer_images" = ["East US 2"] + "staging_packer_images" = ["East US 2"] + "dev_packer_images" = ["East US 2"] } windows_disk_size_gb = 150 # Must be greater than 127 Gb to allow Azure template to work with provisioning_env_vars = concat( diff --git a/sources.pkr.hcl b/sources.pkr.hcl index e128f31d3..6c9329aca 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -5,8 +5,8 @@ source "azure-arm" "base" { vm_size = local.azure_vm_size[var.architecture] # network defined in https://github.com/jenkins-infra/azure-net/blob/a396b20f676602703c0dbdf4cee977eae2669cd7/vnets.tf#L535 - virtual_network_name = "infra-ci-jenkins-io-sponsorship-vnet" - virtual_network_subnet_name = "infra-ci-jenkins-io-sponsorship-vnet-packer-builds" + virtual_network_name = "infra-ci-jenkins-io-sponsorship-vnet" + virtual_network_subnet_name = "infra-ci-jenkins-io-sponsorship-vnet-packer-builds" virtual_network_resource_group_name = "infra-ci-jenkins-io-sponsorship" # Resource group where to create the VM resources (required to scope permissions into this resource group) @@ -16,12 +16,13 @@ source "azure-arm" "base" { client_id = var.azure_client_id client_secret = var.azure_client_secret subscription_id = var.azure_subscription_id + location = "East US 2" # Where to export the image shared_image_gallery_destination { - subscription = var.azure_gallery_subscription_id - resource_group = local.azure_destination_resource_group - gallery_name = "${var.build_type}_packer_images" + subscription = var.azure_gallery_subscription_id + resource_group = local.azure_destination_resource_group + gallery_name = "${var.build_type}_packer_images" # Not unique name defined in https://github.com/jenkins-infra/azure/blob/bfe56cb4f843b0c8029413090c383f7ac38dde2a/locals.tf#L4-L41 image_name = "${local.image_name}" image_version = var.image_version From 2f2b01c358d68cee64a5d72cee9c23588838d753 Mon Sep 17 00:00:00 2001 From: Damien Duportal Date: Thu, 21 Dec 2023 14:05:11 +0100 Subject: [PATCH 08/10] Apply suggestions from code review --- Jenkinsfile_k8s | 2 -- updatecli/updatecli.d/xq.yml | 57 ------------------------------------ 2 files changed, 59 deletions(-) diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index 655d599c8..e21d6d10e 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -174,7 +174,6 @@ pipeline { // Defines the following environment variables: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID // Ref. https://plugins.jenkins.io/azure-credentials/#plugin-content-declarative-pipeline AZURE = credentials('packer-azure-serviceprincipal-sponsorship') - AZURE_GALLERY = credentials('packer-azure-serviceprincipal') // // Split packer plugins/configuration for each matrix cell - ref. https://www.packer.io/docs/configure // PACKER_PLUGIN_PATH = "${env.WORKSPACE}/plugins" // Define Packer Input variables through environment variables prefixed with 'PKR_VAR_' @@ -186,7 +185,6 @@ pipeline { PKR_VAR_agent_os_version = "${env.agent_type.split('-')[1]}" PKR_VAR_architecture = "${env.cpu_architecture}" PKR_VAR_image_type = "${env.compute_type}" - PKR_VAR_azure_gallery_subscription_id = "${env.AZURE_GALLERY_SUBSCRIPTION_ID}" PATH = "${WORKSPACE}/.bin:${env.PATH}" // Required if packer needs to be installed } stages { diff --git a/updatecli/updatecli.d/xq.yml b/updatecli/updatecli.d/xq.yml index 526940d31..e69de29bb 100644 --- a/updatecli/updatecli.d/xq.yml +++ b/updatecli/updatecli.d/xq.yml @@ -1,57 +0,0 @@ ---- -name: Bump `xq` version - -scms: - default: - kind: github - spec: - user: "{{ .github.user }}" - email: "{{ .github.email }}" - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: "{{ requiredEnv .github.token }}" - username: "{{ .github.username }}" - branch: "{{ .github.branch }}" - -sources: - lastReleaseVersion: - kind: githubrelease - name: Get the latest `xq` version - spec: - owner: "sibprogrammer" - repository: "xq" - token: "{{ requiredEnv .github.token }}" - username: "{{ .github.username }}" - versionfilter: - kind: latest - transformers: - - trimprefix: v - -targets: - updateVersion: - name: Update the `xq` version in the tools-versions.yml file - sourceid: lastReleaseVersion - kind: yaml - spec: - file: "provisioning/tools-versions.yml" - key: "$.xq_version" - scmid: default - updateVersionInGoss: - name: "Update the `xq` version in the goss test" - kind: yaml - spec: - files: - - goss/goss-linux.yaml - # - goss/goss-windows.yaml not installed on windows yet - key: $.command.xq.stdout[0] - scmid: default - -actions: - default: - kind: github/pullrequest - scmid: default - title: Bump the `xq` CLI version to {{ source "lastReleaseVersion" }} - spec: - labels: - - enhancement - - xq From e4a21fd084161f7c1f1288770247f37ca03aec49 Mon Sep 17 00:00:00 2001 From: Damien Duportal Date: Thu, 21 Dec 2023 14:05:26 +0100 Subject: [PATCH 09/10] Delete updatecli/updatecli.d/xq.yml --- updatecli/updatecli.d/xq.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 updatecli/updatecli.d/xq.yml diff --git a/updatecli/updatecli.d/xq.yml b/updatecli/updatecli.d/xq.yml deleted file mode 100644 index e69de29bb..000000000 From 03f2ad28b1e745eaecc19efefeac7b82325ceec6 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Thu, 21 Dec 2023 14:37:19 +0100 Subject: [PATCH 10/10] remove location as we user build_resource_group_name --- sources.pkr.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 6c9329aca..7d7b0e582 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -16,7 +16,6 @@ source "azure-arm" "base" { client_id = var.azure_client_id client_secret = var.azure_client_secret subscription_id = var.azure_subscription_id - location = "East US 2" # Where to export the image shared_image_gallery_destination {