diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index c1feb591f..000000000 --- a/.editorconfig +++ /dev/null @@ -1,2 +0,0 @@ -[Jenkinsfile] -indent_size = 2 \ No newline at end of file diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 1dc3a962b..d5e8d6e0d 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -15,4 +15,3 @@ jobs: env: # This token is generated automatically by default in GitHub Actions: no need to create it manually GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -... diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 5916ea230..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,11 +0,0 @@ -# Contributing - -If you have: - -* Any question about this repository, please ask us on the IRC channel at #jenkins-infra or on - -* A problem, a feature request or an unwanted behavior with this code or its artifacts, please open an issue on the GitHub issue tracker of this repository. - -* A contribution to make, please fork the GitHub repository to your own account, create a named branch and open a Pull Request with your proposed change/fix - * Commit messages should follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) messages - * If your PR is not associated to an issue (JIRA, GitHub), there is not guarantee that we will read or review it: please write a message explaining the intent in this case. diff --git a/CiPodTemplate.yaml b/CiPodTemplate.yaml deleted file mode 100644 index 135171ba3..000000000 --- a/CiPodTemplate.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Pod -spec: - automountServiceAccountToken: false - containers: - - name: jnlp - image: jenkinsciinfra/hashicorp-tools:0.5.15 diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index 208013a9b..73cdb0af5 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -7,6 +7,16 @@ if (env.BRANCH_IS_PRIMARY) { ]) } +final String podAgentDefinition = ''' +apiVersion: v1 +kind: Pod +spec: + automountServiceAccountToken: false + containers: + - name: jnlp + image: jenkinsciinfra/hashicorp-tools:latest +''' + pipeline { agent none options { @@ -15,18 +25,20 @@ pipeline { } stages { stage('Side Tasks') { + environment { + DRYRUN = "${env.BRANCH_IS_PRIMARY ? 'false' : 'true'}" + } parallel { stage('GC on AWS us-east-2') { agent { kubernetes { - yamlFile 'CiPodTemplate.yaml' + yaml podAgentDefinition } } environment { AWS_ACCESS_KEY_ID = credentials('packer-aws-access-key-id') AWS_SECRET_ACCESS_KEY = credentials('packer-aws-secret-access-key') AWS_DEFAULT_REGION = 'us-east-2' - DRYRUN = "${env.BRANCH_NAME == 'main' ? 'false' : 'true'}" } steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -36,17 +48,14 @@ pipeline { } } stage('GC on Azure') { - environment { - PACKER_AZURE = credentials('packer-azure-serviceprincipal') - DRYRUN = "${env.BRANCH_IS_PRIMARY ? 'false' : 'true'}" - } - agent { kubernetes { - yamlFile 'CiPodTemplate.yaml' + yaml podAgentDefinition } } - + environment { + PACKER_AZURE = credentials('packer-azure-serviceprincipal') + } steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh 'az login --service-principal -u "$PACKER_AZURE_CLIENT_ID" -p "$PACKER_AZURE_CLIENT_SECRET" -t "$PACKER_AZURE_TENANT_ID"' @@ -73,15 +82,15 @@ pipeline { matrix { axes { axis { - name 'PKR_VAR_architecture' + name 'cpu_architecture' values 'amd64', 'arm64' } axis { - name 'PKR_VAR_agent' - values 'ubuntu-20', 'windows-2019' + name 'agent_type' + values 'ubuntu-20.04', 'windows-2019' } axis { - name 'PKR_VAR_image_type' + name 'compute_type' // "azure-arm" stands for "Azure Resource Manager", unrelated to arm64 CPU values 'amazon-ebs', 'azure-arm', 'docker' } @@ -90,105 +99,90 @@ pipeline { // Do NOT build ARM images in Azure exclude { axis { - name 'PKR_VAR_architecture' + name 'cpu_architecture' values 'arm64' } axis { - name 'PKR_VAR_image_type' + name 'compute_type' values 'azure-arm' } } // Only build Ubuntu images for arm64 CPU in AWS exclude { axis { - name 'PKR_VAR_architecture' + name 'cpu_architecture' values 'arm64' } axis { - name 'PKR_VAR_agent' - notValues 'ubuntu-20' + name 'agent_type' + notValues 'ubuntu-20.04' } axis { - name 'PKR_VAR_image_type' + name 'compute_type' values 'amazon-ebs' } } // No build on Windows or Docker, not yet implemented exclude { axis { - name 'PKR_VAR_agent' + name 'agent_type' values 'windows-2019' } axis { - name 'PKR_VAR_image_type' + name 'compute_type' values 'docker' } } } agent { - // Temporary forcing exception for windows/docker which is not finished yet - label "${env.PKR_VAR_agent == 'windows-2019' && env.PKR_VAR_image_type == 'docker' ? 'windows' : 'linux'}-${PKR_VAR_architecture}-docker" + label "linux-${env.cpu_architecture}-docker" } environment { AWS_ACCESS_KEY_ID = credentials('packer-aws-access-key-id') AWS_SECRET_ACCESS_KEY = credentials('packer-aws-secret-access-key') - PACKER_HOME_DIR = "/tmp/packer.d.${PKR_VAR_image_type}.${PKR_VAR_architecture}.${PKR_VAR_agent}" - PACKER_PLUGIN_PATH = "${PACKER_HOME_DIR}/plugins" - PACKER_VARS_FILE = ".auto.pkrvars.hcl" - // Reuse the updatecli Github token to ensure that packer init does not hit the rate limit - GITHUB_IAT_TOKEN = credentials('github-app-infra') + // Define Packer Input variables through environment variables prefixed with 'PKR_VAR_' + // Re.f https://www.packer.io/docs/templates/hcl_templates/variables#assigning-values-to-build-variables + PKR_VAR_build_type = "${env.TAG_NAME ? 'prod' : (env.BRANCH_IS_PRIMARY ? 'staging' : 'dev') }" + PKR_VAR_image_version = "${env.TAG_NAME ?: ''}" + PKR_VAR_scm_ref = "${env.GIT_COMMIT}" + PKR_VAR_agent_os_type = "${env.agent_type.split('-')[0]}" + PKR_VAR_agent_os_version = "${env.agent_type.split('-')[1]}" + PKR_VAR_architecture = "${env.cpu_architecture}" + PKR_VAR_image_type = "${env.compute_type}" + PATH = "${WORKSPACE}/.bin:${env.PATH}" // Required if packer needs to be installed } stages { - stage('Prepare on primary branch') { - when { - expression { env.BRANCH_IS_PRIMARY } - } - steps { - sh ''' - echo 'build_type = "staging"' >> "${PACKER_VARS_FILE}" - ''' - } - } - stage('Prepare on Tag') { - when { - buildingTag() - } - steps { - sh ''' - set -eu - echo 'build_type = "prod"' >> "${PACKER_VARS_FILE}" - echo 'image_version = "'${TAG_NAME}'"' >> "${PACKER_VARS_FILE}" - ''' - } - } - stage('Prepare and Report') { - steps { - withEnv(["PACKER_GITHUB_API_TOKEN=${GITHUB_IAT_TOKEN_PSW}"]) { - sh ''' - echo 'scm_ref = "'"$(git rev-parse --short --verify HEAD)"'"' >> "${PACKER_VARS_FILE}" - ./run-packer.sh report - ''' - } - } - } - stage('Validate') { - steps { - withEnv(["PACKER_GITHUB_API_TOKEN=${GITHUB_IAT_TOKEN_PSW}"]) { - sh './run-packer.sh validate' - } - } - } - stage('Build') { + stage('Build Template') { steps { + // Pass the Azure credentials through Packer Input variables, as environment variables prefixed with 'PKR_VAR_' + // Re.f https://www.packer.io/docs/templates/hcl_templates/variables#assigning-values-to-build-variables withCredentials([azureServicePrincipal( credentialsId: 'packer-azure-serviceprincipal', clientIdVariable: 'PKR_VAR_azure_client_id', clientSecretVariable: 'PKR_VAR_azure_client_secret', subscriptionIdVariable: 'PKR_VAR_azure_subscription_id' - )]) { - withEnv(["PACKER_GITHUB_API_TOKEN=${GITHUB_IAT_TOKEN_PSW}"]) { - sh './run-packer.sh build' + ),]) { + // Required if packer needs to be installed + sh 'command -v packer >/dev/null 2>&1 || bash ./install-packer.sh "${WORKSPACE}/.bin" "1.8.0"' + + // Help auditing pipeline execution + sh ''' + echo "= Current Packer environment:" + env | grep -i PKR_VAR + env | grep -i PACKER + ''' + + // Initialize the project. To avoid hitting GitHub APi rate limit, Packer authenticates + // with an API token (auto-generated IAT, valid for 1 hour) provided to the environment variable PACKER_GITHUB_API_TOKEN + withCredentials([usernamePassword(credentialsId: 'github-app-infra',usernameVariable: 'UNUSED',passwordVariable: 'PACKER_GITHUB_API_TOKEN')]) { + sh 'packer init ./' } + + // Validate template (for all elements) + sh 'packer validate ./' + + // Execute build only for this matrix cell's setup + sh 'packer build -timestamp-ui -force -only="${PKR_VAR_image_type}.${PKR_VAR_agent_os_type}" ./' } } } diff --git a/README.md b/README.md index f4acfe53a..33463b609 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,15 @@ of the jenkins-infra project, such as ci.jenkins.io, infra.ci.jenkins.io, etc. ## Contributing -See the [CONTRIBUTING.md document](./CONTRIBUTING.md). +If you have: + +* Any question about this repository, please ask us on the IRC channel at #jenkins-infra or on + +* A problem, a feature request or an unwanted behavior with this code or its artifacts, please open an issue on the GitHub issue tracker of this repository. + +* A contribution to make, please fork the GitHub repository to your own account, create a named branch and open a Pull Request with your proposed change/fix + * Commit messages should follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) messages + * If your PR is not associated to an issue (JIRA, GitHub), there is not guarantee that we will read or review it: please write a message explaining the intent in this case. ## Continuous Integration/Delivery/Deployment @@ -50,39 +58,36 @@ In some cases, you might want to execute the build locally (opposed to opening a * A shell (sh, bash, ash, zsh) prompt * Packer (check template's version constraints to know which version to use) -* Define the environment variables `PKR_VAR_image_type`, `PKR_VAR_agent` and `PKR_VAR_architecture` to the target you want to build -* Configure the cloud (defined on the variable `PKR_VAR_image_type`) credentials (e.g. API key, or clouds own CLI access such as `az` or `aws` commands)) +* Define the environment variables `PKR_VAR_*` to the target you want to build (hint: look at the Jenkinsfile or to the `variable` blocks in the packer template HCL files) +* Configure *all* the clouds (defined by the possible values of the variable `PKR_VAR_image_type`) credentials (e.g. API key, or clouds own CLI access such as `az` or `aws` commands)) +* Ensure that you have a Docker Engine available With the requirements verified locally, execute the following command: * Define the target to build: ```bash -# Means: "Build the ubuntu-20 agent for AWS, on ARM64 CPU -export PKR_VAR_image_type=amazon-ebs -export PKR_VAR_agent=ubuntu-20 -export PKR_VAR_architecture=arm64 - -# Verify that you can use AWS API: the command must succeed -aws ec2 describe-instances +# Means: "Build the ubuntu-20.04 agent for Docker +export PKR_VAR_image_type=docker +export PKR_VAR_agent_os_type=ubuntu +export PKR_VAR_agent_os_version=20.04 ``` * Validate the template: ```bash -./run-packer.sh validate +packer validate ./ ``` -* Validate the template: +* Build only one template (example: Docker Ubuntu): ```bash -./run-packer.sh build +packer build -timestamp-ui -force -only="docker.ubuntu" ./ ``` ### Azure -* Retrieve the resource group name from the source to build in the template file (directive `managed_image_resource_group_name`) - from `*.auto.pkrvars.hcl` file associated to azure/. +* Retrieve the resource group name from the source to build in the template files (directive `managed_image_resource_group_name`). * Create a new resource group on your account with this retrieved values @@ -90,11 +95,4 @@ aws ec2 describe-instances az group create -n myResourceGroup -l eastus ``` -* Define the cloud credentials with the 3 additional variables `PKR_VAR_subscription_id`, `PKR_VAR_client_id` and `PKR_VAR_client_secret`: - -```bash -PKR_VAR_subscription_id="$(az account show --query id -o tsv)" -export PKR_VAR_subscription_id - -PKR_VAR_image_type=azure-arm PKR_VAR_agent=windows-2019 PKR_VAR_architecture=amd64 PKR_VAR_client_id= PKR_VAR_client_secret= ./run-packer.sh build -``` +* Define the cloud credentials with the 3 additional environment variables `PKR_VAR_subscription_id`, `PKR_VAR_client_id` and `PKR_VAR_client_secret` diff --git a/build-jenkins-agent-ubuntu.pkr.hcl b/build-jenkins-agent-ubuntu.pkr.hcl new file mode 100644 index 000000000..14ebfb371 --- /dev/null +++ b/build-jenkins-agent-ubuntu.pkr.hcl @@ -0,0 +1,46 @@ +build { + source "docker.base" { + name = "ubuntu" + } + + source "amazon-ebs.base" { + name = "ubuntu" + ssh_username = "ubuntu" + } + + source "azure-arm.base" { + name = "ubuntu" + image_offer = "0001-com-ubuntu-server-focal" + image_publisher = "canonical" + image_sku = "${local.agent_os_version_safe}-lts-gen2" + os_type = "Linux" + vm_size = local.azure_vm_size + } + + # Docker Ubuntu image are missing required tools: let's install it as a preliminary + provisioner "shell" { + only = ["docker.ubuntu"] + environment_vars = local.provisioning_env_vars + inline = [ + "export DEBIAN_FRONTEND=noninteractive", # Avoid APT or dpkg asking questions + "apt-get update --quiet", + "apt-get install --yes --no-install-recommends ca-certificates curl sudo software-properties-common", + ] + } + + provisioner "file" { + source = "./provisioning/add_auth_key_to_user.sh" + destination = "/tmp/add_auth_key_to_user.sh" + } + + provisioner "file" { + source = "./provisioning/docker.gpg" + destination = "/tmp/docker.gpg" + } + + provisioner "shell" { + environment_vars = local.provisioning_env_vars + execute_command = "chmod +x {{ .Path }}; {{ .Vars }} sudo -E bash '{{ .Path }}'" + script = "./provisioning/ubuntu-provision.sh" + } +} diff --git a/build-jenkins-agent-windows.pkr.hcl b/build-jenkins-agent-windows.pkr.hcl new file mode 100644 index 000000000..affb6c09a --- /dev/null +++ b/build-jenkins-agent-windows.pkr.hcl @@ -0,0 +1,76 @@ +build { + source "amazon-ebs.base" { + name = "windows" + communicator = "winrm" + user_data_file = "./provisioning/setupWinRM.ps1" + winrm_insecure = true + winrm_timeout = "20m" + winrm_use_ssl = true + winrm_username = local.windows_winrm_user[var.image_type] + } + + source "azure-arm.base" { + name = "windows" + communicator = "winrm" + image_offer = "WindowsServer" + image_publisher = "MicrosoftWindowsServer" + image_sku = "${var.agent_os_version}-datacenter-core-with-containers-smalldisk-g2" + vm_size = local.azure_vm_size + os_type = "Windows" + os_disk_size_gb = local.windows_disk_size_gb + winrm_insecure = true + winrm_timeout = "20m" + winrm_use_ssl = true + winrm_username = local.windows_winrm_user[var.image_type] + } + + ## Why repeating? https://github.com/rgl/packer-plugin-windows-update/issues/90#issuecomment-842569865 + # Note that restarts are only done when required by windows updates + provisioner "windows-update" { pause_before = "1m" } + provisioner "windows-update" { pause_before = "1m" } + provisioner "windows-update" { pause_before = "1m" } + + provisioner "file" { + pause_before = "1m" + source = "./provisioning/addSSHPubKey.ps1" + destination = "C:/" + } + + provisioner "powershell" { + environment_vars = local.provisioning_env_vars + elevated_user = local.windows_winrm_user[var.image_type] + elevated_password = build.Password + script = "./provisioning/windows-provision.ps1" + } + + # Recommended (and sometimes required) before running deprovisioning (sysprep or AWS scripts) + # ref. https:#www.packer.io/docs/builders/azure/arm#windows + provisioner "windows-restart" { + max_retries = 3 + } + provisioner "powershell" { + only = ["azure-arm.windows"] + elevated_user = local.windows_winrm_user[var.image_type] + elevated_password = build.Password + inline = [ + "& $env:SystemRoot\\System32\\Sysprep\\Sysprep.exe /oobe /generalize /quiet /quit /mode:vm", + "while($true) { $imageState = Get-ItemProperty HKLM:\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup\\State | Select ImageState; if($imageState.ImageState -ne 'IMAGE_STATE_GENERALIZE_RESEAL_TO_OOBE') { Write-Output $imageState.ImageState; Start-Sleep -s 10 } else { break } }" + ] + } + provisioner "file" { + only = ["amazon-ebs.windows"] + source = "./provisioning/EC2-LaunchConfig.json" + destination = "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Config\\LaunchConfig.json" + } + provisioner "powershell" { + only = ["amazon-ebs.windows"] + elevated_user = local.windows_winrm_user[var.image_type] + elevated_password = build.Password + # Ref. https:#docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2-windows-user-data.html#user-data-scripts-subsequent + inline = [ + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SendWindowsIsReady.ps1 -Schedule", + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule", + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown" + ] + } +} diff --git a/datasources.pkr.hcl b/datasources.pkr.hcl new file mode 100644 index 000000000..e67efe24b --- /dev/null +++ b/datasources.pkr.hcl @@ -0,0 +1,22 @@ +# Data sources are always treated BEFORE locals and sources. +data "amazon-ami" "ubuntu-20_04" { + filters = { + name = format("ubuntu/images/hvm-ssd/ubuntu-*-20.04-%s-server-*", var.architecture) + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + region = var.aws_region +} + +data "amazon-ami" "windows-2019" { + filters = { + name = "Windows_Server-2019-English-Core-ContainersLatest-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["amazon"] + region = var.aws_region +} diff --git a/install-packer.sh b/install-packer.sh new file mode 100755 index 000000000..5a09cbec7 --- /dev/null +++ b/install-packer.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# install-packer: install packer for the current architecture, +# in the directory specified as 1st argument, in the version specified as 2nd argument + +packer_install_dir="${1:?First argument missing: directory where to install packer.}" +packer_version="${2:?Second argument missing: version of packer to install.}" + +set -eu -o pipefail + +packer_cmd="packer" +temp_dir="$(mktemp -d)" +mkdir -p "${packer_install_dir}" + +## check if packer exists or install it +echo "====================================" + +## Check for presence of requirements or fail fast +for cli in curl unzip +do + if ! command -v $cli >/dev/null 2>&1 + then + echo "ERROR: command line ${cli} required but not found. Exiting." + exit 1 + fi +done + +echo "= Installing Packer version ${packer_version} to ${packer_install_dir}" + +if ! command -v ${packer_cmd} >/dev/null 2>&1 +then + if test -x "${packer_install_dir}/${packer_cmd}" + then + packer_cmd="${packer_install_dir}/packer" + else + echo "Detecting CPU architecture..." + arch=$(uname -m) + if [[ $arch == x86_64* ]]; then + echo "X64 Architecture" + packer_download_url="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_amd64.zip" + elif [[ $arch == i*86 ]]; then + echo "X32 Architecture" + packer_download_url="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_386.zip" + elif [[ $arch == arm* ]]; then + echo "ARM Architecture 32b" + packer_download_url="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_arm.zip" + elif [[ $arch == aarch64 ]]; then + echo "ARM Architecture 64b" + packer_download_url="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_arm64.zip" + else + echo "ERROR: unknwon architecture (${arch}). Exiting." + exit 2 + fi + + zip_file="${temp_dir}/packer.zip" + curl -sSL -o "${zip_file}" "${packer_download_url}" + unzip "${zip_file}" -d "${packer_install_dir}" + packer_cmd="${packer_install_dir}/packer" + fi +fi + +echo "= Packer installed, running sanity check (command '${packer_cmd} version')..." +"${packer_cmd}" version +echo "====================================" + +exit 0 diff --git a/jenkins-agent.pkr.hcl b/jenkins-agent.pkr.hcl deleted file mode 100644 index 37acbb9b6..000000000 --- a/jenkins-agent.pkr.hcl +++ /dev/null @@ -1,307 +0,0 @@ -packer { - required_version = ">= 1.7.2, < 1.8.0" - required_plugins { - amazon = { - version = "1.0.0" - source = "github.com/hashicorp/amazon" - } - windows-update = { - version = "0.14.0" - source = "github.com/rgl/windows-update" - } - azure = { - version = "1.0.4" - source = "github.com/hashicorp/azure" - } - docker = { - version = ">= 0.0.7" - source = "github.com/hashicorp/docker" - } - } -} - -variable "agent" { - type = string - description = "Which agent to build: ubuntu-20 (default), windows-2019." - default = "ubuntu-20" -} -variable "architecture" { - type = string - description = "CPU architecture ID of the build with the following possible values: [amd64 (default), arm64]" - default = "amd64" -} -variable "aws_region" { - type = string - default = "us-east-2" -} -variable "azure_client_id" { - type = string - default = "" -} -variable "azure_client_secret" { - type = string - default = "" -} -variable "azure_subscription_id" { - default = "" - type = string -} -variable "image_version" { - type = string - default = "0.0.1" # Default is a valid version to not fail azure validation -} -variable "image_type" { - type = string - description = "Which kind of Packer builder to use (e.g. cloud platform): [amazon-ebs (default), azure-arm, docker]" - default = "amazon-ebs" -} -variable "build_type" { - type = string - description = "Type of build e.g. is it a development build (from a contributor machine), a ci build (pull request or branch build) or a production build (principal branch build on ci)?" - default = "dev" # value in ["dev", "ci", "prod"] -} -variable "scm_ref" { - type = string - description = "SCM (e.g. Git...) reference of the current build. Can be a commit hash (short or long), a branch name or a tag name." -} -variable "provision_env_file" { - type = string - description = "Path (absolute or relative to this packer template) to the YAML file with the list of environment variables forwarded to provisionners (mainly tools versions)" - default = "provision-env.yml" -} - -locals { - now_unix_timestamp = formatdate("YYYYMMDDhhmmss", timestamp()) - image_name = "jenkins-agent-${var.agent}" - aws_instance_type = { - "amd64" = "t3.large" # 2 CPU / 1 GB / $0.0104 - "arm64" = "t4g.micro" # 2 CPU / 1 GB / $0.0084 - } - windows_winrm_user = { - "azure-arm" = "packer" - "amazon-ebs" = "Administrator" - } - azure_vm_size = "Standard_D4s_v3" # Huge size requires - avoid https://docs.microsoft.com/en-us/azure/virtual-machines/linux/image-builder-troubleshoot#sysprep-timing and avoid full disk (DS2v2 only have 14 Gb SSD for system) - azure_resource_group = "${var.build_type}-packer-images" - azure_galleries = { - "prod_packer_images" = ["East US", "East US 2"] - "staging_packer_images" = ["East US", "East US 2"] # Only the "main" branch, should map the production as much as possible - "dev_packer_images" = ["East US"] # Faster builds for branches, pull requests or local development - } - os_disk_size_gb = 90 # less than 100 Gb to allow more instance sizes that have a temp cache < 100 Gb such as DS4_v3 - provisionning_env_vars = [for key, value in yamldecode(file(var.provision_env_file)) : "${upper(key)}=${value}"] -} - -data "amazon-ami" "ubuntu-20" { - filters = { - name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-${var.architecture}-server-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["099720109477"] - region = var.aws_region -} - -data "amazon-ami" "windows-2019" { - filters = { - name = "Windows_Server-2019-English-Core-ContainersLatest-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["amazon"] - region = var.aws_region -} - -source "amazon-ebs" "base" { - ami_name = "${local.image_name}-${var.architecture}-${local.now_unix_timestamp}" - instance_type = local.aws_instance_type[var.architecture] - # Define custom rootfs for build to avoid later filesystem extension during agent startups - launch_block_device_mappings { - delete_on_termination = true - device_name = "/dev/sda1" - volume_size = local.os_disk_size_gb - volume_type = "gp2" - } - # Where to build the VM - region = var.aws_region - # Where to export the AMI - ami_regions = [ - var.aws_region - ] - source_ami = data.amazon-ami[var.agent].id - ssh_username = "ubuntu" - tags = { - imageplatform = var.architecture - imagetype = local.image_name - timestamp = local.now_unix_timestamp - version = var.image_version - scm_ref = var.scm_ref - build_type = var.build_type - } -} - -source "azure-arm" "base" { - azure_tags = { - imageplatform = var.architecture - imagetype = local.image_name - timestamp = local.now_unix_timestamp - version = var.image_version - scm_ref = var.scm_ref - build_type = var.build_type - } - client_id = var.azure_client_id - client_secret = var.azure_client_secret - location = "East US" - managed_image_name = local.image_name - subscription_id = var.azure_subscription_id - managed_image_resource_group_name = local.azure_resource_group - - shared_image_gallery_destination { - subscription = var.azure_subscription_id - resource_group = local.azure_resource_group - gallery_name = "${var.build_type}_packer_images" - image_name = local.image_name - image_version = var.image_version - replication_regions = lookup(local.azure_galleries, "${var.build_type}_packer_images", []) - } - - async_resourcegroup_delete = true # Faster builds, but no deletion error reporting -} - -source "docker" "base" { - # only amd64 supported - image = "ubuntu:20.04" - # persist image on local docker engine - commit = true - # some label to stay compliant - changes = [ - "LABEL timestamp = ${local.now_unix_timestamp}", - "LABEL version = ${var.image_version}", - "LABEL scm_ref = ${var.scm_ref}", - "LABEL build_type = ${var.build_type}", - ] -} - -build { - source "docker.base" { - name = "ubuntu-20" - } - - source "amazon-ebs.base" { - name = "ubuntu-20" - } - - source "azure-arm.base" { - name = "ubuntu-20" - image_offer = "0001-com-ubuntu-server-focal" - image_publisher = "canonical" - image_sku = "20_04-lts-gen2" - os_type = "Linux" - vm_size = local.azure_vm_size - } - - provisioner "shell" { - # steps for docker images - only = ["docker.ubuntu-20"] - environment_vars = concat(local.provisionning_env_vars, ["DEBIAN_FRONTEND=noninteractive", "ARCHITECTURE=${var.architecture}"]) - # adding sudo and bash for docker software-properties-common - inline = [ - "apt update && apt install -y sudo curl bash software-properties-common", - ] - } - - provisioner "file" { - source = "./scripts/add_auth_key_to_user.sh" - destination = "/tmp/add_auth_key_to_user.sh" - } - - provisioner "file" { - source = "./scripts/docker.gpg" - destination = "/tmp/docker.gpg" - } - - provisioner "shell" { - environment_vars = concat(local.provisionning_env_vars, ["DEBIAN_FRONTEND=noninteractive", "ARCHITECTURE=${var.architecture}"]) - execute_command = "chmod +x {{ .Path }}; {{ .Vars }} sudo -E bash '{{ .Path }}'" - script = "./scripts/ubuntu-20-provision.sh" - } -} - -build { - source "amazon-ebs.base" { - name = "windows-2019" - communicator = "winrm" - user_data_file = "./scripts/setupWinRM.ps1" - winrm_insecure = true - winrm_timeout = "20m" - winrm_use_ssl = true - winrm_username = local.windows_winrm_user[var.image_type] - } - - source "azure-arm.base" { - name = "windows-2019" - communicator = "winrm" - image_offer = "WindowsServer" - image_publisher = "MicrosoftWindowsServer" - image_sku = "2019-datacenter-core-with-containers-smalldisk-g2" - vm_size = local.azure_vm_size - os_type = "Windows" - os_disk_size_gb = local.os_disk_size_gb - winrm_insecure = true - winrm_timeout = "20m" - winrm_use_ssl = true - winrm_username = local.windows_winrm_user[var.image_type] - } - - ## Why repeating? https://github.com/rgl/packer-plugin-windows-update/issues/90#issuecomment-842569865 - # Note that restarts are only done when required by windows updates - provisioner "windows-update" { pause_before = "1m" } - provisioner "windows-update" { pause_before = "1m" } - provisioner "windows-update" { pause_before = "1m" } - - provisioner "file" { - pause_before = "1m" - source = "./scripts/addSSHPubKey.ps1" - destination = "C:/" - } - - provisioner "powershell" { - environment_vars = concat(local.provisionning_env_vars, ["CLOUD_TYPE=${var.image_type}"]) - elevated_user = local.windows_winrm_user[var.image_type] - elevated_password = build.Password - script = "./scripts/windows-2019-provision.ps1" - } - - # Recommended (and sometimes required) before running deprovisioning (sysprep or AWS scripts) - # ref. https://www.packer.io/docs/builders/azure/arm#windows - provisioner "windows-restart" { - max_retries = 3 - } - provisioner "powershell" { - only = ["azure-arm.windows-2019"] - elevated_user = local.windows_winrm_user[var.image_type] - elevated_password = build.Password - inline = [ - "& $env:SystemRoot\\System32\\Sysprep\\Sysprep.exe /oobe /generalize /quiet /quit /mode:vm", - "while($true) { $imageState = Get-ItemProperty HKLM:\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup\\State | Select ImageState; if($imageState.ImageState -ne 'IMAGE_STATE_GENERALIZE_RESEAL_TO_OOBE') { Write-Output $imageState.ImageState; Start-Sleep -s 10 } else { break } }" - ] - } - provisioner "file" { - only = ["amazon-ebs.windows-2019"] - source = "./scripts/EC2-LaunchConfig.json" - destination = "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Config\\LaunchConfig.json" - } - provisioner "powershell" { - only = ["amazon-ebs.windows-2019"] - elevated_user = local.windows_winrm_user[var.image_type] - elevated_password = build.Password - # Ref. https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2-windows-user-data.html#user-data-scripts-subsequent - inline = [ - "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SendWindowsIsReady.ps1 -Schedule", - "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule", - "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown" - ] - } -} diff --git a/locals.pkr.hcl b/locals.pkr.hcl new file mode 100644 index 000000000..39bf1449f --- /dev/null +++ b/locals.pkr.hcl @@ -0,0 +1,32 @@ +locals { + now_unix_timestamp = formatdate("YYYYMMDDhhmmss", timestamp()) + agent = format("%s-%s", var.agent_os_type, var.agent_os_version) + agent_os_version_safe = replace(var.agent_os_version, ".", "_") + image_name = format("jenkins-agent-%s-%s", var.agent_os_type, var.agent_os_version) + aws_instance_type = { + "amd64" = "t3.large" # 2 CPU / 8 GB / $0.0835 - https://aws.amazon.com/fr/ec2/instance-types/t3/#Product_Details + "arm64" = "t4g.micro" # 2 CPU / 1 GB / $0.0084 + } + windows_winrm_user = { + "azure-arm" = "packer" + "amazon-ebs" = "Administrator" # In AWS EC2, WinRM super admin must be the "Administrator" account + "docker" = "packer" + } + azure_vm_size = "Standard_D4s_v3" # 4 CPU / 16 GB / Huge size required to avoid https:#docs.microsoft.com/en-us/azure/virtual-machines/linux/image-builder-troubleshoot#sysprep-timing and avoid full disk (DS2v2 only have 14 Gb SSD for system) + azure_resource_group = "${var.build_type}-packer-images" + azure_galleries = { + "prod_packer_images" = ["East US", "East US 2"] + "staging_packer_images" = ["East US", "East US 2"] # Only the "main" branch, should map the production as much as possible + "dev_packer_images" = ["East US"] # Faster builds for branches, pull requests or local development + } + windows_disk_size_gb = 128 # Must be greater than 127 Gb to allow Azure template to work with + provisioning_env_vars = concat( + [for key, value in yamldecode(file(var.provision_env_file)) : "${upper(key)}=${value}"], + [ + "CLOUD_TYPE=${var.image_type}", + "ARCHITECTURE=${var.architecture}", + "AGENT_OS_TYPE=${var.agent_os_type}", + "AGENT_OS_VERSION=${var.agent_os_version}", + ], + ) +} diff --git a/main.pkr.hcl b/main.pkr.hcl new file mode 100644 index 000000000..d9bacd491 --- /dev/null +++ b/main.pkr.hcl @@ -0,0 +1,21 @@ +packer { + required_version = ">= 1.7.2, < 1.9.0" + required_plugins { + amazon = { + version = "1.1.0" + source = "github.com/hashicorp/amazon" + } + windows-update = { + version = "0.14.0" + source = "github.com/rgl/windows-update" + } + azure = { + version = "1.0.7" + source = "github.com/hashicorp/azure" + } + docker = { + version = "1.0.3" + source = "github.com/hashicorp/docker" + } + } +} diff --git a/scripts/EC2-LaunchConfig.json b/provisioning/EC2-LaunchConfig.json similarity index 100% rename from scripts/EC2-LaunchConfig.json rename to provisioning/EC2-LaunchConfig.json diff --git a/scripts/addSSHPubKey.ps1 b/provisioning/addSSHPubKey.ps1 similarity index 100% rename from scripts/addSSHPubKey.ps1 rename to provisioning/addSSHPubKey.ps1 diff --git a/scripts/add_auth_key_to_user.sh b/provisioning/add_auth_key_to_user.sh similarity index 100% rename from scripts/add_auth_key_to_user.sh rename to provisioning/add_auth_key_to_user.sh diff --git a/scripts/docker.gpg b/provisioning/docker.gpg similarity index 100% rename from scripts/docker.gpg rename to provisioning/docker.gpg diff --git a/scripts/setupWinRM.ps1 b/provisioning/setupWinRM.ps1 similarity index 100% rename from scripts/setupWinRM.ps1 rename to provisioning/setupWinRM.ps1 diff --git a/provision-env.yml b/provisioning/tools-versions.yml similarity index 100% rename from provision-env.yml rename to provisioning/tools-versions.yml diff --git a/scripts/ubuntu-20-provision.sh b/provisioning/ubuntu-provision.sh similarity index 100% rename from scripts/ubuntu-20-provision.sh rename to provisioning/ubuntu-provision.sh diff --git a/scripts/windows-2019-provision.ps1 b/provisioning/windows-provision.ps1 similarity index 100% rename from scripts/windows-2019-provision.ps1 rename to provisioning/windows-provision.ps1 diff --git a/run-packer.sh b/run-packer.sh deleted file mode 100755 index 33eed6646..000000000 --- a/run-packer.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# run-packer: execute the packer action passed as argument -# This script could be replaced by a Makefile -# But make is not installed in the Docker image hashicorp/packer - -set -eux -o pipefail - -: "${1:?First argument - packer action to execute- not defined.}" - -packer_cmd="packer" -packer_template_dir="./" -packer_install_dir="/$HOME" -packer_version="1.7.8" - -export PKR_VAR_scm_ref PKR_VAR_image_type PKR_VAR_agent - - -## check if packer exists or install it -if ! command -v ${packer_cmd} >/dev/null 2>&1 -then - if test -x "${packer_install_dir}/${packer_cmd}" - then - packer_cmd="${packer_install_dir}/packer" - else - echo "Packer not installed, installing it" - arch=$(uname -i) - if [[ $arch == x86_64* ]]; then - echo "X64 Architecture" - packer_download_path="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_amd64.zip" - elif [[ $arch == i*86 ]]; then - echo "X32 Architecture" - packer_download_path="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_386.zip" - elif [[ $arch == arm* ]]; then - echo "ARM Architecture 32b" - packer_download_path="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_arm.zip" - elif [[ $arch == aarch64 ]]; then - echo "ARM Architecture 64b" - packer_download_path="https://releases.hashicorp.com/packer/${packer_version}/packer_${packer_version}_linux_arm64.zip" - else - echo "Architecture not found" - exit 2 - fi - curl -sSL -o /tmp/packer.zip $packer_download_path - unzip /tmp/packer.zip -d "${packer_install_dir}" - packer_cmd="${packer_install_dir}/packer" - fi -fi - -## Always run initialization to ensure plugins are download and workspace is set up -$packer_cmd init "${packer_template_dir}" - -## Define Packer flags based on the current environment (look at the `Jenkinsfile` to diagnose the pipeline) -PACKER_COMMON_FLAGS=("${packer_template_dir}") -set +u -if test -n "${PKR_VAR_image_type}" && test -n "${PKR_VAR_agent}" -then - PACKER_COMMON_FLAGS=("--only=${PKR_VAR_image_type}.${PKR_VAR_agent}" "${PACKER_COMMON_FLAGS[@]}") -fi -set -u - -echo "== Running action $1 with packer: ==" -case $1 in - validate) - $packer_cmd fmt -recursive . - $packer_cmd validate "${PACKER_COMMON_FLAGS[@]}" - echo "Validation Success." - ;; - build) - $packer_cmd build -timestamp-ui --force "${PACKER_COMMON_FLAGS[@]}" - echo "Build Success." - ;; - report) - echo "= Current Packer environment:" - env | grep -i PKR_VAR - env | grep -i PACKER - PACKER_VARS_FILE=.auto.pkrvars.hcl - echo "= Current Packer var file ${PACKER_VARS_FILE} ==" - cat "${PACKER_VARS_FILE}" - ;; - *) - echo "Error: Packer action '$1' is unknown." - ;; -esac -echo "====================================" - -exit 0 diff --git a/sources.pkr.hcl b/sources.pkr.hcl new file mode 100644 index 000000000..1679da2b3 --- /dev/null +++ b/sources.pkr.hcl @@ -0,0 +1,85 @@ +# This source defines all the common settings for any AWS AMI (whatever Operating System) +source "amazon-ebs" "base" { + ami_name = "${local.image_name}-${var.architecture}-${local.now_unix_timestamp}" + instance_type = local.aws_instance_type[var.architecture] + + # Define custom rootfs for build to avoid later filesystem extension during agent startups + launch_block_device_mappings { + delete_on_termination = true + device_name = "/dev/sda1" + volume_size = local.windows_disk_size_gb + volume_type = "gp2" + } + + # Where to build the VM + region = var.aws_region + + # Where to export the AMI + ami_regions = [ + var.aws_region + ] + + # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? + source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id + + # To improve audit and garbage collecting, we provide tags + tags = { + imageplatform = var.architecture + imagetype = local.image_name + timestamp = local.now_unix_timestamp + version = var.image_version + scm_ref = var.scm_ref + build_type = var.build_type + } +} + +# This source defines all the common settings for any Azure image (whatever Operating System) +source "azure-arm" "base" { + managed_image_name = local.image_name + managed_image_resource_group_name = local.azure_resource_group + async_resourcegroup_delete = true # Faster builds, but no deletion error reporting + + # Azure API connection + client_id = var.azure_client_id + client_secret = var.azure_client_secret + subscription_id = var.azure_subscription_id + + # Where to build the VM + location = "East US" + + # Where to export the image + shared_image_gallery_destination { + subscription = var.azure_subscription_id + resource_group = local.azure_resource_group + gallery_name = "${var.build_type}_packer_images" + image_name = local.image_name + image_version = var.image_version + replication_regions = lookup(local.azure_galleries, "${var.build_type}_packer_images", []) + } + + # To improve audit and garbage collecting, we provide tags + azure_tags = { + imageplatform = var.architecture + imagetype = local.image_name + timestamp = local.now_unix_timestamp + version = var.image_version + scm_ref = var.scm_ref + build_type = var.build_type + } +} + +# This source defines all the common settings for any Azure image (whatever Operating System) +source "docker" "base" { + image = "${var.agent_os_type}:${var.agent_os_version}" + + # Persist image on local docker engine + commit = true + + # To improve audit and garbage collecting, we provide "labels" to the image + changes = [ + "LABEL timestamp = ${local.now_unix_timestamp}", + "LABEL version = ${var.image_version}", + "LABEL scm_ref = ${var.scm_ref}", + "LABEL build_type = ${var.build_type}", + ] +} diff --git a/updatecli/updatecli.d/azure-cli.yml b/updatecli/updatecli.d/azure-cli.yml index 4038871c2..5e7ed93b3 100644 --- a/updatecli/updatecli.d/azure-cli.yml +++ b/updatecli/updatecli.d/azure-cli.yml @@ -35,7 +35,7 @@ targets: sourceid: lastReleaseVersion kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "azurecli_version" scmid: default diff --git a/updatecli/updatecli.d/ciPodTemplate.yml b/updatecli/updatecli.d/ciPodTemplate.yml deleted file mode 100644 index 59aba372a..000000000 --- a/updatecli/updatecli.d/ciPodTemplate.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: Bump `hashicorp-tools` docker image of ci-pod-template.yml - -scms: - default: - kind: github - spec: - user: "{{ .github.user }}" - email: "{{ .github.email }}" - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: "{{ requiredEnv .github.token }}" - username: "{{ .github.username }}" - branch: "{{ .github.branch }}" - -sources: - dockerHashicorpToolsImageVersion: - kind: githubrelease - spec: - owner: "jenkins-infra" - repository: "docker-hashicorp-tools" - token: "{{ requiredEnv .github.token }}" - username: "{{ .github.username }}" - -conditions: - isContainerHashicorpToolsDefined: - name: "Is there a container named 'hashicorp-tools' defined as the first container?" - kind: yaml - disablesourceinput: true - spec: - file: "CiPodTemplate.yaml" - key: spec.containers[0].name - value: jnlp - -targets: - updateImageOnPodTemplateForHashicorpTools: - name: "Update the value of the key 'image' of the first container ('hashicorp-tools')" - kind: yaml - scmid: default - transformers: - - addPrefix: "jenkinsciinfra/hashicorp-tools:" - spec: - file: "CiPodTemplate.yaml" - key: spec.containers[0].image - -pullrequests: - default: - kind: github - scmid: default - targets: - - updateImageOnPodTemplateForHashicorpTools - spec: - labels: - - dependencies diff --git a/updatecli/updatecli.d/compose.yml b/updatecli/updatecli.d/compose.yml index 535aeb2cf..a132bed6f 100644 --- a/updatecli/updatecli.d/compose.yml +++ b/updatecli/updatecli.d/compose.yml @@ -31,7 +31,7 @@ targets: name: Update the DockerCompose version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "compose_version" scmid: default diff --git a/updatecli/updatecli.d/container-structure-test.yml b/updatecli/updatecli.d/container-structure-test.yml index fa557a33e..d43a88044 100644 --- a/updatecli/updatecli.d/container-structure-test.yml +++ b/updatecli/updatecli.d/container-structure-test.yml @@ -31,7 +31,7 @@ targets: sourceid: lastReleaseVersion kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "cst_version" scmid: default diff --git a/updatecli/updatecli.d/git-lfs.yml b/updatecli/updatecli.d/git-lfs.yml index 1c45ef6ba..db483e809 100644 --- a/updatecli/updatecli.d/git-lfs.yml +++ b/updatecli/updatecli.d/git-lfs.yml @@ -31,7 +31,7 @@ targets: name: Update the Git-LFS version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "git_lfs_version" scmid: default diff --git a/updatecli/updatecli.d/git-linux.yml b/updatecli/updatecli.d/git-linux.yml index 53ebedd7a..6a1ed6d90 100644 --- a/updatecli/updatecli.d/git-linux.yml +++ b/updatecli/updatecli.d/git-linux.yml @@ -31,7 +31,7 @@ targets: name: Update the Git version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "git_linux_version" scmid: default diff --git a/updatecli/updatecli.d/git-windows.yml b/updatecli/updatecli.d/git-windows.yml index 5afb5bf8e..e77998eee 100644 --- a/updatecli/updatecli.d/git-windows.yml +++ b/updatecli/updatecli.d/git-windows.yml @@ -35,7 +35,7 @@ targets: name: Update the Git Windows version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "git_windows_version" scmid: default diff --git a/updatecli/updatecli.d/github-cli.yml b/updatecli/updatecli.d/github-cli.yml index df7d539f4..5fb045684 100644 --- a/updatecli/updatecli.d/github-cli.yml +++ b/updatecli/updatecli.d/github-cli.yml @@ -31,7 +31,7 @@ targets: sourceid: lastReleaseVersion kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "gh_version" scmid: default diff --git a/updatecli/updatecli.d/hadolint.yml b/updatecli/updatecli.d/hadolint.yml index b69404ad5..98f15d239 100644 --- a/updatecli/updatecli.d/hadolint.yml +++ b/updatecli/updatecli.d/hadolint.yml @@ -31,7 +31,7 @@ targets: sourceid: lastReleaseVersion kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "hadolint_version" scmid: default diff --git a/updatecli/updatecli.d/jdk11.yml b/updatecli/updatecli.d/jdk11.yml index 22feaca06..8581706a8 100644 --- a/updatecli/updatecli.d/jdk11.yml +++ b/updatecli/updatecli.d/jdk11.yml @@ -40,7 +40,7 @@ targets: name: Update the JDK11 version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "jdk11_version" scmid: default diff --git a/updatecli/updatecli.d/jdk17.yml b/updatecli/updatecli.d/jdk17.yml index 3ac0438a3..bf3c53a6a 100644 --- a/updatecli/updatecli.d/jdk17.yml +++ b/updatecli/updatecli.d/jdk17.yml @@ -40,7 +40,7 @@ targets: name: Update the JDK17 version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "jdk17_version" scmid: default diff --git a/updatecli/updatecli.d/jdk8.yml b/updatecli/updatecli.d/jdk8.yml index 61a2eeb46..7c91e67d0 100644 --- a/updatecli/updatecli.d/jdk8.yml +++ b/updatecli/updatecli.d/jdk8.yml @@ -40,7 +40,7 @@ targets: name: Update the JDK8 version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "jdk8_version" scmid: default diff --git a/updatecli/updatecli.d/jq.yml b/updatecli/updatecli.d/jq.yml index d0d41455e..7ed6a421e 100644 --- a/updatecli/updatecli.d/jq.yml +++ b/updatecli/updatecli.d/jq.yml @@ -35,7 +35,7 @@ targets: sourceid: lastReleaseVersion kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "jq_version" scmid: default diff --git a/updatecli/updatecli.d/jx-release-version.yml b/updatecli/updatecli.d/jx-release-version.yml index 37f4e881d..4c6bbdeac 100644 --- a/updatecli/updatecli.d/jx-release-version.yml +++ b/updatecli/updatecli.d/jx-release-version.yml @@ -31,7 +31,7 @@ targets: sourceid: lastReleaseVersion kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "jxreleaseversion_version" scmid: default diff --git a/updatecli/updatecli.d/maven.yml b/updatecli/updatecli.d/maven.yml index 93651a05a..4fb3813d7 100644 --- a/updatecli/updatecli.d/maven.yml +++ b/updatecli/updatecli.d/maven.yml @@ -39,7 +39,7 @@ targets: name: Update the Maven version in the Packer default values kind: yaml spec: - file: "provision-env.yml" + file: "provisioning/tools-versions.yml" key: "maven_version" scmid: default diff --git a/updatecli/updatecli.d/packer-plugins/amazon.yaml b/updatecli/updatecli.d/packer-plugins/amazon.yaml new file mode 100644 index 000000000..c7e41e93b --- /dev/null +++ b/updatecli/updatecli.d/packer-plugins/amazon.yaml @@ -0,0 +1,48 @@ +title: "Bump packer's amazon plugin version" + +scms: + default: + kind: github + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + branch: "{{ .github.branch }}" + +sources: + lastReleaseVersion: + kind: githubrelease + name: Get the latest packer's amazon plugin version + spec: + owner: "hashicorp" + repository: "packer-plugin-amazon" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + versionfilter: + kind: semver + +targets: + updateVersion: + name: "Update the amazon plugin version in main.pkr.hcl" + sourceid: lastReleaseVersion + kind: file + spec: + file: main.pkr.hcl + matchpattern: 'amazon = \{((\r\n|\r|\n)(\s+))version = ".*"' + replacepattern: >- + amazon = {${1}version = "{{ source "lastReleaseVersion" }}" + scmid: default + +pullrequests: + default: + kind: github + scmid: default + targets: + - updateVersion + spec: + labels: + - dependencies + - packer-amazon-plugin diff --git a/updatecli/updatecli.d/packer-plugins/azure.yaml b/updatecli/updatecli.d/packer-plugins/azure.yaml new file mode 100644 index 000000000..8560b238a --- /dev/null +++ b/updatecli/updatecli.d/packer-plugins/azure.yaml @@ -0,0 +1,48 @@ +title: "Bump packer's azure plugin version" + +scms: + default: + kind: github + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + branch: "{{ .github.branch }}" + +sources: + lastReleaseVersion: + kind: githubrelease + name: Get the latest packer's azure plugin version + spec: + owner: "hashicorp" + repository: "packer-plugin-azure" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + versionfilter: + kind: semver + +targets: + updateVersion: + name: "Update the azure plugin version in main.pkr.hcl" + sourceid: lastReleaseVersion + kind: file + spec: + file: main.pkr.hcl + matchpattern: 'azure = \{((\r\n|\r|\n)(\s+))version = ".*"' + replacepattern: >- + azure = {${1}version = "{{ source "lastReleaseVersion" }}" + scmid: default + +pullrequests: + default: + kind: github + scmid: default + targets: + - updateVersion + spec: + labels: + - dependencies + - packer-azure-plugin diff --git a/updatecli/updatecli.d/packer-plugins/docker.yaml b/updatecli/updatecli.d/packer-plugins/docker.yaml new file mode 100644 index 000000000..e58a29304 --- /dev/null +++ b/updatecli/updatecli.d/packer-plugins/docker.yaml @@ -0,0 +1,48 @@ +title: "Bump packer's docker plugin version" + +scms: + default: + kind: github + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + branch: "{{ .github.branch }}" + +sources: + lastReleaseVersion: + kind: githubrelease + name: Get the latest packer's docker plugin version + spec: + owner: "hashicorp" + repository: "packer-plugin-docker" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + versionfilter: + kind: semver + +targets: + updateVersion: + name: "Update the docker plugin version in main.pkr.hcl" + sourceid: lastReleaseVersion + kind: file + spec: + file: main.pkr.hcl + matchpattern: 'docker = \{((\r\n|\r|\n)(\s+))version = ".*"' + replacepattern: >- + docker = {${1}version = "{{ source "lastReleaseVersion" }}" + scmid: default + +pullrequests: + default: + kind: github + scmid: default + targets: + - updateVersion + spec: + labels: + - dependencies + - packer-docker-plugin diff --git a/updatecli/updatecli.d/packer-plugins/windows-update.yaml b/updatecli/updatecli.d/packer-plugins/windows-update.yaml new file mode 100644 index 000000000..67fce50d5 --- /dev/null +++ b/updatecli/updatecli.d/packer-plugins/windows-update.yaml @@ -0,0 +1,48 @@ +title: "Bump packer's windows-update plugin version" + +scms: + default: + kind: github + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + branch: "{{ .github.branch }}" + +sources: + lastReleaseVersion: + kind: githubrelease + name: Get the latest packer's windows-update plugin version + spec: + owner: "rgl" + repository: "packer-plugin-windows-update" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + versionfilter: + kind: semver + +targets: + updateVersion: + name: "Update the windows-update plugin version in main.pkr.hcl" + sourceid: lastReleaseVersion + kind: file + spec: + file: main.pkr.hcl + matchpattern: 'windows-update = \{((\r\n|\r|\n)(\s+))version = ".*"' + replacepattern: >- + windows-update = {${1}version = "{{ source "lastReleaseVersion" }}" + scmid: default + +pullrequests: + default: + kind: github + scmid: default + targets: + - updateVersion + spec: + labels: + - dependencies + - packer-windows-update-plugin diff --git a/variables.pkr.hcl b/variables.pkr.hcl new file mode 100644 index 000000000..8c12aa504 --- /dev/null +++ b/variables.pkr.hcl @@ -0,0 +1,55 @@ +# Defines these 2 elements as "variables" instead of "locals" to allow using them in data sources +# as per https:#github.com/hashicorp/packer/issues/11011 +variable "agent_os_type" { + type = string + description = "Which operating system to use for this agent template build?" +} +variable "agent_os_version" { + type = string + description = "Which version of the operating system 'var.agent_os_type' to use for this agent template build?" +} +variable "architecture" { + type = string + description = "CPU architecture ID of the build with the following possible values: [amd64 (default), arm64]" + default = "amd64" +} +variable "aws_region" { + type = string + default = "us-east-2" +} +variable "azure_client_id" { + type = string + default = "" +} +variable "azure_client_secret" { + type = string + default = "" +} +variable "azure_subscription_id" { + default = "" + type = string +} +variable "image_version" { + type = string + default = "0.0.1" # Default is a valid version to not fail azure validation +} +variable "image_type" { + type = string + description = "Which kind of Packer builder to use (e.g. cloud platform): [amazon-ebs (default), azure-arm, docker]" + default = "amazon-ebs" +} +variable "build_type" { + type = string + description = "Type of build e.g. is it a development build (from a contributor machine), a ci build (pull request or branch build) or a production build (principal branch build on ci)?" + default = "dev" # value in ["dev", "ci", "prod"] +} +variable "scm_ref" { + type = string + description = "SCM (e.g. Git...) reference of the current build. Can be a commit hash (short or long), a branch name or a tag name." + default = "HEAD" +} +variable "provision_env_file" { + type = string + description = "Path (absolute or relative to this packer template) to the YAML file with the list of environment variables forwarded to provisioners (mainly tools versions)" + default = "provisioning/tools-versions.yml" +}