From c1263bccedd46deca60a6393a8dac4b3f994c27f Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 4 Jun 2024 08:36:06 +0200 Subject: [PATCH 1/8] WIP --- locals.tf | 7 +- privatek8s-sponsored.tf | 288 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 privatek8s-sponsored.tf diff --git a/locals.tf b/locals.tf index 68dfb0a6..673790b8 100644 --- a/locals.tf +++ b/locals.tf @@ -42,9 +42,10 @@ locals { admin_username = "jenkins-infra-team" kubernetes_versions = { - "privatek8s" = "1.27.9" - "publick8s" = "1.27.9" - "cijenkinsio_agents_1" = "1.27.9" + "privatek8s" = "1.27.9" + "privatek8s-sponsorship" = "1.27.9" + "publick8s" = "1.27.9" + "cijenkinsio_agents_1" = "1.27.9" } ci_jenkins_io_fqdn = "ci.jenkins.io" ci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" diff --git a/privatek8s-sponsored.tf b/privatek8s-sponsored.tf new file mode 100644 index 00000000..fe23b336 --- /dev/null +++ b/privatek8s-sponsored.tf @@ -0,0 +1,288 @@ +resource "azurerm_resource_group" "privatek8s_sponsorship" { + provider = azurerm.jenkins-sponsorship + name = "prod-privatek8s-sponsorship" + location = var.location + tags = local.default_tags +} + +data "azurerm_subnet" "privatek8s_tier" { + name = "privatek8s-tier" + resource_group_name = data.azurerm_resource_group.private.name + virtual_network_name = data.azurerm_virtual_network.private.name +} + +data "azurerm_subnet" "privatek8s_release_tier" { + name = "privatek8s-release-tier" + resource_group_name = data.azurerm_resource_group.private.name + virtual_network_name = data.azurerm_virtual_network.private.name +} + +data "azurerm_subnet" "privatek8s_infra_ci_controller_tier" { + name = "privatek8s-infraci-ctrl-tier" + resource_group_name = data.azurerm_resource_group.private.name + virtual_network_name = data.azurerm_virtual_network.private.name +} + +data "azurerm_subnet" "privatek8s_release_ci_controller_tier" { + name = "privatek8s-releaseci-ctrl-tier" + resource_group_name = data.azurerm_resource_group.private.name + virtual_network_name = data.azurerm_virtual_network.private.name +} + +#trivy:ignore:azure-container-logging #trivy:ignore:azure-container-limit-authorized-ips +resource "azurerm_kubernetes_cluster" "privatek8s_sponsorship" { + name = "privatek8s-sponsorship" + location = azurerm_resource_group.privatek8s_sponsorship.location + resource_group_name = azurerm_resource_group.privatek8s_sponsorship.name + kubernetes_version = local.kubernetes_versions["privatek8s_sponsorship"] + dns_prefix = "privatek8s-sponsorship" + role_based_access_control_enabled = true # default value but made explicit to please trivy + + api_server_access_profile { + authorized_ip_ranges = setunion( + formatlist( + "%s/32", + flatten( + concat( + [for key, value in module.jenkins_infra_shared_data.admin_public_ips : value], + # privatek8s_sponsorship outbound IPs (traffic routed through gateways or outbound LBs) + module.jenkins_infra_shared_data.outbound_ips["privatek8s.jenkins.io"], + ) + ) + ), + data.azurerm_subnet.private_vnet_data_tier.address_prefixes, + ) + } + + network_profile { + network_plugin = "azure" + network_policy = "azure" + outbound_type = "loadBalancer" + load_balancer_sku = "standard" + load_balancer_profile { + outbound_ports_allocated = "1088" # Max 58 Nodes, <64000 total + idle_timeout_in_minutes = "4" + managed_outbound_ip_count = "1" + } + } + + default_node_pool { + name = "syspool" + vm_size = "Standard_D2as_v4" + os_sku = "Ubuntu" + os_disk_type = "Ephemeral" + os_disk_size_gb = 50 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dav4-dasv4-series#dasv4-series (depends on the instance size) + orchestrator_version = local.kubernetes_versions["privatek8s-sponsorship"] + kubelet_disk_type = "OS" + enable_auto_scaling = true + min_count = 1 + max_count = 3 + vnet_subnet_id = data.azurerm_subnet.privatek8s_tier.id + tags = local.default_tags + zones = [3] + } + + identity { + type = "SystemAssigned" + } + + lifecycle { + ignore_changes = [default_node_pool[0].node_count] + } + + tags = local.default_tags +} + +resource "azurerm_kubernetes_cluster_node_pool" "infraciarm64_sponsorship" { + name = "arm64smallSp" + vm_size = "Standard_D4pds_v5" # 4 vCPU, 16 GB RAM, local disk: 150 GB and 19000 IOPS + os_disk_type = "Ephemeral" + os_disk_size_gb = 150 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dpsv5-dpdsv5-series#dpdsv5-series (depends on the instance size) + orchestrator_version = local.kubernetes_versions["privatek8s-sponsorship"] + kubernetes_cluster_id = azurerm_kubernetes_cluster.privatek8s_sponsorship.id + enable_auto_scaling = true + min_count = 1 + max_count = 10 + zones = [1] # Linux arm64 VMs are only available in the Zone 1 in this region (undocumented by Azure) + vnet_subnet_id = data.azurerm_subnet.privatek8s_tier.id + + # Spot instances + priority = "Spot" + eviction_policy = "Delete" + spot_max_price = "-1" # in $, -1 = On demand pricing + # Note: label and taint added automatically when in "Spot" priority, putting it here to explicit them + node_labels = { + "kubernetes.azure.com/scalesetpriority" = "spot" + } + node_taints = [ + "jenkins=infra.ci.jenkins.io:NoSchedule", + "kubernetes.azure.com/scalesetpriority=spot:NoSchedule", + ] + lifecycle { + ignore_changes = [node_count] + } + + tags = local.default_tags +} + + + + + + + + + + + + + + + + +# Allow cluster to manage LBs in the privatek8s-tier subnet (Public LB) +resource "azurerm_role_assignment" "privatek8s_networkcontributor" { + scope = data.azurerm_subnet.privatek8s_tier.id + role_definition_name = "Network Contributor" + principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id + skip_service_principal_aad_check = true +} + +# Allow cluster to manage LBs in the data-tier subnet (internal LBs) +resource "azurerm_role_assignment" "datatier_networkcontributor" { + scope = data.azurerm_subnet.private_vnet_data_tier.id + role_definition_name = "Network Contributor" + principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id + skip_service_principal_aad_check = true +} + +# Allow cluster to manage LBs in the data-tier subnet (internal LBs) +resource "azurerm_role_assignment" "publicip_networkcontributor" { + scope = azurerm_public_ip.public_privatek8s.id + role_definition_name = "Network Contributor" + principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id + skip_service_principal_aad_check = true +} + +# Allow cluster to manage get.jenkins.io storage account +resource "azurerm_role_assignment" "getjenkinsio_storage_account_contributor" { + scope = azurerm_storage_account.get_jenkins_io.id + role_definition_name = "Storage Account Contributor" + principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id + skip_service_principal_aad_check = true +} + +resource "kubernetes_storage_class" "managed_csi_premium_retain" { + metadata { + name = "managed-csi-premium-retain" + } + storage_provisioner = "disk.csi.azure.com" + reclaim_policy = "Retain" + parameters = { + skuname = "Premium_LRS" + } + provider = kubernetes.privatek8s +} + +resource "kubernetes_storage_class" "azurefile_csi_premium_retain" { + metadata { + name = "azurefile-csi-premium-retain" + } + storage_provisioner = "file.csi.azure.com" + reclaim_policy = "Retain" + parameters = { + skuname = "Premium_LRS" + } + mount_options = ["dir_mode=0777", "file_mode=0777", "uid=1000", "gid=1000", "mfsymlinks", "nobrl"] + provider = kubernetes.privatek8s +} + +resource "kubernetes_storage_class" "managed_csi_premium_ZRS_retain_private" { + metadata { + name = "managed-csi-premium-zrs-retain" + } + storage_provisioner = "disk.csi.azure.com" + reclaim_policy = "Retain" + parameters = { + skuname = "Premium_ZRS" + } + provider = kubernetes.privatek8s + allow_volume_expansion = true +} + +# https://learn.microsoft.com/en-us/java/api/com.microsoft.azure.management.storage.skuname?view=azure-java-legacy#field-summary +resource "kubernetes_storage_class" "managed_csi_standard_ZRS_retain_private" { + metadata { + name = "managed-csi-standard-zrs-retain" + } + storage_provisioner = "disk.csi.azure.com" + reclaim_policy = "Retain" + parameters = { + skuname = " Standard_ZRS" + } + provider = kubernetes.privatek8s + allow_volume_expansion = true +} + +# Used later by the load balancer deployed on the cluster, see https://github.com/jenkins-infra/kubernetes-management/config/privatek8s.yaml +resource "azurerm_public_ip" "public_privatek8s" { + name = "public-privatek8s" + resource_group_name = azurerm_resource_group.prod_public_ips.name + location = var.location + allocation_method = "Static" + sku = "Standard" # Needed to fix the error "PublicIPAndLBSkuDoNotMatch" + tags = local.default_tags +} +resource "azurerm_management_lock" "public_privatek8s_publicip" { + name = "public-privatek8s-publicip" + scope = azurerm_public_ip.public_privatek8s.id + lock_level = "CanNotDelete" + notes = "Locked because this is a sensitive resource that should not be removed when privatek8s is removed" +} + +resource "azurerm_dns_a_record" "public_privatek8s" { + name = "public.privatek8s" + zone_name = data.azurerm_dns_zone.jenkinsio.name + resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name + ttl = 300 + records = [azurerm_public_ip.public_privatek8s.ip_address] + tags = local.default_tags +} + +resource "azurerm_dns_a_record" "private_privatek8s" { + name = "private.privatek8s" + zone_name = data.azurerm_dns_zone.jenkinsio.name + resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name + ttl = 300 + records = ["10.248.1.5"] # External IP of the private-nginx ingress LoadBalancer, created by https://github.com/jenkins-infra/kubernetes-management/blob/54a0d4aa72b15f4236abcfbde00a080905bbb890/clusters/privatek8s.yaml#L112-L118 + tags = local.default_tags +} + +output "privatek8s_kube_config" { + value = azurerm_kubernetes_cluster.privatek8s.kube_config_raw + sensitive = true +} + +output "privatek8s_public_ip_address" { + value = azurerm_public_ip.public_privatek8s.ip_address +} + +# Configure the jenkins-infra/kubernetes-management admin service account +module "privatek8s_admin_sa" { + providers = { + kubernetes = kubernetes.privatek8s + } + source = "./.shared-tools/terraform/modules/kubernetes-admin-sa" + cluster_name = azurerm_kubernetes_cluster.privatek8s.name + cluster_hostname = azurerm_kubernetes_cluster.privatek8s.kube_config.0.host + cluster_ca_certificate_b64 = azurerm_kubernetes_cluster.privatek8s.kube_config.0.cluster_ca_certificate +} + +output "kubeconfig_privatek8s" { + sensitive = true + value = module.privatek8s_admin_sa.kubeconfig +} + +output "privatek8s_kube_config_command" { + value = "az aks get-credentials --name ${azurerm_kubernetes_cluster.privatek8s.name} --resource-group ${azurerm_kubernetes_cluster.privatek8s.resource_group_name}" +} From 809a7c97cf2e41758e63fbd3dbb2c7c9c688f601 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 4 Jun 2024 14:10:54 +0200 Subject: [PATCH 2/8] wip --- ....jenkins.io-kubernetes-sponsored-agents.tf | 173 ++++++++++++++++++ locals.tf | 7 +- ...nsored.tf => privatek8s-sponsored-DRAFT.tf | 0 3 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 infraci.jenkins.io-kubernetes-sponsored-agents.tf rename privatek8s-sponsored.tf => privatek8s-sponsored-DRAFT.tf (100%) diff --git a/infraci.jenkins.io-kubernetes-sponsored-agents.tf b/infraci.jenkins.io-kubernetes-sponsored-agents.tf new file mode 100644 index 00000000..6be0abb8 --- /dev/null +++ b/infraci.jenkins.io-kubernetes-sponsored-agents.tf @@ -0,0 +1,173 @@ +resource "azurerm_resource_group" "infracijio_kubernetes_agents_sponsorship" { + provider = azurerm.jenkins-sponsorship + name = "infra-ci-jenkins-io-kubernetes-agents" + location = var.location + tags = local.default_tags +} + +data "azurerm_subnet" "infraci_jenkins_io_kubernetes_agent_sponsorship" { + provider = azurerm.jenkins-sponsorship + name = "${data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.name}-infraci_jenkins_io_kubernetes-agent-sponsorship" + resource_group_name = data.azurerm_resource_group.public_jenkins_sponsorship.name + virtual_network_name = data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.name +} + +#trivy:ignore:avd-azu-0040 # No need to enable oms_agent for Azure monitoring as we already have datadog +resource "azurerm_kubernetes_cluster" "infracijenkinsio_agents_1" { + provider = azurerm.jenkins-sponsorship + name = "infracijenkinsio-agents-1" + sku_tier = "standard" + ## Private cluster requires network setup to allow API access from: + # - infra.ci.jenkins.io agents (for both terraform job agents and kubernetes-management agents) + # - private.vpn.jenkins.io to allow admin management (either Azure UI or kube tools from admin machines) + private_cluster_enabled = true + private_cluster_public_fqdn_enabled = true + dns_prefix = "infracijenkinsioagents1" # Avoid hyphens in this DNS host + location = azurerm_resource_group.infracijio_kubernetes_agents_sponsorship.location + resource_group_name = azurerm_resource_group.infracijio_kubernetes_agents_sponsorship.name + kubernetes_version = local.kubernetes_versions["infracijenkinsio_agents_1"] + role_based_access_control_enabled = true # default value but made explicit to please trivy + + network_profile { + network_plugin = "azure" + network_plugin_mode = "overlay" + network_policy = "azure" + outbound_type = "userAssignedNATGateway" + load_balancer_sku = "standard" # Required to customize the outbound type + pod_cidr = local.infraci_jenkins_io_agents_1_pod_cidr + } + + identity { + type = "SystemAssigned" + } + + default_node_pool { + name = "systempool1" + only_critical_addons_enabled = true # This property is the only valid way to add the "CriticalAddonsOnly=true:NoSchedule" taint to the default node pool + vm_size = "Standard_D4pds_v5" # At least 4 vCPUS/4 Gb as per AKS best practises + os_sku = "AzureLinux" + os_disk_type = "Ephemeral" + os_disk_size_gb = 150 # Ref. Cache storage size athttps://learn.microsoft.com/fr-fr/azure/virtual-machines/dasv5-dadsv5-series#dadsv5-series (depends on the instance size) + orchestrator_version = local.kubernetes_versions["infracijenkinsio_agents_1"] + kubelet_disk_type = "OS" + enable_auto_scaling = false + node_count = 3 # 3 nodes for HA as per AKS best practises + vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id + tags = local.default_tags + zones = local.cijenkinsio_agents_1_compute_zones + } + + tags = local.default_tags +} + +# Node pool to host "jenkins-infra" applications required on this cluster such as ACP or datadog's cluster-agent, e.g. "Not agent, neither AKS System tools" +resource "azurerm_kubernetes_cluster_node_pool" "linux_arm64_n2_applications" { + provider = azurerm.jenkins-sponsorship + name = "la64n2app" + vm_size = "Standard_D4pds_v5" + os_disk_type = "Ephemeral" + os_disk_size_gb = 150 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dsv3-series (depends on the instance size) + orchestrator_version = local.kubernetes_versions["infracijenkinsio_agents_1"] + kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.id + enable_auto_scaling = true + min_count = 1 + max_count = 3 # 2 nodes always up for HA, a 3rd one is allowed for surge upgrades + zones = local.cijenkinsio_agents_1_compute_zones + vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id + + node_labels = { + "jenkins" = "ci.jenkins.io" + "role" = "applications" + } + node_taints = [ + "ci.jenkins.io/applications=true:NoSchedule", + ] + + lifecycle { + ignore_changes = [node_count] + } + + tags = local.default_tags +} + +# Node pool to host ci.jenkins.io agents for usual builds +resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_agents_1" { + provider = azurerm.jenkins-sponsorship + name = "lx86n3agt1" + vm_size = "Standard_D16ads_v5" + os_disk_type = "Ephemeral" + os_disk_size_gb = 600 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dsv3-series (depends on the instance size) + orchestrator_version = local.kubernetes_versions["infracijenkinsio_agents_1"] + kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.id + enable_auto_scaling = true + min_count = 0 + max_count = 40 # 3 pods per nodes, max 120 pods - due to quotas + zones = local.infracijenkinsio_agents_1_compute_zones + vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id + + node_labels = { + "jenkins" = "ci.jenkins.io" + "role" = "jenkins-agents" + } + node_taints = [ + "ci.jenkins.io/agents=true:NoSchedule", + ] + + lifecycle { + ignore_changes = [node_count] + } + + tags = local.default_tags +} + +# Node pool to host ci.jenkins.io agents for BOM builds +resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_bom_1" { + provider = azurerm.jenkins-sponsorship + name = "lx86n3bom1" + vm_size = "Standard_D16ads_v5" + os_disk_type = "Ephemeral" + os_disk_size_gb = 600 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dsv3-series (depends on the instance size) + orchestrator_version = local.kubernetes_versions["cijenkinsio_agents_1"] + kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.id + enable_auto_scaling = true + min_count = 0 + max_count = 50 + zones = local.cijenkinsio_agents_1_compute_zones + vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id + + node_labels = { + "jenkins" = "ci.jenkins.io" + "role" = "jenkins-agents-bom" + } + node_taints = [ + "ci.jenkins.io/agents=true:NoSchedule", + "ci.jenkins.io/bom=true:NoSchedule", + ] + + lifecycle { + ignore_changes = [node_count] + } + + tags = local.default_tags +} + + + + +# Configure the jenkins-infra/kubernetes-management admin service account +module "cijenkinsio_agents_1_admin_sa" { + providers = { + kubernetes = kubernetes.cijenkinsio_agents_1 + } + source = "./.shared-tools/terraform/modules/kubernetes-admin-sa" + cluster_name = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.name + cluster_hostname = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.fqdn + cluster_ca_certificate_b64 = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.kube_config.0.cluster_ca_certificate +} +output "kubeconfig_cijenkinsio_agents_1" { + sensitive = true + value = module.cijenkinsio_agents_1_admin_sa.kubeconfig +} +output "cijenkinsio_agents_1_kube_config_command" { + value = "az aks get-credentials --name ${azurerm_kubernetes_cluster.infracijenkinsio_agents_1.name} --resource-group ${azurerm_kubernetes_cluster.infracijenkinsio_agents_1.resource_group_name}" +} diff --git a/locals.tf b/locals.tf index 673790b8..914d6efa 100644 --- a/locals.tf +++ b/locals.tf @@ -43,15 +43,18 @@ locals { kubernetes_versions = { "privatek8s" = "1.27.9" - "privatek8s-sponsorship" = "1.27.9" + "infracijenkinsio_agents_1" = "1.27.9" "publick8s" = "1.27.9" "cijenkinsio_agents_1" = "1.27.9" } ci_jenkins_io_fqdn = "ci.jenkins.io" - ci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" + ci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 publick8s_compute_zones = [3] cijenkinsio_agents_1_compute_zones = [1] + infracijenkinsio_agents_1_compute_zones = [1] + + infraci_jenkins_io_agents_1_pod_cidr = "10.110.0.0/14" # 10.110.0.1 - 10.113.0.255 or 10.108.0.1 - 10.111.255.255 weekly_ci_disk_size = 8 weekly_ci_access_modes = ["ReadWriteOnce"] diff --git a/privatek8s-sponsored.tf b/privatek8s-sponsored-DRAFT.tf similarity index 100% rename from privatek8s-sponsored.tf rename to privatek8s-sponsored-DRAFT.tf From cb26d42a5e6c51bffbb8d8ead07d14c3333de11e Mon Sep 17 00:00:00 2001 From: smerle33 Date: Tue, 4 Jun 2024 16:20:32 +0200 Subject: [PATCH 3/8] use the same ip class for pods as ci --- locals.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locals.tf b/locals.tf index 914d6efa..d6d144d7 100644 --- a/locals.tf +++ b/locals.tf @@ -54,7 +54,7 @@ locals { cijenkinsio_agents_1_compute_zones = [1] infracijenkinsio_agents_1_compute_zones = [1] - infraci_jenkins_io_agents_1_pod_cidr = "10.110.0.0/14" # 10.110.0.1 - 10.113.0.255 or 10.108.0.1 - 10.111.255.255 + infraci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 weekly_ci_disk_size = 8 weekly_ci_access_modes = ["ReadWriteOnce"] From 7a24b41985fab18aaaac32257bf6024c71ce2a7a Mon Sep 17 00:00:00 2001 From: smerle33 Date: Fri, 7 Jun 2024 16:28:39 +0200 Subject: [PATCH 4/8] cleanup --- .gitignore | 2 + privatek8s-sponsored-DRAFT.tf | 288 ---------------------------------- 2 files changed, 2 insertions(+), 288 deletions(-) delete mode 100644 privatek8s-sponsored-DRAFT.tf diff --git a/.gitignore b/.gitignore index 73b33fdf..05c447b6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ backend-config terraform-plan-output.txt tfplan +# temporary and local test +.tmp/ # sensitive files from terraform outputs .env* *.zip diff --git a/privatek8s-sponsored-DRAFT.tf b/privatek8s-sponsored-DRAFT.tf deleted file mode 100644 index fe23b336..00000000 --- a/privatek8s-sponsored-DRAFT.tf +++ /dev/null @@ -1,288 +0,0 @@ -resource "azurerm_resource_group" "privatek8s_sponsorship" { - provider = azurerm.jenkins-sponsorship - name = "prod-privatek8s-sponsorship" - location = var.location - tags = local.default_tags -} - -data "azurerm_subnet" "privatek8s_tier" { - name = "privatek8s-tier" - resource_group_name = data.azurerm_resource_group.private.name - virtual_network_name = data.azurerm_virtual_network.private.name -} - -data "azurerm_subnet" "privatek8s_release_tier" { - name = "privatek8s-release-tier" - resource_group_name = data.azurerm_resource_group.private.name - virtual_network_name = data.azurerm_virtual_network.private.name -} - -data "azurerm_subnet" "privatek8s_infra_ci_controller_tier" { - name = "privatek8s-infraci-ctrl-tier" - resource_group_name = data.azurerm_resource_group.private.name - virtual_network_name = data.azurerm_virtual_network.private.name -} - -data "azurerm_subnet" "privatek8s_release_ci_controller_tier" { - name = "privatek8s-releaseci-ctrl-tier" - resource_group_name = data.azurerm_resource_group.private.name - virtual_network_name = data.azurerm_virtual_network.private.name -} - -#trivy:ignore:azure-container-logging #trivy:ignore:azure-container-limit-authorized-ips -resource "azurerm_kubernetes_cluster" "privatek8s_sponsorship" { - name = "privatek8s-sponsorship" - location = azurerm_resource_group.privatek8s_sponsorship.location - resource_group_name = azurerm_resource_group.privatek8s_sponsorship.name - kubernetes_version = local.kubernetes_versions["privatek8s_sponsorship"] - dns_prefix = "privatek8s-sponsorship" - role_based_access_control_enabled = true # default value but made explicit to please trivy - - api_server_access_profile { - authorized_ip_ranges = setunion( - formatlist( - "%s/32", - flatten( - concat( - [for key, value in module.jenkins_infra_shared_data.admin_public_ips : value], - # privatek8s_sponsorship outbound IPs (traffic routed through gateways or outbound LBs) - module.jenkins_infra_shared_data.outbound_ips["privatek8s.jenkins.io"], - ) - ) - ), - data.azurerm_subnet.private_vnet_data_tier.address_prefixes, - ) - } - - network_profile { - network_plugin = "azure" - network_policy = "azure" - outbound_type = "loadBalancer" - load_balancer_sku = "standard" - load_balancer_profile { - outbound_ports_allocated = "1088" # Max 58 Nodes, <64000 total - idle_timeout_in_minutes = "4" - managed_outbound_ip_count = "1" - } - } - - default_node_pool { - name = "syspool" - vm_size = "Standard_D2as_v4" - os_sku = "Ubuntu" - os_disk_type = "Ephemeral" - os_disk_size_gb = 50 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dav4-dasv4-series#dasv4-series (depends on the instance size) - orchestrator_version = local.kubernetes_versions["privatek8s-sponsorship"] - kubelet_disk_type = "OS" - enable_auto_scaling = true - min_count = 1 - max_count = 3 - vnet_subnet_id = data.azurerm_subnet.privatek8s_tier.id - tags = local.default_tags - zones = [3] - } - - identity { - type = "SystemAssigned" - } - - lifecycle { - ignore_changes = [default_node_pool[0].node_count] - } - - tags = local.default_tags -} - -resource "azurerm_kubernetes_cluster_node_pool" "infraciarm64_sponsorship" { - name = "arm64smallSp" - vm_size = "Standard_D4pds_v5" # 4 vCPU, 16 GB RAM, local disk: 150 GB and 19000 IOPS - os_disk_type = "Ephemeral" - os_disk_size_gb = 150 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dpsv5-dpdsv5-series#dpdsv5-series (depends on the instance size) - orchestrator_version = local.kubernetes_versions["privatek8s-sponsorship"] - kubernetes_cluster_id = azurerm_kubernetes_cluster.privatek8s_sponsorship.id - enable_auto_scaling = true - min_count = 1 - max_count = 10 - zones = [1] # Linux arm64 VMs are only available in the Zone 1 in this region (undocumented by Azure) - vnet_subnet_id = data.azurerm_subnet.privatek8s_tier.id - - # Spot instances - priority = "Spot" - eviction_policy = "Delete" - spot_max_price = "-1" # in $, -1 = On demand pricing - # Note: label and taint added automatically when in "Spot" priority, putting it here to explicit them - node_labels = { - "kubernetes.azure.com/scalesetpriority" = "spot" - } - node_taints = [ - "jenkins=infra.ci.jenkins.io:NoSchedule", - "kubernetes.azure.com/scalesetpriority=spot:NoSchedule", - ] - lifecycle { - ignore_changes = [node_count] - } - - tags = local.default_tags -} - - - - - - - - - - - - - - - - -# Allow cluster to manage LBs in the privatek8s-tier subnet (Public LB) -resource "azurerm_role_assignment" "privatek8s_networkcontributor" { - scope = data.azurerm_subnet.privatek8s_tier.id - role_definition_name = "Network Contributor" - principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id - skip_service_principal_aad_check = true -} - -# Allow cluster to manage LBs in the data-tier subnet (internal LBs) -resource "azurerm_role_assignment" "datatier_networkcontributor" { - scope = data.azurerm_subnet.private_vnet_data_tier.id - role_definition_name = "Network Contributor" - principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id - skip_service_principal_aad_check = true -} - -# Allow cluster to manage LBs in the data-tier subnet (internal LBs) -resource "azurerm_role_assignment" "publicip_networkcontributor" { - scope = azurerm_public_ip.public_privatek8s.id - role_definition_name = "Network Contributor" - principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id - skip_service_principal_aad_check = true -} - -# Allow cluster to manage get.jenkins.io storage account -resource "azurerm_role_assignment" "getjenkinsio_storage_account_contributor" { - scope = azurerm_storage_account.get_jenkins_io.id - role_definition_name = "Storage Account Contributor" - principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id - skip_service_principal_aad_check = true -} - -resource "kubernetes_storage_class" "managed_csi_premium_retain" { - metadata { - name = "managed-csi-premium-retain" - } - storage_provisioner = "disk.csi.azure.com" - reclaim_policy = "Retain" - parameters = { - skuname = "Premium_LRS" - } - provider = kubernetes.privatek8s -} - -resource "kubernetes_storage_class" "azurefile_csi_premium_retain" { - metadata { - name = "azurefile-csi-premium-retain" - } - storage_provisioner = "file.csi.azure.com" - reclaim_policy = "Retain" - parameters = { - skuname = "Premium_LRS" - } - mount_options = ["dir_mode=0777", "file_mode=0777", "uid=1000", "gid=1000", "mfsymlinks", "nobrl"] - provider = kubernetes.privatek8s -} - -resource "kubernetes_storage_class" "managed_csi_premium_ZRS_retain_private" { - metadata { - name = "managed-csi-premium-zrs-retain" - } - storage_provisioner = "disk.csi.azure.com" - reclaim_policy = "Retain" - parameters = { - skuname = "Premium_ZRS" - } - provider = kubernetes.privatek8s - allow_volume_expansion = true -} - -# https://learn.microsoft.com/en-us/java/api/com.microsoft.azure.management.storage.skuname?view=azure-java-legacy#field-summary -resource "kubernetes_storage_class" "managed_csi_standard_ZRS_retain_private" { - metadata { - name = "managed-csi-standard-zrs-retain" - } - storage_provisioner = "disk.csi.azure.com" - reclaim_policy = "Retain" - parameters = { - skuname = " Standard_ZRS" - } - provider = kubernetes.privatek8s - allow_volume_expansion = true -} - -# Used later by the load balancer deployed on the cluster, see https://github.com/jenkins-infra/kubernetes-management/config/privatek8s.yaml -resource "azurerm_public_ip" "public_privatek8s" { - name = "public-privatek8s" - resource_group_name = azurerm_resource_group.prod_public_ips.name - location = var.location - allocation_method = "Static" - sku = "Standard" # Needed to fix the error "PublicIPAndLBSkuDoNotMatch" - tags = local.default_tags -} -resource "azurerm_management_lock" "public_privatek8s_publicip" { - name = "public-privatek8s-publicip" - scope = azurerm_public_ip.public_privatek8s.id - lock_level = "CanNotDelete" - notes = "Locked because this is a sensitive resource that should not be removed when privatek8s is removed" -} - -resource "azurerm_dns_a_record" "public_privatek8s" { - name = "public.privatek8s" - zone_name = data.azurerm_dns_zone.jenkinsio.name - resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name - ttl = 300 - records = [azurerm_public_ip.public_privatek8s.ip_address] - tags = local.default_tags -} - -resource "azurerm_dns_a_record" "private_privatek8s" { - name = "private.privatek8s" - zone_name = data.azurerm_dns_zone.jenkinsio.name - resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name - ttl = 300 - records = ["10.248.1.5"] # External IP of the private-nginx ingress LoadBalancer, created by https://github.com/jenkins-infra/kubernetes-management/blob/54a0d4aa72b15f4236abcfbde00a080905bbb890/clusters/privatek8s.yaml#L112-L118 - tags = local.default_tags -} - -output "privatek8s_kube_config" { - value = azurerm_kubernetes_cluster.privatek8s.kube_config_raw - sensitive = true -} - -output "privatek8s_public_ip_address" { - value = azurerm_public_ip.public_privatek8s.ip_address -} - -# Configure the jenkins-infra/kubernetes-management admin service account -module "privatek8s_admin_sa" { - providers = { - kubernetes = kubernetes.privatek8s - } - source = "./.shared-tools/terraform/modules/kubernetes-admin-sa" - cluster_name = azurerm_kubernetes_cluster.privatek8s.name - cluster_hostname = azurerm_kubernetes_cluster.privatek8s.kube_config.0.host - cluster_ca_certificate_b64 = azurerm_kubernetes_cluster.privatek8s.kube_config.0.cluster_ca_certificate -} - -output "kubeconfig_privatek8s" { - sensitive = true - value = module.privatek8s_admin_sa.kubeconfig -} - -output "privatek8s_kube_config_command" { - value = "az aks get-credentials --name ${azurerm_kubernetes_cluster.privatek8s.name} --resource-group ${azurerm_kubernetes_cluster.privatek8s.resource_group_name}" -} From 0b144a945ba767f2d3d3a7854561aeae4e17fe10 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Fri, 7 Jun 2024 16:57:22 +0200 Subject: [PATCH 5/8] wip --- ....jenkins.io-kubernetes-sponsored-agents.tf | 20 +++++++++---------- locals.tf | 12 +++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/infraci.jenkins.io-kubernetes-sponsored-agents.tf b/infraci.jenkins.io-kubernetes-sponsored-agents.tf index 6be0abb8..ef24b20d 100644 --- a/infraci.jenkins.io-kubernetes-sponsored-agents.tf +++ b/infraci.jenkins.io-kubernetes-sponsored-agents.tf @@ -7,8 +7,8 @@ resource "azurerm_resource_group" "infracijio_kubernetes_agents_sponsorship" { data "azurerm_subnet" "infraci_jenkins_io_kubernetes_agent_sponsorship" { provider = azurerm.jenkins-sponsorship - name = "${data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.name}-infraci_jenkins_io_kubernetes-agent-sponsorship" - resource_group_name = data.azurerm_resource_group.public_jenkins_sponsorship.name + name = "${data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.name}-infraci_jenkins_io_kubernetes-agent" + resource_group_name = data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.resource_group_name virtual_network_name = data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.name } @@ -16,7 +16,7 @@ data "azurerm_subnet" "infraci_jenkins_io_kubernetes_agent_sponsorship" { resource "azurerm_kubernetes_cluster" "infracijenkinsio_agents_1" { provider = azurerm.jenkins-sponsorship name = "infracijenkinsio-agents-1" - sku_tier = "standard" + sku_tier = "Standard" ## Private cluster requires network setup to allow API access from: # - infra.ci.jenkins.io agents (for both terraform job agents and kubernetes-management agents) # - private.vpn.jenkins.io to allow admin management (either Azure UI or kube tools from admin machines) @@ -61,7 +61,7 @@ resource "azurerm_kubernetes_cluster" "infracijenkinsio_agents_1" { } # Node pool to host "jenkins-infra" applications required on this cluster such as ACP or datadog's cluster-agent, e.g. "Not agent, neither AKS System tools" -resource "azurerm_kubernetes_cluster_node_pool" "linux_arm64_n2_applications" { +resource "azurerm_kubernetes_cluster_node_pool" "linux_arm64_n2_applications_sponsorship" { provider = azurerm.jenkins-sponsorship name = "la64n2app" vm_size = "Standard_D4pds_v5" @@ -91,7 +91,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "linux_arm64_n2_applications" { } # Node pool to host ci.jenkins.io agents for usual builds -resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_agents_1" { +resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_agents_1_sponsorship" { provider = azurerm.jenkins-sponsorship name = "lx86n3agt1" vm_size = "Standard_D16ads_v5" @@ -121,7 +121,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_agents_1" { } # Node pool to host ci.jenkins.io agents for BOM builds -resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_bom_1" { +resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_bom_1_sponsorship" { provider = azurerm.jenkins-sponsorship name = "lx86n3bom1" vm_size = "Standard_D16ads_v5" @@ -155,7 +155,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_bom_1" { # Configure the jenkins-infra/kubernetes-management admin service account -module "cijenkinsio_agents_1_admin_sa" { +module "infracijenkinsio_agents_1_admin_sa_sponsorship" { providers = { kubernetes = kubernetes.cijenkinsio_agents_1 } @@ -164,10 +164,10 @@ module "cijenkinsio_agents_1_admin_sa" { cluster_hostname = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.fqdn cluster_ca_certificate_b64 = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.kube_config.0.cluster_ca_certificate } -output "kubeconfig_cijenkinsio_agents_1" { +output "kubeconfig_infracijenkinsio_agents_1" { sensitive = true - value = module.cijenkinsio_agents_1_admin_sa.kubeconfig + value = module.infracijenkinsio_agents_1_admin_sa_sponsorship.kubeconfig } -output "cijenkinsio_agents_1_kube_config_command" { +output "infracijenkinsio_agents_1_kube_config_command" { value = "az aks get-credentials --name ${azurerm_kubernetes_cluster.infracijenkinsio_agents_1.name} --resource-group ${azurerm_kubernetes_cluster.infracijenkinsio_agents_1.resource_group_name}" } diff --git a/locals.tf b/locals.tf index d6d144d7..6ec23698 100644 --- a/locals.tf +++ b/locals.tf @@ -42,16 +42,16 @@ locals { admin_username = "jenkins-infra-team" kubernetes_versions = { - "privatek8s" = "1.27.9" - "infracijenkinsio_agents_1" = "1.27.9" - "publick8s" = "1.27.9" - "cijenkinsio_agents_1" = "1.27.9" + "privatek8s" = "1.27.9" + "infracijenkinsio_agents_1" = "1.27.9" + "publick8s" = "1.27.9" + "cijenkinsio_agents_1" = "1.27.9" } ci_jenkins_io_fqdn = "ci.jenkins.io" ci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 - publick8s_compute_zones = [3] - cijenkinsio_agents_1_compute_zones = [1] + publick8s_compute_zones = [3] + cijenkinsio_agents_1_compute_zones = [1] infracijenkinsio_agents_1_compute_zones = [1] infraci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 From 3160315551300cc769cb82f104a519f5d42b64e8 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Mon, 10 Jun 2024 14:14:06 +0200 Subject: [PATCH 6/8] feat(infra.ci/agent): new cluster in azure sponsored --- ....jenkins.io-kubernetes-sponsored-agents.tf | 112 ------------------ 1 file changed, 112 deletions(-) diff --git a/infraci.jenkins.io-kubernetes-sponsored-agents.tf b/infraci.jenkins.io-kubernetes-sponsored-agents.tf index ef24b20d..b561b1d4 100644 --- a/infraci.jenkins.io-kubernetes-sponsored-agents.tf +++ b/infraci.jenkins.io-kubernetes-sponsored-agents.tf @@ -59,115 +59,3 @@ resource "azurerm_kubernetes_cluster" "infracijenkinsio_agents_1" { tags = local.default_tags } - -# Node pool to host "jenkins-infra" applications required on this cluster such as ACP or datadog's cluster-agent, e.g. "Not agent, neither AKS System tools" -resource "azurerm_kubernetes_cluster_node_pool" "linux_arm64_n2_applications_sponsorship" { - provider = azurerm.jenkins-sponsorship - name = "la64n2app" - vm_size = "Standard_D4pds_v5" - os_disk_type = "Ephemeral" - os_disk_size_gb = 150 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dsv3-series (depends on the instance size) - orchestrator_version = local.kubernetes_versions["infracijenkinsio_agents_1"] - kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.id - enable_auto_scaling = true - min_count = 1 - max_count = 3 # 2 nodes always up for HA, a 3rd one is allowed for surge upgrades - zones = local.cijenkinsio_agents_1_compute_zones - vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id - - node_labels = { - "jenkins" = "ci.jenkins.io" - "role" = "applications" - } - node_taints = [ - "ci.jenkins.io/applications=true:NoSchedule", - ] - - lifecycle { - ignore_changes = [node_count] - } - - tags = local.default_tags -} - -# Node pool to host ci.jenkins.io agents for usual builds -resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_agents_1_sponsorship" { - provider = azurerm.jenkins-sponsorship - name = "lx86n3agt1" - vm_size = "Standard_D16ads_v5" - os_disk_type = "Ephemeral" - os_disk_size_gb = 600 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dsv3-series (depends on the instance size) - orchestrator_version = local.kubernetes_versions["infracijenkinsio_agents_1"] - kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.id - enable_auto_scaling = true - min_count = 0 - max_count = 40 # 3 pods per nodes, max 120 pods - due to quotas - zones = local.infracijenkinsio_agents_1_compute_zones - vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id - - node_labels = { - "jenkins" = "ci.jenkins.io" - "role" = "jenkins-agents" - } - node_taints = [ - "ci.jenkins.io/agents=true:NoSchedule", - ] - - lifecycle { - ignore_changes = [node_count] - } - - tags = local.default_tags -} - -# Node pool to host ci.jenkins.io agents for BOM builds -resource "azurerm_kubernetes_cluster_node_pool" "linux_x86_64_n4_bom_1_sponsorship" { - provider = azurerm.jenkins-sponsorship - name = "lx86n3bom1" - vm_size = "Standard_D16ads_v5" - os_disk_type = "Ephemeral" - os_disk_size_gb = 600 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series#dsv3-series (depends on the instance size) - orchestrator_version = local.kubernetes_versions["cijenkinsio_agents_1"] - kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.id - enable_auto_scaling = true - min_count = 0 - max_count = 50 - zones = local.cijenkinsio_agents_1_compute_zones - vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id - - node_labels = { - "jenkins" = "ci.jenkins.io" - "role" = "jenkins-agents-bom" - } - node_taints = [ - "ci.jenkins.io/agents=true:NoSchedule", - "ci.jenkins.io/bom=true:NoSchedule", - ] - - lifecycle { - ignore_changes = [node_count] - } - - tags = local.default_tags -} - - - - -# Configure the jenkins-infra/kubernetes-management admin service account -module "infracijenkinsio_agents_1_admin_sa_sponsorship" { - providers = { - kubernetes = kubernetes.cijenkinsio_agents_1 - } - source = "./.shared-tools/terraform/modules/kubernetes-admin-sa" - cluster_name = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.name - cluster_hostname = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.fqdn - cluster_ca_certificate_b64 = azurerm_kubernetes_cluster.infracijenkinsio_agents_1.kube_config.0.cluster_ca_certificate -} -output "kubeconfig_infracijenkinsio_agents_1" { - sensitive = true - value = module.infracijenkinsio_agents_1_admin_sa_sponsorship.kubeconfig -} -output "infracijenkinsio_agents_1_kube_config_command" { - value = "az aks get-credentials --name ${azurerm_kubernetes_cluster.infracijenkinsio_agents_1.name} --resource-group ${azurerm_kubernetes_cluster.infracijenkinsio_agents_1.resource_group_name}" -} From 1e8f37277d86ac7b8e6d2d687b622eb19e5e4439 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Mon, 10 Jun 2024 17:05:11 +0200 Subject: [PATCH 7/8] fix compute zone for infra and sort --- infraci.jenkins.io-kubernetes-sponsored-agents.tf | 2 +- locals.tf | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/infraci.jenkins.io-kubernetes-sponsored-agents.tf b/infraci.jenkins.io-kubernetes-sponsored-agents.tf index b561b1d4..279945b1 100644 --- a/infraci.jenkins.io-kubernetes-sponsored-agents.tf +++ b/infraci.jenkins.io-kubernetes-sponsored-agents.tf @@ -54,7 +54,7 @@ resource "azurerm_kubernetes_cluster" "infracijenkinsio_agents_1" { node_count = 3 # 3 nodes for HA as per AKS best practises vnet_subnet_id = data.azurerm_subnet.infraci_jenkins_io_kubernetes_agent_sponsorship.id tags = local.default_tags - zones = local.cijenkinsio_agents_1_compute_zones + zones = local.infracijenkinsio_agents_1_compute_zones } tags = local.default_tags diff --git a/locals.tf b/locals.tf index 6ec23698..d12f1b5b 100644 --- a/locals.tf +++ b/locals.tf @@ -47,14 +47,15 @@ locals { "publick8s" = "1.27.9" "cijenkinsio_agents_1" = "1.27.9" } - ci_jenkins_io_fqdn = "ci.jenkins.io" - ci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 - publick8s_compute_zones = [3] - cijenkinsio_agents_1_compute_zones = [1] + ci_jenkins_io_fqdn = "ci.jenkins.io" + cijenkinsio_agents_1_compute_zones = [1] + ci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 + infracijenkinsio_agents_1_compute_zones = [1] + infraci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 - infraci_jenkins_io_agents_1_pod_cidr = "10.100.0.0/14" # 10.100.0.1 - 10.103.255.255 + publick8s_compute_zones = [3] weekly_ci_disk_size = 8 weekly_ci_access_modes = ["ReadWriteOnce"] From ee7facc2b6edcbba467915f86cf8f4443f46e5f0 Mon Sep 17 00:00:00 2001 From: smerle33 Date: Mon, 10 Jun 2024 17:06:32 +0200 Subject: [PATCH 8/8] sorting --- locals.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locals.tf b/locals.tf index d12f1b5b..a01670ac 100644 --- a/locals.tf +++ b/locals.tf @@ -42,10 +42,10 @@ locals { admin_username = "jenkins-infra-team" kubernetes_versions = { - "privatek8s" = "1.27.9" + "cijenkinsio_agents_1" = "1.27.9" "infracijenkinsio_agents_1" = "1.27.9" + "privatek8s" = "1.27.9" "publick8s" = "1.27.9" - "cijenkinsio_agents_1" = "1.27.9" } ci_jenkins_io_fqdn = "ci.jenkins.io"