From 6c8dfe4249c2a707c91223555f6c7eeb9b0f16c0 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Wed, 8 Jan 2025 10:59:43 +0100 Subject: [PATCH] various fixes and updates - updated msi pull controller to 0.1.7 - updated pull binding CRD for the additional workload identity fields - updated podmonitor to use the az apigroup (we don't have core prometheus CRDs installed) - updated VAP parameterNotFoundAction to `Allow` to mitigate a potential bug - updated servicegroup name in pipeline.yaml for acrpull - serialize image-puller MI federation setup (only one update at a time is supported) Signed-off-by: Gerd Oberlechner --- acrpull/Makefile | 4 +- ...acrpull.microsoft.com_acrpullbindings.yaml | 52 ++++++++++++++++++- .../helm/acrpull/templates/namespace.yaml | 7 --- .../helm/acrpull/templates/podmonitor.yaml | 2 +- .../validatingadmissionpolicybindings.yaml | 6 +-- acrpull/pipeline.yaml | 6 +-- config/config.msft.yaml | 1 + config/config.yaml | 2 +- config/public-cloud-cs-pr.json | 1 + config/public-cloud-dev.json | 1 + config/public-cloud-msft-int.json | 1 + config/public-cloud-personal-dev.json | 1 + .../modules/aks-cluster-base.bicep | 6 +++ .../frontend/templates/acrpullbinding.yaml | 4 +- 14 files changed, 75 insertions(+), 19 deletions(-) delete mode 100644 acrpull/deploy/helm/acrpull/templates/namespace.yaml diff --git a/acrpull/Makefile b/acrpull/Makefile index 996892a2e..28b8e7e1d 100644 --- a/acrpull/Makefile +++ b/acrpull/Makefile @@ -1,8 +1,10 @@ -include ../setup-env.mk +-include ../helm-cmd.mk +HELM_CMD ?= helm upgrade --install deploy: kubectl create namespace acrpull --dry-run=client -o json | kubectl apply -f - && \ - helm upgrade --install ${HELM_DRY_RUN} acrpull \ + ${HELM_CMD} acrpull \ deploy/helm/acrpull/ \ --set image=mcr.microsoft.com/aks/msi-acrpull@${ACRPULL_DIGEST} \ --namespace acrpull diff --git a/acrpull/deploy/helm/acrpull/templates/acrpull.microsoft.com_acrpullbindings.yaml b/acrpull/deploy/helm/acrpull/templates/acrpull.microsoft.com_acrpullbindings.yaml index efabf5cc3..6e444be51 100644 --- a/acrpull/deploy/helm/acrpull/templates/acrpull.microsoft.com_acrpullbindings.yaml +++ b/acrpull/deploy/helm/acrpull/templates/acrpull.microsoft.com_acrpullbindings.yaml @@ -17,7 +17,34 @@ spec: singular: acrpullbinding scope: Namespaced versions: - - name: v1beta2 + - additionalPrinterColumns: + - description: FQDN for the ACR. + jsonPath: .spec.acr.server + name: Server + type: string + - description: Scope for the ACR token. + jsonPath: .spec.acr.scope + name: Scope + priority: 1 + type: string + - description: ServiceAccount to which the pull credentials are attached. + jsonPath: .spec.serviceAccountName + name: Target + type: string + - description: Time the token was last refreshed. + jsonPath: .status.lastTokenRefreshTime + name: Last Refresh + priority: 1 + type: date + - description: Time the current token expires. + jsonPath: .status.tokenExpirationTime + name: Expiration + type: date + - description: Errors encountered during token generation, if any. + jsonPath: .status.error + name: Error + type: string + name: v1beta2 schema: openAPIV3Schema: description: AcrPullBinding is the Schema for the acrpullbindings API @@ -136,12 +163,35 @@ spec: description: WorkloadIdentity uses Azure Workload Identity to authenticate with Azure. properties: + clientID: + description: |- + ClientID holds an optional client identifier of a federated identity. + Specify this identifier if multiple identities are federated with the + service account and the identity to use for image pulling is not the + default identity stored in the service account's annotations. The + client and tenant ID must be specified together. + example: 1b461305-28be-5271-beda-bd9fd2e24251 + type: string serviceAccountRef: description: |- ServiceAccountName specifies the name of the service account that should be used when authenticating with WorkloadIdentity. type: string + tenantID: + description: |- + TenantID holds an optional tenant identifier of a federated identity. + Specify this identifier if multiple identities are federated with the + service account and the identity to use for image pulling is not the + default identity stored in the service account's annotations. The + client and tenant ID must be specified together. + example: 72f988bf-86f1-41af-91ab-2d7cd011db47 + type: string type: object + x-kubernetes-validations: + - message: custom client and tenant identifiers must be provided + together, if at all + rule: (has(self.clientID) && has(self.tenantID)) || (!has(self.clientID) + && !has(self.tenantID)) type: object x-kubernetes-validations: - message: only one authentication type can be set diff --git a/acrpull/deploy/helm/acrpull/templates/namespace.yaml b/acrpull/deploy/helm/acrpull/templates/namespace.yaml deleted file mode 100644 index 5ffff4a27..000000000 --- a/acrpull/deploy/helm/acrpull/templates/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/name: acrpull - app.kubernetes.io/managed-by: Helm - name: acrpull diff --git a/acrpull/deploy/helm/acrpull/templates/podmonitor.yaml b/acrpull/deploy/helm/acrpull/templates/podmonitor.yaml index 4ced676f8..3dec16cb3 100644 --- a/acrpull/deploy/helm/acrpull/templates/podmonitor.yaml +++ b/acrpull/deploy/helm/acrpull/templates/podmonitor.yaml @@ -1,4 +1,4 @@ -apiVersion: monitoring.coreos.com/v1 +apiVersion: azmonitoring.coreos.com/v1 kind: PodMonitor metadata: labels: diff --git a/acrpull/deploy/helm/acrpull/templates/validatingadmissionpolicybindings.yaml b/acrpull/deploy/helm/acrpull/templates/validatingadmissionpolicybindings.yaml index 6fe97342d..21d281b81 100644 --- a/acrpull/deploy/helm/acrpull/templates/validatingadmissionpolicybindings.yaml +++ b/acrpull/deploy/helm/acrpull/templates/validatingadmissionpolicybindings.yaml @@ -11,7 +11,7 @@ spec: paramRef: name: "admission-policies-controller-config" namespace: {{ .Values.namespace }} - parameterNotFoundAction: "Deny" + parameterNotFoundAction: "Allow" --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding @@ -26,7 +26,7 @@ spec: paramRef: name: "admission-policies-controller-config" namespace: {{ .Values.namespace }} - parameterNotFoundAction: "Deny" + parameterNotFoundAction: "Allow" --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding @@ -41,4 +41,4 @@ spec: paramRef: name: "admission-policies-controller-config" namespace: {{ .Values.namespace }} - parameterNotFoundAction: "Deny" + parameterNotFoundAction: "Allow" diff --git a/acrpull/pipeline.yaml b/acrpull/pipeline.yaml index a4ea72131..90e1a843a 100644 --- a/acrpull/pipeline.yaml +++ b/acrpull/pipeline.yaml @@ -1,5 +1,5 @@ $schema: "pipeline.schema.v1" -serviceGroup: Microsoft.Azure.ARO.HCP.RP.Frontend +serviceGroup: Microsoft.Azure.ARO.HCP.ACRPull rolloutName: ACRPull Controller Rollout resourceGroups: - name: {{ .svc.rg }} @@ -11,8 +11,8 @@ resourceGroups: command: make deploy dryRun: variables: - - name: HELM_DRY_RUN - value: "--dry-run=server --debug" + - name: DRY_RUN + value: "true" variables: - name: ACRPULL_DIGEST configRef: acrPullImageDigest diff --git a/config/config.msft.yaml b/config/config.msft.yaml index 5d59c95f8..9b768c0e0 100644 --- a/config/config.msft.yaml +++ b/config/config.msft.yaml @@ -18,6 +18,7 @@ defaults: subnetPrefix: "10.128.8.0/21" podSubnetPrefix: "10.128.64.0/18" aksName: aro-hcp-aks + acrPullImageDigest: sha256:1d18e828564dcd509a8551185808549bd8bfddec1fcc4a2783914dc2103bc2ca #v0.1.7 # Hypershift hypershift: diff --git a/config/config.yaml b/config/config.yaml index 413ee05db..f28db8e8a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -16,7 +16,7 @@ defaults: subnetPrefix: "10.128.8.0/21" podSubnetPrefix: "10.128.64.0/18" aksName: aro-hcp-aks - acrPullImageDigest: sha256:9816561e7ee91a0814a482564d202288f2e5401ca2387a56641f144d04fa3535 #v0.1.5 + acrPullImageDigest: sha256:1d18e828564dcd509a8551185808549bd8bfddec1fcc4a2783914dc2103bc2ca #v0.1.7 # Hypershift hypershift: diff --git a/config/public-cloud-cs-pr.json b/config/public-cloud-cs-pr.json index 2db4731e7..0f63782d7 100644 --- a/config/public-cloud-cs-pr.json +++ b/config/public-cloud-cs-pr.json @@ -1,4 +1,5 @@ { + "acrPullImageDigest": "sha256:1d18e828564dcd509a8551185808549bd8bfddec1fcc4a2783914dc2103bc2ca", "aksName": "aro-hcp-aks", "armHelperClientId": "2c6ca254-36bd-43c8-a7a8-fe880bc2c489", "armHelperFPAPrincipalId": "bc17c825-6cf8-40d0-8bd6-5536a993115e", diff --git a/config/public-cloud-dev.json b/config/public-cloud-dev.json index 914fe6be3..fa5b5e1f2 100644 --- a/config/public-cloud-dev.json +++ b/config/public-cloud-dev.json @@ -1,4 +1,5 @@ { + "acrPullImageDigest": "sha256:1d18e828564dcd509a8551185808549bd8bfddec1fcc4a2783914dc2103bc2ca", "aksName": "aro-hcp-aks", "armHelperClientId": "2c6ca254-36bd-43c8-a7a8-fe880bc2c489", "armHelperFPAPrincipalId": "bc17c825-6cf8-40d0-8bd6-5536a993115e", diff --git a/config/public-cloud-msft-int.json b/config/public-cloud-msft-int.json index ca4546a82..d86eb90a5 100644 --- a/config/public-cloud-msft-int.json +++ b/config/public-cloud-msft-int.json @@ -1,4 +1,5 @@ { + "acrPullImageDigest": "sha256:1d18e828564dcd509a8551185808549bd8bfddec1fcc4a2783914dc2103bc2ca", "aksName": "aro-hcp-aks", "armHelperClientId": "", "armHelperFPAPrincipalId": "", diff --git a/config/public-cloud-personal-dev.json b/config/public-cloud-personal-dev.json index ae9585522..688da9403 100644 --- a/config/public-cloud-personal-dev.json +++ b/config/public-cloud-personal-dev.json @@ -1,4 +1,5 @@ { + "acrPullImageDigest": "sha256:1d18e828564dcd509a8551185808549bd8bfddec1fcc4a2783914dc2103bc2ca", "aksName": "aro-hcp-aks", "armHelperClientId": "2c6ca254-36bd-43c8-a7a8-fe880bc2c489", "armHelperFPAPrincipalId": "bc17c825-6cf8-40d0-8bd6-5536a993115e", diff --git a/dev-infrastructure/modules/aks-cluster-base.bicep b/dev-infrastructure/modules/aks-cluster-base.bicep index 62107af77..f949b7af5 100644 --- a/dev-infrastructure/modules/aks-cluster-base.bicep +++ b/dev-infrastructure/modules/aks-cluster-base.bicep @@ -469,6 +469,10 @@ resource uami_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federate } ] +// +// A C R P U L L C O N T R O L L E R +// + resource pullerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { location: location name: 'image-puller' @@ -485,6 +489,7 @@ module acrPullerRoles 'acr/acr-permissions.bicep' = [ } ] +@batchSize(1) resource puller_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = [ for i in range(0, length(workloadIdentities)): { parent: pullerIdentity @@ -499,6 +504,7 @@ resource puller_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federa } ] + // grant aroDevopsMsi the aksClusterAdmin role on the aksCluster so it can // deploy services to the cluster resource aroDevopsMSIClusterAdmin 'Microsoft.Authorization/roleAssignments@2022-04-01' = { diff --git a/frontend/deploy/helm/frontend/templates/acrpullbinding.yaml b/frontend/deploy/helm/frontend/templates/acrpullbinding.yaml index 90101407e..6c509b939 100644 --- a/frontend/deploy/helm/frontend/templates/acrpullbinding.yaml +++ b/frontend/deploy/helm/frontend/templates/acrpullbinding.yaml @@ -5,8 +5,8 @@ metadata: spec: acr: environment: PublicCloud - scope: {{ .Values.pullBinding.registry }} - server: {{ .Values.pullBinding.scope }} + server: {{ .Values.pullBinding.registry }} + scope: {{ .Values.pullBinding.scope }} auth: workloadIdentity: serviceAccountRef: frontend