introduce pipeline.yaml file for CS

.yamllint.yml

@@ -3,8 +3,7 @@ yaml-files:
   - '*.yaml'
   - '*.yml'
   - '.yamllint'
-ignore:
-  - 'cluster-service/deploy/helm/templates/azure-operators-managed-identities-config.configmap.yaml'
+ignore: []
 
 rules:
   brackets: enable

cluster-service/Makefile

@@ -1,7 +1,7 @@
-SHELL = /bin/bash
-DEPLOY_ENV ?= personal-dev
-$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
-include config.mk
+-include ../setup-env.mk
+
+ZONE_NAME ?= "${REGIONAL_DNS_SUBDOMAIN}.${BASE_DNS_ZONE_NAME}"
+
 
 deploy: provision-shard
 	@ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \
@@ -14,8 +14,17 @@ deploy: provision-shard
 	OIDC_ISSUER_BASE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.web -o tsv) && \
 	OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \
 	OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \
-	helm upgrade --install cluster-service --namespace cluster-service \
-	  deploy/helm/ \
+	DB_HOST=$$(if [ "${USE_AZURE_DB}" = "true" ]; then az postgres flexible-server show -g ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv; else echo "ocm-cs-db"; fi) && \
+	OVERRIDES=$$(if [ "${USE_AZURE_DB}" = "true" ]; then echo "azuredb.values.yaml"; else echo "containerdb.values.yaml"; fi) && \
+	OP_CLOUD_CONTROLLER_MANAGER_ROLE_ID=$(shell az role definition list --name "${OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME}" --query "[].name" -o tsv) && \
+	OP_INGRESS_ROLE_ID=$(shell az role definition list --name "${OP_INGRESS_ROLE_NAME}" --query "[].name" -o tsv) && \
+	OP_DISK_CSI_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_DISK_CSI_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
+	OP_FILE_CSI_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_FILE_CSI_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
+	OP_IMAGE_REGISTRY_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
+	OP_CLOUD_NETWORK_CONFIG_ROLE_ID=$(shell az role definition list --name "${OP_CLOUD_NETWORK_CONFIG_ROLE_NAME}" --query "[].name" -o tsv) && \
+	helm upgrade --install --wait ${HELM_DRY_RUN} cluster-service deploy/helm \
+	  --namespace cluster-service \
+	  -f deploy/helm/$${OVERRIDES} \
 	  --set azureCsMiClientId=$${AZURE_CS_MI_CLIENT_ID} \
 	  --set oidcIssuerBlobServiceUrl=$${OIDC_BLOB_SERVICE_ENDPOINT} \
 	  --set oidcIssuerBaseUrl=$${OIDC_ISSUER_BASE_ENDPOINT} \
@@ -30,21 +39,26 @@ deploy: provision-shard
 	  --set fpaCertName=${FPA_CERT_NAME} \
 	  --set ocpAcrResourceId=$${OCP_ACR_RESOURCE_ID} \
 	  --set ocpAcrUrl=$${OCP_ACR_URL} \
-	  --set databaseDisableTls=${DATABASE_DISABLE_TLS} \
-	  --set databaseAuthMethod=${DATABASE_AUTH_METHOD} \
 	  --set provisionShardsConfig="$(shell base64 -i deploy/provisioning-shards.yml | tr -d '\n')" \
-	  --set deployLocalDatabase=${DEPLOY_LOCAL_DB} \
-	  --set databaseHost=${DB_HOST} \
-	  --set databaseName=${DB_NAME} \
-	  --set databaseUser=${DB_USERNAME} \
-	  --set databasePassword=${DB_PASSWORD} \
+	  --set databaseHost=$${DB_HOST} \
 	  --set azureMiMockServicePrincipalPrincipalId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID} \
 	  --set azureMiMockServicePrincipalClientId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID} \
 	  --set azureMiMockServicePrincipalCertName=${MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME} \
 	  --set azureArmHelperIdentityCertName=${ARM_HELPER_CERT_NAME} \
 	  --set azureArmHelperIdentityClientId=${AZURE_ARM_HELPER_IDENTITY_CLIENT_ID} \
 	  --set azureArmHelperMockFpaPrincipalId=${AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID} \
-	  --set azureOperatorsManagedIdentitiesConfig=${AZURE_OPERATORS_MANAGED_IDENTITIES_CONFIG}
+	  --set azureOperatorsMI.cloudControllerManager.roleName="${OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME}" \
+	  --set azureOperatorsMI.cloudControllerManager.roleId="$${OP_CLOUD_CONTROLLER_MANAGER_ROLE_ID}" \
+	  --set azureOperatorsMI.ingress.roleName="${OP_INGRESS_ROLE_NAME}" \
+	  --set azureOperatorsMI.ingress.roleId="$${OP_INGRESS_ROLE_ID}" \
+	  --set azureOperatorsMI.diskCsiDriver.roleName="${OP_DISK_CSI_DRIVER_ROLE_NAME}" \
+	  --set azureOperatorsMI.diskCsiDriver.roleId="$${OP_DISK_CSI_DRIVER_ROLE_ID}" \
+	  --set azureOperatorsMI.fileCsiDriver.roleName="${OP_FILE_CSI_DRIVER_ROLE_NAME}" \
+	  --set azureOperatorsMI.fileCsiDriver.roleId="$${OP_FILE_CSI_DRIVER_ROLE_ID}" \
+	  --set azureOperatorsMI.imageRegistry.roleName="${OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME}" \
+	  --set azureOperatorsMI.imageRegistry.roleId="$${OP_IMAGE_REGISTRY_DRIVER_ROLE_ID}" \
+	  --set azureOperatorsMI.cloudNetworkConfig.roleName="${OP_CLOUD_NETWORK_CONFIG_ROLE_NAME}" \
+	  --set azureOperatorsMI.cloudNetworkConfig.roleId="$${OP_CLOUD_NETWORK_CONFIG_ROLE_ID}"
 
 deploy-pr-env-deps:
 	AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \

cluster-service/deploy/helm/azuredb.values.yaml

@@ -0,0 +1,6 @@
+databaseDisableTls: false
+databaseAuthMethod: az-entra
+deployLocalDatabase: false
+databaseName: clusters-service
+databaseUser: clusters-service
+databasePassword: ''

cluster-service/deploy/helm/containerdb.values.yaml

@@ -0,0 +1,6 @@
+databaseDisableTls: true
+databaseAuthMethod: postgres
+deployLocalDatabase: true
+databaseName: ocm-cs-db
+databaseUser: ocm
+databasePassword: TheBlurstOfTimes

cluster-service/deploy/helm/templates/azure-operators-managed-identities-config.configmap.yaml

@@ -6,4 +6,83 @@ metadata:
   namespace: {{ .Release.Namespace }}
 data:
   azure-operators-managed-identities-config.yaml: |
-{{ .Values.azureOperatorsManagedIdentitiesConfig | b64dec | indent 4 }}
+    controlPlaneOperatorsIdentities:
+      cloud-controller-manager:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudControllerManager.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudControllerManager.roleName }}'
+        optional: false
+      ingress:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.ingress.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.ingress.roleName }}'
+        optional: false
+      disk-csi-driver:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.diskCsiDriver.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.diskCsiDriver.roleName }}'
+        optional: false
+      file-csi-driver:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.fileCsiDriver.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.fileCsiDriver.roleName }}'
+        optional: false
+      image-registry:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.imageRegistry.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.imageRegistry.roleName }}'
+        optional: false
+      cloud-network-config:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleName }}'
+        optional: false
+    dataPlaneOperatorsIdentities:
+      disk-csi-driver:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.diskCsiDriver.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.diskCsiDriver.roleName }}'
+        k8sServiceAccounts:
+          - name: 'azure-disk-csi-driver-operator'
+            namespace: 'openshift-cluster-csi-drivers'
+          - name: 'azure-disk-csi-driver-controller-sa'
+            namespace: 'openshift-cluster-csi-drivers'
+        optional: false
+      image-registry:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.imageRegistry.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.imageRegistry.roleName }}'
+        k8sServiceAccounts:
+          - name: 'cluster-image-registry-operator'
+            namespace: 'openshift-image-registry'
+          - name: 'registry'
+            namespace: 'openshift-image-registry'
+        optional: false
+      file-csi-driver:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.fileCsiDriver.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.fileCsiDriver.roleName }}'
+        k8sServiceAccounts:
+          - name: 'azure-file-csi-driver-operator'
+            namespace: 'openshift-cluster-csi-drivers'
+          - name: 'azure-file-csi-driver-controller-sa'
+            namespace: 'openshift-cluster-csi-drivers'
+          - name: 'azure-file-csi-driver-node-sa'
+            namespace: 'openshift-cluster-csi-drivers'
+        optional: false
+      ingress:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.ingress.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.ingress.roleName }}'
+        k8sServiceAccounts:
+          - name: 'ingress-operator'
+            namespace: 'openshift-ingress-operator'
+        optional: false
+      cloud-network-config:
+        minOpenShiftVersion: 4.17
+        azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleId }}'
+        azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleName }}'
+        k8sServiceAccounts:
+          - name: 'cloud-network-config-controller'
+            namespace: 'openshift-cloud-network-config-controller'
+        optional: false

cluster-service/deploy/helm/templates/deployment.yaml

@@ -16,6 +16,14 @@ spec:
       labels:
         app: clusters-service
         azure.workload.identity/use: "true"
+      annotations:
+        checksum/db: {{ include (print $.Template.BasePath "/database.secret.yaml") . | sha256sum }}
+        checksum/azurecreds: {{ include (print $.Template.BasePath "/azure-credentials.secret.yaml") . | sha256sum }}
+        checksum/operatorcfg: {{ include (print $.Template.BasePath "/azure-operators-managed-identities-config.configmap.yaml") . | sha256sum }}
+        checksum/cskv: {{ include (print $.Template.BasePath "/cs-keyvault.secret.yaml") . | sha256sum }}
+        checksum/provisionshard: {{ include (print $.Template.BasePath "/provisioning-shards.secret.yaml") . | sha256sum }}
+        checksum/cs: {{ include (print $.Template.BasePath "/clusters-service.secret.yaml") . | sha256sum }}
+        checksum/runtime: {{ include (print $.Template.BasePath "/azure-runtime-config.configmap.yaml") . | sha256sum }}
     spec:
       serviceAccount: clusters-service
       serviceAccountName: clusters-service

cluster-service/deploy/helm/values.yaml

@@ -262,4 +262,22 @@ databasePort: "5432"
 managedIdentitiesDataPlaneAudienceResource: "https://dummy.org"
 
 # The Azure Operator Managed Identities.
-azureOperatorsManagedIdentitiesConfig: ""
+azureOperatorsMI:
+  cloudControllerManager:
+    roleName: ''
+    roleId: ''
+  ingress:
+    roleName: ''
+    roleId: ''
+  diskCsiDriver:
+    roleName: ''
+    roleId: ''
+  fileCsiDriver:
+    roleName: ''
+    roleId: ''
+  imageRegistry:
+    roleName: ''
+    roleId: ''
+  cloudNetworkConfig:
+    roleName: ''
+    roleId: ''

cluster-service/pipeline.yaml

@@ -0,0 +1,87 @@
+$schema: "pipeline.schema.v1"
+serviceGroup: Microsoft.Azure.ARO.HCP.ClusterService
+rolloutName: Cluster Service Rollout
+resourceGroups:
+- name: {{ .svc.rg }}
+  subscription: {{ .svc.subscription }}
+  aksCluster: {{ .aksName }}
+  steps:
+  - name: deploy
+    action: Shell
+    command: make deploy
+    dryRun:
+      variables:
+      - name: HELM_DRY_RUN
+        value: "--dry-run=server --debug"
+      - name: KUBECTL_DRY_RUN
+        value: "--dry-run=server"
+    variables:
+    - name: REGION
+      configRef: region
+    - name: RESOURCEGROUP
+      configRef: svc.rg
+    - name: AKS_NAME
+      configRef: aksName
+    - name: SERVICE_KV
+      configRef: serviceKeyVault.name
+    - name: OIDC_STORAGE_ACCOUNT
+      configRef: oidcStorageAccountName
+    - name: IMAGE_REPO
+      configRef: clusterService.imageRepo
+    - name: IMAGE_TAG
+      configRef: clusterService.imageTag
+    - name: ACR_NAME
+      configRef: svcAcrName
+    - name: OCP_ACR_NAME
+      configRef: ocpAcrName
+    - name: AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID
+      configRef: firstPartyAppClientId
+    - name: FPA_CERT_NAME
+      value: firstPartyCert
+    - name: AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID
+      configRef: miMockPrincipalId
+    - name: AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID
+      configRef: miMockClientId
+    - name: AZURE_ARM_HELPER_IDENTITY_CLIENT_ID
+      configRef: armHelperClientId
+    - name: AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID
+      configRef: armHelperFPAPrincipalId
+    - name: MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME
+      value: msiMockCert
+    - name: ARM_HELPER_CERT_NAME
+      value: armHelperCert
+    - name: BASE_DNS_ZONE_NAME
+      configRef: baseDnsZoneName
+    - name: REGIONAL_DNS_SUBDOMAIN
+      configRef: regionalDNSSubdomain
+    - name: USE_AZURE_DB
+      configRef: clusterService.postgres.deploy
+    - name: DATABASE_SERVER_NAME
+      configRef: clusterService.postgres.name
+    - name: DEVOPS_MSI_ID
+      configRef: aroDevopsMsiId
+    - name: OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME
+      configRef: clusterService.azureOperatorsManagedIdentities.cloudControllerManager.roleName
+    - name: OP_INGRESS_ROLE_NAME
+      configRef: clusterService.azureOperatorsManagedIdentities.ingress.roleName
+    - name: OP_DISK_CSI_DRIVER_ROLE_NAME
+      configRef: clusterService.azureOperatorsManagedIdentities.diskCsiDriver.roleName
+    - name: OP_FILE_CSI_DRIVER_ROLE_NAME
+      configRef: clusterService.azureOperatorsManagedIdentities.fileCsiDriver.roleName
+    - name: OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME
+      configRef: clusterService.azureOperatorsManagedIdentities.imageRegistry.roleName
+    - name: OP_CLOUD_NETWORK_CONFIG_ROLE_NAME
+      configRef: clusterService.azureOperatorsManagedIdentities.cloudNetworkConfig.roleName
+
+    # this is maestro consumer registration stuff
+    # this goes away when we have a real registration process
+    - name: CONSUMER_NAME
+      configRef: maestro.consumerName
+    - name: REGIONAL_RESOURCEGROUP
+      configRef: regionRG
+    - name: MGMT_RESOURCEGROUP
+      configRef: mgmt.rg
+    - name: CX_SECRETS_KV_NAME
+      configRef: cxKeyVault.name
+    - name: CX_MI_KV_NAME
+      configRef: msiKeyVault.name

config/config.msft.yaml

@@ -138,6 +138,19 @@ clouds:
       clusterService:
         imageTag: ecd15ad
         imageRepo: app-sre/uhc-clusters-service
+        azureOperatorsManagedIdentities:
+          cloudControllerManager:
+            roleName: Azure Red Hat OpenShift Cloud Controller Manager Role
+          ingress:
+            roleName: Azure Red Hat OpenShift Cluster Ingress Operator Role
+          diskCsiDriver:
+            roleName: Azure Red Hat OpenShift Disk Storage Operator Role
+          fileCsiDriver:
+            roleName: Azure Red Hat OpenShift File Storage Operator Role
+          imageRegistry:
+            roleName: Azure Red Hat OpenShift Image Registry Operator Role
+          cloudNetworkConfig:
+            roleName: Azure Red Hat OpenShift Network Operator Role
       hypershiftOperator:
         imageTag: 9aca808
       imageSync: