From c9327909ae9a713611e3183be6eb664df6d10f3c Mon Sep 17 00:00:00 2001 From: morvencao Date: Mon, 4 Nov 2024 13:46:31 +0000 Subject: [PATCH] deploy aro-hcp env using upstream. Signed-off-by: morvencao --- .gitignore | 4 +- test/e2e/setup/aro/Makefile | 62 ++ test/e2e/setup/aro/README.md | 20 - test/e2e/setup/aro/cluster/Makefile | 135 ----- .../setup/aro/cluster/configurations/dev.mk | 3 - .../configurations/mgmt-cluster.bicepparam | 28 - .../cluster/configurations/region.bicepparam | 9 - .../configurations/svc-cluster.bicepparam | 24 - .../cluster/modules/aks-cluster-base.bicep | 437 -------------- .../aro/cluster/modules/key-vault-cert.bicep | 45 -- .../cluster/modules/keyvault/keyvault.bicep | 40 -- .../modules/maestro/maestro-consumer.bicep | 25 - .../maestro/maestro-eventgrid-access.bicep | 133 ---- .../modules/maestro/maestro-infra.bicep | 251 -------- .../modules/maestro/maestro-server.bicep | 146 ----- .../modules/postgres/postgres-access.bicep | 48 -- .../modules/postgres/postgres-sql.bicep | 66 -- .../cluster/modules/postgres/postgres.bicep | 231 ------- .../aro/cluster/scripts/aks-admin-access.sh | 11 - .../scripts/cleanup-orphaned-rolebindings.sh | 18 - .../scripts/digicert-global-root-g3.crt | 16 - .../aro/cluster/scripts/key-vault-cert.ps1 | 111 ---- .../templates/feature-registration.bicep | 18 - .../aro/cluster/templates/mgmt-cluster.bicep | 156 ----- .../setup/aro/cluster/templates/region.bicep | 48 -- .../aro/cluster/templates/svc-cluster.bicep | 158 ----- test/e2e/setup/aro/maestro/Makefile | 68 --- .../aro/maestro/deploy/helm/agent/Chart.yaml | 7 - ...anagement.io.customresourcedefinition.yaml | 128 ---- .../maestro-agent.agent.clusterrole.yaml | 42 -- ...aestro-agent.agent.clusterrolebinding.yaml | 12 - .../templates/maestro-agent.agent.role.yaml | 38 -- .../maestro-agent.agent.rolebinding.yaml | 13 - ...-agent.agent:extension-apiserver.role.yaml | 16 - ...agent:extension-apiserver.rolebinding.yaml | 13 - .../templates/maestro-agent.deployment.yaml | 47 -- ...nt.execution-admin.clusterrolebinding.yaml | 12 - ...ro-agent.execution.clusterrolebinding.yaml | 12 - .../helm/agent/templates/maestro.secret.yaml | 15 - .../maestro.secretproviderclass.yaml | 19 - .../templates/maestro.serviceaccount.yaml | 7 - ...anagement.io.customresourcedefinition.yaml | 570 ------------------ .../aro/maestro/deploy/helm/agent/values.yaml | 13 - .../deploy/helm/registration/Chart.yaml | 7 - .../templates/registration.job.yaml | 22 - .../helm/registration/templates/script.yaml | 52 -- .../deploy/helm/registration/values.yaml | 3 - .../aro/maestro/deploy/helm/server/Chart.yaml | 7 - .../helm/server/templates/grpc.service.yaml | 15 - .../helm/server/templates/health.service.yaml | 14 - .../helm/server/templates/http.service.yaml | 15 - .../server/templates/maestro.deployment.yaml | 136 ----- .../templates/maestro.peerauthentication.yaml | 12 - .../helm/server/templates/maestro.secret.yaml | 17 - .../maestro.secretproviderclass.yaml | 19 - .../templates/maestro.serviceaccount.yaml | 7 - ...anagement.io.customresourcedefinition.yaml | 570 ------------------ .../server/templates/metrics.service.yaml | 15 - .../helm/server/templates/pg.deployment.yaml | 70 --- .../deploy/helm/server/templates/pg.pvc.yaml | 13 - .../helm/server/templates/pg.secret.yaml | 11 - .../helm/server/templates/pg.service.yaml | 20 - .../maestro/deploy/helm/server/values.yaml | 46 -- 63 files changed, 65 insertions(+), 4281 deletions(-) create mode 100644 test/e2e/setup/aro/Makefile delete mode 100644 test/e2e/setup/aro/cluster/Makefile delete mode 100644 test/e2e/setup/aro/cluster/configurations/dev.mk delete mode 100644 test/e2e/setup/aro/cluster/configurations/mgmt-cluster.bicepparam delete mode 100644 test/e2e/setup/aro/cluster/configurations/region.bicepparam delete mode 100644 test/e2e/setup/aro/cluster/configurations/svc-cluster.bicepparam delete mode 100644 test/e2e/setup/aro/cluster/modules/aks-cluster-base.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/key-vault-cert.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/keyvault/keyvault.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/maestro/maestro-consumer.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/maestro/maestro-eventgrid-access.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/maestro/maestro-infra.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/maestro/maestro-server.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/postgres/postgres-access.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/postgres/postgres-sql.bicep delete mode 100644 test/e2e/setup/aro/cluster/modules/postgres/postgres.bicep delete mode 100755 test/e2e/setup/aro/cluster/scripts/aks-admin-access.sh delete mode 100755 test/e2e/setup/aro/cluster/scripts/cleanup-orphaned-rolebindings.sh delete mode 100644 test/e2e/setup/aro/cluster/scripts/digicert-global-root-g3.crt delete mode 100644 test/e2e/setup/aro/cluster/scripts/key-vault-cert.ps1 delete mode 100644 test/e2e/setup/aro/cluster/templates/feature-registration.bicep delete mode 100644 test/e2e/setup/aro/cluster/templates/mgmt-cluster.bicep delete mode 100644 test/e2e/setup/aro/cluster/templates/region.bicep delete mode 100644 test/e2e/setup/aro/cluster/templates/svc-cluster.bicep delete mode 100644 test/e2e/setup/aro/maestro/Makefile delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/agent/Chart.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secret.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml delete mode 100755 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/agent/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/agent/values.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/registration/Chart.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/registration/templates/registration.job.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/registration/templates/script.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/registration/values.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/Chart.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/grpc.service.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/health.service.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/http.service.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.deployment.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secret.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/metrics.service.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.deployment.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.pvc.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.secret.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.service.yaml delete mode 100644 test/e2e/setup/aro/maestro/deploy/helm/server/values.yaml diff --git a/.gitignore b/.gitignore index 4c06cbde..6df1bed4 100755 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,6 @@ test/e2e/.consumer_name test/e2e/.external_host_ip test/e2e/report/* unit-test-results.json -integration-test-results.json \ No newline at end of file +integration-test-results.json + +test/e2e/setup/aro/aro-hcp \ No newline at end of file diff --git a/test/e2e/setup/aro/Makefile b/test/e2e/setup/aro/Makefile new file mode 100644 index 00000000..25042f7a --- /dev/null +++ b/test/e2e/setup/aro/Makefile @@ -0,0 +1,62 @@ +# define the variables +REPO_URL = git@github.com:Azure/ARO-HCP.git +BRANCH = maestro-light-setup +CLONE_DIR = aro-hcp + +# clone the repo +clone: + @if [ -d $(CLONE_DIR) ]; then \ + echo "Removing existing directory $(CLONE_DIR)..."; \ + rm -rf $(CLONE_DIR); \ + fi; \ + echo "Cloning repository..."; \ + git clone $(REPO_URL) -b $(BRANCH) $(CLONE_DIR) +.PHONY: clone + +# create the cluster (svc-cluster or mgmt-cluster) +cluster: clone +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure cluster +.PHONY: cluster + +# grant admin access to the cluster +aks.admin-access: +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure aks.admin-access +.PHONY: aks.admin-access + +# retrieve the kubeconfig +aks.kubeconfig: +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure aks.kubeconfig +.PHONY: aks.kubeconfig + +# deploy the maestro server +deploy-server: + @AKSCONFIG=svc-cluster $(MAKE) -C $(CLONE_DIR)/maestro deploy-server +.PHONY: deploy-server + +# deploy the maestro agent +deploy-agent: + @AKSCONFIG=mgmt-cluster $(MAKE) -C $(CLONE_DIR)/maestro deploy-agent +.PHONY: deploy-agent + +# register the maestro agent +register-agent: + @AKSCONFIG=svc-cluster $(MAKE) -C $(CLONE_DIR)/maestro register-agent +.PHONY: register-agent + +# clean up the resources +clean: +ifndef AKSCONFIG + $(error "Must set AKSCONFIG") +endif + @$(MAKE) -C $(CLONE_DIR)/dev-infrastructure clean + @rm -rf $(CLONE_DIR) +.PHONY: clean diff --git a/test/e2e/setup/aro/README.md b/test/e2e/setup/aro/README.md index ceacd0a4..4323fe7a 100644 --- a/test/e2e/setup/aro/README.md +++ b/test/e2e/setup/aro/README.md @@ -1,31 +1,15 @@ # Maestro ARO-HCP Env Setup -## Background - -The idea of this directory is to provide means to create a development or testing environment that resemble the (future) production setup in a repeatable way. In order to do so, the creation of all infrastructure resources is based on bicep templates and parameter files. - ## Prerequisites * `az` version >= 2.60, `jq`, `make`, [kubelogin](https://azure.github.io/kubelogin/install.html), `kubectl` version >= 1.30, `helm` * `az login` with service principal (azure AD user support is WIP) -* Register the needed [AFEC](https://aka.ms/afec) feature flags using `cd cluster && make feature-registration` - * __NOTE:__ This will take awhile, you will have to wait until they're in a registered state. - -## Cluster Creation Procedure - -There are a few variants to chose from when creating an AKS cluster: - -* Service Cluster: Public AKS cluster with optional params that can be modified to include all Azure resources needed to run a Service cluster -* Management Cluster: Public AKS cluster with optional params that can be modified to include all Azure resources needed to run a Management cluster - -When creating a cluster, also supporting infrastructure is created, e.g. managed identities, permissions, databases, keyvaults, ... ### Create Service Cluster Change those flags accordingly and then run the following command. Depending on the selected features, this may take a while: ```bash - cd cluster AKSCONFIG=svc-cluster make cluster ``` @@ -34,7 +18,6 @@ Change those flags accordingly and then run the following command. Depending on A Management Cluster depends on certain resources found in the resource group of the Service Cluster. Therefore, a standalone Management Cluster can't be created right now and requires a Service Cluster ```bash - cd cluster AKSCONFIG=mgmt-cluster make cluster ``` @@ -54,7 +37,6 @@ A Management Cluster depends on certain resources found in the resource group of Setting the correct `AKSCONFIG`, this will cleanup all resources created in Azure ```bash - cd cluster AKSCONFIG=svc-cluster make clean ``` @@ -69,7 +51,6 @@ Setting the correct `AKSCONFIG`, this will cleanup all resources created in Azur > The service cluster has no ingress. To interact with the services you deploy use `kubectl port-forward` ```bash - cd maestro AKSCONFIG=svc-cluster make deploy-server ``` @@ -97,7 +78,6 @@ If you need to restart the maestro server during testing and don't want the port First install the agent ```bash - cd maestro AKSCONFIG=mgmt-cluster make deploy-agent ``` diff --git a/test/e2e/setup/aro/cluster/Makefile b/test/e2e/setup/aro/cluster/Makefile deleted file mode 100644 index 19b57dc1..00000000 --- a/test/e2e/setup/aro/cluster/Makefile +++ /dev/null @@ -1,135 +0,0 @@ -SHELL = /bin/bash - -include configurations/dev.mk - -# CURRENTUSER = $(shell az ad signed-in-user show | jq -r '.id') -CURRENTUSER = $(shell az account show -o json | jq -r '.name') -DEPLOYMENTNAME = $(RESOURCEGROUP) -DEPLOYMENTNAMEREGION = $(REGIONAL_RESOURCEGROUP)-REGION - -# Set SKIP_CONFIRM to a non-empty value to skip "what-if" confirmation prompts. -ifndef SKIP_CONFIRM -PROMPT_TO_CONFIRM = "--confirm-with-what-if" -else -PROMPT_TO_CONFIRM_CLEANUP = "--yes" -endif - -list: - @grep '^[^#[:space:]].*:' Makefile -.PHONY: list - -fmt: - set -e; \ - FILES="$$(find . -type f -name "*.bicep*")"; \ - for file in $$FILES; do \ - echo "az bicep format --file $${file}"; \ - az bicep format --file $$file; \ - done -.PHONY: fmt - -lint: - set -e; \ - FILES="$$(find . -type f -name "*.bicep*")"; \ - for file in $$FILES; do \ - echo "az bicep lint --file $${file}"; \ - az bicep lint --file $$file; \ - done -.PHONY: lint - -feature-registration: # hardcoded to eastus as this is a subscription deployment, not a resource group - @az deployment sub create \ - --name "feature-registration" \ - --location eastus \ - --template-file templates/feature-registration.bicep \ - $(PROMPT_TO_CONFIRM) -.PHONY: feature-registration - -rg: - @if [ "$$(az group exists --name $(RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: rg - -regionalRg: - @if [ "$$(az group exists --name $(REGIONAL_RESOURCEGROUP))" = "false" ]; then \ - az group create \ - --name $(REGIONAL_RESOURCEGROUP) \ - --location $(REGION) \ - --output none; \ - fi -.PHONY: regionalRg - -cleanup-orphaned-rolebindings: - @scripts/cleanup-orphaned-rolebindings.sh $(RESOURCEGROUP) - @scripts/cleanup-orphaned-rolebindings.sh $(REGIONAL_RESOURCEGROUP) -.PHONY: cleanup-orphaned-rolebindings - -region: regionalRg - az deployment group create \ - --name "$(DEPLOYMENTNAMEREGION)" \ - --resource-group $(REGIONAL_RESOURCEGROUP) \ - --template-file templates/region.bicep \ - $(PROMPT_TO_CONFIRM) \ - --parameters \ - configurations/region.bicepparam \ - --parameters \ - currentUserId=$(CURRENTUSER) -.PHONY: region - -# cluster: rg cleanup-orphaned-rolebindings region acr -cluster: rg cleanup-orphaned-rolebindings region -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - az deployment group create \ - --name "$(DEPLOYMENTNAME)" \ - --resource-group $(RESOURCEGROUP) \ - --template-file templates/$(AKSCONFIG).bicep \ - $(PROMPT_TO_CONFIRM) \ - --parameters \ - configurations/$(AKSCONFIG).bicepparam \ - --parameters \ - currentUserId=$(CURRENTUSER) \ - regionalResourceGroup=$(REGIONAL_RESOURCEGROUP) -.PHONY: cluster - -cluster-what-if: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - az deployment group what-if \ - --name "$(DEPLOYMENTNAME)" \ - --resource-group $(RESOURCEGROUP) \ - --template-file templates/$(AKSCONFIG).bicep \ - --parameters \ - configurations/$(AKSCONFIG).bicepparam \ - --parameters \ - currentUserId=$(CURRENTUSER) \ -.PHONY: cluster-what-if - -aks.admin-access: rg -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - @scripts/aks-admin-access.sh $(RESOURCEGROUP) -.PHONY: aks.admin-access - -aks.kubeconfig: -ifndef AKSCONFIG - $(error "Must set AKSCONFIG") -endif - AKS_NAME="$$(az aks list --query "[?tags.clusterType == '$(AKSCONFIG)'].name" -ojson -g $(RESOURCEGROUP) | jq -r '.[0]')"; \ - az aks get-credentials -n "$${AKS_NAME}" -g $(RESOURCEGROUP) -f "${HOME}/.kube/$(AKSCONFIG).kubeconfig" - kubelogin convert-kubeconfig -l azurecli --kubeconfig "${HOME}/.kube/$(AKSCONFIG).kubeconfig" -.PHONY: aks.kubeconfig - -clean: - az group delete -g $(RESOURCEGROUP) $(PROMPT_TO_CONFIRM_CLEANUP) - KEY_VAULT_NAME=$(shell az keyvault list-deleted --query "[?properties.tags.resourceGroup == '$(RESOURCEGROUP)'].name" -ojson | jq -r '.[0]') - if [ -n "$${KEY_VAULT_NAME}" ]; then \ - az keyvault purge --name "$${KEY_VAULT_NAME}"; \ - fi -.PHONY: clean diff --git a/test/e2e/setup/aro/cluster/configurations/dev.mk b/test/e2e/setup/aro/cluster/configurations/dev.mk deleted file mode 100644 index 3cba4048..00000000 --- a/test/e2e/setup/aro/cluster/configurations/dev.mk +++ /dev/null @@ -1,3 +0,0 @@ -REGION ?= westus3 -RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION)-$(AKSCONFIG) -REGIONAL_RESOURCEGROUP ?= aro-hcp-$(USER)-$(REGION) diff --git a/test/e2e/setup/aro/cluster/configurations/mgmt-cluster.bicepparam b/test/e2e/setup/aro/cluster/configurations/mgmt-cluster.bicepparam deleted file mode 100644 index bf170641..00000000 --- a/test/e2e/setup/aro/cluster/configurations/mgmt-cluster.bicepparam +++ /dev/null @@ -1,28 +0,0 @@ -using '../templates/mgmt-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param vnetAddressPrefix = '10.132.0.0/14' -param subnetPrefix = '10.132.8.0/21' -param podSubnetPrefix = '10.132.64.0/18' -param aksClusterName = 'aro-hcp-mgmt-cluster' -param aksKeyVaultName = take('aks-kv-mgmt-cluster-${uniqueString(currentUserId)}', 24) -param aksEtcdKVEnableSoftDelete = false -param systemAgentMinCount = 2 -param systemAgentMaxCount = 3 -param systemAgentVMSize = 'Standard_D2s_v3' -param aksSystemOsDiskSizeGB = 32 -param userAgentMinCount = 1 -param userAgentMaxCount = 6 -param userAgentVMSize = 'Standard_D4s_v3' -param aksUserOsDiskSizeGB = 100 -param userAgentPoolAZCount = 3 -param persist = false - -param deployMaestroConsumer = true -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-int.azure.com' - -// These parameters are always overriden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/test/e2e/setup/aro/cluster/configurations/region.bicepparam b/test/e2e/setup/aro/cluster/configurations/region.bicepparam deleted file mode 100644 index f2fc62fa..00000000 --- a/test/e2e/setup/aro/cluster/configurations/region.bicepparam +++ /dev/null @@ -1,9 +0,0 @@ -using '../templates/region.bicep' - -// maestro -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroEventGridMaxClientSessionsPerAuthName = 4 - -// These parameters are always overriden in the Makefile -param currentUserId = '' diff --git a/test/e2e/setup/aro/cluster/configurations/svc-cluster.bicepparam b/test/e2e/setup/aro/cluster/configurations/svc-cluster.bicepparam deleted file mode 100644 index a5b10d28..00000000 --- a/test/e2e/setup/aro/cluster/configurations/svc-cluster.bicepparam +++ /dev/null @@ -1,24 +0,0 @@ -using '../templates/svc-cluster.bicep' - -param kubernetesVersion = '1.30.4' -param istioVersion = ['asm-1-21'] -param vnetAddressPrefix = '10.128.0.0/14' -param subnetPrefix = '10.128.8.0/21' -param podSubnetPrefix = '10.128.64.0/18' -param persist = false -param aksClusterName = 'aro-hcp-svc-cluster' -param aksKeyVaultName = take('aks-kv-svc-cluster-${uniqueString(currentUserId)}', 24) -param aksEtcdKVEnableSoftDelete = false - -param maestroKeyVaultName = take('maestro-kv-${uniqueString(currentUserId)}', 24) -param maestroEventGridNamespacesName = take('maestro-eg-${uniqueString(currentUserId)}', 24) -param maestroCertDomain = 'selfsigned.maestro.keyvault.aro-int.azure.com' -param maestroPostgresServerName = take('maestro-pg-${uniqueString(currentUserId)}', 60) -param maestroPostgresServerVersion = '15' -param maestroPostgresServerStorageSizeGB = 32 -param deployMaestroPostgres = false -param maestroPostgresPrivate = false - -// These parameters are always overriden in the Makefile -param currentUserId = '' -param regionalResourceGroup = '' diff --git a/test/e2e/setup/aro/cluster/modules/aks-cluster-base.bicep b/test/e2e/setup/aro/cluster/modules/aks-cluster-base.bicep deleted file mode 100644 index aabcc7a3..00000000 --- a/test/e2e/setup/aro/cluster/modules/aks-cluster-base.bicep +++ /dev/null @@ -1,437 +0,0 @@ -// Constants -param aksClusterName string -param aksNodeResourceGroupName string -param aksEtcdKVEnableSoftDelete bool - -// System agentpool spec(Infra) -param systemAgentMinCount int = 2 -param systemAgentMaxCount int = 3 -param systemAgentVMSize string = 'Standard_D2s_v3' - -// User agentpool spec (Worker) -param userAgentMinCount int = 1 -param userAgentMaxCount int = 3 -param userAgentVMSize string = 'Standard_D2s_v3' -param userAgentPoolAZCount int = 3 - -param serviceCidr string = '10.130.0.0/16' -param dnsServiceIP string = '10.130.0.10' - -// Passed Params and Overrides -param location string - -@description('Set to true to prevent resources from being pruned after 48 hours') -param persist bool = false - -param kubernetesVersion string -param deployIstio bool -param istioVersion array = ['asm-1-21'] -param vnetAddressPrefix string -param subnetPrefix string -param podSubnetPrefix string -param clusterType string -param workloadIdentities array - -@maxLength(24) -param aksKeyVaultName string - -// Local Params -@description('Optional DNS prefix to use with hosted Kubernetes API server FQDN.') -param dnsPrefix string = aksClusterName - -@description('Disk size (in GB) to provision for each of the agent pool nodes. This value ranges from 0 to 1023. Specifying 0 will apply the default disk size for that agentVMSize.') -@minValue(0) -@maxValue(1023) -param systemOsDiskSizeGB int -param userOsDiskSizeGB int - -@description('Perform cryptographic operations using keys. Only works for key vaults that use the Azure role-based access control permission model.') -var keyVaultCryptoUserId = subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - '12338af0-0e69-4776-bea7-57ae8d297424' -) - -var aksClusterAdminRoleId = subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions/', - '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8' -) -var networkContributorRoleId = subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions/', - '4d97b98b-1d4f-4787-a291-c67834d212e7' -) - -module aks_keyvault_builder '../modules/keyvault/keyvault.bicep' = { - name: aksKeyVaultName - params: { - location: location - keyVaultName: aksKeyVaultName - // todo: change for higher environments - private: false - enableSoftDelete: aksEtcdKVEnableSoftDelete - purpose: 'etcd-encryption' - } -} - -resource aks_keyvault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { - name: aks_keyvault_builder.name -} - -resource aks_etcd_kms 'Microsoft.KeyVault/vaults/keys@2023-07-01' = { - parent: aks_keyvault - name: 'aks-etcd-encryption' - properties: { - kty: 'RSA' - keyOps: [ - 'encrypt' - 'decrypt' - ] - keySize: 2048 - rotationPolicy: { - lifetimeActions: [ - { - action: { - type: 'notify' - } - trigger: { - timeBeforeExpiry: 'P30D' - } - } - ] - } - } -} - -resource aks_keyvault_crypto_user 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(aksClusterUserDefinedManagedIdentity.id, keyVaultCryptoUserId, aks_keyvault.id) - scope: aks_keyvault - properties: { - roleDefinitionId: keyVaultCryptoUserId - principalId: aksClusterUserDefinedManagedIdentity.properties.principalId - principalType: 'ServicePrincipal' - } -} - -resource vnet 'Microsoft.Network/virtualNetworks@2023-11-01' = { - location: location - name: 'aks-net' - properties: { - addressSpace: { - addressPrefixes: [ - vnetAddressPrefix - ] - } - } -} - -resource aksNodeSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' = { - parent: vnet - name: 'ClusterSubnet-001' - properties: { - addressPrefix: subnetPrefix - privateEndpointNetworkPolicies: 'Disabled' - serviceEndpoints: [ - { - service: 'Microsoft.AzureCosmosDB' - } - { - service: 'Microsoft.ContainerRegistry' - } - { - service: 'Microsoft.Storage' - } - { - service: 'Microsoft.KeyVault' - } - ] - } -} - -resource aksPodSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' = { - parent: vnet - name: 'PodSubnet-001' - properties: { - addressPrefix: podSubnetPrefix - privateEndpointNetworkPolicies: 'Disabled' - serviceEndpoints: [ - { - service: 'Microsoft.Storage' - } - ] - delegations: [ - { - name: 'AKS' - properties: { - serviceName: 'Microsoft.ContainerService/managedClusters' - } - } - ] - } - dependsOn: [ - aksNodeSubnet - ] -} - -resource aksClusterUserDefinedManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: '${aksClusterName}-msi' - location: location -} - -resource aksNetworkContributorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: vnet - name: guid(aksClusterUserDefinedManagedIdentity.id, networkContributorRoleId, aksNodeSubnet.id) - properties: { - roleDefinitionId: networkContributorRoleId - principalId: aksClusterUserDefinedManagedIdentity.properties.principalId - principalType: 'ServicePrincipal' - } -} - -resource aksClusterAdminRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: aksCluster - name: guid(aksClusterUserDefinedManagedIdentity.id, aksClusterAdminRoleId, aksCluster.id) - properties: { - roleDefinitionId: aksClusterAdminRoleId - principalId: aksClusterUserDefinedManagedIdentity.properties.principalId - principalType: 'ServicePrincipal' - } -} - -resource aksCluster 'Microsoft.ContainerService/managedClusters@2024-04-02-preview' = { - location: location - kind: 'Base' - name: aksClusterName - sku: { - name: 'Base' - tier: 'Standard' - } - tags: { - persist: toLower(string(persist)) - clusterType: clusterType - } - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${aksClusterUserDefinedManagedIdentity.id}': {} - } - } - properties: { - aadProfile: { - managed: true - enableAzureRBAC: true - } - addonProfiles: { - azureKeyvaultSecretsProvider: { - enabled: true - config: { - enableSecretRotation: 'true' - rotationPollInterval: '5m' - } - } - } - agentPoolProfiles: [ - { - name: 'system' - osType: 'Linux' - osSKU: 'AzureLinux' - mode: 'System' - enableAutoScaling: true - // enableEncryptionAtHost: true - enableEncryptionAtHost: false - enableFIPS: true - enableNodePublicIP: false - kubeletDiskType: 'OS' - osDiskType: 'Ephemeral' - osDiskSizeGB: systemOsDiskSizeGB - count: systemAgentMinCount - minCount: systemAgentMinCount - maxCount: systemAgentMaxCount - vmSize: systemAgentVMSize - type: 'VirtualMachineScaleSets' - upgradeSettings: { - maxSurge: '10%' - } - vnetSubnetID: aksNodeSubnet.id - podSubnetID: aksPodSubnet.id - maxPods: 100 - availabilityZones: [ - '1' - '2' - '3' - ] - securityProfile: { - enableSecureBoot: false - enableVTPM: false - sshAccess: 'Disabled' - } - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - } - ] - autoScalerProfile: { - 'balance-similar-node-groups': 'true' - 'daemonset-eviction-for-occupied-nodes': true - 'scan-interval': '10s' - 'scale-down-delay-after-add': '10m' - 'scale-down-delay-after-delete': '20s' - 'scale-down-delay-after-failure': '3m' - 'scale-down-unneeded-time': '10m' - 'scale-down-unready-time': '20m' - 'scale-down-utilization-threshold': '0.5' - 'skip-nodes-with-local-storage': 'false' - 'max-graceful-termination-sec': '600' - 'max-node-provision-time': '15m' - } - autoUpgradeProfile: { - nodeOSUpgradeChannel: 'NodeImage' - upgradeChannel: 'patch' - } - disableLocalAccounts: true - dnsPrefix: dnsPrefix - enableRBAC: true - kubernetesVersion: kubernetesVersion - metricsProfile: { - costAnalysis: { - enabled: false - } - } - networkProfile: { - ipFamilies: ['IPv4'] - loadBalancerSku: 'standard' - networkDataplane: 'cilium' - networkPolicy: 'cilium' - networkPlugin: 'azure' - podLinkLocalAccess: 'IMDS' - serviceCidr: serviceCidr - serviceCidrs: [serviceCidr] - dnsServiceIP: dnsServiceIP - } - nodeProvisioningProfile: { - mode: 'Manual' - } - nodeResourceGroup: aksNodeResourceGroupName - oidcIssuerProfile: { - enabled: true - } - securityProfile: { - azureKeyVaultKms: { - enabled: false - keyId: aks_etcd_kms.properties.keyUriWithVersion - keyVaultNetworkAccess: 'Public' - } - imageCleaner: { - enabled: true - intervalHours: 24 - } - workloadIdentity: { - enabled: true - } - } - servicePrincipalProfile: { - clientId: 'msi' - } - serviceMeshProfile: (deployIstio) - ? { - mode: 'Istio' - istio: { - components: { - ingressGateways: [ - { - enabled: true - mode: 'External' - } - ] - } - revisions: istioVersion - } - } - : null - storageProfile: { - diskCSIDriver: { - enabled: true - version: 'v1' - } - fileCSIDriver: { - enabled: true - } - snapshotController: { - enabled: true - } - } - supportPlan: 'KubernetesOfficial' - } -} - -resource userAgentPools 'Microsoft.ContainerService/managedClusters/agentPools@2024-04-02-preview' = [ - for i in range(0, userAgentPoolAZCount): { - parent: aksCluster - name: 'user${take(string(i+1), 8)}' - properties: { - osType: 'Linux' - osSKU: 'AzureLinux' - mode: 'User' - enableAutoScaling: true - // enableEncryptionAtHost: true - enableEncryptionAtHost: false - enableFIPS: true - enableNodePublicIP: false - kubeletDiskType: 'OS' - osDiskType: 'Ephemeral' - osDiskSizeGB: userOsDiskSizeGB - count: userAgentMinCount - minCount: userAgentMinCount - maxCount: userAgentMaxCount - vmSize: userAgentVMSize - type: 'VirtualMachineScaleSets' - upgradeSettings: { - maxSurge: '10%' - } - vnetSubnetID: aksNodeSubnet.id - podSubnetID: aksPodSubnet.id - maxPods: 225 - availabilityZones: [ - '${(i + 1)}' - ] - securityProfile: { - enableSecureBoot: false - enableVTPM: false - sshAccess: 'Disabled' - } - } - } -] - -resource uami 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = [ - for wi in workloadIdentities: { - location: location - name: wi.value.uamiName - } -] - -resource uami_fedcred 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = [ - for i in range(0, length(workloadIdentities)): { - parent: uami[i] - name: '${workloadIdentities[i].value.uamiName}-${location}-fedcred' - properties: { - audiences: [ - 'api://AzureADTokenExchange' - ] - issuer: aksCluster.properties.oidcIssuerProfile.issuerURL - subject: 'system:serviceaccount:${workloadIdentities[i].value.namespace}:${workloadIdentities[i].value.serviceAccountName}' - } - } -] - -// Outputs -output userAssignedIdentities array = [ - for i in range(0, length(workloadIdentities)): { - uamiID: uami[i].id - uamiName: workloadIdentities[i].value.uamiName - uamiClientID: uami[i].properties.clientId - uamiPrincipalID: uami[i].properties.principalId - } -] -output aksVnetId string = vnet.id -output aksNodeSubnetId string = aksNodeSubnet.id -output aksOidcIssuerUrl string = aksCluster.properties.oidcIssuerProfile.issuerURL -output aksClusterName string = aksClusterName diff --git a/test/e2e/setup/aro/cluster/modules/key-vault-cert.bicep b/test/e2e/setup/aro/cluster/modules/key-vault-cert.bicep deleted file mode 100644 index 7225b0fd..00000000 --- a/test/e2e/setup/aro/cluster/modules/key-vault-cert.bicep +++ /dev/null @@ -1,45 +0,0 @@ -/* -Creating certificates in Azure Key Vault is not supported by Bicep yet. -This module leverages a deploymentscript to solve this for the time beeing. -Proudly stolen from https://github.com/Azure/bicep/discussions/8457 - -We might not need certificates for MQTT authentication altogether if -Entra autentication can be leveraged: https://redhat-external.slack.com/archives/C03F6AA3HDH/p1713340078776669 -*/ - -param keyVaultName string -param certName string -param subjectName string -param issuerName string -param dnsNames array -param now string = utcNow('F') -param keyVaultManagedIdentityId string -param location string -param force bool = false -var boolstring = force == false ? '$false' : '$true' -param validityInMonths int = 12 - -resource newCertwithRotationKV 'Microsoft.Resources/deploymentScripts@2023-08-01' = { - name: 'newCertwithRotationKV-${certName}' - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${keyVaultManagedIdentityId}': {} - } - } - location: location - kind: 'AzurePowerShell' - properties: { - azPowerShellVersion: '7.5.0' - arguments: ' -VaultName ${keyVaultName} -ValidityInMonths ${validityInMonths} -IssuerName ${issuerName} -CertName ${certName} -SubjectName ${subjectName} -DnsNames ${join(dnsNames,'_')} -Force ${boolstring}' - scriptContent: loadTextContent('../scripts/key-vault-cert.ps1') - forceUpdateTag: now - cleanupPreference: 'Always' - retentionInterval: 'P1D' - timeout: 'PT5M' - } -} - -output Thumbprint string = newCertwithRotationKV.properties.outputs.Thumbprint -output CACert string = newCertwithRotationKV.properties.outputs.CACert -output KeyVaultCertId string = newCertwithRotationKV.properties.outputs.KeyVaultCertId diff --git a/test/e2e/setup/aro/cluster/modules/keyvault/keyvault.bicep b/test/e2e/setup/aro/cluster/modules/keyvault/keyvault.bicep deleted file mode 100644 index 169996e7..00000000 --- a/test/e2e/setup/aro/cluster/modules/keyvault/keyvault.bicep +++ /dev/null @@ -1,40 +0,0 @@ -@description('Location of the keyvault.') -param location string - -@description('Name of the key vault.') -param keyVaultName string - -@description('Toggle to enable soft delete.') -param enableSoftDelete bool - -@description('Toggle to make the keyvault private.') -param private bool - -@description('Purpose of the keyvault.') -param purpose string - -resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = { - location: location - name: keyVaultName - tags: { - resourceGroup: resourceGroup().name - aroHCPPurpose: purpose - } - properties: { - enableRbacAuthorization: true - enabledForDeployment: false - enabledForDiskEncryption: false - enabledForTemplateDeployment: false - enableSoftDelete: enableSoftDelete - publicNetworkAccess: private ? 'Disabled' : 'Enabled' - sku: { - name: 'standard' - family: 'A' - } - tenantId: subscription().tenantId - } -} - -output kvId string = keyVault.id - -output kvName string = keyVault.name diff --git a/test/e2e/setup/aro/cluster/modules/maestro/maestro-consumer.bicep b/test/e2e/setup/aro/cluster/modules/maestro/maestro-consumer.bicep deleted file mode 100644 index b7b1beef..00000000 --- a/test/e2e/setup/aro/cluster/modules/maestro/maestro-consumer.bicep +++ /dev/null @@ -1,25 +0,0 @@ -param maestroServerManagedIdentityPrincipalId string -@minLength(1) -param maestroConsumerName string -param maestroInfraResourceGroup string -param maestroEventGridNamespaceName string -param maestroKeyVaultName string -param maestroKeyVaultOfficerManagedIdentityName string -param maestroKeyVaultCertificateDomain string - -param location string - -module evengGridAccess './maestro-eventgrid-access.bicep' = { - name: 'event-grid-access-${uniqueString(maestroConsumerName)}' - scope: resourceGroup(maestroInfraResourceGroup) - params: { - eventGridNamespaceName: maestroEventGridNamespaceName - keyVaultName: maestroKeyVaultName - kvCertOfficerManagedIdentityName: maestroKeyVaultOfficerManagedIdentityName - certDomain: maestroKeyVaultCertificateDomain - clientName: maestroConsumerName - clientRole: 'consumer' - certificateAccessManagedIdentityPrincipalId: maestroServerManagedIdentityPrincipalId - location: location - } -} diff --git a/test/e2e/setup/aro/cluster/modules/maestro/maestro-eventgrid-access.bicep b/test/e2e/setup/aro/cluster/modules/maestro/maestro-eventgrid-access.bicep deleted file mode 100644 index c3b982f4..00000000 --- a/test/e2e/setup/aro/cluster/modules/maestro/maestro-eventgrid-access.bicep +++ /dev/null @@ -1,133 +0,0 @@ -/* -This module manages access to EventGrid for a maestro client, which -can be the server or a consumer. - -- Creates a certificate in Key Vault signed by the specified issuer. - For dev environments `Self` is used as issuer, for higher environments - OneCertV2 Private will be used. -- An MQTT client is registered within eventgrid. Depending on the certificate - issuer, the certificate validation schema will be thumbprint based for - self-signed certificates and DNS based for OneCertV2 Private certificates. -- The MQTT client is placed into the right MQTT client group based on the - client role. This defines the topic access permissions for the client. -- The specified managed identity `certificateAccessManagedIdentityPrincipalId` - is granted access to the certificate in Key Vault. This will be leveraged - with CSI secret store to access the certificate from the maestro pods. - -Execution scope: the resourcegroup of the maestro infrastructure -*/ - -@description('The EventGrid Namespace name where access will be managed') -param eventGridNamespaceName string - -@description('The Key Vault name where the certificate for Event Grid access will be stored') -param keyVaultName string - -@description('The name of the managed identity that will be used to manage the certificate in Key Vault') -param kvCertOfficerManagedIdentityName string - -@description('The base domain name to be used for the certificates DNS name.') -param certDomain string - -@description('The name of the client that will be created in the EventGrid Namespace') -param clientName string - -@description('The role of the client in the EventGrid Namespace.') -@allowed([ - 'server' - 'consumer' -]) -param clientRole string - -@description('Grant this managed identity access to the certificate in Key Vault.') -param certificateAccessManagedIdentityPrincipalId string - -@description('The issuer of the certificate.') -param certificateIssuer string = 'Self' - -param location string - -var clientAuthenticationName = '${clientName}.${certDomain}' - -resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2023-12-15-preview' existing = { - name: eventGridNamespaceName -} - -resource kvCertOfficerManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { - name: kvCertOfficerManagedIdentityName -} - -// certificate for MQTT authentication -module clientCertificate '../key-vault-cert.bicep' = { - name: '${deployment().name}-client-cert' - params: { - location: location - keyVaultName: keyVaultName - subjectName: 'CN=${clientName}' - certName: clientName - keyVaultManagedIdentityId: kvCertOfficerManagedIdentity.id - dnsNames: [ - clientAuthenticationName - ] - // todo - use Private OnceCertV2 in higher environments - issuerName: certificateIssuer - } -} - -// D O N ' T U S E T H I S I N P R O D U C T I O N -// eventgrid MQTT client trusting the certificate by thumbprint if -// Key Vault self-signed certificates are used. trusting self-signed certificates -// as CAs is not supported in EventGrid -resource mqttClient 'Microsoft.EventGrid/namespaces/clients@2023-12-15-preview' = if (certificateIssuer == 'Self') { - name: clientName - parent: eventGridNamespace - properties: { - authenticationName: clientAuthenticationName - attributes: { - role: clientRole - consumer_name: clientName - } - clientCertificateAuthentication: { - allowedThumbprints: [ - clientCertificate.outputs.Thumbprint - ] - validationScheme: 'ThumbprintMatch' - } - state: 'Enabled' - } -} - -// TODO - implement issuer CA registration with EventGrid + register the mqtt client with -// the DnsMatchesAuthenticationName authentication validation scheme - -var keyVaultSecretUserRoleId = subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions/', - '4633458b-17de-408a-b874-0445c86b69e6' -) - -resource kv 'Microsoft.KeyVault/vaults@2023-07-01' existing = { - name: keyVaultName -} - -// grant permissions on the secret that contains the certificate - -resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' existing = { - parent: kv - name: clientName -} - -resource secretAccessPermission 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: secret - name: guid(certificateAccessManagedIdentityPrincipalId, keyVaultSecretUserRoleId, kv.id) - properties: { - roleDefinitionId: keyVaultSecretUserRoleId - principalId: certificateAccessManagedIdentityPrincipalId - principalType: 'ServicePrincipal' - } -} - -// output - -output KeyVaultCertId string = clientCertificate.outputs.KeyVaultCertId -output KeyVaultCertName string = clientName -output EventGridHostname string = eventGridNamespace.properties.topicSpacesConfiguration.hostname diff --git a/test/e2e/setup/aro/cluster/modules/maestro/maestro-infra.bicep b/test/e2e/setup/aro/cluster/modules/maestro/maestro-infra.bicep deleted file mode 100644 index 5798247e..00000000 --- a/test/e2e/setup/aro/cluster/modules/maestro/maestro-infra.bicep +++ /dev/null @@ -1,251 +0,0 @@ -/* -This module creates the infrastructure required by maestro to run. This includes: - -- A KeyVault where the client certificates for EventGrid MQTT broker access - are generated and stored -- A managed identity to create and manage certificates in Key Vault. This is - used by the maestro-eventgrid-access bicep module deploymentscripts. - - Why is this needed? There are no bicep modules for KeyVault certificate management, - so we need deploymentscripts + a managed identity with Key Vault access to run them. - -- Create an EventGrid namespaces instance with MQTT enabled. -- Create EventGrid client groups for the server and consumers and define topic - access permissions. - -Execution scope: the resourcegroup of the maestro infrastructure - -TODO: -- Key Vault network access restrictions (e.g. privatelink) -- EventGrid network access restrictions (e.g. privatelink) -*/ - -@description('The Maestro Event Grid Namespaces name') -param eventGridNamespaceName string - -@description('The location of the EventGrid Namespace') -param location string - -@description('The maximum client sessions per authentication name for the EventGrid MQTT broker') -param maxClientSessionsPerAuthName int - -@description('The name for the Key Vault for Maestro certificates') -param maestroKeyVaultName string - -@description('The name for the Managed Identity that will be created for Key Vault Certificate management.') -param kvCertOfficerManagedIdentityName string - -// -// K E Y V A U L T -// - -resource kv 'Microsoft.KeyVault/vaults@2023-07-01' = { - name: maestroKeyVaultName - location: location - tags: { - resourceGroup: resourceGroup().name - } - properties: { - accessPolicies: [] - enableRbacAuthorization: true - enabledForDeployment: false - enabledForDiskEncryption: false - enabledForTemplateDeployment: false - enableSoftDelete: false - networkAcls: { - bypass: 'AzureServices' - defaultAction: 'Allow' - ipRules: [ - { - // TODO: restrict in higher environments - value: '0.0.0.0/0' - } - ] - } - // TODO: disabled in higher environments - publicNetworkAccess: 'Enabled' - sku: { - family: 'A' - name: 'standard' - } - tenantId: subscription().tenantId - } -} - -// -// C E R T I F I C A T E O F F I C E R M S I -// - -resource kvCertOfficerManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: kvCertOfficerManagedIdentityName - location: location -} - -var keyVaultCertificateOfficerRoleId = subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions/', - 'a4417e6f-fecd-4de8-b567-7b0420556985' -) - -resource kvManagedIdentityRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - scope: kv - name: guid(kvCertOfficerManagedIdentity.id, keyVaultCertificateOfficerRoleId, kv.id) - properties: { - roleDefinitionId: keyVaultCertificateOfficerRoleId - principalId: kvCertOfficerManagedIdentity.properties.principalId - principalType: 'ServicePrincipal' - } -} - -// -// E V E N T G R I D -// - -// create an event grid namespace with MQTT enabled -resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2024-06-01-preview' = { - name: eventGridNamespaceName - location: location - sku: { - name: 'Standard' - capacity: 1 - } - properties: { - isZoneRedundant: true - publicNetworkAccess: 'Enabled' - topicSpacesConfiguration: { - state: 'Enabled' - maximumSessionExpiryInHours: 1 - maximumClientSessionsPerAuthenticationName: maxClientSessionsPerAuthName - clientAuthentication: { - alternativeAuthenticationNameSources: [ - 'ClientCertificateDns' - ] - } - } - } -} - -// -// E V E N T G R I D M A E S T R O S E R V E R C O N F I G -// - -// an MQTT client group to hold the maestro server client -resource maestroServerMqttClientGroup 'Microsoft.EventGrid/namespaces/clientGroups@2023-12-15-preview' = { - name: 'maestro-server' - parent: eventGridNamespace - properties: { - query: 'attributes.role IN [\'server\']' - } -} - -// create a topic space for the maestro server to subscribe to -resource maestroServerSubscribeTopicspace 'Microsoft.EventGrid/namespaces/topicSpaces@2023-12-15-preview' = { - name: 'maestro-server-subscribe' - parent: eventGridNamespace - properties: { - topicTemplates: [ - 'sources/maestro/consumers/+/agentevents' - ] - } -} - -// ... and grant the maestro server client permission to subscribe to the topic space -resource maestroServerPermissionBindingSubscribe 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { - name: 'maestro-server-subscribe-binding' - parent: eventGridNamespace - properties: { - clientGroupName: maestroServerMqttClientGroup.name - permission: 'Subscriber' - topicSpaceName: maestroServerSubscribeTopicspace.name - } -} - -// create a topic space for the maestro server to publish to -resource maestroServerPublishTopicspace 'Microsoft.EventGrid/namespaces/topicSpaces@2023-12-15-preview' = { - name: 'maestro-server-publish' - parent: eventGridNamespace - properties: { - topicTemplates: [ - 'sources/maestro/consumers/+/sourceevents' - ] - } - dependsOn: [ - maestroServerSubscribeTopicspace // this dependency prevents concurrent topicspace updates - ] -} - -// ... and grant the maestro server client permission to publish to the topic space -resource maestroServerPermissionBindingPublish 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { - name: 'maestro-server-publish-binding' - parent: eventGridNamespace - properties: { - clientGroupName: maestroServerMqttClientGroup.name - permission: 'Publisher' - topicSpaceName: maestroServerPublishTopicspace.name - } -} - -// -// E V E N T G R I D M A E S T R O C O N S U M E R C O N F I G -// - -// an MQTT client group to hold the maestro consumer clients -resource maestroConsumerMqttClientGroup 'Microsoft.EventGrid/namespaces/clientGroups@2023-12-15-preview' = { - name: 'maestro-consumers' - parent: eventGridNamespace - properties: { - query: 'attributes.role IN [\'consumer\']' - } -} - -// create a topic space for the maestro consumers to subscribe to -resource maestroConsumersSubscribeTopicspace 'Microsoft.EventGrid/namespaces/topicSpaces@2023-12-15-preview' = { - name: 'maestro-consumer-subscribe' - parent: eventGridNamespace - properties: { - topicTemplates: [ - 'sources/maestro/consumers/\${client.attributes.consumer_name}/sourceevents' - ] - } - dependsOn: [ - maestroServerPublishTopicspace // this dependency prevents concurrent topicspace updates - ] -} - -// ... and grant the maestro consumer client group permission to subscribe to the topic space -resource maestroConsumersSubscribeTopicspacePermissionBinding 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { - name: 'maestro-consumer-subscribe' - parent: eventGridNamespace - properties: { - clientGroupName: maestroConsumerMqttClientGroup.name - permission: 'Subscriber' - topicSpaceName: maestroConsumersSubscribeTopicspace.name - } -} - -// create a topic space for the maestro consumers to publish to -resource maestroConsumersPublishTopicspace 'Microsoft.EventGrid/namespaces/topicSpaces@2023-12-15-preview' = { - name: 'maestro-consumer-publish' - parent: eventGridNamespace - properties: { - topicTemplates: [ - 'sources/maestro/consumers/\${client.attributes.consumer_name}/agentevents' - ] - } - dependsOn: [ - maestroConsumersSubscribeTopicspace // this dependency prevents concurrent topicspace updates - ] -} - -// ... and grant the maestro consumer client group permission to publish to the topic space -resource maestroConsumersPublishTopicspacePermissionBinding 'Microsoft.EventGrid/namespaces/permissionBindings@2023-12-15-preview' = { - name: 'maestro-consumer-publish' - parent: eventGridNamespace - properties: { - clientGroupName: maestroConsumerMqttClientGroup.name - permission: 'Publisher' - topicSpaceName: maestroConsumersPublishTopicspace.name - } -} - -output keyVaultName string = kv.name -output eventGridNamespaceName string = eventGridNamespace.name diff --git a/test/e2e/setup/aro/cluster/modules/maestro/maestro-server.bicep b/test/e2e/setup/aro/cluster/modules/maestro/maestro-server.bicep deleted file mode 100644 index 1cb94082..00000000 --- a/test/e2e/setup/aro/cluster/modules/maestro/maestro-server.bicep +++ /dev/null @@ -1,146 +0,0 @@ -/* -This module is responsible for: - - setting up EventGrid access for the maestro server - -Execution scope: the resourcegroup of the AKS cluster where the maestro server -will be deployed. - -TODO: -- once Key Vault and EventGrid have network access restrictions enabled, - this module needs to be enhanced to manage access to both (e.g. privatelink) -*/ - -param maestroInfraResourceGroup string -param maestroEventGridNamespaceName string -param maestroKeyVaultName string -param maestroKeyVaultOfficerManagedIdentityName string -param maestroKeyVaultCertificateDomain string - -@description('Whether to deploy the Postgres server for Maestro') -param deployPostgres bool - -@description('The name of the Postgres server for Maestro') -param postgresServerName string - -@description('The version of the Postgres server for Maestro') -param postgresServerVersion string - -@description('The size of the Postgres server storage for Maestro') -@allowed([ - 32 - 64 - 128 - 256 - 512 - 1024 - 2048 - 4096 - 8192 - 16384 - 32768 -]) -param postgresServerStorageSizeGB int - -param postgresServerPrivate bool - -param privateEndpointSubnetId string = '' - -param privateEndpointVnetId string = '' - -@description('The name of the database to create for Maestro') -param maestroDatabaseName string = 'maestro' - -@description('The name of the Managed Identity for the Maestro cluster service') -param maestroServerManagedIdentityName string - -@description('The principal ID of the Managed Identity for the Maestro cluster service') -param maestroServerManagedIdentityPrincipalId string - -param location string - -// -// P O S T G R E S -// - -resource postgresAdminManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: '${postgresServerName}-db-admin-msi' - location: location -} - -module postgres '../postgres/postgres.bicep' = if (deployPostgres) { - name: '${deployment().name}-postgres' - params: { - name: postgresServerName - databaseAdministrators: [ - // add the dedicated admin managed identity as administrator - // this one is going to be used to manage DB access - { - principalId: postgresAdminManagedIdentity.properties.principalId - principalName: postgresAdminManagedIdentity.name - principalType: 'ServicePrincipal' - } - ] - version: postgresServerVersion - configurations: [ - { - source: 'log_min_duration_statement' - value: '3000' - } - { - source: 'log_statement' - value: 'all' - } - ] - databases: [ - { - name: maestroDatabaseName - charset: 'UTF8' - collation: 'en_US.utf8' - } - ] - maintenanceWindow: { - customWindow: 'Enabled' - dayOfWeek: 0 - startHour: 1 - startMinute: 12 - } - storageSizeGB: postgresServerStorageSizeGB - private: postgresServerPrivate - subnetId: privateEndpointSubnetId - vnetId: privateEndpointVnetId - managedPrivateEndpoint: true - } -} - -module csManagedIdentityDatabaseAccess '../postgres/postgres-access.bicep' = if (deployPostgres) { - name: '${deployment().name}-maestro-db-access' - params: { - postgresServerName: postgresServerName - postgresAdminManagedIdentityName: postgresAdminManagedIdentity.name - databaseName: maestroDatabaseName - newUserName: maestroServerManagedIdentityName - newUserPrincipalId: maestroServerManagedIdentityPrincipalId - } - dependsOn: [ - postgres - ] -} - -// -// E V E N T G R I D -// - -module evengGridAccess './maestro-eventgrid-access.bicep' = { - name: '${deployment().name}-event-grid-access' - scope: resourceGroup(maestroInfraResourceGroup) - params: { - eventGridNamespaceName: maestroEventGridNamespaceName - keyVaultName: maestroKeyVaultName - kvCertOfficerManagedIdentityName: maestroKeyVaultOfficerManagedIdentityName - certDomain: maestroKeyVaultCertificateDomain - clientName: 'maestro-server' - clientRole: 'server' - certificateAccessManagedIdentityPrincipalId: maestroServerManagedIdentityPrincipalId - location: location - } -} diff --git a/test/e2e/setup/aro/cluster/modules/postgres/postgres-access.bicep b/test/e2e/setup/aro/cluster/modules/postgres/postgres-access.bicep deleted file mode 100644 index 617bff66..00000000 --- a/test/e2e/setup/aro/cluster/modules/postgres/postgres-access.bicep +++ /dev/null @@ -1,48 +0,0 @@ -/* -Manages access to a postgres database by creating a new user and granting access to a database. -The user will also be enabled for entra authentication. -*/ - -@description('The name of the postgres server that will be managed') -param postgresServerName string - -@description('The name of the managed identity that will be used to manage access in the database') -param postgresAdminManagedIdentityName string - -@description('The principal ID / object ID of the managed identity that will be granted access to') -param newUserPrincipalId string - -@description('The name of the managed identity that will be granted access to') -param newUserName string - -@description('The name of the database, the new new user will be granted access to') -param databaseName string - -resource postgres 'Microsoft.DBforPostgreSQL/flexibleServers@2023-03-01-preview' existing = { - name: postgresServerName -} - -var sqlScriptLines = [ - 'do' - '$$' - 'begin' - ' if not exists (select * from pg_user where usename = \'${newUserName}\') then' - ' create user "${newUserName}";' - ' end if;' - 'end' - '$$' - ';' - 'SECURITY LABEL for "pgaadauth" on role "${newUserName}" is \'aadauth,oid=${newUserPrincipalId},type=service\';' - 'GRANT ALL PRIVILEGES ON DATABASE ${databaseName} TO "${newUserName}";' - 'GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO "${newUserName}";' -] - -module csManagedIdentityDatabaseAccess 'postgres-sql.bicep' = { - name: '${deployment().name}-db-access' - params: { - postgresServerName: postgres.properties.fullyQualifiedDomainName - databaseName: 'postgres' // access configuration is managed in the postgres DB - postgresAdminManagedIdentityName: postgresAdminManagedIdentityName - sqlScript: string(join(sqlScriptLines, '\n')) - } -} diff --git a/test/e2e/setup/aro/cluster/modules/postgres/postgres-sql.bicep b/test/e2e/setup/aro/cluster/modules/postgres/postgres-sql.bicep deleted file mode 100644 index 9744f8cd..00000000 --- a/test/e2e/setup/aro/cluster/modules/postgres/postgres-sql.bicep +++ /dev/null @@ -1,66 +0,0 @@ -/* -Executes an SQL script on a PostgreSQL server using a user-assigned managed identity. -*/ - -@description('The postgres server name where an SQL script will be executed') -param postgresServerName string - -@description('The database name where an SQL script will be executed') -param databaseName string - -@description('The name of the user-assigned managed identity that will be used to execute the SQL script') -param postgresAdminManagedIdentityName string - -@description('The SQL script to execute on the PostgreSQL server') -param sqlScript string - -param forceUpdateTag string = guid('${sqlScript}/${postgresServerName}/${databaseName}+${postgresAdminManagedIdentityName}') - -resource postgresAdminManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { - name: postgresAdminManagedIdentityName -} - -resource deploymentScript 'Microsoft.Resources/deploymentScripts@2023-08-01' = { - name: deployment().name - location: resourceGroup().location - kind: 'AzureCLI' - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${postgresAdminManagedIdentity.id}': {} - } - } - - properties: { - azCliVersion: '2.30.0' - cleanupPreference: 'OnSuccess' - retentionInterval: 'P1D' - scriptContent: ''' - az login --identity - export PGPASSWORD=$(az account get-access-token --resource-type oss-rdbms -o json | jq .accessToken -r) - echo "${SQL_SCRIPT}" | base64 -d > script.sql - apk add postgresql-client - psql -f script.sql - ''' - forceUpdateTag: forceUpdateTag - environmentVariables: [ - { - name: 'SQL_SCRIPT' - value: base64(string(sqlScript)) - } - { - name: 'PGHOST' - value: postgresServerName - } - { - name: 'PGDATABASE' - value: databaseName - } - { - name: 'PGUSER' - value: postgresAdminManagedIdentity.name - } - ] - timeout: 'PT30M' - } -} diff --git a/test/e2e/setup/aro/cluster/modules/postgres/postgres.bicep b/test/e2e/setup/aro/cluster/modules/postgres/postgres.bicep deleted file mode 100644 index 0b9766e4..00000000 --- a/test/e2e/setup/aro/cluster/modules/postgres/postgres.bicep +++ /dev/null @@ -1,231 +0,0 @@ -/* -This module creates a postgres flexible server, firewall rules, administrators, configurations, and databases. -*/ - -@description('The name of the Postgres server.') -param name string - -param location string = resourceGroup().location - -param sku string = 'Standard_D2s_v3' -param tier string = 'GeneralPurpose' - -type DatabaseAdministrators = { - principalId: string - principalName: string - principalType: string -} - -@description('The database administrators to create on the server.') -param databaseAdministrators DatabaseAdministrators[] = [] - -type DatabaseConfigurations = { - source: string - value: string -} -@description('The configuration options to set on the server.') -param configurations DatabaseConfigurations[] = [] - -type DatabaseProperties = { - name: string - charset: string - collation: string -} -@description('The databases to create on the server.') -param databases DatabaseProperties[] = [] - -type MaintenanceWindow = { - customWindow: string - dayOfWeek: int - startHour: int - startMinute: int -} -@description('The maintenance window for the server.') -param maintenanceWindow MaintenanceWindow - -@description('The number of days to retain backups for.') -param backupRetentionDays int = 7 - -@allowed([ - 32 - 64 - 128 - 256 - 512 - 1024 - 2048 - 4096 - 8192 - 16384 - 32768 -]) -param storageSizeGB int - -param version string - -param private bool - -param managedPrivateEndpoint bool = true - -param subnetId string = '' - -param vnetId string = '' - -resource postgres 'Microsoft.DBforPostgreSQL/flexibleServers@2023-12-01-preview' = { - name: name - location: resourceGroup().location - sku: { - name: sku - tier: tier - } - properties: { - administratorLogin: '' - administratorLoginPassword: '' - version: version - createMode: 'Default' - network: { - publicNetworkAccess: private ? 'Disabled' : 'Enabled' - } - authConfig: { - activeDirectoryAuth: 'Enabled' - passwordAuth: 'Disabled' - tenantId: subscription().tenantId - } - backup: { - backupRetentionDays: backupRetentionDays - geoRedundantBackup: 'Disabled' - } - dataEncryption: { - type: 'SystemManaged' - } - highAvailability: { - mode: 'ZoneRedundant' - } - maintenanceWindow: maintenanceWindow - storage: { - autoGrow: 'Enabled' - storageSizeGB: storageSizeGB - } - } -} - -resource postgres_allow_azure_firewall 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2023-12-01-preview' = { - name: 'AllowAllAzureServicesAndResourcesWithinAzureIps' - parent: postgres - properties: { - startIpAddress: '0.0.0.0' - endIpAddress: '0.0.0.0' - } -} - -@batchSize(1) -resource postgres_admin 'Microsoft.DBforPostgreSQL/flexibleServers/administrators@2023-12-01-preview' = [ - for admin in filter(databaseAdministrators, a => a.principalId != ''): { - name: admin.principalId - parent: postgres - properties: { - principalName: admin.principalName - principalType: admin.principalType - tenantId: subscription().tenantId - } - dependsOn: [postgres_allow_azure_firewall] - } -] - -@batchSize(1) -resource postgres_config 'Microsoft.DBforPostgreSQL/flexibleServers/configurations@2023-12-01-preview' = [ - for config in configurations: { - name: config.source - parent: postgres - properties: { - source: 'user-override' - value: config.value - } - dependsOn: [postgres_admin] - } -] - -@batchSize(1) -resource postgres_database 'Microsoft.DBforPostgreSQL/flexibleServers/databases@2023-12-01-preview' = [ - for database in databases: { - name: database.name - parent: postgres - properties: { - charset: database.charset - collation: database.collation - } - dependsOn: [postgres_config] - } -] - -output hostname string = postgres.properties.fullyQualifiedDomainName -output port int = 5432 - -// -// P R I V A T E E N D P O I N T -// - -var privateDnsZoneName = 'privatelink.postgres.database.azure.com' - -resource postgresPrivateEndpoint 'Microsoft.Network/privateEndpoints@2024-01-01' = if (managedPrivateEndpoint) { - name: '${name}-pe' - location: location - properties: { - privateLinkServiceConnections: [ - { - name: '${name}-pe' - properties: { - groupIds: [ - 'postgresqlServer' - ] - privateLinkServiceId: postgres.id - } - } - ] - subnet: { - id: subnetId - } - } - dependsOn: [ - postgres_database - ] -} - -resource postgresPrivateEndpointDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = if (managedPrivateEndpoint) { - name: privateDnsZoneName - location: 'global' - properties: {} - dependsOn: [ - postgresPrivateEndpoint - ] -} - -resource postgresPrivateDnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = if (managedPrivateEndpoint) { - parent: postgresPrivateEndpointDnsZone - name: 'postgres' - location: 'global' - properties: { - registrationEnabled: false - virtualNetwork: { - id: vnetId - } - } -} - -resource privateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-09-01' = if (managedPrivateEndpoint) { - parent: postgresPrivateEndpoint - name: '${name}-dns-group' - properties: { - privateDnsZoneConfigs: [ - { - name: 'config1' - properties: { - privateDnsZoneId: postgresPrivateEndpointDnsZone.id - } - } - ] - } - dependsOn: [ - postgresPrivateDnsZoneVnetLink - ] -} diff --git a/test/e2e/setup/aro/cluster/scripts/aks-admin-access.sh b/test/e2e/setup/aro/cluster/scripts/aks-admin-access.sh deleted file mode 100755 index b213c983..00000000 --- a/test/e2e/setup/aro/cluster/scripts/aks-admin-access.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -RESOURCEGROUP=$1 -# CURRENTUSER_CLIENT_ID=$(az ad signed-in-user show -o json | jq -r '.id') -CURRENTUSER_CLIENT_ID=$(az account show -o json | jq -r '.user.name') -CLUSTER_ID=$(az aks list -g $RESOURCEGROUP -o json | jq -r .[0].id) - -az role assignment create --assignee $CURRENTUSER_CLIENT_ID --role "Azure Kubernetes Service RBAC Cluster Admin" --scope $CLUSTER_ID -az role assignment create --assignee $CURRENTUSER_CLIENT_ID --role "Azure Kubernetes Service Cluster Admin Role" --scope $CLUSTER_ID -echo "It might take a couple of minutes for the permissions to take effect" diff --git a/test/e2e/setup/aro/cluster/scripts/cleanup-orphaned-rolebindings.sh b/test/e2e/setup/aro/cluster/scripts/cleanup-orphaned-rolebindings.sh deleted file mode 100755 index 8b1e732d..00000000 --- a/test/e2e/setup/aro/cluster/scripts/cleanup-orphaned-rolebindings.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -RESOURCEGROUP=$1 - -# List all role assignments and filter for 'ServicePrincipal' -roleAssignments=$(az role assignment list -g ${RESOURCEGROUP} --query "[?principalType=='ServicePrincipal'].{id:id, principalId:principalId}" -o tsv) - -if [ -n "$roleAssignments" ]; then - while IFS=$'\t' read -r id principalId; do - # Check if the Managed Identity exists - identityExists=$(az ad sp show --id $principalId --query id -o tsv 2>/dev/null) - - if [ -z "$identityExists" ]; then - echo "Role Assignment ID $id is bound to a non-existent Managed Identity $principalId... deleting" - az role assignment delete --ids "$id" - fi - done <<< "$roleAssignments" -fi diff --git a/test/e2e/setup/aro/cluster/scripts/digicert-global-root-g3.crt b/test/e2e/setup/aro/cluster/scripts/digicert-global-root-g3.crt deleted file mode 100644 index 66433b3e..00000000 --- a/test/e2e/setup/aro/cluster/scripts/digicert-global-root-g3.crt +++ /dev/null @@ -1,16 +0,0 @@ -# DigiCert Global Root G3 ------BEGIN CERTIFICATE----- -MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw -CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu -ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe -Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw -EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x -IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF -K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG -fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO -Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd -BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx -AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ -oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 -sycX ------END CERTIFICATE----- diff --git a/test/e2e/setup/aro/cluster/scripts/key-vault-cert.ps1 b/test/e2e/setup/aro/cluster/scripts/key-vault-cert.ps1 deleted file mode 100644 index 9ce6b8df..00000000 --- a/test/e2e/setup/aro/cluster/scripts/key-vault-cert.ps1 +++ /dev/null @@ -1,111 +0,0 @@ -param ( - [string]$IssuerName, - - [string]$VaultName, - - [string]$CertName, - - [string]$SubjectName, - - [string]$DnsNames, - - [int]$ValidityInMonths = 12, - - [int]$RenewAtPercentageLifetime = 24, - - [string]$SecretContentType = 'application/x-pkcs12', - - [switch]$Disabled, - - [bool]$Force -) - -try -{ - Write-Output "`nUTC is: $(Get-Date)" - - $DNSNamesArray = $DnsNames -split '_' - - Write-Output $DNSNamesArray - - $c = Get-AzContext -ErrorAction stop - if ($c) - { - Write-Output "`nContext is: " - $c | Select-Object Account, Subscription, Tenant, Environment | Format-List | Out-String - - $DNSNamesArray = $DnsNames -split '_' - - Write-Output $DNSNamesArray - - $PolicyParams = @{ - RenewAtPercentageLifetime = $RenewAtPercentageLifetime - SecretContentType = $SecretContentType - ValidityInMonths = $ValidityInMonths - IssuerName = $IssuerName - Disabled = $Disabled - SubjectName = $SubjectName - DnsNames = $DNSNamesArray - KeyUsage = @('DigitalSignature', 'KeyEncipherment') - } - - $Cert = Get-AzKeyVaultCertificate -VaultName $VaultName -Name $CertName - If ($Cert) - { - $Policy = $Cert | Get-AzKeyVaultCertificatePolicy | Where-Object SubjectName -EQ $SubjectName - } - - if ($Policy) - { - Write-Warning -Message "Policy exists [$($policy.SubjectName)]" - if ($Force) - { - Write-Warning -Message "Force Policy [$($policy.SubjectName)] settings" - $Policy = New-AzKeyVaultCertificatePolicy @PolicyParams - } - } - else - { - Write-Warning -Message "Creating Policy [$SubjectName]" - $Policy = New-AzKeyVaultCertificatePolicy @PolicyParams - } - - if ($Cert -and (-not $Force)) - { - Write-Warning -Message "Certificate exists [$($Cert.Name)]" - } - else - { - Write-Warning -Message "Creating Certificate [$CertName]" - $Result = Add-AzKeyVaultCertificate -VaultName $VaultName -Name $CertName -CertificatePolicy $Policy - $Result.StatusDetails - while ($New.Enabled -ne $true) - { - $New = Get-AzKeyVaultCertificate -VaultName $VaultName -Name $CertName - Start-Sleep -Seconds 30 - } - } - - $out = $cert ?? $new - - $DeploymentScriptOutputs = @{} - $DeploymentScriptOutputs['KeyVaultCertId'] = $out.Id - $DeploymentScriptOutputs['Thumbprint'] = $out.Thumbprint - - if ($IssuerName -eq 'Self') - { - $base64Cert = [System.Convert]::ToBase64String($out.Certificate.Export('Cert')) - $pemCert = "-----BEGIN CERTIFICATE-----`n$base64Cert`n-----END CERTIFICATE-----" - $DeploymentScriptOutputs['CACert'] = $pemCert - } - } - else - { - throw 'Cannot get a context' - } -} -catch -{ - Write-Warning $_ - Write-Warning $_.exception -} diff --git a/test/e2e/setup/aro/cluster/templates/feature-registration.bicep b/test/e2e/setup/aro/cluster/templates/feature-registration.bicep deleted file mode 100644 index c3b240ad..00000000 --- a/test/e2e/setup/aro/cluster/templates/feature-registration.bicep +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Registers AFEC feature flags. This does not wait for registration to complete. - */ - -targetScope = 'subscription' - -@description('The namespace and name of an AFEC feature flag in a `Microsoft.ResourceProvider/FeatureName` format') -param features array = [ - 'Microsoft.ContainerService/DisableSSHPreview' - 'Microsoft.ContainerService/IstioNativeSidecarModePreview' - 'Microsoft.Compute/EncryptionAtHost' -] - -resource featureReg 'Microsoft.Features/featureProviders/subscriptionFeatureRegistrations@2021-07-01' = [ - for feature in features: { - name: feature - } -] diff --git a/test/e2e/setup/aro/cluster/templates/mgmt-cluster.bicep b/test/e2e/setup/aro/cluster/templates/mgmt-cluster.bicep deleted file mode 100644 index ceaf1a3b..00000000 --- a/test/e2e/setup/aro/cluster/templates/mgmt-cluster.bicep +++ /dev/null @@ -1,156 +0,0 @@ -@description('Azure Region Location') -param location string = resourceGroup().location - -@description('Set to true to prevent resources from being pruned after 48 hours') -param persist bool = false - -@description('Captures logged in users UID') -param currentUserId string - -@description('AKS cluster name') -param aksClusterName string = 'aro-hcp-aks' - -@description('Disk size for the AKS system nodes') -param aksSystemOsDiskSizeGB int - -@description('Disk size for the AKS user nodes') -param aksUserOsDiskSizeGB int - -@description('Name of the resource group for the AKS nodes') -param aksNodeResourceGroupName string = '${resourceGroup().name}-aks1' - -@description('VNET address prefix') -param vnetAddressPrefix string - -@description('Min replicas for the worker nodes') -param userAgentMinCount int = 1 - -@description('Max replicas for the worker nodes') -param userAgentMaxCount int = 3 - -@description('VM instance type for the worker nodes') -param userAgentVMSize string = 'Standard_D2s_v3' - -@description('Availability Zone count for worker nodes') -param userAgentPoolAZCount int = 3 - -@description('Min replicas for the system nodes') -param systemAgentMinCount int = 2 - -@description('Max replicas for the system nodes') -param systemAgentMaxCount int = 3 - -@description('VM instance type for the system nodes') -param systemAgentVMSize string = 'Standard_D2s_v3' - -@description('Subnet address prefix') -param subnetPrefix string - -@description('Specifies the address prefix of the subnet hosting the pods of the AKS cluster.') -param podSubnetPrefix string - -@description('Kuberentes version to use with AKS') -param kubernetesVersion string - -@description('The name of the keyvault for AKS.') -@maxLength(24) -param aksKeyVaultName string - -@description('Manage soft delete setting for AKS etcd key-value store') -param aksEtcdKVEnableSoftDelete bool = true - -@description('Deploys a Maestro Consumer to the management cluster if set to true.') -param deployMaestroConsumer bool - -@description('The domain to use to use for the maestro certificate. Relevant only for environments where OneCert can be used.') -param maestroCertDomain string - -@description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) -param maestroKeyVaultName string - -@description('The name of the managed identity that will manage certificates in maestros keyvault.') -param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-officer-msi' - -@description('The name of the eventgrid namespace for Maestro.') -param maestroEventGridNamespacesName string - -@description('The resource group that hosts the regional zone') -param regionalResourceGroup string - -func isValidMaestroConsumerName(input string) bool => length(input) <= 90 && contains(input, '[^a-zA-Z0-9_-]') == false - -// Tags the resource group -resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { - name: 'default' - scope: resourceGroup() - properties: { - tags: { - persist: toLower(string(persist)) - deployedBy: currentUserId - } - } -} - -module mgmtCluster '../modules/aks-cluster-base.bicep' = { - name: 'mgmt-cluster' - scope: resourceGroup() - params: { - location: location - persist: persist - aksClusterName: aksClusterName - aksNodeResourceGroupName: aksNodeResourceGroupName - aksEtcdKVEnableSoftDelete: aksEtcdKVEnableSoftDelete - deployIstio: false - kubernetesVersion: kubernetesVersion - vnetAddressPrefix: vnetAddressPrefix - subnetPrefix: subnetPrefix - podSubnetPrefix: podSubnetPrefix - clusterType: 'mgmt-cluster' - workloadIdentities: items({ - maestro_wi: { - uamiName: 'maestro-consumer' - namespace: 'maestro' - serviceAccountName: 'maestro' - } - external_dns_wi: { - uamiName: 'external-dns' - namespace: 'hypershift' - serviceAccountName: 'external-dns' - } - }) - aksKeyVaultName: aksKeyVaultName - userAgentMinCount: userAgentMinCount - userAgentPoolAZCount: userAgentPoolAZCount - userAgentMaxCount: userAgentMaxCount - userAgentVMSize: userAgentVMSize - systemAgentMinCount: systemAgentMinCount - systemAgentMaxCount: systemAgentMaxCount - systemAgentVMSize: systemAgentVMSize - systemOsDiskSizeGB: aksSystemOsDiskSizeGB - userOsDiskSizeGB: aksUserOsDiskSizeGB - } -} - -output aksClusterName string = mgmtCluster.outputs.aksClusterName - -// -// M A E S T R O -// - -module maestroConsumer '../modules/maestro/maestro-consumer.bicep' = if (deployMaestroConsumer) { - name: 'maestro-consumer' - params: { - maestroServerManagedIdentityPrincipalId: filter( - mgmtCluster.outputs.userAssignedIdentities, - id => id.uamiName == 'maestro-consumer' - )[0].uamiPrincipalID - maestroInfraResourceGroup: regionalResourceGroup - maestroConsumerName: isValidMaestroConsumerName(resourceGroup().name) ? resourceGroup().name : '' - maestroEventGridNamespaceName: maestroEventGridNamespacesName - maestroKeyVaultName: maestroKeyVaultName - maestroKeyVaultOfficerManagedIdentityName: maestroKeyVaultCertOfficerMSIName - maestroKeyVaultCertificateDomain: maestroCertDomain - location: location - } -} diff --git a/test/e2e/setup/aro/cluster/templates/region.bicep b/test/e2e/setup/aro/cluster/templates/region.bicep deleted file mode 100644 index f730bc39..00000000 --- a/test/e2e/setup/aro/cluster/templates/region.bicep +++ /dev/null @@ -1,48 +0,0 @@ -@description('Azure Region Location') -param location string = resourceGroup().location - -@description('Captures logged in users UID') -param currentUserId string - -@description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) -param maestroKeyVaultName string - -@description('The name of the managed identity that will manage certificates in maestros keyvault.') -param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-officer-msi' - -@description('The name of the eventgrid namespace for Maestro.') -param maestroEventGridNamespacesName string - -@description('The maximum client sessions per authentication name for the EventGrid MQTT broker') -param maestroEventGridMaxClientSessionsPerAuthName int - -@description('Set to true to prevent resources from being pruned after 48 hours') -param persist bool = false - -// Tags the resource group -resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { - name: 'default' - scope: resourceGroup() - properties: { - tags: { - persist: toLower(string(persist)) - deployedBy: currentUserId - } - } -} - -// -// M A E S T R O -// - -module maestroInfra '../modules/maestro/maestro-infra.bicep' = { - name: 'maestro-infra' - params: { - eventGridNamespaceName: maestroEventGridNamespacesName - location: location - maxClientSessionsPerAuthName: maestroEventGridMaxClientSessionsPerAuthName - maestroKeyVaultName: maestroKeyVaultName - kvCertOfficerManagedIdentityName: maestroKeyVaultCertOfficerMSIName - } -} diff --git a/test/e2e/setup/aro/cluster/templates/svc-cluster.bicep b/test/e2e/setup/aro/cluster/templates/svc-cluster.bicep deleted file mode 100644 index 87e58f7f..00000000 --- a/test/e2e/setup/aro/cluster/templates/svc-cluster.bicep +++ /dev/null @@ -1,158 +0,0 @@ -@description('Azure Region Location') -param location string = resourceGroup().location - -@description('Set to true to prevent resources from being pruned after 48 hours') -param persist bool = false - -@description('Captures logged in users UID') -param currentUserId string - -@description('AKS cluster name') -param aksClusterName string - -@description('Disk size for the AKS system nodes') -param aksSystemOsDiskSizeGB int = 32 - -@description('Disk size for the AKS user nodes') -param aksUserOsDiskSizeGB int = 32 - -@description('Name of the resource group for the AKS nodes') -param aksNodeResourceGroupName string = '${resourceGroup().name}-aks1' - -@description('VNET address prefix') -param vnetAddressPrefix string - -@description('Subnet address prefix') -param subnetPrefix string - -@description('Specifies the address prefix of the subnet hosting the pods of the AKS cluster.') -param podSubnetPrefix string - -@description('Kuberentes version to use with AKS') -param kubernetesVersion string - -@description('Istio control plane version to use with AKS') -param istioVersion array - -@description('The name of the keyvault for AKS.') -@maxLength(24) -param aksKeyVaultName string - -@description('Manage soft delete setting for AKS etcd key-value store') -param aksEtcdKVEnableSoftDelete bool = true - -@description('The resourcegroup for regional infrastructure') -param regionalResourceGroup string - -@description('The domain to use to use for the maestro certificate. Relevant only for environments where OneCert can be used.') -param maestroCertDomain string - -@description('The name of the eventgrid namespace for Maestro.') -param maestroEventGridNamespacesName string - -@description('The name of the keyvault for Maestro Eventgrid namespace certificates.') -@maxLength(24) -param maestroKeyVaultName string - -@description('The name of the managed identity that will manage certificates in maestros keyvault.') -param maestroKeyVaultCertOfficerMSIName string = '${maestroKeyVaultName}-cert-officer-msi' - -@description('Deploy ARO HCP Maestro Postgres if true') -param deployMaestroPostgres bool = false - -@description('If true, make the Maestro Postgres instance private') -param maestroPostgresPrivate bool = true - -@description('The name of the Postgres server for Maestro') -@maxLength(60) -param maestroPostgresServerName string - -@description('The version of the Postgres server for Maestro') -param maestroPostgresServerVersion string - -@description('The size of the Postgres server for Maestro') -param maestroPostgresServerStorageSizeGB int - -// Tags the resource group -resource subscriptionTags 'Microsoft.Resources/tags@2024-03-01' = { - name: 'default' - scope: resourceGroup() - properties: { - tags: { - persist: toLower(string(persist)) - deployedBy: currentUserId - } - } -} - -module svcCluster '../modules/aks-cluster-base.bicep' = { - name: 'svc-cluster' - scope: resourceGroup() - params: { - location: location - persist: persist - aksClusterName: aksClusterName - aksNodeResourceGroupName: aksNodeResourceGroupName - aksEtcdKVEnableSoftDelete: aksEtcdKVEnableSoftDelete - kubernetesVersion: kubernetesVersion - deployIstio: false - istioVersion: istioVersion - vnetAddressPrefix: vnetAddressPrefix - subnetPrefix: subnetPrefix - podSubnetPrefix: podSubnetPrefix - clusterType: 'svc-cluster' - systemOsDiskSizeGB: aksSystemOsDiskSizeGB - userOsDiskSizeGB: aksUserOsDiskSizeGB - workloadIdentities: items({ - frontend_wi: { - uamiName: 'frontend' - namespace: 'aro-hcp' - serviceAccountName: 'frontend' - } - backend_wi: { - uamiName: 'backend' - namespace: 'aro-hcp' - serviceAccountName: 'backend' - } - maestro_wi: { - uamiName: 'maestro-server' - namespace: 'maestro' - serviceAccountName: 'maestro' - } - }) - aksKeyVaultName: aksKeyVaultName - } -} - -output aksClusterName string = svcCluster.outputs.aksClusterName - -// -// M A E S T R O -// - -module maestroServer '../modules/maestro/maestro-server.bicep' = { - name: 'maestro-server' - params: { - maestroInfraResourceGroup: regionalResourceGroup - maestroEventGridNamespaceName: maestroEventGridNamespacesName - maestroKeyVaultName: maestroKeyVaultName - maestroKeyVaultOfficerManagedIdentityName: maestroKeyVaultCertOfficerMSIName - maestroKeyVaultCertificateDomain: maestroCertDomain - deployPostgres: deployMaestroPostgres - postgresServerName: maestroPostgresServerName - postgresServerVersion: maestroPostgresServerVersion - postgresServerStorageSizeGB: maestroPostgresServerStorageSizeGB - privateEndpointSubnetId: svcCluster.outputs.aksNodeSubnetId - privateEndpointVnetId: svcCluster.outputs.aksVnetId - postgresServerPrivate: maestroPostgresPrivate - maestroServerManagedIdentityPrincipalId: filter( - svcCluster.outputs.userAssignedIdentities, - id => id.uamiName == 'maestro-server' - )[0].uamiPrincipalID - maestroServerManagedIdentityName: filter( - svcCluster.outputs.userAssignedIdentities, - id => id.uamiName == 'maestro-server' - )[0].uamiName - location: location - } -} diff --git a/test/e2e/setup/aro/maestro/Makefile b/test/e2e/setup/aro/maestro/Makefile deleted file mode 100644 index 2ebd621e..00000000 --- a/test/e2e/setup/aro/maestro/Makefile +++ /dev/null @@ -1,68 +0,0 @@ -SHELL = /bin/bash - -include ../cluster/configurations/dev.mk - -CONSUMER_NAME ?= $(shell az aks list --query "[?tags.clusterType == 'mgmt-cluster' && starts_with(resourceGroup, '$(REGIONAL_RESOURCEGROUP)')].resourceGroup" -o tsv) -EVENTGRID_ID = $(shell az resource list -g ${REGIONAL_RESOURCEGROUP} --resource-type "Microsoft.EventGrid/namespaces" --query "[].id" -o tsv) - -RESTRICT_ISTIO_INGRESS ?= true - -MAESTRO_BASE_IMAGE ?= quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro -MAESTRO_TAG ?= ea066c250a002f0cc458711945165591bc9f6d3f - -deploy-server: - MAESTRO_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n maestro-server \ - --query clientId) && \ - TENANT_ID=$(shell az account show --query tenantId --output tsv) && \ - MAESTRO_KV_NAME=$(shell az keyvault list --query "[?starts_with(name, 'maestro-kv')].name" -g ${REGIONAL_RESOURCEGROUP} --output tsv) && \ - EVENTGRID_CA_BASE64='$(shell cat ../cluster/scripts/digicert-global-root-g3.crt | base64 | tr -d '\n')' && \ - EVENTGRID_HOSTNAME=$(shell az resource show --ids ${EVENTGRID_ID} --query properties.topicSpacesConfiguration.hostname -o tsv) && \ - ISTO_VERSION=$(shell az aks list --query "[?tags.clusterType == 'svc-cluster' && starts_with(resourceGroup, '${RESOURCEGROUP}')].serviceMeshProfile.istio.revisions[-1]" -o tsv) && \ - kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - && \ - kubectl label namespace maestro "istio.io/rev=$${ISTO_VERSION}" --overwrite=true && \ - helm upgrade --install maestro-server deploy/helm/server \ - --namespace maestro \ - --set broker.host=$${EVENTGRID_HOSTNAME} \ - --set credsKeyVault.name=$${MAESTRO_KV_NAME} \ - --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ - --set azure.tenantId=$${TENANT_ID} \ - --set istio.restrictIngress=${RESTRICT_ISTIO_INGRESS} \ - --set image.base=${MAESTRO_BASE_IMAGE} \ - --set image.tag=${MAESTRO_TAG} \ - --set database.containerizedDb=true \ - --set database.ssl=disable - -deploy-agent: - MAESTRO_MI_CLIENT_ID=$(shell az identity show \ - -g ${RESOURCEGROUP} \ - -n maestro-consumer \ - --query clientId -o tsv) && \ - TENANT_ID=$(shell az account show --query tenantId -o tsv) && \ - MAESTRO_KV_NAME=$(shell az keyvault list --query "[?starts_with(name, 'maestro-kv')].name" -g ${REGIONAL_RESOURCEGROUP} -o tsv) && \ - EVENTGRID_HOSTNAME=$(shell az resource show --ids ${EVENTGRID_ID} --query properties.topicSpacesConfiguration.hostname -o tsv) && \ - helm upgrade --install maestro-agent deploy/helm/agent \ - --create-namespace --namespace maestro \ - --set consumerName=${CONSUMER_NAME} \ - --set broker.host=$${EVENTGRID_HOSTNAME} \ - --set credsKeyVault.name=$${MAESTRO_KV_NAME} \ - --set credsKeyVault.secret=${CONSUMER_NAME} \ - --set azure.clientId=$${MAESTRO_MI_CLIENT_ID} \ - --set azure.tenantId=$${TENANT_ID} \ - --set image.base=${MAESTRO_BASE_IMAGE} \ - --set image.tag=${MAESTRO_TAG} - -register-agent: -ifeq ($(strip ${CONSUMER_NAME}),) - $(error "Must set CONSUMER_NAME") -endif - @if ! kubectl get service maestro -n maestro > /dev/null 2>&1; then \ - echo "Error: Service 'maestro' not found in namespace 'maestro'"; \ - exit 1; \ - fi - helm upgrade --install ${CONSUMER_NAME} deploy/helm/registration \ - --namespace maestro \ - --set consumerName=${CONSUMER_NAME} - -.PHONY: deploy-server deploy-agent register-agent diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/Chart.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/Chart.yaml deleted file mode 100644 index fb95b195..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -name: maestro-agent -description: A Helm chart for Maestro Agent -type: application - -version: 0.1.0 -appVersion: "ae149df618cb0812d2072b20658a9cff84c087eb" diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml deleted file mode 100755 index bd1e2781..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/appliedmanifestworks.work.open-cluster-management.io.customresourcedefinition.yaml +++ /dev/null @@ -1,128 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: appliedmanifestworks.work.open-cluster-management.io -spec: - group: work.open-cluster-management.io - names: - kind: AppliedManifestWork - listKind: AppliedManifestWorkList - plural: appliedmanifestworks - singular: appliedmanifestwork - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: |- - AppliedManifestWork represents an applied manifestwork on managed cluster that is placed - on a managed cluster. An AppliedManifestWork links to a manifestwork on a hub recording resources - deployed in the managed cluster. - When the agent is removed from managed cluster, cluster-admin on managed cluster - can delete appliedmanifestwork to remove resources deployed by the agent. - The name of the appliedmanifestwork must be in the format of - {hash of hub's first kube-apiserver url}-{manifestwork name} - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec represents the desired configuration of AppliedManifestWork. - properties: - agentID: - description: AgentID represents the ID of the work agent who is to - handle this AppliedManifestWork. - type: string - hubHash: - description: |- - HubHash represents the hash of the first hub kube apiserver to identify which hub - this AppliedManifestWork links to. - type: string - manifestWorkName: - description: ManifestWorkName represents the name of the related manifestwork - on the hub. - type: string - type: object - status: - description: Status represents the current status of AppliedManifestWork. - properties: - appliedResources: - description: |- - AppliedResources represents a list of resources defined within the manifestwork that are applied. - Only resources with valid GroupVersionResource, namespace, and name are suitable. - An item in this slice is deleted when there is no mapped manifest in manifestwork.Spec or by finalizer. - The resource relating to the item will also be removed from managed cluster. - The deleted resource may still be present until the finalizers for that resource are finished. - However, the resource will not be undeleted, so it can be removed from this list and eventual consistency is preserved. - items: - description: |- - AppliedManifestResourceMeta represents the group, version, resource, name and namespace of a resource. - Since these resources have been created, they must have valid group, version, resource, namespace, and name. - properties: - group: - description: |- - Group is the API Group of the Kubernetes resource, - empty string indicates it is in core group. - type: string - name: - description: Name is the name of the Kubernetes resource. - type: string - namespace: - description: |- - Name is the namespace of the Kubernetes resource, empty string indicates - it is a cluster scoped resource. - type: string - resource: - description: Resource is the resource name of the Kubernetes - resource. - type: string - uid: - description: |- - UID is set on successful deletion of the Kubernetes resource by controller. The - resource might be still visible on the managed cluster after this field is set. - It is not directly settable by a client. - type: string - version: - description: Version is the version of the Kubernetes resource. - type: string - required: - - name - - resource - - version - type: object - type: array - evictionStartTime: - description: |- - EvictionStartTime represents the current appliedmanifestwork will be evicted after a grace period. - An appliedmanifestwork will be evicted from the managed cluster in the following two scenarios: - - the manifestwork of the current appliedmanifestwork is missing on the hub, or - - the appliedmanifestwork hub hash does not match the current hub hash of the work agent. - format: date-time - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml deleted file mode 100755 index ddd89b5a..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrole.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: maestro-agent:agent-{{ .Release.Name }} -rules: -- apiGroups: - - work.open-cluster-management.io - resources: - - appliedmanifestworks - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - work.open-cluster-management.io - resources: - - appliedmanifestworks/status - verbs: - - patch - - update -- apiGroups: - - work.open-cluster-management.io - resources: - - appliedmanifestworks/finalizers - verbs: - - update -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - impersonate diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml deleted file mode 100755 index 601df0e4..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: maestro-agent:agent-{{ .Release.Name }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: maestro-agent:agent-{{ .Release.Name }} -subjects: -- kind: ServiceAccount - name: maestro - namespace: {{ .Release.Namespace }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml deleted file mode 100755 index bb4227a3..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.role.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: maestro-agent:agent - namespace: {{ .Release.Namespace }} -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - delete - - update - - patch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - update - - watch - - patch -- apiGroups: - - "" - - events.k8s.io - resources: - - events - verbs: - - create - - patch - - update diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml deleted file mode 100755 index b6c0b472..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent.rolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: maestro-agent:agent - namespace: {{ .Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: maestro-agent:agent -subjects: -- kind: ServiceAccount - name: maestro - namespace: {{ .Release.Namespace }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml deleted file mode 100755 index 60a72537..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.role.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: maestro-agent:agent:extension-apiserver-{{ .Release.Name }} - namespace: kube-system -rules: -- apiGroups: - - "" - resourceNames: - - extension-apiserver-authentication - resources: - - configmaps - verbs: - - get - - list - - watch diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml deleted file mode 100755 index d718c7bd..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.agent:extension-apiserver.rolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: maestro-agent:agent:extension-apiserver-{{ .Release.Name }} - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: maestro-agent:agent:extension-apiserver-{{ .Release.Name }} -subjects: -- kind: ServiceAccount - name: maestro - namespace: {{ .Release.Namespace }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml deleted file mode 100755 index 9151a18c..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.deployment.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: maestro-agent - name: maestro-agent - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: maestro-agent - template: - metadata: - labels: - app: maestro-agent - annotations: - checksum/credsstore: {{ include (print $.Template.BasePath "/maestro.secretproviderclass.yaml") . | sha256sum }} - checksum/config: {{ include (print $.Template.BasePath "/maestro.secret.yaml") . | sha256sum }} - spec: - containers: - - command: - - /usr/local/bin/maestro - - agent - - --consumer-name={{ .Values.consumerName }} - - --workload-source-driver=mqtt - - --workload-source-config=/secrets/maestro/config.yaml - - --cloudevents-client-id={{ .Values.consumerName }}-work-agent - image: "{{ .Values.image.base }}:{{ .Values.image.tag }}" - imagePullPolicy: IfNotPresent - name: maestro-agent - volumeMounts: - - mountPath: /secrets/maestro - name: maestro - - mountPath: /secrets/mqtt-creds - name: mqtt-creds - readOnly: true - serviceAccountName: maestro - volumes: - - name: maestro - secret: - secretName: maestro - - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: maestro - name: mqtt-creds diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml deleted file mode 100755 index 8cd42d55..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution-admin.clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: maestro-agent:execution-admin-{{ .Release.Name }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: maestro - namespace: {{ .Release.Namespace }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml deleted file mode 100755 index af455098..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro-agent.execution.clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: maestro-agent:execution -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: maestro-agent:execution -subjects: -- kind: ServiceAccount - name: maestro - namespace: {{ .Release.Namespace }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secret.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secret.yaml deleted file mode 100755 index fff3df1c..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: maestro - namespace: {{ .Release.Namespace }} -stringData: - config.yaml: | - brokerHost: "{{ .Values.broker.host }}:{{ .Values.broker.port }}" - username: "" - password: "" - clientCertFile: /secrets/mqtt-creds/maestro.crt - clientKeyFile: /secrets/mqtt-creds/maestro.key - topics: - sourceEvents: sources/maestro/consumers/{{ .Values.consumerName }}/sourceevents - agentEvents: sources/maestro/consumers/{{ .Values.consumerName }}/agentevents diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml deleted file mode 100755 index 4950f015..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.secretproviderclass.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: maestro - namespace: {{ .Release.Namespace }} -spec: - parameters: - clientID: {{ .Values.azure.clientId }} - cloudName: AzurePublicCloud - keyvaultName: {{ .Values.credsKeyVault.name }} - objects: |- - array: - - | - objectName: {{ .Values.credsKeyVault.secret }} - objectType: secret - objectAlias: maestro - tenantId: {{ .Values.azure.tenantId }} - usePodIdentity: "false" - provider: azure diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml deleted file mode 100755 index efe19745..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/maestro.serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - azure.workload.identity/client-id: {{ .Values.azure.clientId }} - name: maestro - namespace: {{ .Release.Namespace }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml deleted file mode 100644 index efb889f9..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml +++ /dev/null @@ -1,570 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: manifestworks.work.open-cluster-management.io -spec: - group: work.open-cluster-management.io - names: - kind: ManifestWork - listKind: ManifestWorkList - plural: manifestworks - singular: manifestwork - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: |- - ManifestWork represents a manifests workload that hub wants to deploy on the managed cluster. - A manifest workload is defined as a set of Kubernetes resources. - ManifestWork must be created in the cluster namespace on the hub, so that agent on the - corresponding managed cluster can access this resource and deploy on the managed - cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec represents a desired configuration of work to be deployed - on the managed cluster. - properties: - deleteOption: - description: |- - DeleteOption represents deletion strategy when the manifestwork is deleted. - Foreground deletion strategy is applied to all the resource in this manifestwork if it is not set. - properties: - propagationPolicy: - default: Foreground - description: |- - propagationPolicy can be Foreground, Orphan or SelectivelyOrphan - SelectivelyOrphan should be rarely used. It is provided for cases where particular resources is transfering - ownership from one ManifestWork to another or another management unit. - Setting this value will allow a flow like - 1. create manifestwork/2 to manage foo - 2. update manifestwork/1 to selectively orphan foo - 3. remove foo from manifestwork/1 without impacting continuity because manifestwork/2 adopts it. - enum: - - Foreground - - Orphan - - SelectivelyOrphan - type: string - selectivelyOrphans: - description: selectivelyOrphan represents a list of resources - following orphan deletion stratecy - properties: - orphaningRules: - description: |- - orphaningRules defines a slice of orphaningrule. - Each orphaningrule identifies a single resource included in this manifestwork - items: - description: OrphaningRule identifies a single resource - included in this manifestwork to be orphaned - properties: - group: - description: |- - Group is the API Group of the Kubernetes resource, - empty string indicates it is in core group. - type: string - name: - description: Name is the name of the Kubernetes resource. - type: string - namespace: - description: |- - Name is the namespace of the Kubernetes resource, empty string indicates - it is a cluster scoped resource. - type: string - resource: - description: Resource is the resource name of the Kubernetes - resource. - type: string - required: - - name - - resource - type: object - type: array - type: object - type: object - executor: - description: |- - Executor is the configuration that makes the work agent to perform some pre-request processing/checking. - e.g. the executor identity tells the work agent to check the executor has sufficient permission to write - the workloads to the local managed cluster. - Note that nil executor is still supported for backward-compatibility which indicates that the work agent - will not perform any additional actions before applying resources. - properties: - subject: - description: |- - Subject is the subject identity which the work agent uses to talk to the - local cluster when applying the resources. - properties: - serviceAccount: - description: |- - ServiceAccount is for identifying which service account to use by the work agent. - Only required if the type is "ServiceAccount". - properties: - name: - description: Name is the name of the service account. - maxLength: 253 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ - type: string - namespace: - description: Namespace is the namespace of the service - account. - maxLength: 253 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ - type: string - required: - - name - - namespace - type: object - type: - description: |- - Type is the type of the subject identity. - Supported types are: "ServiceAccount". - enum: - - ServiceAccount - type: string - required: - - type - type: object - type: object - manifestConfigs: - description: ManifestConfigs represents the configurations of manifests - defined in workload field. - items: - description: ManifestConfigOption represents the configurations - of a manifest defined in workload field. - properties: - feedbackRules: - description: |- - FeedbackRules defines what resource status field should be returned. If it is not set or empty, - no feedback rules will be honored. - items: - properties: - jsonPaths: - description: JsonPaths defines the json path under status - field to be synced. - items: - properties: - name: - description: Name represents the alias name for - this field - type: string - path: - description: |- - Path represents the json path of the field under status. - The path must point to a field with single value in the type of integer, bool or string. - If the path points to a non-existing field, no value will be returned. - If the path points to a structure, map or slice, no value will be returned and the status conddition - of StatusFeedBackSynced will be set as false. - Ref to https://kubernetes.io/docs/reference/kubectl/jsonpath/ on how to write a jsonPath. - type: string - version: - description: |- - Version is the version of the Kubernetes resource. - If it is not specified, the resource with the semantically latest version is - used to resolve the path. - type: string - required: - - name - - path - type: object - type: array - type: - description: |- - Type defines the option of how status can be returned. - It can be jsonPaths or wellKnownStatus. - If the type is JSONPaths, user should specify the jsonPaths field - If the type is WellKnownStatus, certain common fields of status defined by a rule only - for types in in k8s.io/api and open-cluster-management/api will be reported, - If these status fields do not exist, no values will be reported. - enum: - - WellKnownStatus - - JSONPaths - type: string - required: - - type - type: object - type: array - resourceIdentifier: - description: |- - ResourceIdentifier represents the group, resource, name and namespace of a resoure. - iff this refers to a resource not created by this manifest work, the related rules will not be executed. - properties: - group: - description: |- - Group is the API Group of the Kubernetes resource, - empty string indicates it is in core group. - type: string - name: - description: Name is the name of the Kubernetes resource. - type: string - namespace: - description: |- - Name is the namespace of the Kubernetes resource, empty string indicates - it is a cluster scoped resource. - type: string - resource: - description: Resource is the resource name of the Kubernetes - resource. - type: string - required: - - name - - resource - type: object - updateStrategy: - description: |- - UpdateStrategy defines the strategy to update this manifest. UpdateStrategy is Update - if it is not set. - properties: - serverSideApply: - description: |- - serverSideApply defines the configuration for server side apply. It is honored only when - type of updateStrategy is ServerSideApply - properties: - fieldManager: - default: work-agent - description: |- - FieldManager is the manager to apply the resource. It is work-agent by default, but can be other name with work-agent - as the prefix. - pattern: ^work-agent - type: string - force: - description: Force represents to force apply the manifest. - type: boolean - type: object - type: - default: Update - description: |- - type defines the strategy to update this manifest, default value is Update. - Update type means to update resource by an update call. - CreateOnly type means do not update resource based on current manifest. - ServerSideApply type means to update resource using server side apply with work-controller as the field manager. - If there is conflict, the related Applied condition of manifest will be in the status of False with the - reason of ApplyConflict. - ReadOnly type means the agent will only check the existence of the resource based on its metadata, - statusFeedBackRules can still be used to get feedbackResults. - enum: - - Update - - CreateOnly - - ServerSideApply - - ReadOnly - type: string - required: - - type - type: object - required: - - resourceIdentifier - type: object - type: array - workload: - description: Workload represents the manifest workload to be deployed - on a managed cluster. - properties: - manifests: - description: Manifests represents a list of kuberenetes resources - to be deployed on a managed cluster. - items: - description: Manifest represents a resource to be deployed on - managed cluster. - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - type: array - type: object - type: object - status: - description: Status represents the current status of work. - properties: - conditions: - description: |- - Conditions contains the different condition statuses for this work. - Valid condition types are: - 1. Applied represents workload in ManifestWork is applied successfully on managed cluster. - 2. Progressing represents workload in ManifestWork is being applied on managed cluster. - 3. Available represents workload in ManifestWork exists on the managed cluster. - 4. Degraded represents the current state of workload does not match the desired - state for a certain period. - items: - description: "Condition contains details for one aspect of the current - state of this API Resource.\n---\nThis struct is intended for - direct use as an array at the field path .status.conditions. For - example,\n\n\n\ttype FooStatus struct{\n\t // Represents the - observations of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // - +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t - \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - resourceStatus: - description: |- - ResourceStatus represents the status of each resource in manifestwork deployed on a - managed cluster. The Klusterlet agent on managed cluster syncs the condition from the managed cluster to the hub. - properties: - manifests: - description: |- - Manifests represents the condition of manifests deployed on managed cluster. - Valid condition types are: - 1. Progressing represents the resource is being applied on managed cluster. - 2. Applied represents the resource is applied successfully on managed cluster. - 3. Available represents the resource exists on the managed cluster. - 4. Degraded represents the current state of resource does not match the desired - state for a certain period. - items: - description: |- - ManifestCondition represents the conditions of the resources deployed on a - managed cluster. - properties: - conditions: - description: Conditions represents the conditions of this - resource on a managed cluster. - items: - description: "Condition contains details for one aspect - of the current state of this API Resource.\n---\nThis - struct is intended for direct use as an array at the - field path .status.conditions. For example,\n\n\n\ttype - FooStatus struct{\n\t // Represents the observations - of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t - \ // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t - \ // +listType=map\n\t // +listMapKey=type\n\t - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, - False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - resourceMeta: - description: ResourceMeta represents the group, version, - kind, name and namespace of a resoure. - properties: - group: - description: Group is the API Group of the Kubernetes - resource. - type: string - kind: - description: Kind is the kind of the Kubernetes resource. - type: string - name: - description: Name is the name of the Kubernetes resource. - type: string - namespace: - description: Name is the namespace of the Kubernetes - resource. - type: string - ordinal: - description: Ordinal represents the index of the manifest - on spec. - format: int32 - type: integer - resource: - description: Resource is the resource name of the Kubernetes - resource. - type: string - version: - description: Version is the version of the Kubernetes - resource. - type: string - type: object - statusFeedback: - description: StatusFeedback represents the values of the - feild synced back defined in statusFeedbacks - properties: - values: - description: Values represents the synced value of the - interested field. - items: - properties: - fieldValue: - description: |- - Value is the value of the status field. - The value of the status field can only be integer, string or boolean. - properties: - boolean: - description: Boolean is bool value when type - is boolean. - type: boolean - integer: - description: Integer is the integer value - when type is integer. - format: int64 - type: integer - jsonRaw: - description: JsonRaw is a json string when - type is a list or object - maxLength: 1024 - type: string - string: - description: String is the string value when - type is string. - type: string - type: - description: Type represents the type of the - value, it can be integer, string or boolean. - enum: - - Integer - - String - - Boolean - - JsonRaw - type: string - required: - - type - type: object - name: - description: |- - Name represents the alias name for this field. It is the same as what is specified - in StatuFeedbackRule in the spec. - type: string - required: - - fieldValue - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: object - type: array - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/test/e2e/setup/aro/maestro/deploy/helm/agent/values.yaml b/test/e2e/setup/aro/maestro/deploy/helm/agent/values.yaml deleted file mode 100644 index 9abf9ff9..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/agent/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -broker: - host: "" - port: 8883 -azure: - clientId: "" - tenantId: "" -image: - base: "" - tag: "" -credsKeyVault: - name: "" - secret: "" -consumerName: "" diff --git a/test/e2e/setup/aro/maestro/deploy/helm/registration/Chart.yaml b/test/e2e/setup/aro/maestro/deploy/helm/registration/Chart.yaml deleted file mode 100644 index 5b231723..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/registration/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -name: maestro-agent-registration -description: A Helm chart for Maestro Agent Registration -type: application - -version: 0.1.0 -appVersion: "ae149df618cb0812d2072b20658a9cff84c087eb" diff --git a/test/e2e/setup/aro/maestro/deploy/helm/registration/templates/registration.job.yaml b/test/e2e/setup/aro/maestro/deploy/helm/registration/templates/registration.job.yaml deleted file mode 100644 index 9dcfff2f..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/registration/templates/registration.job.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: registration-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - ttlSecondsAfterFinished: 60 - template: - spec: - containers: - - name: python - image: "{{ .Values.pythonImage }}" - command: ["python3.9", "/scripts/registration.py", "{{ .Values.maestroBaseUrl }}", "{{ .Values.consumerName }}"] - volumeMounts: - - name: script-volume - mountPath: /scripts - restartPolicy: Never - volumes: - - name: script-volume - configMap: - name: registration-{{ .Release.Name }} - backoffLimit: 4 diff --git a/test/e2e/setup/aro/maestro/deploy/helm/registration/templates/script.yaml b/test/e2e/setup/aro/maestro/deploy/helm/registration/templates/script.yaml deleted file mode 100644 index 33bcb024..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/registration/templates/script.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: registration-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} -data: - registration.py: | - import sys - import http.client - import json - import urllib.parse - - def main(): - if len(sys.argv) != 3: - print("Usage: register-maestro-consumer.py ") - sys.exit(1) - - maestro_base_url = sys.argv[1] - consumer_name = sys.argv[2] - - # Parse the base URL - parsed_url = urllib.parse.urlparse(maestro_base_url) - conn = http.client.HTTPConnection(parsed_url.netloc) - - # Check if the consumer is already registered - search_query = urllib.parse.urlencode({"search": f"name='{consumer_name}'"}) - conn.request("GET", f"{parsed_url.path}/api/maestro/v1/consumers?{search_query}") - response = conn.getresponse() - if response.status != 200: - print(f"Error checking consumer: {response.status} {response.reason}") - sys.exit(1) - - data = response.read() - found = json.loads(data).get('total', 0) - - if found == 0: - # Register the consumer - headers = {"Content-Type": "application/json"} - consumer_data = json.dumps({"name": consumer_name}) - conn.request("POST", f"{parsed_url.path}/api/maestro/v1/consumers", body=consumer_data, headers=headers) - response = conn.getresponse() - if response.status != 201: - print(f"Error registering consumer: {response.status} {response.reason}") - sys.exit(1) - print(f"Maestro consumer {consumer_name} registered successfully") - else: - print(f"Maestro consumer {consumer_name} already registered") - - conn.close() - - if __name__ == "__main__": - main() diff --git a/test/e2e/setup/aro/maestro/deploy/helm/registration/values.yaml b/test/e2e/setup/aro/maestro/deploy/helm/registration/values.yaml deleted file mode 100644 index c3613f14..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/registration/values.yaml +++ /dev/null @@ -1,3 +0,0 @@ -pythonImage: "mcr.microsoft.com/cbl-mariner/base/python:3.9" -consumerName: "" -maestroBaseUrl: "http://maestro.maestro.svc.cluster.local:8000" diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/Chart.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/Chart.yaml deleted file mode 100644 index 267deaad..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -name: maestro-server -description: A Helm chart for Maestro Server -type: application - -version: 0.1.0 -appVersion: "ae149df618cb0812d2072b20658a9cff84c087eb" diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/grpc.service.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/grpc.service.yaml deleted file mode 100644 index bbc9c7e2..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/grpc.service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: maestro-grpc - namespace: {{ .Release.Namespace }} - labels: - app: maestro-grpc - port: grpc -spec: - selector: - app: maestro - ports: - - port: {{ .Values.maestro.grpcBindPort }} - targetPort: {{ .Values.maestro.grpcBindPort }} - protocol: TCP diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/health.service.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/health.service.yaml deleted file mode 100644 index 61948386..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/health.service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: maestro-healthcheck - namespace: {{ .Release.Namespace }} - labels: - app: maestro - port: healthcheck -spec: - selector: - app: maestro - ports: - - port: {{ .Values.maestro.healthCheckBindPort }} - targetPort: {{ .Values.maestro.healthCheckBindPort }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/http.service.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/http.service.yaml deleted file mode 100644 index cfb6c6b2..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/http.service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: maestro - namespace: {{ .Release.Namespace }} - labels: - app: maestro - port: api -spec: - selector: - app: maestro - ports: - - port: {{ .Values.maestro.httpBindPort }} - targetPort: {{ .Values.maestro.httpBindPort }} - protocol: TCP diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.deployment.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.deployment.yaml deleted file mode 100644 index 3ee367a2..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.deployment.yaml +++ /dev/null @@ -1,136 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: maestro - namespace: {{ .Release.Namespace }} - labels: - app: maestro -spec: - selector: - matchLabels: - app: maestro - replicas: {{ .Values.deployment.replicas }} - strategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - app: maestro - annotations: - checksum/credsstore: {{ include (print $.Template.BasePath "/maestro.secretproviderclass.yaml") . | sha256sum }} - checksum/config: {{ include (print $.Template.BasePath "/maestro.secret.yaml") . | sha256sum }} - checksum/db: {{ include (print $.Template.BasePath "/pg.secret.yaml") . | sha256sum }} - spec: - serviceAccountName: {{ .Values.maestro.serviceAccount }} - volumes: - - name: db - secret: - secretName: maestro-db - - name: maestro - secret: - secretName: maestro - - name: mqtt-creds - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "maestro" - initContainers: - - name: migration - image: "{{ .Values.image.base }}:{{ .Values.image.tag }}" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: db - mountPath: /secrets/db - command: - - /usr/local/bin/maestro - - migration - - --db-host-file=/secrets/db/db.host - - --db-port-file=/secrets/db/db.port - - --db-user-file=/secrets/db/db.user - - --db-password-file=/secrets/db/db.password - - --db-name-file=/secrets/db/db.name - - --db-sslmode={{ .Values.database.ssl }} - - --alsologtostderr - - -v={{ .Values.maestro.glog_v }} - containers: - - name: service - image: "{{ .Values.image.base }}:{{ .Values.image.tag }}" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: db - mountPath: /secrets/db - - name: maestro - mountPath: /secrets/maestro - - name: mqtt-creds - mountPath: /secrets/mqtt-creds - readOnly: true - env: - - name: "AMS_ENV" - value: "production" - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - command: - - /usr/local/bin/maestro - - server - - --client-id=maestro-$(POD_NAME) - - --subscription-type=broadcast - - --db-host-file=/secrets/db/db.host - - --db-port-file=/secrets/db/db.port - - --db-user-file=/secrets/db/db.user - - --db-password-file=/secrets/db/db.password - - --db-name-file=/secrets/db/db.name - - --db-rootcert=/secrets/db/db.ca_cert - - --db-sslmode={{ .Values.database.ssl }} - - --db-max-open-connections={{ .Values.database.maxOpenConnections }} - - --message-broker-config-file=/secrets/maestro/config.yaml - - --message-broker-type=mqtt - - --enable-ocm-mock=true - - --enable-jwt=false - - --enable-https=false - - --enable-grpc-server=true - - --disable-grpc-tls=true - - --server-hostname= - - --http-server-bindport={{ .Values.maestro.httpBindPort }} - - --grpc-server-bindport={{ .Values.maestro.grpcBindPort }} - - --health-check-server-bindport={{ .Values.maestro.healthCheckBindPort }} - - --enable-health-check-https=false - - --enable-authz=true - - --enable-db-debug={{ .Values.database.debug }} - - --enable-metrics-https=false - - --enable-sentry=false - - --http-read-timeout={{ .Values.maestro.httpReadTimeout }} - - --http-write-timeout={{ .Values.maestro.httpWriteTimeout }} - - --label-metrics-inclusion-duration={{ .Values.maestro.labelMetricsInclusionDuration }} - - --alsologtostderr - - -v={{ .Values.maestro.glog_v }} - resources: - requests: - cpu: {{ .Values.deployment.requests.cpu }} - memory: {{ .Values.deployment.requests.memory }} - limits: - cpu: {{ .Values.deployment.limits.cpu }} - memory: {{ .Values.deployment.limits.memory }} - livenessProbe: - httpGet: - path: /api/maestro - port: {{ .Values.maestro.httpBindPort }} - scheme: HTTP - initialDelaySeconds: 15 - periodSeconds: 5 - readinessProbe: - httpGet: - path: /healthcheck - port: {{ .Values.maestro.healthCheckBindPort }} - scheme: HTTP - httpHeaders: - - name: User-Agent - value: Probe - initialDelaySeconds: 20 - periodSeconds: 10 diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml deleted file mode 100644 index d94700c3..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.peerauthentication.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# apiVersion: security.istio.io/v1beta1 -# kind: PeerAuthentication -# metadata: -# name: default -# namespace: {{ .Release.Namespace }} -# spec: -# selector: -# matchLabels: -# app: maestro -# portLevelMtls: -# 8080: -# mode: PERMISSIVE diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secret.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secret.yaml deleted file mode 100644 index faeb21b0..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secret.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: maestro - namespace: {{ .Release.Namespace }} - labels: - app: maestro -stringData: - config.yaml: | - brokerHost: "{{ .Values.broker.host }}:{{ .Values.broker.port }}" - username: "" - password: "" - clientCertFile: /secrets/mqtt-creds/maestro.crt - clientKeyFile: /secrets/mqtt-creds/maestro.key - topics: - sourceEvents: sources/maestro/consumers/+/sourceevents - agentEvents: sources/maestro/consumers/+/agentevents diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml deleted file mode 100644 index 4950f015..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.secretproviderclass.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1 -kind: SecretProviderClass -metadata: - name: maestro - namespace: {{ .Release.Namespace }} -spec: - parameters: - clientID: {{ .Values.azure.clientId }} - cloudName: AzurePublicCloud - keyvaultName: {{ .Values.credsKeyVault.name }} - objects: |- - array: - - | - objectName: {{ .Values.credsKeyVault.secret }} - objectType: secret - objectAlias: maestro - tenantId: {{ .Values.azure.tenantId }} - usePodIdentity: "false" - provider: azure diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml deleted file mode 100644 index 7d3faf00..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/maestro.serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: {{ .Values.maestro.serviceAccount }} - namespace: {{ .Release.Namespace }} - annotations: - azure.workload.identity/client-id: {{ .Values.azure.clientId }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml deleted file mode 100644 index efb889f9..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/manifestworks.work.open-cluster-management.io.customresourcedefinition.yaml +++ /dev/null @@ -1,570 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: manifestworks.work.open-cluster-management.io -spec: - group: work.open-cluster-management.io - names: - kind: ManifestWork - listKind: ManifestWorkList - plural: manifestworks - singular: manifestwork - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: |- - ManifestWork represents a manifests workload that hub wants to deploy on the managed cluster. - A manifest workload is defined as a set of Kubernetes resources. - ManifestWork must be created in the cluster namespace on the hub, so that agent on the - corresponding managed cluster can access this resource and deploy on the managed - cluster. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec represents a desired configuration of work to be deployed - on the managed cluster. - properties: - deleteOption: - description: |- - DeleteOption represents deletion strategy when the manifestwork is deleted. - Foreground deletion strategy is applied to all the resource in this manifestwork if it is not set. - properties: - propagationPolicy: - default: Foreground - description: |- - propagationPolicy can be Foreground, Orphan or SelectivelyOrphan - SelectivelyOrphan should be rarely used. It is provided for cases where particular resources is transfering - ownership from one ManifestWork to another or another management unit. - Setting this value will allow a flow like - 1. create manifestwork/2 to manage foo - 2. update manifestwork/1 to selectively orphan foo - 3. remove foo from manifestwork/1 without impacting continuity because manifestwork/2 adopts it. - enum: - - Foreground - - Orphan - - SelectivelyOrphan - type: string - selectivelyOrphans: - description: selectivelyOrphan represents a list of resources - following orphan deletion stratecy - properties: - orphaningRules: - description: |- - orphaningRules defines a slice of orphaningrule. - Each orphaningrule identifies a single resource included in this manifestwork - items: - description: OrphaningRule identifies a single resource - included in this manifestwork to be orphaned - properties: - group: - description: |- - Group is the API Group of the Kubernetes resource, - empty string indicates it is in core group. - type: string - name: - description: Name is the name of the Kubernetes resource. - type: string - namespace: - description: |- - Name is the namespace of the Kubernetes resource, empty string indicates - it is a cluster scoped resource. - type: string - resource: - description: Resource is the resource name of the Kubernetes - resource. - type: string - required: - - name - - resource - type: object - type: array - type: object - type: object - executor: - description: |- - Executor is the configuration that makes the work agent to perform some pre-request processing/checking. - e.g. the executor identity tells the work agent to check the executor has sufficient permission to write - the workloads to the local managed cluster. - Note that nil executor is still supported for backward-compatibility which indicates that the work agent - will not perform any additional actions before applying resources. - properties: - subject: - description: |- - Subject is the subject identity which the work agent uses to talk to the - local cluster when applying the resources. - properties: - serviceAccount: - description: |- - ServiceAccount is for identifying which service account to use by the work agent. - Only required if the type is "ServiceAccount". - properties: - name: - description: Name is the name of the service account. - maxLength: 253 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ - type: string - namespace: - description: Namespace is the namespace of the service - account. - maxLength: 253 - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ - type: string - required: - - name - - namespace - type: object - type: - description: |- - Type is the type of the subject identity. - Supported types are: "ServiceAccount". - enum: - - ServiceAccount - type: string - required: - - type - type: object - type: object - manifestConfigs: - description: ManifestConfigs represents the configurations of manifests - defined in workload field. - items: - description: ManifestConfigOption represents the configurations - of a manifest defined in workload field. - properties: - feedbackRules: - description: |- - FeedbackRules defines what resource status field should be returned. If it is not set or empty, - no feedback rules will be honored. - items: - properties: - jsonPaths: - description: JsonPaths defines the json path under status - field to be synced. - items: - properties: - name: - description: Name represents the alias name for - this field - type: string - path: - description: |- - Path represents the json path of the field under status. - The path must point to a field with single value in the type of integer, bool or string. - If the path points to a non-existing field, no value will be returned. - If the path points to a structure, map or slice, no value will be returned and the status conddition - of StatusFeedBackSynced will be set as false. - Ref to https://kubernetes.io/docs/reference/kubectl/jsonpath/ on how to write a jsonPath. - type: string - version: - description: |- - Version is the version of the Kubernetes resource. - If it is not specified, the resource with the semantically latest version is - used to resolve the path. - type: string - required: - - name - - path - type: object - type: array - type: - description: |- - Type defines the option of how status can be returned. - It can be jsonPaths or wellKnownStatus. - If the type is JSONPaths, user should specify the jsonPaths field - If the type is WellKnownStatus, certain common fields of status defined by a rule only - for types in in k8s.io/api and open-cluster-management/api will be reported, - If these status fields do not exist, no values will be reported. - enum: - - WellKnownStatus - - JSONPaths - type: string - required: - - type - type: object - type: array - resourceIdentifier: - description: |- - ResourceIdentifier represents the group, resource, name and namespace of a resoure. - iff this refers to a resource not created by this manifest work, the related rules will not be executed. - properties: - group: - description: |- - Group is the API Group of the Kubernetes resource, - empty string indicates it is in core group. - type: string - name: - description: Name is the name of the Kubernetes resource. - type: string - namespace: - description: |- - Name is the namespace of the Kubernetes resource, empty string indicates - it is a cluster scoped resource. - type: string - resource: - description: Resource is the resource name of the Kubernetes - resource. - type: string - required: - - name - - resource - type: object - updateStrategy: - description: |- - UpdateStrategy defines the strategy to update this manifest. UpdateStrategy is Update - if it is not set. - properties: - serverSideApply: - description: |- - serverSideApply defines the configuration for server side apply. It is honored only when - type of updateStrategy is ServerSideApply - properties: - fieldManager: - default: work-agent - description: |- - FieldManager is the manager to apply the resource. It is work-agent by default, but can be other name with work-agent - as the prefix. - pattern: ^work-agent - type: string - force: - description: Force represents to force apply the manifest. - type: boolean - type: object - type: - default: Update - description: |- - type defines the strategy to update this manifest, default value is Update. - Update type means to update resource by an update call. - CreateOnly type means do not update resource based on current manifest. - ServerSideApply type means to update resource using server side apply with work-controller as the field manager. - If there is conflict, the related Applied condition of manifest will be in the status of False with the - reason of ApplyConflict. - ReadOnly type means the agent will only check the existence of the resource based on its metadata, - statusFeedBackRules can still be used to get feedbackResults. - enum: - - Update - - CreateOnly - - ServerSideApply - - ReadOnly - type: string - required: - - type - type: object - required: - - resourceIdentifier - type: object - type: array - workload: - description: Workload represents the manifest workload to be deployed - on a managed cluster. - properties: - manifests: - description: Manifests represents a list of kuberenetes resources - to be deployed on a managed cluster. - items: - description: Manifest represents a resource to be deployed on - managed cluster. - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - type: array - type: object - type: object - status: - description: Status represents the current status of work. - properties: - conditions: - description: |- - Conditions contains the different condition statuses for this work. - Valid condition types are: - 1. Applied represents workload in ManifestWork is applied successfully on managed cluster. - 2. Progressing represents workload in ManifestWork is being applied on managed cluster. - 3. Available represents workload in ManifestWork exists on the managed cluster. - 4. Degraded represents the current state of workload does not match the desired - state for a certain period. - items: - description: "Condition contains details for one aspect of the current - state of this API Resource.\n---\nThis struct is intended for - direct use as an array at the field path .status.conditions. For - example,\n\n\n\ttype FooStatus struct{\n\t // Represents the - observations of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // - +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t - \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - resourceStatus: - description: |- - ResourceStatus represents the status of each resource in manifestwork deployed on a - managed cluster. The Klusterlet agent on managed cluster syncs the condition from the managed cluster to the hub. - properties: - manifests: - description: |- - Manifests represents the condition of manifests deployed on managed cluster. - Valid condition types are: - 1. Progressing represents the resource is being applied on managed cluster. - 2. Applied represents the resource is applied successfully on managed cluster. - 3. Available represents the resource exists on the managed cluster. - 4. Degraded represents the current state of resource does not match the desired - state for a certain period. - items: - description: |- - ManifestCondition represents the conditions of the resources deployed on a - managed cluster. - properties: - conditions: - description: Conditions represents the conditions of this - resource on a managed cluster. - items: - description: "Condition contains details for one aspect - of the current state of this API Resource.\n---\nThis - struct is intended for direct use as an array at the - field path .status.conditions. For example,\n\n\n\ttype - FooStatus struct{\n\t // Represents the observations - of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t - \ // +patchMergeKey=type\n\t // +patchStrategy=merge\n\t - \ // +listType=map\n\t // +listMapKey=type\n\t - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, - False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - resourceMeta: - description: ResourceMeta represents the group, version, - kind, name and namespace of a resoure. - properties: - group: - description: Group is the API Group of the Kubernetes - resource. - type: string - kind: - description: Kind is the kind of the Kubernetes resource. - type: string - name: - description: Name is the name of the Kubernetes resource. - type: string - namespace: - description: Name is the namespace of the Kubernetes - resource. - type: string - ordinal: - description: Ordinal represents the index of the manifest - on spec. - format: int32 - type: integer - resource: - description: Resource is the resource name of the Kubernetes - resource. - type: string - version: - description: Version is the version of the Kubernetes - resource. - type: string - type: object - statusFeedback: - description: StatusFeedback represents the values of the - feild synced back defined in statusFeedbacks - properties: - values: - description: Values represents the synced value of the - interested field. - items: - properties: - fieldValue: - description: |- - Value is the value of the status field. - The value of the status field can only be integer, string or boolean. - properties: - boolean: - description: Boolean is bool value when type - is boolean. - type: boolean - integer: - description: Integer is the integer value - when type is integer. - format: int64 - type: integer - jsonRaw: - description: JsonRaw is a json string when - type is a list or object - maxLength: 1024 - type: string - string: - description: String is the string value when - type is string. - type: string - type: - description: Type represents the type of the - value, it can be integer, string or boolean. - enum: - - Integer - - String - - Boolean - - JsonRaw - type: string - required: - - type - type: object - name: - description: |- - Name represents the alias name for this field. It is the same as what is specified - in StatuFeedbackRule in the spec. - type: string - required: - - fieldValue - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - type: object - type: array - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/metrics.service.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/metrics.service.yaml deleted file mode 100644 index 7e073d85..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/metrics.service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: maestro-metrics - namespace: {{ .Release.Namespace }} - labels: - app: maestro - port: metrics -spec: - selector: - app: maestro - ports: - - port: 8080 - targetPort: 8080 - name: metrics diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.deployment.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.deployment.yaml deleted file mode 100644 index dac3f598..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.deployment.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# {{- if .Values.database.containerizedDb }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: maestro-db - namespace: {{ .Release.Namespace }} -spec: - replicas: 1 - selector: - matchLabels: - name: maestro-db - strategy: - type: Recreate - template: - metadata: - labels: - name: maestro-db - spec: - containers: - - env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - key: db.user - name: maestro-db - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - key: db.password - name: maestro-db - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - key: db.name - name: maestro-db - - name: PGDATA - value: /var/lib/pgsql/data/pgdata - image: {{ .Values.database.containerizedDbImage }} - imagePullPolicy: IfNotPresent - livenessProbe: - tcpSocket: - port: 5432 - initialDelaySeconds: 120 - timeoutSeconds: 10 - name: postgresql - ports: - - containerPort: 5432 - protocol: TCP - readinessProbe: - exec: - command: - - /bin/sh - - -c - - exec /usr/bin/pg_isready -U $POSTGRES_USER -d $POSTGRES_DB -h localhost -p 5432 - initialDelaySeconds: 5 - timeoutSeconds: 1 - securityContext: - capabilities: {} - privileged: false - terminationMessagePath: /dev/termination-log - volumeMounts: - - mountPath: /var/lib/pgsql/data - name: maestro-db-data - dnsPolicy: ClusterFirst - restartPolicy: Always - volumes: - - name: maestro-db-data - persistentVolumeClaim: - claimName: maestro-db -# {{- end }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.pvc.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.pvc.yaml deleted file mode 100644 index c5037087..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.pvc.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# {{- if .Values.database.containerizedDb }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: maestro-db - namespace: {{ .Release.Namespace }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.database.containerizedDbPvcCapacity }} -# {{- end }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.secret.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.secret.yaml deleted file mode 100644 index bb500d77..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: maestro-db - namespace: {{ .Release.Namespace }} -stringData: - db.host: {{ .Values.database.host }} - db.port: "{{ .Values.database.port }}" - db.name: {{ .Values.database.name }} - db.user: {{ .Values.database.user }} - db.password: {{ .Values.database.password }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.service.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.service.yaml deleted file mode 100644 index 13771240..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/templates/pg.service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# {{- if .Values.database.containerizedDb }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.database.host }} - namespace: {{ .Release.Namespace }} -spec: - ports: - - name: postgresql - nodePort: 0 - port: {{ .Values.database.port }} - protocol: TCP - targetPort: 5432 - selector: - name: maestro-db - sessionAffinity: None - type: ClusterIP -status: - loadBalancer: {} -# {{- end }} diff --git a/test/e2e/setup/aro/maestro/deploy/helm/server/values.yaml b/test/e2e/setup/aro/maestro/deploy/helm/server/values.yaml deleted file mode 100644 index c23d1547..00000000 --- a/test/e2e/setup/aro/maestro/deploy/helm/server/values.yaml +++ /dev/null @@ -1,46 +0,0 @@ -deployment: - replicas: 3 - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: 1 - memory: 1Gi -broker: - host: "" - port: 8883 -database: - containerizedDb: false - containerizedDbImage: docker.io/library/postgres:14.2 - containerizedDbPvcCapacity: 512Mi - host: "maestro-db" - port: 5432 - name: "maestro" - user: "maestro" - password: "TheBlurstOfTimes" - ssl: "require" - maxOpenConnections: 50 - debug: false -maestro: - serviceAccount: maestro - httpReadTimeout: 60s - httpWriteTimeout: 60s - labelMetricsInclusionDuration: 168h - httpBindPort: 8000 - grpcBindPort: 8090 - healthCheckBindPort: 8083 - glog_v: 10 -clusterService: - namespace: cluster-service - serviceAccount: clusters-service -istio: - restrictIngress: true -azure: - clientId: "" - tenantId: "" -image: - base: "" - tag: "" -credsKeyVault: - name: "" - secret: "maestro-server"