diff --git a/docs/dev.md b/docs/dev.md index 108ec81d4..1204c3648 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -171,3 +171,55 @@ manually with: ``` CLUSTER_NAME=example-e2e-test make dev-aws-nuke ``` + +## Credential propagation + +The following is the notes on provider specific CCM credentials delivery process + +### Azure + +Azure CCM/CSI controllers expect well-known `azure.json` to be provided though +Secret or by placing it on host file system. + +The 2A controller will create Secret named `azure-cloud-provider` in the +`kube-system` namespace (where all controllers reside). The name is passed to +controllers via helm values. + +The `azure.json` parameters are documented in detail in the +[official docs](https://cloud-provider-azure.sigs.k8s.io/install/configs) + +Most parameters are obtained from CAPZ objects. Rest parameters are either +omitted or set to sane defaults. + +### vSphere + +#### CCM + +cloud-provider-vsphere expects configuration to be passed in ConfigMap. The +credentials are located in the secret which is referenced in the configuration. + +The config itself is a yaml file and it's not very well documented (the +[spec docs](https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/cloud_config.md) +haven't been updated for years). + +Most options however has similar names and could be inferred. + +All optional parameters are omitted in the configuration created by 2A +controller. + +Some options are hardcoded (since values are hard/impossible to get from CAPV +objects). For example: + +- `insecureFlag` is set to `true` to omit certificate management parameters. This + is also a default in the official charts since most vcenters are using + self-signed or signed by internal authority certificates. +- `port` is set to `443` (HTTPS) +- [Multi-vcenter](https://cloud-provider-vsphere.sigs.k8s.io/tutorials/deploying_cpi_with_multi_dc_vc_aka_zones.html) + labels are set to default values of region and zone (`k8s-region` and + `k8s-zone`) + +#### CSI + +CSI expects single Secret with configuration in `ini` format +([documented here](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-BFF39F1D-F70A-4360-ABC9-85BDAFBE8864.html)). +Options are similar to CCM and same defaults/considerations are applicable. diff --git a/internal/controller/managedcluster_controller.go b/internal/controller/managedcluster_controller.go index b0a6a1b93..d75ac3381 100644 --- a/internal/controller/managedcluster_controller.go +++ b/internal/controller/managedcluster_controller.go @@ -15,13 +15,11 @@ package controller import ( - "bytes" "context" "encoding/json" "errors" "fmt" "strings" - texttemplate "text/template" "time" hcv2 "github.com/fluxcd/helm-controller/api/v2" @@ -40,19 +38,15 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" - capv "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/yaml" hmc "github.com/Mirantis/hmc/api/v1alpha1" + "github.com/Mirantis/hmc/internal/credspropagation" "github.com/Mirantis/hmc/internal/helm" "github.com/Mirantis/hmc/internal/sveltos" "github.com/Mirantis/hmc/internal/telemetry" @@ -646,7 +640,7 @@ func (r *ManagedClusterReconciler) reconcileCredentialPropagation(ctx context.Co providers, err := r.getInfraProvidersNames(ctx, managedCluster.Namespace, managedCluster.Spec.Template) if err != nil { - return fmt.Errorf("failed to get cluster providers for cluster %s/%s: %s", managedCluster.Namespace, managedCluster.Name, err) + return fmt.Errorf("failed to get cluster providers for cluster %s/%s: %w", managedCluster.Namespace, managedCluster.Name, err) } kubeconfSecret := &corev1.Secret{} @@ -654,7 +648,7 @@ func (r *ManagedClusterReconciler) reconcileCredentialPropagation(ctx context.Co Name: managedCluster.Name + "-kubeconfig", Namespace: managedCluster.Namespace, }, kubeconfSecret); err != nil { - return fmt.Errorf("failed to get kubeconfig secret for cluster %s/%s: %s", managedCluster.Namespace, managedCluster.Name, err) + return fmt.Errorf("failed to get kubeconfig secret for cluster %s/%s: %w", managedCluster.Namespace, managedCluster.Name, err) } for _, provider := range providers { @@ -663,7 +657,13 @@ func (r *ManagedClusterReconciler) reconcileCredentialPropagation(ctx context.Co l.Info("Skipping creds propagation for AWS") case "azure": l.Info("Azure creds propagation start") - if err := r.propagateAzureSecrets(ctx, managedCluster, kubeconfSecret); err != nil { + if err := credspropagation.PropagateAzureSecrets(credspropagation.PropagationCfg{ + Ctx: ctx, + Client: r.Client, + ManagedCluster: managedCluster, + KubeconfSecret: kubeconfSecret, + SystemNamespace: r.SystemNamespace, + }); err != nil { errMsg := fmt.Sprintf("failed to create Azure CCM credentials: %s", err) apimeta.SetStatusCondition(managedCluster.GetConditions(), metav1.Condition{ Type: hmc.CredentialsPropagatedCondition, @@ -683,7 +683,13 @@ func (r *ManagedClusterReconciler) reconcileCredentialPropagation(ctx context.Co }) case "vsphere": l.Info("vSphere creds propagation start") - if err := r.propagateVSphereSecrets(ctx, managedCluster, kubeconfSecret); err != nil { + if err := credspropagation.PropagateVSphereSecrets(credspropagation.PropagationCfg{ + Ctx: ctx, + Client: r.Client, + ManagedCluster: managedCluster, + KubeconfSecret: kubeconfSecret, + SystemNamespace: r.SystemNamespace, + }); err != nil { errMsg := fmt.Sprintf("failed to create vSphere CCM credentials: %s", err) apimeta.SetStatusCondition(managedCluster.GetConditions(), metav1.Condition{ Type: hmc.CredentialsPropagatedCondition, @@ -715,270 +721,6 @@ func (r *ManagedClusterReconciler) reconcileCredentialPropagation(ctx context.Co return nil } -func (r *ManagedClusterReconciler) propagateAzureSecrets(ctx context.Context, managedCluster *hmc.ManagedCluster, kubeconfSecret *corev1.Secret) error { - azureCluster := &capz.AzureCluster{} - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: managedCluster.Name, - Namespace: managedCluster.Namespace, - }, azureCluster); err != nil { - return fmt.Errorf("failed to get AzureCluster %s: %s", managedCluster.Name, err) - } - - azureClIdty := &capz.AzureClusterIdentity{} - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: azureCluster.Spec.IdentityRef.Name, - Namespace: azureCluster.Spec.IdentityRef.Namespace, - }, azureClIdty); err != nil { - return fmt.Errorf("failed to get AzureClusterIdentity %s: %s", azureCluster.Spec.IdentityRef.Name, err) - } - - azureSecret := &corev1.Secret{} - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: azureClIdty.Spec.ClientSecret.Name, - Namespace: azureClIdty.Spec.ClientSecret.Namespace, - }, azureSecret); err != nil { - return fmt.Errorf("failed to get azure Secret %s: %s", azureClIdty.Spec.ClientSecret.Name, err) - } - - ccmSecret, err := generateAzureCCMSecret(azureCluster, azureClIdty, azureSecret) - if err != nil { - return fmt.Errorf("failed to generate Azure CCM secret: %s", err) - } - - if err := applyCCMConfigs(ctx, kubeconfSecret, ccmSecret); err != nil { - return fmt.Errorf("failed to apply Azure CCM secret: %s", err) - } - - return nil -} - -func generateAzureCCMSecret(azureCluster *capz.AzureCluster, azureClIdty *capz.AzureClusterIdentity, azureSecret *corev1.Secret) (*corev1.Secret, error) { - azureJSONMap := map[string]any{ - "cloud": azureCluster.Spec.AzureEnvironment, - "tenantId": azureClIdty.Spec.TenantID, - "subscriptionId": azureCluster.Spec.SubscriptionID, - "aadClientId": azureClIdty.Spec.ClientID, - "aadClientSecret": string(azureSecret.Data["clientSecret"]), - "resourceGroup": azureCluster.Spec.ResourceGroup, - "securityGroupName": azureCluster.Spec.NetworkSpec.Subnets[0].SecurityGroup.Name, - "securityGroupResourceGroup": azureCluster.Spec.NetworkSpec.Vnet.ResourceGroup, - "location": azureCluster.Spec.Location, - "vmType": "vmss", - "vnetName": azureCluster.Spec.NetworkSpec.Vnet.Name, - "vnetResourceGroup": azureCluster.Spec.NetworkSpec.Vnet.ResourceGroup, - "subnetName": azureCluster.Spec.NetworkSpec.Subnets[0].Name, - "loadBalancerSku": "Standard", - "loadBalancerName": "", - "maximumLoadBalancerRuleCount": 250, - "useManagedIdentityExtension": false, - "useInstanceMetadata": true, - } - azureJSON, err := json.Marshal(azureJSONMap) - if err != nil { - return nil, fmt.Errorf("error marshalling azure.json: %s", err) - } - - secretData := map[string][]byte{ - "cloud-config": azureJSON, - } - - return makeSecret("azure-cloud-provider", metav1.NamespaceSystem, secretData), nil -} - -func (r *ManagedClusterReconciler) propagateVSphereSecrets(ctx context.Context, managedCluster *hmc.ManagedCluster, kubeconfSecret *corev1.Secret) error { - vsphereCluster := &capv.VSphereCluster{} - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: managedCluster.Name, - Namespace: managedCluster.Namespace, - }, vsphereCluster); err != nil { - return fmt.Errorf("failed to get VSphereCluster %s: %s", managedCluster.Name, err) - } - - vsphereClIdty := &capv.VSphereClusterIdentity{} - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: vsphereCluster.Spec.IdentityRef.Name, - }, vsphereClIdty); err != nil { - return fmt.Errorf("failed to get VSphereClusterIdentity %s: %s", vsphereCluster.Spec.IdentityRef.Name, err) - } - - vsphereSecret := &corev1.Secret{} - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: vsphereClIdty.Spec.SecretName, - Namespace: r.SystemNamespace, - }, vsphereSecret); err != nil { - return fmt.Errorf("failed to get VSphere Secret %s: %s", vsphereClIdty.Spec.SecretName, err) - } - - vsphereMachines := &capv.VSphereMachineList{} - if err := r.Client.List( - ctx, - vsphereMachines, - &client.ListOptions{ - Namespace: managedCluster.Namespace, - LabelSelector: labels.SelectorFromSet(map[string]string{ - hmc.ClusterNameLabelKey: managedCluster.Name, - }), - Limit: 1, - }, - ); err != nil { - return fmt.Errorf("failed to list VSphereMachines for cluster %s: %s", managedCluster.Name, err) - } - ccmSecret, ccmConfig, err := generateVSphereCCMConfigs(vsphereCluster, vsphereSecret, &vsphereMachines.Items[0]) - if err != nil { - return fmt.Errorf("failed to generate VSphere CCM config: %s", err) - } - csiSecret, err := generateVSphereCSISecret(vsphereCluster, vsphereSecret, &vsphereMachines.Items[0]) - if err != nil { - return fmt.Errorf("failed to generate VSphere CSI secret: %s", err) - } - - if err := applyCCMConfigs(ctx, kubeconfSecret, ccmSecret, ccmConfig, csiSecret); err != nil { - return fmt.Errorf("failed to apply VSphere CCM/CSI secrets: %s", err) - } - - return nil -} - -func generateVSphereCCMConfigs(vCl *capv.VSphereCluster, vScrt *corev1.Secret, vMa *capv.VSphereMachine) (*corev1.Secret, *corev1.ConfigMap, error) { - secretName := "vsphere-cloud-secret" - secretData := map[string][]byte{ - vCl.Spec.Server + ".username": vScrt.Data["username"], - vCl.Spec.Server + ".password": vScrt.Data["password"], - } - ccmCfg := map[string]any{ - "global": map[string]any{ - "port": 443, - "insecureFlag": true, - "secretName": secretName, - "secretNamespace": metav1.NamespaceSystem, - }, - "vcenter": map[string]any{ - vCl.Spec.Server: map[string]any{ - "server": vCl.Spec.Server, - "datacenters": []string{ - vMa.Spec.Datacenter, - }, - }, - }, - "labels": map[string]any{ - "region": "k8s-region", - "zone": "k8s-zone", - }, - } - - ccmCfgYaml, err := yaml.Marshal(ccmCfg) - if err != nil { - return nil, nil, fmt.Errorf("failed to marshal CCM config: %s", err) - } - - cmData := map[string]string{ - "vsphere.conf": string(ccmCfgYaml), - } - return makeSecret(secretName, metav1.NamespaceSystem, secretData), - makeConfigMap("cloud-config", metav1.NamespaceSystem, cmData), - nil -} - -func generateVSphereCSISecret(vCl *capv.VSphereCluster, vScrt *corev1.Secret, vMa *capv.VSphereMachine) (*corev1.Secret, error) { - csiCfg := ` -[Global] -cluster-id = "{{ .ClusterID }}" - -[VirtualCenter "{{ .Vcenter }}"] -insecure-flag = "true" -user = "{{ .Username }}" -password = "{{ .Password }}" -port = "443" -datacenters = "{{ .Datacenter }}" -` - type CSIFields struct { - ClusterID, Vcenter, Username, Password, Datacenter string - } - - fields := CSIFields{ - ClusterID: vCl.Name, - Vcenter: vCl.Spec.Server, - Username: string(vScrt.Data["username"]), - Password: string(vScrt.Data["password"]), - Datacenter: vMa.Spec.Datacenter, - } - - tmpl, err := texttemplate.New("csiCfg").Parse(csiCfg) - if err != nil { - return nil, fmt.Errorf("failed to generate CSI secret (tmpl parse): %s", err) - } - var buf bytes.Buffer - if err := tmpl.Execute(&buf, fields); err != nil { - return nil, fmt.Errorf("failed to generate CSI secret (tmpl execute): %s", err) - } - - secretData := map[string][]byte{ - "csi-vsphere.conf": buf.Bytes(), - } - - return makeSecret("vcenter-config-secret", metav1.NamespaceSystem, secretData), nil -} - -func applyCCMConfigs(ctx context.Context, kubeconfSecret *corev1.Secret, objects ...client.Object) error { - clnt, err := makeClientFromSecret(kubeconfSecret) - if err != nil { - return fmt.Errorf("failed to create k8s client: %s", err) - } - for _, object := range objects { - if err := clnt.Patch( - ctx, - object, - client.Apply, - client.FieldOwner("hmc-controller"), - ); err != nil { - return fmt.Errorf("failed to apply CCM config object %s: %s", object.GetName(), err) - } - } - return nil -} - -func makeSecret(name, namespace string, data map[string][]byte) *corev1.Secret { - s := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: data, - } - s.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Secret")) - return s -} - -func makeConfigMap(name, namespace string, data map[string]string) *corev1.ConfigMap { - c := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: data, - } - c.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) - return c -} - -func makeClientFromSecret(kubeconfSecret *corev1.Secret) (client.Client, error) { - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - return nil, err - } - restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfSecret.Data["value"]) - if err != nil { - return nil, err - } - cl, err := client.New(restConfig, client.Options{ - Scheme: scheme, - }) - if err != nil { - return nil, err - } - return cl, nil -} - func setIdentityHelmValues(values *apiextensionsv1.JSON, idRef *corev1.ObjectReference) (*apiextensionsv1.JSON, error) { var valuesJSON map[string]any err := json.Unmarshal(values.Raw, &valuesJSON) diff --git a/internal/credspropagation/azure.go b/internal/credspropagation/azure.go new file mode 100644 index 000000000..f2b216c1e --- /dev/null +++ b/internal/credspropagation/azure.go @@ -0,0 +1,95 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credspropagation + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func PropagateAzureSecrets(cfg PropagationCfg) error { + azureCluster := &capz.AzureCluster{} + if err := cfg.Client.Get(cfg.Ctx, client.ObjectKey{ + Name: cfg.ManagedCluster.Name, + Namespace: cfg.ManagedCluster.Namespace, + }, azureCluster); err != nil { + return fmt.Errorf("failed to get AzureCluster %s: %w", cfg.ManagedCluster.Name, err) + } + + azureClIdty := &capz.AzureClusterIdentity{} + if err := cfg.Client.Get(cfg.Ctx, client.ObjectKey{ + Name: azureCluster.Spec.IdentityRef.Name, + Namespace: azureCluster.Spec.IdentityRef.Namespace, + }, azureClIdty); err != nil { + return fmt.Errorf("failed to get AzureClusterIdentity %s: %w", azureCluster.Spec.IdentityRef.Name, err) + } + + azureSecret := &corev1.Secret{} + if err := cfg.Client.Get(cfg.Ctx, client.ObjectKey{ + Name: azureClIdty.Spec.ClientSecret.Name, + Namespace: azureClIdty.Spec.ClientSecret.Namespace, + }, azureSecret); err != nil { + return fmt.Errorf("failed to get azure Secret %s: %w", azureClIdty.Spec.ClientSecret.Name, err) + } + + ccmSecret, err := generateAzureCCMSecret(azureCluster, azureClIdty, azureSecret) + if err != nil { + return fmt.Errorf("failed to generate Azure CCM secret: %s", err) + } + + if err := applyCCMConfigs(cfg.Ctx, cfg.KubeconfSecret, ccmSecret); err != nil { + return fmt.Errorf("failed to apply Azure CCM secret: %s", err) + } + + return nil +} + +func generateAzureCCMSecret(azureCluster *capz.AzureCluster, azureClIdty *capz.AzureClusterIdentity, azureSecret *corev1.Secret) (*corev1.Secret, error) { + azureJSONMap := map[string]any{ + "cloud": azureCluster.Spec.AzureEnvironment, + "tenantId": azureClIdty.Spec.TenantID, + "subscriptionId": azureCluster.Spec.SubscriptionID, + "aadClientId": azureClIdty.Spec.ClientID, + "aadClientSecret": string(azureSecret.Data["clientSecret"]), + "resourceGroup": azureCluster.Spec.ResourceGroup, + "securityGroupName": azureCluster.Spec.NetworkSpec.Subnets[0].SecurityGroup.Name, + "securityGroupResourceGroup": azureCluster.Spec.NetworkSpec.Vnet.ResourceGroup, + "location": azureCluster.Spec.Location, + "vmType": "vmss", + "vnetName": azureCluster.Spec.NetworkSpec.Vnet.Name, + "vnetResourceGroup": azureCluster.Spec.NetworkSpec.Vnet.ResourceGroup, + "subnetName": azureCluster.Spec.NetworkSpec.Subnets[0].Name, + "loadBalancerSku": "Standard", + "loadBalancerName": "", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true, + } + azureJSON, err := json.Marshal(azureJSONMap) + if err != nil { + return nil, fmt.Errorf("error marshalling azure.json: %s", err) + } + + secretData := map[string][]byte{ + "cloud-config": azureJSON, + } + + return makeSecret("azure-cloud-provider", metav1.NamespaceSystem, secretData), nil +} diff --git a/internal/credspropagation/common.go b/internal/credspropagation/common.go new file mode 100644 index 000000000..f32091f5b --- /dev/null +++ b/internal/credspropagation/common.go @@ -0,0 +1,97 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credspropagation + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + hmc "github.com/Mirantis/hmc/api/v1alpha1" +) + +type PropagationCfg struct { + Ctx context.Context + Client client.Client + ManagedCluster *hmc.ManagedCluster + KubeconfSecret *corev1.Secret + SystemNamespace string +} + +func applyCCMConfigs(ctx context.Context, kubeconfSecret *corev1.Secret, objects ...client.Object) error { + clnt, err := makeClientFromSecret(kubeconfSecret) + if err != nil { + return fmt.Errorf("failed to create k8s client: %w", err) + } + for _, object := range objects { + if err := clnt.Patch( + ctx, + object, + client.Apply, + client.FieldOwner("hmc-controller"), + ); err != nil { + return fmt.Errorf("failed to apply CCM config object %s: %w", object.GetName(), err) + } + } + return nil +} + +func makeSecret(name, namespace string, data map[string][]byte) *corev1.Secret { + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } + s.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Secret")) + return s +} + +func makeConfigMap(name, namespace string, data map[string]string) *corev1.ConfigMap { + c := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } + c.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) + return c +} + +func makeClientFromSecret(kubeconfSecret *corev1.Secret) (client.Client, error) { + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, err + } + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfSecret.Data["value"]) + if err != nil { + return nil, err + } + cl, err := client.New(restConfig, client.Options{ + Scheme: scheme, + }) + if err != nil { + return nil, err + } + return cl, nil +} diff --git a/internal/credspropagation/vsphere.go b/internal/credspropagation/vsphere.go new file mode 100644 index 000000000..595ad397c --- /dev/null +++ b/internal/credspropagation/vsphere.go @@ -0,0 +1,164 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credspropagation + +import ( + "bytes" + "fmt" + texttemplate "text/template" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + capv "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + hmc "github.com/Mirantis/hmc/api/v1alpha1" +) + +func PropagateVSphereSecrets(cfg PropagationCfg) error { + vsphereCluster := &capv.VSphereCluster{} + if err := cfg.Client.Get(cfg.Ctx, client.ObjectKey{ + Name: cfg.ManagedCluster.Name, + Namespace: cfg.ManagedCluster.Namespace, + }, vsphereCluster); err != nil { + return fmt.Errorf("failed to get VSphereCluster %s: %w", cfg.ManagedCluster.Name, err) + } + + vsphereClIdty := &capv.VSphereClusterIdentity{} + if err := cfg.Client.Get(cfg.Ctx, client.ObjectKey{ + Name: vsphereCluster.Spec.IdentityRef.Name, + }, vsphereClIdty); err != nil { + return fmt.Errorf("failed to get VSphereClusterIdentity %s: %w", vsphereCluster.Spec.IdentityRef.Name, err) + } + + vsphereSecret := &corev1.Secret{} + if err := cfg.Client.Get(cfg.Ctx, client.ObjectKey{ + Name: vsphereClIdty.Spec.SecretName, + Namespace: cfg.SystemNamespace, + }, vsphereSecret); err != nil { + return fmt.Errorf("failed to get VSphere Secret %s: %w", vsphereClIdty.Spec.SecretName, err) + } + + vsphereMachines := &capv.VSphereMachineList{} + if err := cfg.Client.List( + cfg.Ctx, + vsphereMachines, + &client.ListOptions{ + Namespace: cfg.ManagedCluster.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + hmc.ClusterNameLabelKey: cfg.ManagedCluster.Name, + }), + Limit: 1, + }, + ); err != nil { + return fmt.Errorf("failed to list VSphereMachines for cluster %s: %w", cfg.ManagedCluster.Name, err) + } + ccmSecret, ccmConfig, err := generateVSphereCCMConfigs(vsphereCluster, vsphereSecret, &vsphereMachines.Items[0]) + if err != nil { + return fmt.Errorf("failed to generate VSphere CCM config: %s", err) + } + csiSecret, err := generateVSphereCSISecret(vsphereCluster, vsphereSecret, &vsphereMachines.Items[0]) + if err != nil { + return fmt.Errorf("failed to generate VSphere CSI secret: %s", err) + } + + if err := applyCCMConfigs(cfg.Ctx, cfg.KubeconfSecret, ccmSecret, ccmConfig, csiSecret); err != nil { + return fmt.Errorf("failed to apply VSphere CCM/CSI secrets: %s", err) + } + + return nil +} + +func generateVSphereCCMConfigs(vCl *capv.VSphereCluster, vScrt *corev1.Secret, vMa *capv.VSphereMachine) (*corev1.Secret, *corev1.ConfigMap, error) { + const secretName = "vsphere-cloud-secret" + secretData := map[string][]byte{ + vCl.Spec.Server + ".username": vScrt.Data["username"], + vCl.Spec.Server + ".password": vScrt.Data["password"], + } + ccmCfg := map[string]any{ + "global": map[string]any{ + "port": 443, + "insecureFlag": true, + "secretName": secretName, + "secretNamespace": metav1.NamespaceSystem, + }, + "vcenter": map[string]any{ + vCl.Spec.Server: map[string]any{ + "server": vCl.Spec.Server, + "datacenters": []string{ + vMa.Spec.Datacenter, + }, + }, + }, + "labels": map[string]any{ + "region": "k8s-region", + "zone": "k8s-zone", + }, + } + + ccmCfgYaml, err := yaml.Marshal(ccmCfg) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal CCM config: %s", err) + } + + cmData := map[string]string{ + "vsphere.conf": string(ccmCfgYaml), + } + return makeSecret(secretName, metav1.NamespaceSystem, secretData), + makeConfigMap("cloud-config", metav1.NamespaceSystem, cmData), + nil +} + +func generateVSphereCSISecret(vCl *capv.VSphereCluster, vScrt *corev1.Secret, vMa *capv.VSphereMachine) (*corev1.Secret, error) { + csiCfg := ` +[Global] +cluster-id = "{{ .ClusterID }}" + +[VirtualCenter "{{ .Vcenter }}"] +insecure-flag = "true" +user = "{{ .Username }}" +password = "{{ .Password }}" +port = "443" +datacenters = "{{ .Datacenter }}" +` + type CSIFields struct { + ClusterID, Vcenter, Username, Password, Datacenter string + } + + fields := CSIFields{ + ClusterID: vCl.Name, + Vcenter: vCl.Spec.Server, + Username: string(vScrt.Data["username"]), + Password: string(vScrt.Data["password"]), + Datacenter: vMa.Spec.Datacenter, + } + + tmpl, err := texttemplate.New("csiCfg").Parse(csiCfg) + if err != nil { + return nil, fmt.Errorf("failed to generate CSI secret (tmpl parse): %s", err) + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, fields); err != nil { + return nil, fmt.Errorf("failed to generate CSI secret (tmpl execute): %s", err) + } + + secretData := map[string][]byte{ + "csi-vsphere.conf": buf.Bytes(), + } + + return makeSecret("vcenter-config-secret", metav1.NamespaceSystem, secretData), nil +}