diff --git a/api/v1alpha1/multiclusterservice_types.go b/api/v1alpha1/multiclusterservice_types.go index 74916e7ef..3ed1c18db 100644 --- a/api/v1alpha1/multiclusterservice_types.go +++ b/api/v1alpha1/multiclusterservice_types.go @@ -19,6 +19,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + MultiClusterServiceFinalizer = "hmc.mirantis.com/multicluster-service" +) + // ServiceSpec represents a Service to be managed type ServiceSpec struct { // Values is the helm values to be passed to the template. @@ -27,6 +31,12 @@ type ServiceSpec struct { // +kubebuilder:validation:MinLength=1 // Template is a reference to a Template object located in the same namespace. + // wahab: + // -------- + // I think we should be able to reference Template from another namespace + // in a MultiClusterService obj, so we need to also have another filed + // to specify namespace. So maybe we should have separate ServiceSpec + // structs for ManagedCluster and MultiClusterService? Template string `json:"template"` // +kubebuilder:validation:MinLength=1 @@ -57,6 +67,9 @@ type MultiClusterServiceSpec struct { // In case of conflict with another object managing the service, // the one with higher priority will get to deploy its services. Priority int32 `json:"priority,omitempty"` + + // +kubebuilder:default:=false + // StopOnConflict specifies what to do in case of a conflict. // E.g. If another object is already managing a service. // By default the remaining services will be deployed even if conflict is detected. @@ -70,8 +83,8 @@ type MultiClusterServiceSpec struct { // If this status ends up being common with ManagedClusterStatus, // then make a common status struct that can be shared by both. type MultiClusterServiceStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file + // ObservedGeneration is the last observed generation. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` } // +kubebuilder:object:root=true diff --git a/config/dev/aws-managedcluster.yaml b/config/dev/aws-managedcluster.yaml index dd303141c..7f95f7011 100644 --- a/config/dev/aws-managedcluster.yaml +++ b/config/dev/aws-managedcluster.yaml @@ -1,7 +1,7 @@ apiVersion: hmc.mirantis.com/v1alpha1 kind: ManagedCluster metadata: - name: aws-dev + name: wali-aws-dev namespace: ${NAMESPACE} spec: credential: aws-cluster-identity-cred @@ -10,15 +10,18 @@ spec: name: aws-cluster-identity namespace: ${NAMESPACE} controlPlane: + amiID: ami-0eb9fdcf0d07bd5ef instanceType: t3.small controlPlaneNumber: 1 publicIP: true - region: us-west-2 + region: ca-central-1 worker: + amiID: ami-0eb9fdcf0d07bd5ef instanceType: t3.small workersNumber: 1 installBeachHeadServices: false template: aws-standalone-cp-0-0-1 + priority: 1000 services: - template: kyverno-3-2-6 name: kyverno diff --git a/internal/controller/managedcluster_controller.go b/internal/controller/managedcluster_controller.go index 16af664ec..376f6bd1a 100644 --- a/internal/controller/managedcluster_controller.go +++ b/internal/controller/managedcluster_controller.go @@ -429,10 +429,6 @@ func (r *ManagedClusterReconciler) updateServices(ctx context.Context, mc *hmc.M } if _, err := sveltos.ReconcileProfile(ctx, r.Client, l, mc.Namespace, mc.Name, - map[string]string{ - hmc.FluxHelmChartNamespaceKey: mc.Namespace, - hmc.FluxHelmChartNameKey: mc.Name, - }, sveltos.ReconcileProfileOpts{ OwnerReference: &metav1.OwnerReference{ APIVersion: hmc.GroupVersion.String(), @@ -440,6 +436,10 @@ func (r *ManagedClusterReconciler) updateServices(ctx context.Context, mc *hmc.M Name: mc.Name, UID: mc.UID, }, + MatchLabels: map[string]string{ + hmc.FluxHelmChartNamespaceKey: mc.Namespace, + hmc.FluxHelmChartNameKey: mc.Name, + }, HelmChartOpts: opts, Priority: mc.Spec.Priority, StopOnConflict: mc.Spec.StopOnConflict, diff --git a/internal/controller/multiclusterservice_controller.go b/internal/controller/multiclusterservice_controller.go index 46e5ab497..0ca1984e1 100644 --- a/internal/controller/multiclusterservice_controller.go +++ b/internal/controller/multiclusterservice_controller.go @@ -16,24 +16,212 @@ package controller import ( "context" + "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/yaml" hmc "github.com/Mirantis/hmc/api/v1alpha1" + "github.com/Mirantis/hmc/internal/sveltos" + "github.com/Mirantis/hmc/internal/utils" + sourcev1 "github.com/fluxcd/source-controller/api/v1" + "github.com/go-logr/logr" ) +/* +apiVersion: hmc.mirantis.com/v1alpha1 +kind: MultiClusterService +metadata: + name: wali-global-ingress +spec: + clusterSelector: + matchLabels: + app.kubernetes.io/managed-by: Helm + services: + - template: ingress-nginx-4-11-0 + name: ingress-nginx + namespace: ingress-nginx +*/ + // MultiClusterServiceReconciler reconciles a MultiClusterService object type MultiClusterServiceReconciler struct { client.Client } // Reconcile reconciles a MultiClusterService object. -func (*MultiClusterServiceReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) { - _ = ctrl.LoggerFrom(ctx) +func (r *MultiClusterServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + l := ctrl.LoggerFrom(ctx).WithValues("MultiClusterServiceController", req.NamespacedName.String()) + l.Info("Reconciling MultiClusterService") + + // 1. Get the MultiClusterService obj from kube based on its name only (cluster-scope) + mcsvc := &hmc.MultiClusterService{} + err := r.Get(ctx, req.NamespacedName, mcsvc) + + fmt.Printf("\n>>>>>>>>>>>> [Reconcile] req.Namespace=%s, req=Name=%s, req.NamespacedName=%s, err=%s\n", req.Namespace, req.Name, req.NamespacedName, err) + + if apierrors.IsNotFound(err) { + l.Info("MultiClusterService not found, ignoring since object must be deleted") + return ctrl.Result{}, nil + } + if err != nil { + l.Error(err, "Failed to get MultiClusterService") + return ctrl.Result{}, err + } + + b, _ := yaml.Marshal(mcsvc) + fmt.Printf("\n>>>>>>>>>>>> [Reconcile] object=\n%s\n", string(b)) + + // ================================================================================================================ + + // 2. If its DeletionTimestamp.IsZero() then reconcile its deletion + if !mcsvc.DeletionTimestamp.IsZero() { + l.Info("Deleting MultiClusterService") + return r.reconcileDelete(ctx) + } + + // ================================================================================================================ + + // 3. If ObservedGeneration == 0 then make a creation event for telemetry + + // ================================================================================================================ + + // 4. Now reconcile + // 4a. Add finalizer if doesn't exist + // 4b. Reconcile the ClusterProfile object + return r.reconcile(ctx, l, mcsvc) +} + +func (r *MultiClusterServiceReconciler) reconcile(ctx context.Context, l logr.Logger, mcsvc *hmc.MultiClusterService) (ctrl.Result, error) { + isUpdated := controllerutil.AddFinalizer(mcsvc, hmc.MultiClusterServiceFinalizer) + if isUpdated { + if err := r.Client.Update(ctx, mcsvc); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update MultiClusterService %s/%s with finalizer %s: %w", mcsvc.Namespace, mcsvc.Name, hmc.MultiClusterServiceFinalizer, err) + } + return ctrl.Result{}, nil + } + + opts, err := r.getHelmChartOpts(ctx, l, mcsvc) + if err != nil { + return ctrl.Result{}, err + } + + if _, err := sveltos.ReconcileClusterProfile(ctx, r.Client, l, mcsvc.Namespace, mcsvc.Name, + sveltos.ReconcileProfileOpts{ + OwnerReference: &metav1.OwnerReference{ + APIVersion: hmc.GroupVersion.String(), + Kind: hmc.ManagedClusterKind, + Name: mcsvc.Name, + UID: mcsvc.UID, + }, + MatchLabels: map[string]string{ + hmc.FluxHelmChartNamespaceKey: mcsvc.Namespace, + hmc.FluxHelmChartNameKey: mcsvc.Name, + }, + HelmChartOpts: opts, + Priority: mcsvc.Spec.Priority, + StopOnConflict: mcsvc.Spec.StopOnConflict, + }); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile Profile: %w", err) + } - // TODO(https://github.com/Mirantis/hmc/issues/455): Implement me. + return ctrl.Result{RequeueAfter: DefaultRequeueInterval}, nil +} + +// wahab: needs to be shared somehow +func (r *MultiClusterServiceReconciler) getHelmChartOpts(ctx context.Context, l logr.Logger, mcsvc *hmc.MultiClusterService) ([]sveltos.HelmChartOpts, error) { + opts := []sveltos.HelmChartOpts{} + + // NOTE: The Profile object will be updated with no helm + // charts if len(mc.Spec.Services) == 0. This will result in the + // helm charts being uninstalled on matching clusters if + // Profile originally had len(m.Spec.Sevices) > 0. + for _, svc := range mcsvc.Spec.Services { + if svc.Disable { + l.Info(fmt.Sprintf("Skip adding Template (%s) to Profile (%s) because Disable=true", svc.Template, mcsvc.Name)) + continue + } + + tmpl := &hmc.ServiceTemplate{} + tmplRef := types.NamespacedName{Name: svc.Template, Namespace: utils.DefaultSystemNamespace} + if err := r.Get(ctx, tmplRef, tmpl); err != nil { + return nil, fmt.Errorf("failed to get Template (%s): %w", tmplRef.String(), err) + } + + source, err := r.getServiceTemplateSource(ctx, tmpl) + if err != nil { + return nil, fmt.Errorf("could not get repository url: %w", err) + } + + opts = append(opts, sveltos.HelmChartOpts{ + Values: svc.Values, + RepositoryURL: source.Spec.URL, + // We don't have repository name so chart name becomes repository name. + RepositoryName: tmpl.Spec.Helm.ChartName, + ChartName: func() string { + if source.Spec.Type == utils.RegistryTypeOCI { + return tmpl.Spec.Helm.ChartName + } + // Sveltos accepts ChartName in / format for non-OCI. + // We don't have a repository name, so we can use / instead. + // See: https://projectsveltos.github.io/sveltos/addons/helm_charts/. + return fmt.Sprintf("%s/%s", tmpl.Spec.Helm.ChartName, tmpl.Spec.Helm.ChartName) + }(), + ChartVersion: tmpl.Spec.Helm.ChartVersion, + ReleaseName: svc.Name, + ReleaseNamespace: func() string { + if svc.Namespace != "" { + return svc.Namespace + } + return svc.Name + }(), + // The reason it is passed to PlainHTTP instead of InsecureSkipTLSVerify is because + // the source.Spec.Insecure field is meant to be used for connecting to repositories + // over plain HTTP, which is different than what InsecureSkipTLSVerify is meant for. + // See: https://github.com/fluxcd/source-controller/pull/1288 + PlainHTTP: source.Spec.Insecure, + }) + } + + return opts, nil +} + +// wahab: needs to be shared +func (r *MultiClusterServiceReconciler) getServiceTemplateSource(ctx context.Context, tmpl *hmc.ServiceTemplate) (*sourcev1.HelmRepository, error) { + tmplRef := types.NamespacedName{Namespace: tmpl.Namespace, Name: tmpl.Name} + + if tmpl.Status.ChartRef == nil { + return nil, fmt.Errorf("status for ServiceTemplate (%s) has not been updated yet", tmplRef.String()) + } + + hc := &sourcev1.HelmChart{} + if err := r.Get(ctx, types.NamespacedName{ + Namespace: tmpl.Status.ChartRef.Namespace, + Name: tmpl.Status.ChartRef.Name, + }, hc); err != nil { + return nil, fmt.Errorf("failed to get HelmChart (%s): %w", tmplRef.String(), err) + } + + repo := &sourcev1.HelmRepository{} + if err := r.Get(ctx, types.NamespacedName{ + // Using chart's namespace because it's source + // (helm repository in this case) should be within the same namespace. + Namespace: hc.Namespace, + Name: hc.Spec.SourceRef.Name, + }, repo); err != nil { + return nil, fmt.Errorf("failed to get HelmRepository (%s): %w", tmplRef.String(), err) + } + + return repo, nil +} +func (r *MultiClusterServiceReconciler) reconcileDelete(_ context.Context) (ctrl.Result, error) { + // 2a. Handle what you need to handle + // 2b. Remove finalizer return ctrl.Result{}, nil } diff --git a/internal/sveltos/profile.go b/internal/sveltos/profile.go index 500b40dab..da62d5ebf 100644 --- a/internal/sveltos/profile.go +++ b/internal/sveltos/profile.go @@ -31,8 +31,14 @@ import ( "sigs.k8s.io/yaml" ) +const ( + ClusterProfileKind = sveltosv1beta1.ClusterProfileKind + ProfileKind = sveltosv1beta1.ProfileKind +) + type ReconcileProfileOpts struct { OwnerReference *metav1.OwnerReference + MatchLabels map[string]string HelmChartOpts []HelmChartOpts Priority int32 StopOnConflict bool @@ -50,94 +56,145 @@ type HelmChartOpts struct { InsecureSkipTLSVerify bool } -// ReconcileProfile reconciles a Sveltos Profile object. -func ReconcileProfile(ctx context.Context, +func ReconcileClusterProfile( + ctx context.Context, + cl client.Client, + l logr.Logger, + profileKind string, + name string, + opts ReconcileProfileOpts, +) (*sveltosv1beta1.ClusterProfile, error) { + obj := ObjectMeta(opts.OwnerReference) + obj.SetName(name) + + cp := &sveltosv1beta1.ClusterProfile{ + ObjectMeta: obj, + } + + operation, err := ctrl.CreateOrUpdate(ctx, cl, cp, func() error { + spec, err := Spec(&opts) + if err != nil { + return err + } + cp.Spec = *spec + + return nil + }) + if err != nil { + return nil, err + } + + if operation == controllerutil.OperationResultCreated || operation == controllerutil.OperationResultUpdated { + l.Info(fmt.Sprintf("Successfully %s ClusterProfile (%s)", string(operation), cp.Name)) + } + + return cp, nil +} + +func ReconcileProfile( + ctx context.Context, cl client.Client, l logr.Logger, namespace string, name string, - matchLabels map[string]string, opts ReconcileProfileOpts, ) (*sveltosv1beta1.Profile, error) { - cp := &sveltosv1beta1.Profile{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, + obj := ObjectMeta(opts.OwnerReference) + obj.SetNamespace(namespace) + obj.SetName(name) + + p := &sveltosv1beta1.Profile{ + ObjectMeta: obj, + } + + operation, err := ctrl.CreateOrUpdate(ctx, cl, p, func() error { + spec, err := Spec(&opts) + if err != nil { + return err + } + p.Spec = *spec + + return nil + }) + if err != nil { + return nil, err + } + + if operation == controllerutil.OperationResultCreated || operation == controllerutil.OperationResultUpdated { + l.Info(fmt.Sprintf("Successfully %s ClusterProfile (%s)", string(operation), p.Name)) + } + + return p, nil +} + +func ObjectMeta(owner *metav1.OwnerReference) metav1.ObjectMeta { + obj := metav1.ObjectMeta{ + Labels: map[string]string{ + hmc.HMCManagedLabelKey: hmc.HMCManagedLabelValue, }, } + if owner != nil { + obj.OwnerReferences = []metav1.OwnerReference{*owner} + } + + return obj +} + +func Spec(opts *ReconcileProfileOpts) (*sveltosv1beta1.Spec, error) { tier, err := PriorityToTier(opts.Priority) if err != nil { return nil, err } - operation, err := ctrl.CreateOrUpdate(ctx, cl, cp, func() error { - if cp.Labels == nil { - cp.Labels = make(map[string]string) - } - - cp.Labels[hmc.HMCManagedLabelKey] = hmc.HMCManagedLabelValue - if opts.OwnerReference != nil { - cp.OwnerReferences = []metav1.OwnerReference{*opts.OwnerReference} - } + spec := &sveltosv1beta1.Spec{ + ClusterSelector: libsveltosv1beta1.Selector{ + LabelSelector: metav1.LabelSelector{ + MatchLabels: opts.MatchLabels, + }, + }, + Tier: tier, + ContinueOnConflict: !opts.StopOnConflict, + } - cp.Spec = sveltosv1beta1.Spec{ - ClusterSelector: libsveltosv1beta1.Selector{ - LabelSelector: metav1.LabelSelector{ - MatchLabels: matchLabels, - }, + for _, hc := range opts.HelmChartOpts { + helmChart := sveltosv1beta1.HelmChart{ + RepositoryURL: hc.RepositoryURL, + RepositoryName: hc.RepositoryName, + ChartName: hc.ChartName, + ChartVersion: hc.ChartVersion, + ReleaseName: hc.ReleaseName, + ReleaseNamespace: hc.ReleaseNamespace, + HelmChartAction: sveltosv1beta1.HelmChartActionInstall, + RegistryCredentialsConfig: &sveltosv1beta1.RegistryCredentialsConfig{ + PlainHTTP: hc.PlainHTTP, + InsecureSkipTLSVerify: hc.InsecureSkipTLSVerify, }, - Tier: tier, - ContinueOnConflict: !opts.StopOnConflict, } - for _, hc := range opts.HelmChartOpts { - helmChart := sveltosv1beta1.HelmChart{ - RepositoryURL: hc.RepositoryURL, - RepositoryName: hc.RepositoryName, - ChartName: hc.ChartName, - ChartVersion: hc.ChartVersion, - ReleaseName: hc.ReleaseName, - ReleaseNamespace: hc.ReleaseNamespace, - HelmChartAction: sveltosv1beta1.HelmChartActionInstall, - RegistryCredentialsConfig: &sveltosv1beta1.RegistryCredentialsConfig{ - PlainHTTP: hc.PlainHTTP, - InsecureSkipTLSVerify: hc.InsecureSkipTLSVerify, - }, - } + if hc.PlainHTTP { + // InsecureSkipTLSVerify is redundant in this case. + helmChart.RegistryCredentialsConfig.InsecureSkipTLSVerify = false + } - if hc.PlainHTTP { - // InsecureSkipTLSVerify is redundant in this case. - helmChart.RegistryCredentialsConfig.InsecureSkipTLSVerify = false + if hc.Values != nil { + b, err := hc.Values.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal values to JSON for service (%s): %w", hc.RepositoryName, err) } - if hc.Values != nil { - b, err := hc.Values.MarshalJSON() - if err != nil { - return fmt.Errorf("failed to marshal values to JSON for service (%s) in ManagedCluster: %w", hc.RepositoryName, err) - } - - b, err = yaml.JSONToYAML(b) - if err != nil { - return fmt.Errorf("failed to convert values from JSON to YAML for service (%s) in ManagedCluster: %w", hc.RepositoryName, err) - } - - helmChart.Values = string(b) + b, err = yaml.JSONToYAML(b) + if err != nil { + return nil, fmt.Errorf("failed to convert values from JSON to YAML for service (%s): %w", hc.RepositoryName, err) } - cp.Spec.HelmCharts = append(cp.Spec.HelmCharts, helmChart) + helmChart.Values = string(b) } - return nil - }) - if err != nil { - return nil, err - } - if operation == controllerutil.OperationResultCreated || operation == controllerutil.OperationResultUpdated { - l.Info(fmt.Sprintf("Successfully %s Profile (%s/%s)", string(operation), cp.Namespace, cp.Name)) + spec.HelmCharts = append(spec.HelmCharts, helmChart) } - return cp, nil + return spec, nil } func DeleteProfile(ctx context.Context, cl client.Client, namespace string, name string) error { @@ -151,6 +208,16 @@ func DeleteProfile(ctx context.Context, cl client.Client, namespace string, name return client.IgnoreNotFound(err) } +func DeleteClusterProfile(ctx context.Context, cl client.Client, name string) error { + err := cl.Delete(ctx, &sveltosv1beta1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + }) + + return client.IgnoreNotFound(err) +} + // PriorityToTier converts priority value to Sveltos tier value. func PriorityToTier(priority int32) (int32, error) { var mini int32 = 1 diff --git a/multiclusterservice.yaml b/multiclusterservice.yaml new file mode 100644 index 000000000..522ac552a --- /dev/null +++ b/multiclusterservice.yaml @@ -0,0 +1,12 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: MultiClusterService +metadata: + name: wali-global-ingress +spec: + clusterSelector: + matchLabels: + app.kubernetes.io/managed-by: Helm + services: + - template: ingress-nginx-4-11-0 + name: ingress-nginx + namespace: ingress-nginx diff --git a/templates/provider/hmc/templates/rbac/controller/roles.yaml b/templates/provider/hmc/templates/rbac/controller/roles.yaml index 21c49b962..8c58c2071 100644 --- a/templates/provider/hmc/templates/rbac/controller/roles.yaml +++ b/templates/provider/hmc/templates/rbac/controller/roles.yaml @@ -165,6 +165,7 @@ rules: - config.projectsveltos.io resources: - profiles + - clusterprofiles verbs: {{ include "rbac.editorVerbs" . | nindent 4 }} - apiGroups: - hmc.mirantis.com