Skip to content

Commit

Permalink
Get status.conditions from CAPI operator during updateComponentsStatus
Browse files Browse the repository at this point in the history
* Differentiate status into own package

Signed-off-by: Kyle Squizzato <[email protected]>
  • Loading branch information
squizzi committed Oct 16, 2024
1 parent 3f9a2ad commit 3769ed9
Show file tree
Hide file tree
Showing 8 changed files with 221 additions and 87 deletions.
1 change: 1 addition & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ func main() {
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Config: mgr.GetConfig(),
DynamicClient: dc,
SystemNamespace: currentNamespace,
CreateTemplateManagement: createTemplateManagement,
}).SetupWithManager(mgr); err != nil {
Expand Down
55 changes: 15 additions & 40 deletions internal/controller/managedcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
Expand All @@ -49,6 +47,7 @@ import (
"github.com/Mirantis/hmc/internal/helm"
"github.com/Mirantis/hmc/internal/telemetry"
"github.com/Mirantis/hmc/internal/utils"
"github.com/Mirantis/hmc/internal/utils/status"
)

const (
Expand Down Expand Up @@ -115,60 +114,36 @@ func (r *ManagedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque
l.Error(err, "Failed to get Management object")
return ctrl.Result{}, err
}
if err := telemetry.TrackManagedClusterCreate(string(mgmt.UID), string(managedCluster.UID), managedCluster.Spec.Template, managedCluster.Spec.DryRun); err != nil {
if err := telemetry.TrackManagedClusterCreate(
string(mgmt.UID), string(managedCluster.UID), managedCluster.Spec.Template, managedCluster.Spec.DryRun); err != nil {
l.Error(err, "Failed to track ManagedCluster creation")
}
}

return r.Update(ctx, managedCluster)
}

func (r *ManagedClusterReconciler) setStatusFromClusterStatus(ctx context.Context, managedCluster *hmc.ManagedCluster) (requeue bool, _ error) {
func (r *ManagedClusterReconciler) setStatusFromClusterStatus(
ctx context.Context, managedCluster *hmc.ManagedCluster,
) (bool, error) {
l := ctrl.LoggerFrom(ctx)

resourceID := schema.GroupVersionResource{
_, _, conditions, err := status.ConditionsFromResource(ctx, r.DynamicClient, schema.GroupVersionResource{
Group: "cluster.x-k8s.io",
Version: "v1beta1",
Resource: "clusters",
}

list, err := r.DynamicClient.Resource(resourceID).Namespace(managedCluster.Namespace).List(ctx, metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: managedCluster.Name}).String(),
})

if apierrors.IsNotFound(err) || len(list.Items) == 0 {
l.Info("Clusters not found, ignoring since object must be deleted or not yet created")
return true, nil
}

if err != nil {
return true, fmt.Errorf("failed to get cluster information for managedCluster %s in namespace: %s: %w",
managedCluster.Namespace, managedCluster.Name, err)
}
conditions, found, err := unstructured.NestedSlice(list.Items[0].Object, "status", "conditions")
}, labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: managedCluster.Name}).String())
if err != nil {
return true, fmt.Errorf("failed to get cluster information for managedCluster %s in namespace: %s: %w",
managedCluster.Namespace, managedCluster.Name, err)
}
if !found {
return true, fmt.Errorf("failed to get cluster information for managedCluster %s in namespace: %s: status.conditions not found",
managedCluster.Namespace, managedCluster.Name)
notFoundErr := status.ResourceNotFoundError{}
if errors.As(err, &notFoundErr) {
l.Info(err.Error())
return true, nil
}
return false, fmt.Errorf("failed to get conditions: %w", err)
}

allConditionsComplete := true
for _, condition := range conditions {
conditionMap, ok := condition.(map[string]any)
if !ok {
return true, fmt.Errorf("failed to cast condition to map[string]any for managedCluster: %s in namespace: %s: %w",
managedCluster.Namespace, managedCluster.Name, err)
}

var metaCondition metav1.Condition
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(conditionMap, &metaCondition); err != nil {
return true, fmt.Errorf("failed to convert unstructured conditions to metav1.Condition for managedCluster %s in namespace: %s: %w",
managedCluster.Namespace, managedCluster.Name, err)
}

for _, metaCondition := range conditions {
if metaCondition.Status != "True" {
allConditionsComplete = false
}
Expand Down
67 changes: 67 additions & 0 deletions internal/controller/management_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"

fluxv2 "github.com/fluxcd/helm-controller/api/v2"
"github.com/fluxcd/pkg/apis/meta"
Expand All @@ -29,6 +30,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -38,6 +41,7 @@ import (
"github.com/Mirantis/hmc/internal/certmanager"
"github.com/Mirantis/hmc/internal/helm"
"github.com/Mirantis/hmc/internal/utils"
"github.com/Mirantis/hmc/internal/utils/status"
)

// Those are only needed for the initial installation
Expand All @@ -55,6 +59,7 @@ type ManagementReconciler struct {
Scheme *runtime.Scheme
Config *rest.Config
SystemNamespace string
DynamicClient *dynamic.DynamicClient
CreateTemplateManagement bool
}

Expand Down Expand Up @@ -150,6 +155,16 @@ func (r *ManagementReconciler) Update(ctx context.Context, management *hmc.Manag
errs = errors.Join(errs, errors.New(errMsg))
continue
}

if component.Template != hmc.CoreHMCName {
if err := r.checkProviderStatus(ctx, component.Template); err != nil {
errMsg := err.Error()
updateComponentsStatus(detectedComponents, &detectedProviders, component.helmReleaseName, component.Template, template.Status.Providers, errMsg)
errs = errors.Join(errs, errors.New(errMsg))
continue
}
}

updateComponentsStatus(detectedComponents, &detectedProviders, component.helmReleaseName, component.Template, template.Status.Providers, "")
}

Expand Down Expand Up @@ -201,6 +216,58 @@ func (r *ManagementReconciler) ensureTemplateManagement(ctx context.Context, mgm
return fmt.Errorf("failed to create %s TemplateManagement object: %w", hmc.TemplateManagementName, err)
}
l.Info("Successfully created TemplateManagement object")

return nil
}

// checkProviderStatus checks the status of a provider associated with a given
// ProviderTemplate name. Since there's no way to determine resource Kind from
// the given template iterate over all possible provider types.
func (r *ManagementReconciler) checkProviderStatus(ctx context.Context, providerTemplateName string) error {
var errs error

for _, resourceType := range []string{
"coreproviders",
"infrastructureproviders",
"controlplaneproviders",
"bootstrapproviders",
} {
gvr := schema.GroupVersionResource{
Group: "operator.cluster.x-k8s.io",
Version: "v1alpha2",
Resource: resourceType,
}

name, kind, conditions, err := status.ConditionsFromResource(ctx, r.DynamicClient, gvr,
labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: providerTemplateName}).String(),
)
if err != nil {
notFoundErr := status.ResourceNotFoundError{}
if errors.As(err, &notFoundErr) {
// Check the next resource type.
continue
}

return fmt.Errorf("failed to get status for template: %s: %w", providerTemplateName, err)
}

var falseConditionTypes []string
for _, condition := range conditions {
if condition.Status != metav1.ConditionTrue {
falseConditionTypes = append(falseConditionTypes, condition.Type)
}
}

if len(falseConditionTypes) > 0 {
errs = errors.Join(fmt.Errorf("%s: %s is not yet ready, has false conditions: %s",
name, kind, strings.Join(falseConditionTypes, ", ")))
}
}

if errs != nil {
return errs
}

return nil
}

Expand Down
117 changes: 117 additions & 0 deletions internal/utils/status/status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright 2024
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package status

import (
"context"
"fmt"

apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
)

// ConditionsFromUnstructured fetches all of the status.conditions from an
// unstructured object and returns them as a slice of metav1.Condition. The
// status.conditions field is expected to be a slice of map[string]any
// which can be cast into a metav1.Condition.
func ConditionsFromUnstructured(unstrObj *unstructured.Unstructured) ([]metav1.Condition, error) {
objKind, objName := ObjKindName(unstrObj)

// Iterate the status conditions and ensure each condition reports a "Ready"
// status.
unstrConditions, found, err := unstructured.NestedSlice(unstrObj.Object, "status", "conditions")
if !found {
return nil, fmt.Errorf("no status conditions found for %s: %s", objKind, objName)
}
if err != nil {
return nil, fmt.Errorf("failed to get status conditions for %s: %s: %w", objKind, objName, err)
}

conditions := make([]metav1.Condition, 0, len(unstrConditions))

for _, condition := range unstrConditions {
conditionMap, ok := condition.(map[string]any)
if !ok {
return nil, fmt.Errorf("expected %s: %s condition to be type map[string]any, got: %T",
objKind, objName, conditionMap)
}

var c *metav1.Condition

if err := runtime.DefaultUnstructuredConverter.FromUnstructured(conditionMap, &c); err != nil {
return nil, fmt.Errorf("failed to convert condition map to metav1.Condition: %w", err)
}

conditions = append(conditions, *c)
}

return conditions, nil
}

type ResourceNotFoundError struct {
Resource string
}

func (e ResourceNotFoundError) Error() string {
return fmt.Sprintf("no %s found, ignoring since object must be deleted or not yet created", e.Resource)
}

// ConditionsFromResource fetches the conditions from a resource identified by
// the provided GroupVersionResource and labelSelector. The function returns
// the name and kind of the resource and a slice of metav1.Condition. If the
// resource is not found, returns a ResourceNotFoundError which can be checked
// by the caller to prevent reconciliation loops.
//
//nolint:revive
func ConditionsFromResource(
ctx context.Context, dynamicClient dynamic.Interface,
gvr schema.GroupVersionResource, labelSelector string,
) (string, string, []metav1.Condition, error) {
list, err := dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
if err != nil {
if apierrors.IsNotFound(err) {
return "", "", nil, ResourceNotFoundError{Resource: gvr.Resource}
}

return "", "", nil, fmt.Errorf("failed to list %s: %w", gvr.Resource, err)
}

if len(list.Items) == 0 {
return "", "", nil, ResourceNotFoundError{Resource: gvr.Resource}
}

if len(list.Items) > 1 {
return "", "", nil, fmt.Errorf("expected to find only one of resource: %s with label: %q, found: %d",
gvr.Resource, labelSelector, len(list.Items))
}

kind, name := ObjKindName(&list.Items[0])
conditions, err := ConditionsFromUnstructured(&list.Items[0])
if err != nil {
return "", "", nil, fmt.Errorf("failed to get conditions: %w", err)
}

return kind, name, conditions, nil
}

func ObjKindName(unstrObj *unstructured.Unstructured) (name string, kind string) {
kind = unstrObj.GetKind()
name = unstrObj.GetName()
return kind, name
}
10 changes: 10 additions & 0 deletions templates/provider/hmc/templates/rbac/controller/roles.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,16 @@ metadata:
labels:
{{- include "hmc.labels" . | nindent 4 }}
rules:
- apiGroups:
- operator.cluster.x-k8s.io
resources:
- coreproviders
- infrastructureproviders
- bootstrapproviders
- controlplaneproviders
verbs:
- get
- list
- apiGroups:
- cluster.x-k8s.io
resources:
Expand Down
3 changes: 2 additions & 1 deletion test/managedcluster/validate_deleted.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"errors"
"fmt"

"github.com/Mirantis/hmc/internal/utils/status"
"github.com/Mirantis/hmc/test/kubeclient"
"github.com/Mirantis/hmc/test/utils"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -43,7 +44,7 @@ func validateClusterDeleted(ctx context.Context, kc *kubeclient.KubeClient, clus
return fmt.Errorf("cluster: %q exists, but is not in 'Deleting' phase", clusterName)
}

conditions, err := utils.GetConditionsFromUnstructured(cluster)
conditions, err := status.ConditionsFromUnstructured(cluster)
if err != nil {
return fmt.Errorf("failed to get conditions from unstructured object: %w", err)
}
Expand Down
9 changes: 5 additions & 4 deletions test/managedcluster/validate_deployed.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"fmt"
"strings"

"github.com/Mirantis/hmc/internal/utils/status"
"github.com/Mirantis/hmc/test/kubeclient"
"github.com/Mirantis/hmc/test/utils"
. "github.com/onsi/ginkgo/v2"
Expand Down Expand Up @@ -86,22 +87,22 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl
Fail(err.Error())
}

objKind, objName := utils.ObjKindName(&controlPlane)
objKind, objName := status.ObjKindName(&controlPlane)

// k0s does not use the metav1.Condition type for status.conditions,
// instead it uses a custom type so we can't use
// ValidateConditionsTrue here, instead we'll check for "ready: true".
status, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status")
objStatus, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status")
if !found {
return fmt.Errorf("no status found for %s: %s", objKind, objName)
}
if err != nil {
return fmt.Errorf("failed to get status conditions for %s: %s: %w", objKind, objName, err)
}

st, ok := status.(map[string]any)
st, ok := objStatus.(map[string]any)
if !ok {
return fmt.Errorf("expected K0sControlPlane condition to be type map[string]any, got: %T", status)
return fmt.Errorf("expected K0sControlPlane condition to be type map[string]any, got: %T", objStatus)
}

if _, ok := st["ready"]; !ok {
Expand Down
Loading

0 comments on commit 3769ed9

Please sign in to comment.