diff --git a/go.mod b/go.mod index 51b863edd..24da6e5d0 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/ahmetb/gen-crd-api-reference-docs v0.1.5 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/gardener/etcd-druid v0.1.3 - github.com/gardener/gardener v1.1.1-0.20200311075931-7f7e52b986e7 - github.com/gardener/gardener-extensions v1.4.1-0.20200322183545-5329339c95e9 + github.com/gardener/gardener v1.1.1-0.20200323102039-58593d8be86a + github.com/gardener/gardener-extensions v1.5.0 github.com/gardener/machine-controller-manager v0.26.0 github.com/go-logr/logr v0.1.0 github.com/gobuffalo/packr/v2 v2.1.0 diff --git a/go.sum b/go.sum index daa98fff0..4f82bb8e1 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,10 @@ github.com/gardener/external-dns-management v0.7.3 h1:SAW9ur2mjZ+x89xbmcplJgqNUm github.com/gardener/external-dns-management v0.7.3/go.mod h1:Y3om11E865x4aQ7cmcHjknb8RMgCO153huRb/SvP+9o= github.com/gardener/gardener v1.1.1-0.20200311075931-7f7e52b986e7 h1:UD25lsw3fYBK7pUlXkGwUXmlpnksG9JbdwC75XZTBOQ= github.com/gardener/gardener v1.1.1-0.20200311075931-7f7e52b986e7/go.mod h1:lGAx5NkFDWoC4hPIL+lHJamafBxmOt5MrHq9hGtp5VI= -github.com/gardener/gardener-extensions v1.4.1-0.20200322183545-5329339c95e9 h1:v1BPB5ZhUrjhBDBzyV5ItM04gNeor+vO6MDWwLRfJIw= -github.com/gardener/gardener-extensions v1.4.1-0.20200322183545-5329339c95e9/go.mod h1:yCdFgMAz++ex3d1fmhN3Yti9MR9HN9iKTUjz5eI0uIQ= +github.com/gardener/gardener v1.1.1-0.20200323102039-58593d8be86a h1:TkMIvx1xRmd3xLuORXEqsQhpni49+wfuT4keC6d3Tsc= +github.com/gardener/gardener v1.1.1-0.20200323102039-58593d8be86a/go.mod h1:lGAx5NkFDWoC4hPIL+lHJamafBxmOt5MrHq9hGtp5VI= +github.com/gardener/gardener-extensions v1.5.0 h1:6JkU0/DV2bJvwkuPoP7/nPlyCrzPGKfw5j4f+wtXBeI= +github.com/gardener/gardener-extensions v1.5.0/go.mod h1:yCdFgMAz++ex3d1fmhN3Yti9MR9HN9iKTUjz5eI0uIQ= github.com/gardener/gardener-resource-manager v0.10.0 h1:6OUKoWI3oha42F0oJN8OEo3UR+D3onOCel4Th+zgotU= github.com/gardener/gardener-resource-manager v0.10.0/go.mod h1:0pKTHOhvU91eQB0EYr/6Ymd7lXc/5Hi8P8tF/gpV0VQ= github.com/gardener/hvpa-controller v0.0.0-20191014062307-fad3bdf06a25 h1:nOFITmV7vt4fcYPEXgj66Qs83FdDEMvL/LQcR0diRRE= diff --git a/pkg/controller/healthcheck/add.go b/pkg/controller/healthcheck/add.go index 960e04f8d..6a2268590 100644 --- a/pkg/controller/healthcheck/add.go +++ b/pkg/controller/healthcheck/add.go @@ -18,6 +18,7 @@ import ( "time" "github.com/gardener/gardener-extension-provider-openstack/pkg/openstack" + genericcontrolplaneactuator "github.com/gardener/gardener-extensions/pkg/controller/controlplane/genericactuator" "github.com/gardener/gardener-extensions/pkg/controller/healthcheck" healthcheckconfig "github.com/gardener/gardener-extensions/pkg/controller/healthcheck/config" @@ -25,7 +26,6 @@ import ( "github.com/gardener/gardener-extensions/pkg/controller/healthcheck/worker" genericworkeractuator "github.com/gardener/gardener-extensions/pkg/controller/worker/genericactuator" extensionspredicate "github.com/gardener/gardener-extensions/pkg/predicate" - gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,18 +45,26 @@ var ( // RegisterHealthChecks registers health checks for each extension resource // HealthChecks are grouped by extension (e.g worker), extension.type (e.g aws) and Health Check Type (e.g SystemComponentsHealthy) func RegisterHealthChecks(mgr manager.Manager, opts healthcheck.DefaultAddArgs) error { - normalPredicates := []predicate.Predicate{extensionspredicate.HasPurpose(extensionsv1alpha1.Normal)} if err := healthcheck.DefaultRegistration( openstack.Type, extensionsv1alpha1.SchemeGroupVersion.WithKind(extensionsv1alpha1.ControlPlaneResource), func() runtime.Object { return &extensionsv1alpha1.ControlPlane{} }, mgr, opts, - normalPredicates, - map[healthcheck.HealthCheck]string{ - general.NewSeedDeploymentHealthChecker(openstack.CloudControllerManagerName): string(gardencorev1beta1.ShootControlPlaneHealthy), - general.CheckManagedResource(genericcontrolplaneactuator.ControlPlaneShootChartResourceName): string(gardencorev1beta1.ShootSystemComponentsHealthy), - general.CheckManagedResource(genericcontrolplaneactuator.StorageClassesChartResourceName): string(gardencorev1beta1.ShootSystemComponentsHealthy), + []predicate.Predicate{extensionspredicate.HasPurpose(extensionsv1alpha1.Normal)}, + []healthcheck.ConditionTypeToHealthCheck{ + { + ConditionType: string(gardencorev1beta1.ShootControlPlaneHealthy), + HealthCheck: general.NewSeedDeploymentHealthChecker(openstack.CloudControllerManagerName), + }, + { + ConditionType: string(gardencorev1beta1.ShootSystemComponentsHealthy), + HealthCheck: general.CheckManagedResource(genericcontrolplaneactuator.ControlPlaneShootChartResourceName), + }, + { + ConditionType: string(gardencorev1beta1.ShootSystemComponentsHealthy), + HealthCheck: general.CheckManagedResource(genericcontrolplaneactuator.StorageClassesChartResourceName), + }, }); err != nil { return err } @@ -68,10 +76,19 @@ func RegisterHealthChecks(mgr manager.Manager, opts healthcheck.DefaultAddArgs) mgr, opts, nil, - map[healthcheck.HealthCheck]string{ - general.CheckManagedResource(genericworkeractuator.McmShootResourceName): string(gardencorev1beta1.ShootSystemComponentsHealthy), - general.NewSeedDeploymentHealthChecker(openstack.MachineControllerManagerName): string(gardencorev1beta1.ShootControlPlaneHealthy), - worker.NewSufficientNodesChecker(): string(gardencorev1beta1.ShootEveryNodeReady), + []healthcheck.ConditionTypeToHealthCheck{ + { + ConditionType: string(gardencorev1beta1.ShootSystemComponentsHealthy), + HealthCheck: general.CheckManagedResource(genericworkeractuator.McmShootResourceName), + }, + { + ConditionType: string(gardencorev1beta1.ShootControlPlaneHealthy), + HealthCheck: general.NewSeedDeploymentHealthChecker(openstack.MachineControllerManagerName), + }, + { + ConditionType: string(gardencorev1beta1.ShootEveryNodeReady), + HealthCheck: worker.NewSufficientNodesChecker(), + }, }) } diff --git a/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/actuator.go b/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/actuator.go index 4eb2511e7..3ecd84790 100644 --- a/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/actuator.go +++ b/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/actuator.go @@ -17,6 +17,8 @@ package healthcheck import ( "context" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,8 +41,20 @@ import ( More sophisticated checks should be implemented in the extension itself by using the HealthCheck interface. */ -// RegisterExtension returns the extension object that should be registered with the health check controller -type RegisterExtension = func() runtime.Object +// GetExtensionObjectFunc returns the extension object that should be registered with the health check controller +type GetExtensionObjectFunc = func() runtime.Object + +// PreCheckFunc checks whether the health check shall be performed based on the given object and cluster. +type PreCheckFunc = func(runtime.Object, *extensionscontroller.Cluster) bool + +// ConditionTypeToHealthCheck registers a HealthCheck for the given ConditionType. If the PreCheckFunc is not nil it will +// be executed with the given object before the health check if performed. Otherwise, the health check will always be +// performed. +type ConditionTypeToHealthCheck struct { + ConditionType string + PreCheckFunc PreCheckFunc + HealthCheck HealthCheck +} // HealthCheckActuator acts upon registered resources. type HealthCheckActuator interface { diff --git a/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/controller.go b/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/controller.go index 5afdf7318..4d56c18a0 100644 --- a/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/controller.go +++ b/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/controller.go @@ -25,6 +25,7 @@ import ( "github.com/gardener/gardener/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" @@ -72,7 +73,7 @@ type DefaultAddArgs struct { // The field groupVersionKind stores the GroupVersionKind of the extension resource type RegisteredExtension struct { extension extensionsv1alpha1.Object - register RegisterExtension + getExtensionObjFunc GetExtensionObjectFunc healthConditionType []string groupVersionKind schema.GroupVersionKind } @@ -87,7 +88,7 @@ type RegisteredExtension struct { // custom predicates allow for fine-grained control which resources to watch // healthChecks defines the checks to execute mapped to the healthConditionType its contributing to (e.g checkDeployment in Seed -> ControlPlaneHealthy). // register returns a runtime representation of the extension resource to register it with the controller-runtime -func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, register RegisterExtension, mgr manager.Manager, opts DefaultAddArgs, customPredicates []predicate.Predicate, healthChecks map[HealthCheck]string) error { +func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, getExtensionObjFunc GetExtensionObjectFunc, mgr manager.Manager, opts DefaultAddArgs, customPredicates []predicate.Predicate, healthChecks []ConditionTypeToHealthCheck) error { predicates := DefaultPredicates() predicates = append(predicates, customPredicates...) @@ -98,11 +99,11 @@ func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, reg SyncPeriod: opts.HealthCheckConfig.SyncPeriod, } - if err := args.RegisterExtension(register, getHealthCheckTypes(healthChecks), kind); err != nil { + if err := args.RegisterExtension(getExtensionObjFunc, getHealthCheckTypes(healthChecks), kind); err != nil { return err } - healthCheckActuator := NewActuator(args.Type, args.GetExtensionGroupVersionKind().Kind, healthChecks) + healthCheckActuator := NewActuator(args.Type, args.GetExtensionGroupVersionKind().Kind, getExtensionObjFunc, healthChecks) return Register(mgr, args, healthCheckActuator) } @@ -111,8 +112,8 @@ func DefaultRegistration(extensionType string, kind schema.GroupVersionKind, reg // The controller writes the healthCheckTypes as a condition.type into the extension resource. // To contribute to the Shoot's health, the Gardener checks each extension for a Health Condition Type of SystemComponentsHealthy, EveryNodeReady, ControlPlaneHealthy. // However extensions are free to choose any healthCheckType -func (a *AddArgs) RegisterExtension(register RegisterExtension, conditionTypes []string, kind schema.GroupVersionKind) error { - acc, err := extensions.Accessor(register()) +func (a *AddArgs) RegisterExtension(getExtensionObjFunc GetExtensionObjectFunc, conditionTypes []string, kind schema.GroupVersionKind) error { + acc, err := extensions.Accessor(getExtensionObjFunc()) if err != nil { return err } @@ -121,7 +122,7 @@ func (a *AddArgs) RegisterExtension(register RegisterExtension, conditionTypes [ extension: acc, healthConditionType: conditionTypes, groupVersionKind: kind, - register: register, + getExtensionObjFunc: getExtensionObjFunc, } return nil } @@ -162,19 +163,15 @@ func add(mgr manager.Manager, args AddArgs) error { } predicates := extensionspredicate.AddTypePredicate(args.Predicates, args.Type) - log.Log.Info("Registered health check controller", "kind", args.registeredExtension.groupVersionKind.Kind, "type", args.Type, "health check type", args.registeredExtension.healthConditionType, "sync period", args.SyncPeriod.Duration.String()) + log.Log.Info("Registered health check controller", "Kind", args.registeredExtension.groupVersionKind.Kind, "type", args.Type, "health check type", args.registeredExtension.healthConditionType, "sync period", args.SyncPeriod.Duration.String()) - return ctrl.Watch(&source.Kind{Type: args.registeredExtension.register()}, &handler.EnqueueRequestForObject{}, predicates...) + return ctrl.Watch(&source.Kind{Type: args.registeredExtension.getExtensionObjFunc()}, &handler.EnqueueRequestForObject{}, predicates...) } -func getHealthCheckTypes(healthChecks map[HealthCheck]string) []string { - var types []string - typeMap := make(map[string]struct{}) - for _, check := range healthChecks { - if _, ok := typeMap[check]; !ok { - types = append(types, check) - } - typeMap[check] = struct{}{} +func getHealthCheckTypes(healthChecks []ConditionTypeToHealthCheck) []string { + types := sets.NewString() + for _, healthCheck := range healthChecks { + types.Insert(healthCheck.ConditionType) } - return types + return types.UnsortedList() } diff --git a/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/healtcheck_actuator.go b/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/healtcheck_actuator.go index 9ff0e73f3..dc8dc5cd0 100644 --- a/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/healtcheck_actuator.go +++ b/vendor/github.com/gardener/gardener-extensions/pkg/controller/healthcheck/healtcheck_actuator.go @@ -20,6 +20,7 @@ import ( "strings" "sync" + extensionscontroller "github.com/gardener/gardener-extensions/pkg/controller" "github.com/gardener/gardener-extensions/pkg/util" "github.com/go-logr/logr" @@ -36,19 +37,21 @@ type Actuator struct { logger logr.Logger restConfig *rest.Config + seedClient client.Client + scheme *runtime.Scheme + decoder runtime.Decoder - seedClient client.Client - scheme *runtime.Scheme - decoder runtime.Decoder provider string extensionKind string - healthCheckMappings map[HealthCheck]string + getExtensionObjFunc GetExtensionObjectFunc + healthChecks []ConditionTypeToHealthCheck } // NewActuator creates a new Actuator. -func NewActuator(provider, extensionKind string, healthChecks map[HealthCheck]string) HealthCheckActuator { +func NewActuator(provider, extensionKind string, getExtensionObjFunc GetExtensionObjectFunc, healthChecks []ConditionTypeToHealthCheck) HealthCheckActuator { return &Actuator{ - healthCheckMappings: healthChecks, + healthChecks: healthChecks, + getExtensionObjFunc: getExtensionObjFunc, provider: provider, extensionKind: extensionKind, logger: log.Log.WithName(fmt.Sprintf("%s-%s-healthcheck-actuator", provider, extensionKind)), @@ -93,28 +96,76 @@ type checkResultForConditionType struct { func (a *Actuator) ExecuteHealthCheckFunctions(ctx context.Context, request types.NamespacedName) (*[]Result, error) { _, shootClient, err := util.NewClientForShoot(ctx, a.seedClient, request.Namespace, client.Options{}) if err != nil { - msg := fmt.Sprintf("failed to create shoot client in namespace '%s'", request.Namespace) - a.logger.Error(err, msg) - return nil, fmt.Errorf(msg) + msg := fmt.Errorf("failed to create shoot client in namespace '%s'", request.Namespace) + a.logger.Error(err, msg.Error()) + return nil, msg } - channel := make(chan channelResult) - var wg sync.WaitGroup - wg.Add(len(a.healthCheckMappings)) - for healthCheck, healthConditionType := range a.healthCheckMappings { + + var ( + channel = make(chan channelResult) + wg sync.WaitGroup + ) + + wg.Add(len(a.healthChecks)) + for _, hc := range a.healthChecks { // clone to avoid problems during parallel execution - check := healthCheck.DeepCopy() + check := hc.HealthCheck.DeepCopy() check.InjectSeedClient(a.seedClient) check.InjectShootClient(shootClient) check.SetLoggerSuffix(a.provider, a.extensionKind) - go func(ctx context.Context, request types.NamespacedName, check HealthCheck, healthConditionType string) { + + go func(ctx context.Context, request types.NamespacedName, check HealthCheck, preCheckFunc PreCheckFunc, healthConditionType string) { defer wg.Done() + + if preCheckFunc != nil { + obj := a.getExtensionObjFunc() + if err := a.seedClient.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: request.Name}, obj); err != nil { + channel <- channelResult{ + healthCheckResult: &SingleCheckResult{ + IsHealthy: false, + Detail: err.Error(), + Reason: "ReadExtensionObjectFailed", + }, + error: err, + healthConditionType: healthConditionType, + } + return + } + + cluster, err := extensionscontroller.GetCluster(ctx, a.seedClient, request.Namespace) + if err != nil { + channel <- channelResult{ + healthCheckResult: &SingleCheckResult{ + IsHealthy: false, + Detail: err.Error(), + Reason: "ReadClusterObjectFailed", + }, + error: err, + healthConditionType: healthConditionType, + } + return + } + + if !preCheckFunc(obj, cluster) { + a.logger.Info("Skipping health check for condition type %q as pre check function returned false", healthConditionType) + channel <- channelResult{ + healthCheckResult: &SingleCheckResult{ + IsHealthy: true, + }, + error: nil, + healthConditionType: healthConditionType, + } + return + } + } + healthCheckResult, err := check.Check(ctx, request) channel <- channelResult{ healthCheckResult: healthCheckResult, error: err, healthConditionType: healthConditionType, } - }(ctx, request, check, healthConditionType) + }(ctx, request, check, hc.PreCheckFunc, hc.ConditionType) } // close channel when wait group has 0 counter diff --git a/vendor/github.com/gardener/gardener-extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go b/vendor/github.com/gardener/gardener-extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go index b08eeac8a..feb2ccac9 100644 --- a/vendor/github.com/gardener/gardener-extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go +++ b/vendor/github.com/gardener/gardener-extensions/pkg/controller/worker/genericactuator/actuator_reconcile.go @@ -145,7 +145,7 @@ func (a *genericActuator) Reconcile(ctx context.Context, worker *extensionsv1alp timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() - if err := a.waitUntilMachineDeploymentsAvailable(timeoutCtx, cluster, worker, wantedMachineDeployments); err != nil { + if err := a.waitUntilWantedMachineDeploymentsAvailable(timeoutCtx, cluster, worker, wantedMachineDeployments); err != nil { return gardencorev1beta1helper.DetermineError(fmt.Sprintf("Failed while waiting for all machine deployments to be ready: '%s'", err.Error())) } @@ -164,6 +164,14 @@ func (a *genericActuator) Reconcile(ctx context.Context, worker *extensionsv1alp return errors.Wrapf(err, "failed to cleanup the orphaned machine class secrets") } + // Wait until all unwanted machine deployments are deleted from the system. + timeoutCtx2, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + if err := a.waitUntilUnwantedMachineDeploymentsDeleted(timeoutCtx2, worker, wantedMachineDeployments); err != nil { + return errors.Wrapf(err, "error while waiting for all undesired machine deployments to be deleted") + } + // Delete MachineSets having number of desired and actual replicas equaling 0 if err := a.cleanupMachineSets(ctx, worker.Namespace); err != nil { return errors.Wrapf(err, "failed to cleanup the machine sets") @@ -299,9 +307,9 @@ func (a *genericActuator) deployMachineDeployments(ctx context.Context, cluster return nil } -// waitUntilMachineDeploymentsAvailable waits for a maximum of 30 minutes until all the desired -// were marked as healthy/available by the machine-controller-manager. It polls the status every 5 seconds. -func (a *genericActuator) waitUntilMachineDeploymentsAvailable(ctx context.Context, cluster *controller.Cluster, worker *extensionsv1alpha1.Worker, wantedMachineDeployments worker.MachineDeployments) error { +// waitUntilWantedMachineDeploymentsAvailable waits until all the desired were marked as healthy / +// available by the machine-controller-manager. It polls the status every 5 seconds. +func (a *genericActuator) waitUntilWantedMachineDeploymentsAvailable(ctx context.Context, cluster *controller.Cluster, worker *extensionsv1alpha1.Worker, wantedMachineDeployments worker.MachineDeployments) error { return wait.PollUntil(5*time.Second, func() (bool, error) { var numHealthyDeployments, numUpdated, numDesired, numberOfAwakeMachines int32 @@ -313,30 +321,32 @@ func (a *genericActuator) waitUntilMachineDeploymentsAvailable(ctx context.Conte // Collect the numbers of ready and desired replicas. for _, existingMachineDeployment := range existingMachineDeployments.Items { - // If the shoot get hibernated we want to wait until all machine deployments have been deleted entirely. + // Filter out all machine deployments that are not desired (any more). + if !wantedMachineDeployments.HasDeployment(existingMachineDeployment.Name) { + continue + } + + // If the shoot get hibernated we want to wait until all wanted machine deployments have been deleted + // entirely. if controller.IsHibernated(cluster) { numberOfAwakeMachines += existingMachineDeployment.Status.Replicas continue } - // If the Shoot is not hibernated we want to wait until all machine deployments have been as many ready - // replicas as desired (specified in the .spec.replicas). However, if we see any error in the status of - // the deployment then we return it. + // If the Shoot is not hibernated we want to wait until all wanted machine deployments have as many + // ready replicas as desired (specified in the .spec.replicas). However, if we see any error in the + // status of the deployment then we return it. for _, failedMachine := range existingMachineDeployment.Status.FailedMachines { return false, fmt.Errorf("Machine %s failed: %s", failedMachine.Name, failedMachine.LastOperation.Description) } - // If the Shoot is not hibernated we want to wait until all machine deployments have been as many ready - // replicas as desired (specified in the .spec.replicas). - for _, machineDeployment := range wantedMachineDeployments { - if machineDeployment.Name == existingMachineDeployment.Name { - if workerhealthcheck.CheckMachineDeployment(&existingMachineDeployment) == nil { - numHealthyDeployments++ - } - numDesired += existingMachineDeployment.Spec.Replicas - numUpdated += existingMachineDeployment.Status.UpdatedReplicas - } + // If the Shoot is not hibernated we want to wait until all wanted machine deployments have as many + // ready replicas as desired (specified in the .spec.replicas). + if workerhealthcheck.CheckMachineDeployment(&existingMachineDeployment) == nil { + numHealthyDeployments++ } + numDesired += existingMachineDeployment.Spec.Replicas + numUpdated += existingMachineDeployment.Status.UpdatedReplicas } switch { @@ -356,6 +366,27 @@ func (a *genericActuator) waitUntilMachineDeploymentsAvailable(ctx context.Conte }, ctx.Done()) } +// waitUntilUnwantedMachineDeploymentsDeleted waits until all the undesired are deleted from the +// system. It polls the status every 5 seconds. +func (a *genericActuator) waitUntilUnwantedMachineDeploymentsDeleted(ctx context.Context, worker *extensionsv1alpha1.Worker, wantedMachineDeployments worker.MachineDeployments) error { + return wait.PollUntil(5*time.Second, func() (bool, error) { + existingMachineDeployments := &machinev1alpha1.MachineDeploymentList{} + if err := a.client.List(ctx, existingMachineDeployments, client.InNamespace(worker.Namespace)); err != nil { + return false, err + } + + atLeastOneUnwantedMachineDeploymentExists := false + for _, existingMachineDeployment := range existingMachineDeployments.Items { + if !wantedMachineDeployments.HasDeployment(existingMachineDeployment.Name) { + atLeastOneUnwantedMachineDeploymentExists = true + break + } + } + + return !atLeastOneUnwantedMachineDeploymentExists, nil + }, ctx.Done()) +} + func (a *genericActuator) updateWorkerStatusMachineDeployments(ctx context.Context, worker *extensionsv1alpha1.Worker, machineDeployments worker.MachineDeployments) error { var statusMachineDeployments []extensionsv1alpha1.MachineDeployment diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/field_constants.go b/vendor/github.com/gardener/gardener/pkg/apis/core/field_constants.go index be5a19b5f..59aa51e9f 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/field_constants.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/field_constants.go @@ -17,11 +17,17 @@ package core // Field path constants that are specific to the internal API // representation. const ( - // ShootSeedName is the field selector path for finding - // the Seed cluster of a core.gardener.cloud/{v1alpha1,v1beta1} Shoot. - ShootSeedName = "spec.seedName" + // BackupBucketSeedName is the field selector path for finding + // the Seed cluster of a core.gardener.cloud/v1beta1 BackupBucket. + BackupBucketSeedName = "spec.seedName" + // BackupEntrySeedName is the field selector path for finding + // the Seed cluster of a core.gardener.cloud/v1beta1 BackupEntry. + BackupEntrySeedName = "spec.seedName" // ShootCloudProfileName is the field selector path for finding // the CloudProfile name of a core.gardener.cloud/{v1alpha1,v1beta1} Shoot. ShootCloudProfileName = "spec.cloudProfileName" + // ShootSeedName is the field selector path for finding + // the Seed cluster of a core.gardener.cloud/{v1alpha1,v1beta1} Shoot. + ShootSeedName = "spec.seedName" ) diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/v1alpha1/types_shoot.go b/vendor/github.com/gardener/gardener/pkg/apis/core/v1alpha1/types_shoot.go index 79289a473..133072bf4 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/v1alpha1/types_shoot.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/v1alpha1/types_shoot.go @@ -218,12 +218,11 @@ type DNSProvider struct { Primary *bool `json:"primary,omitempty"` // SecretName is a name of a secret containing credentials for the stated domain and the // provider. When not specified, the Gardener will use the cloud provider credentials referenced - // by the Shoot and try to find respective credentials there. Specifying this field may override + // by the Shoot and try to find respective credentials there (primary provider only). Specifying this field may override // this behavior, i.e. forcing the Gardener to only look into the given secret. // +optional SecretName *string `json:"secretName,omitempty"` - // Type is the DNS provider type for the Shoot. Only relevant if not the default domain is used for - // this shoot. + // Type is the DNS provider type. // +optional Type *string `json:"type,omitempty"` // Zones contains information about which hosted zones shall be included/excluded for this provider. diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/conversions.go b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/conversions.go index 49927e6c4..77015e00a 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/conversions.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/conversions.go @@ -24,7 +24,36 @@ import ( ) func addConversionFuncs(scheme *runtime.Scheme) error { - if err := scheme.AddFieldLabelConversionFunc(SchemeGroupVersion.WithKind("Shoot"), + if err := scheme.AddFieldLabelConversionFunc( + SchemeGroupVersion.WithKind("BackupBucket"), + func(label, value string) (string, string, error) { + switch label { + case "metadata.name", "metadata.namespace", core.BackupBucketSeedName: + return label, value, nil + default: + return "", "", fmt.Errorf("field label not supported: %s", label) + } + }, + ); err != nil { + return err + } + + if err := scheme.AddFieldLabelConversionFunc( + SchemeGroupVersion.WithKind("BackupEntry"), + func(label, value string) (string, string, error) { + switch label { + case "metadata.name", "metadata.namespace", core.BackupEntrySeedName: + return label, value, nil + default: + return "", "", fmt.Errorf("field label not supported: %s", label) + } + }, + ); err != nil { + return err + } + + if err := scheme.AddFieldLabelConversionFunc( + SchemeGroupVersion.WithKind("Shoot"), func(label, value string) (string, string, error) { switch label { case "metadata.name", "metadata.namespace", core.ShootSeedName, core.ShootCloudProfileName: diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go index a0bdf860b..0840b3d2c 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/helper/helper.go @@ -153,13 +153,21 @@ func IsResourceSupported(resources []gardencorev1beta1.ControllerResource, resou // IsControllerInstallationSuccessful returns true if a ControllerInstallation has been marked as "successfully" // installed. func IsControllerInstallationSuccessful(controllerInstallation gardencorev1beta1.ControllerInstallation) bool { + var ( + installed bool + healthy bool + ) + for _, condition := range controllerInstallation.Status.Conditions { if condition.Type == gardencorev1beta1.ControllerInstallationInstalled && condition.Status == gardencorev1beta1.ConditionTrue { - return true + installed = true + } + if condition.Type == gardencorev1beta1.ControllerInstallationHealthy && condition.Status == gardencorev1beta1.ConditionTrue { + healthy = true } } - return false + return installed && healthy } // ComputeOperationType checksthe and determines whether is it is Create operation or reconcile operation diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/types_shoot.go b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/types_shoot.go index 49e3b7e70..4d16e8e33 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/types_shoot.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/v1beta1/types_shoot.go @@ -215,12 +215,11 @@ type DNSProvider struct { Primary *bool `json:"primary,omitempty"` // SecretName is a name of a secret containing credentials for the stated domain and the // provider. When not specified, the Gardener will use the cloud provider credentials referenced - // by the Shoot and try to find respective credentials there. Specifying this field may override + // by the Shoot and try to find respective credentials there (primary provider only). Specifying this field may override // this behavior, i.e. forcing the Gardener to only look into the given secret. // +optional SecretName *string `json:"secretName,omitempty"` - // Type is the DNS provider type for the Shoot. Only relevant if not the default domain is used for - // this shoot. + // Type is the DNS provider type. // +optional Type *string `json:"type,omitempty"` // Zones contains information about which hosted zones shall be included/excluded for this provider. diff --git a/vendor/github.com/gardener/gardener/pkg/apis/core/validation/shoot.go b/vendor/github.com/gardener/gardener/pkg/apis/core/validation/shoot.go index 1b3ef06fc..b5d02749a 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/core/validation/shoot.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/core/validation/shoot.go @@ -815,9 +815,6 @@ func ValidateWorker(worker core.Worker, fldPath *field.Path) field.ErrorList { if worker.Maximum < worker.Minimum { allErrs = append(allErrs, field.Forbidden(fldPath.Child("maximum"), "maximum value must not be less or equal than minimum value")) } - if worker.Maximum != 0 && worker.Minimum == 0 { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("minimum"), "minimum value must be >= 1 if maximum value > 0 (cluster-autoscaler cannot handle min=0)")) - } allErrs = append(allErrs, ValidatePositiveIntOrPercent(worker.MaxSurge, fldPath.Child("maxSurge"))...) allErrs = append(allErrs, ValidatePositiveIntOrPercent(worker.MaxUnavailable, fldPath.Child("maxUnavailable"))...) @@ -1035,10 +1032,13 @@ func validateTaintEffect(effect *corev1.TaintEffect, allowEmpty bool, fldPath *f // ValidateWorkers validates worker objects. func ValidateWorkers(workers []core.Worker, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} + var ( + allErrs = field.ErrorList{} - workerNames := make(map[string]bool) - atLeastOneActivePool := false + workerNames = make(map[string]bool) + atLeastOneActivePool = false + atLeastOnePoolWithCompatibleTaints = len(workers) == 0 + ) for i, worker := range workers { if worker.Minimum != 0 && worker.Maximum != 0 { @@ -1049,12 +1049,33 @@ func ValidateWorkers(workers []core.Worker, fldPath *field.Path) field.ErrorList allErrs = append(allErrs, field.Duplicate(fldPath.Index(i).Child("name"), worker.Name)) } workerNames[worker.Name] = true + + switch { + case atLeastOnePoolWithCompatibleTaints: + case len(worker.Taints) == 0: + atLeastOnePoolWithCompatibleTaints = true + case !atLeastOnePoolWithCompatibleTaints: + onlyPreferNoScheduleEffectTaints := true + for _, taint := range worker.Taints { + if taint.Effect != corev1.TaintEffectPreferNoSchedule { + onlyPreferNoScheduleEffectTaints = false + break + } + } + if onlyPreferNoScheduleEffectTaints { + atLeastOnePoolWithCompatibleTaints = true + } + } } if !atLeastOneActivePool { allErrs = append(allErrs, field.Forbidden(fldPath, "at least one worker pool with min>0 and max> 0 needed")) } + if !atLeastOnePoolWithCompatibleTaints { + allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("at least one worker pool must exist having either no taints or only the %q taint", corev1.TaintEffectPreferNoSchedule))) + } + return allErrs } diff --git a/vendor/github.com/gardener/gardener/pkg/apis/extensions/v1alpha1/types_operatingsystemconfig.go b/vendor/github.com/gardener/gardener/pkg/apis/extensions/v1alpha1/types_operatingsystemconfig.go index 17172121d..6373cbdc4 100644 --- a/vendor/github.com/gardener/gardener/pkg/apis/extensions/v1alpha1/types_operatingsystemconfig.go +++ b/vendor/github.com/gardener/gardener/pkg/apis/extensions/v1alpha1/types_operatingsystemconfig.go @@ -69,7 +69,7 @@ type OperatingSystemConfigList struct { type OperatingSystemConfigSpec struct { // CRI config is a structure contains configurations of the CRI library // +optional - CRIConfig *CRIConfig `json:"criConfig,omitempty"` + CRIConfig *CRIConfig `json:"criConfig,omitempty"` // DefaultSpec is a structure containing common fields used by all extension resources. DefaultSpec `json:",inline"` // Purpose describes how the result of this OperatingSystemConfig is used by Gardener. Either it diff --git a/vendor/github.com/gardener/gardener/pkg/client/kubernetes/chartapplier.go b/vendor/github.com/gardener/gardener/pkg/client/kubernetes/chartapplier.go index 95a6698d9..8bc628827 100644 --- a/vendor/github.com/gardener/gardener/pkg/client/kubernetes/chartapplier.go +++ b/vendor/github.com/gardener/gardener/pkg/client/kubernetes/chartapplier.go @@ -62,7 +62,9 @@ func (c *chartApplier) Apply(ctx context.Context, chartPath, namespace, name str applyOpts := &ApplyOptions{} for _, o := range opts { - o.MutateApplyOptions(applyOpts) + if o != nil { + o.MutateApplyOptions(applyOpts) + } } if len(applyOpts.MergeFuncs) == 0 { @@ -92,7 +94,9 @@ func (c *chartApplier) Delete(ctx context.Context, chartPath, namespace, name st deleteOpts := &DeleteOptions{} for _, o := range opts { - o.MutateDeleteOptions(deleteOpts) + if o != nil { + o.MutateDeleteOptions(deleteOpts) + } } manifestReader, err := c.manifestReader(chartPath, namespace, name, deleteOpts.Values) diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go index f647ba4dc..a8b588514 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/botanist.go @@ -20,15 +20,12 @@ import ( "sort" "strings" - "github.com/gardener/gardener/pkg/apis/core" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" - extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/operation" "github.com/gardener/gardener/pkg/operation/common" + shootpkg "github.com/gardener/gardener/pkg/operation/shoot" - dnsv1alpha1 "github.com/gardener/external-dns-management/pkg/apis/dns/v1alpha1" - "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -64,13 +61,23 @@ func New(o *operation.Operation) (*Botanist, error) { } // RequiredExtensionsExist checks whether all required extensions needed for an shoot operation exist. -func (b *Botanist) RequiredExtensionsExist() error { +func (b *Botanist) RequiredExtensionsExist(ctx context.Context) error { + controllerRegistrationList := &gardencorev1beta1.ControllerRegistrationList{} + if err := b.K8sGardenClient.Client().List(ctx, controllerRegistrationList); err != nil { + return err + } + controllerInstallationList := &gardencorev1beta1.ControllerInstallationList{} - if err := b.K8sGardenClient.Client().List(context.TODO(), controllerInstallationList); err != nil { + if err := b.K8sGardenClient.Client().List(ctx, controllerInstallationList); err != nil { return err } - requiredExtensions := b.computeRequiredExtensions() + var controllerRegistrations []*gardencorev1beta1.ControllerRegistration + for _, controllerRegistration := range controllerRegistrationList.Items { + controllerRegistrations = append(controllerRegistrations, controllerRegistration.DeepCopy()) + } + + requiredExtensions := shootpkg.ComputeRequiredExtensions(b.Shoot.Info, b.Seed.Info, controllerRegistrations, b.Garden.InternalDomain, b.Shoot.ExternalDomain) for _, controllerInstallation := range controllerInstallationList.Items { if controllerInstallation.Spec.SeedRef.Name != b.Seed.Info.Name { @@ -78,18 +85,19 @@ func (b *Botanist) RequiredExtensionsExist() error { } controllerRegistration := &gardencorev1beta1.ControllerRegistration{} - if err := b.K8sGardenClient.Client().Get(context.TODO(), client.ObjectKey{Name: controllerInstallation.Spec.RegistrationRef.Name}, controllerRegistration); err != nil { + if err := b.K8sGardenClient.Client().Get(ctx, client.ObjectKey{Name: controllerInstallation.Spec.RegistrationRef.Name}, controllerRegistration); err != nil { return err } - for extensionKind, extensionTypes := range requiredExtensions { - for extensionType := range extensionTypes { - if helper.IsResourceSupported(controllerRegistration.Spec.Resources, extensionKind, extensionType) && helper.IsControllerInstallationSuccessful(controllerInstallation) { - extensionTypes.Delete(extensionType) - } + for _, kindType := range requiredExtensions.UnsortedList() { + split := strings.Split(kindType, "/") + if len(split) != 2 { + return fmt.Errorf("unexpected required extension: %q", kindType) } - if extensionTypes.Len() == 0 { - delete(requiredExtensions, extensionKind) + extensionKind, extensionType := split[0], split[1] + + if helper.IsResourceSupported(controllerRegistration.Spec.Resources, extensionKind, extensionType) && helper.IsControllerInstallationSuccessful(controllerInstallation) { + requiredExtensions.Delete(kindType) } } } @@ -100,53 +108,3 @@ func (b *Botanist) RequiredExtensionsExist() error { return nil } - -func (b *Botanist) computeRequiredExtensions() map[string]sets.String { - requiredExtensions := make(map[string]sets.String) - - machineImagesSet := sets.NewString() - for _, worker := range b.Shoot.Info.Spec.Provider.Workers { - if worker.Machine.Image != nil { - machineImagesSet.Insert(string(worker.Machine.Image.Name)) - } - } - requiredExtensions[extensionsv1alpha1.OperatingSystemConfigResource] = machineImagesSet - - if !b.Shoot.DisableDNS { - requiredExtensions[dnsv1alpha1.DNSProviderKind] = sets.NewString() - if b.Garden.InternalDomain.Provider != "unmanaged" { - requiredExtensions[dnsv1alpha1.DNSProviderKind].Insert(b.Garden.InternalDomain.Provider) - } - - if b.Shoot.ExternalDomain != nil && b.Shoot.ExternalDomain.Provider != "unmanaged" { - requiredExtensions[dnsv1alpha1.DNSProviderKind].Insert(b.Shoot.ExternalDomain.Provider) - } - - if b.Shoot.Info.Spec.DNS != nil { - for _, provider := range b.Shoot.Info.Spec.DNS.Providers { - if provider.Type != nil && *provider.Type != core.DNSUnmanaged { - requiredExtensions[dnsv1alpha1.DNSProviderKind].Insert(*provider.Type) - } - } - } - } - - for extensionType := range b.Shoot.Extensions { - if requiredExtensions[extensionsv1alpha1.ExtensionResource] == nil { - requiredExtensions[extensionsv1alpha1.ExtensionResource] = sets.NewString() - } - requiredExtensions[extensionsv1alpha1.ExtensionResource].Insert(extensionType) - } - - requiredExtensions[extensionsv1alpha1.InfrastructureResource] = sets.NewString(string(b.Shoot.Info.Spec.Provider.Type)) - requiredExtensions[extensionsv1alpha1.ControlPlaneResource] = sets.NewString(string(b.Shoot.Info.Spec.Provider.Type)) - requiredExtensions[extensionsv1alpha1.NetworkResource] = sets.NewString(b.Shoot.Info.Spec.Networking.Type) - requiredExtensions[extensionsv1alpha1.WorkerResource] = sets.NewString(string(b.Shoot.Info.Spec.Provider.Type)) - - if b.Seed.Info.Spec.Backup != nil { - requiredExtensions[extensionsv1alpha1.BackupBucketResource] = sets.NewString(string(b.Seed.Info.Spec.Backup.Provider)) - requiredExtensions[extensionsv1alpha1.BackupEntryResource] = sets.NewString(string(b.Seed.Info.Spec.Backup.Provider)) - } - - return requiredExtensions -} diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go index ba1e17d86..53fb9165b 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/controlplane.go @@ -136,10 +136,6 @@ func (b *Botanist) DeployClusterAutoscaler(ctx context.Context) error { var workerPools []map[string]interface{} for _, worker := range b.Shoot.MachineDeployments { - // Skip worker pools for which min=0. Auto scaler cannot handle worker pools having a min count of 0. - if worker.Minimum == 0 { - continue - } workerPools = append(workerPools, map[string]interface{}{ "name": worker.Name, @@ -556,40 +552,56 @@ func init() { } // getResourcesForAPIServer returns the cpu and memory requirements for API server based on nodeCount -func getResourcesForAPIServer(nodeCount int32) (string, string, string, string) { +func getResourcesForAPIServer(nodeCount int32, scalingClass string) (string, string, string, string) { var ( - cpuRequest string - memoryRequest string - cpuLimit string - memoryLimit string + validScalingClasses = sets.NewString("small", "medium", "large", "xlarge", "2xlarge") + cpuRequest string + memoryRequest string + cpuLimit string + memoryLimit string ) + if !validScalingClasses.Has(scalingClass) { + switch { + case nodeCount <= 2: + scalingClass = "small" + case nodeCount <= 10: + scalingClass = "medium" + case nodeCount <= 50: + scalingClass = "large" + case nodeCount <= 100: + scalingClass = "xlarge" + default: + scalingClass = "2xlarge" + } + } + switch { - case nodeCount <= 2: + case scalingClass == "small": cpuRequest = "800m" memoryRequest = "800Mi" cpuLimit = "1000m" memoryLimit = "1200Mi" - case nodeCount <= 10: + case scalingClass == "medium": cpuRequest = "1000m" memoryRequest = "1100Mi" cpuLimit = "1200m" memoryLimit = "1900Mi" - case nodeCount <= 50: + case scalingClass == "large": cpuRequest = "1200m" memoryRequest = "1600Mi" cpuLimit = "1500m" memoryLimit = "3900Mi" - case nodeCount <= 100: + case scalingClass == "xlarge": cpuRequest = "2500m" memoryRequest = "5200Mi" cpuLimit = "3000m" memoryLimit = "5900Mi" - default: + case scalingClass == "2xlarge": cpuRequest = "3000m" memoryRequest = "5200Mi" @@ -768,9 +780,9 @@ func (b *Botanist) DeployKubeAPIServer(ctx context.Context) error { var cpuRequest, memoryRequest, cpuLimit, memoryLimit string if hvpaEnabled { - cpuRequest, memoryRequest, cpuLimit, memoryLimit = getResourcesForAPIServer(b.Shoot.GetMinNodeCount()) + cpuRequest, memoryRequest, cpuLimit, memoryLimit = getResourcesForAPIServer(b.Shoot.GetMinNodeCount(), b.Shoot.Info.Annotations[common.ShootAlphaScalingAPIServerClass]) } else { - cpuRequest, memoryRequest, cpuLimit, memoryLimit = getResourcesForAPIServer(b.Shoot.GetMaxNodeCount()) + cpuRequest, memoryRequest, cpuLimit, memoryLimit = getResourcesForAPIServer(b.Shoot.GetMaxNodeCount(), b.Shoot.Info.Annotations[common.ShootAlphaScalingAPIServerClass]) } defaultValues["apiServerResources"] = map[string]interface{}{ "limits": map[string]interface{}{ diff --git a/vendor/github.com/gardener/gardener/pkg/operation/botanist/resource_manager.go b/vendor/github.com/gardener/gardener/pkg/operation/botanist/resource_manager.go index 9764a2646..8524ee779 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/botanist/resource_manager.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/botanist/resource_manager.go @@ -19,11 +19,9 @@ import ( "fmt" "time" - kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/retry" resourcesv1alpha1 "github.com/gardener/gardener-resource-manager/pkg/apis/resources/v1alpha1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -34,35 +32,8 @@ const ( ManagedResourceLabelValueGardener = "gardener" ) -// LabelWellKnownManagedResources labels well-known MRs that were potentially created without origin label. -// TODO: This code can be removed in a future version. -func (b *Botanist) LabelWellKnownManagedResources(ctx context.Context) error { - for _, name := range []string{"shoot-cloud-config-execution", "shoot-core", "shoot-core-namespaces", "addons"} { - obj := &resourcesv1alpha1.ManagedResource{} - if err := b.K8sSeedClient.Client().Get(ctx, kutil.Key(b.Shoot.SeedNamespace, name), obj); err != nil { - if !apierrors.IsNotFound(err) { - return err - } - continue - } - - objCopy := obj.DeepCopy() - kutil.SetMetaDataLabel(objCopy, ManagedResourceLabelKeyOrigin, ManagedResourceLabelValueGardener) - if err := b.K8sSeedClient.Client().Patch(ctx, objCopy, client.MergeFrom(obj)); err != nil { - return err - } - } - - return nil -} - // DeleteManagedResources deletes all managed resources from the Shoot namespace in the Seed. func (b *Botanist) DeleteManagedResources(ctx context.Context) error { - // TODO: This labelling code can be removed in a future version. - if err := b.LabelWellKnownManagedResources(ctx); err != nil { - return err - } - return b.K8sSeedClient.Client().DeleteAllOf( ctx, &resourcesv1alpha1.ManagedResource{}, diff --git a/vendor/github.com/gardener/gardener/pkg/operation/common/types.go b/vendor/github.com/gardener/gardener/pkg/operation/common/types.go index 6d2139000..d01e755b2 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/common/types.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/common/types.go @@ -290,6 +290,13 @@ const ( // SecretRefChecksumAnnotation is the annotation key for checksum of referred secret in resource spec. SecretRefChecksumAnnotation = "checksum/secret.data" + // ShootAlphaScalingAPIServerClass is a constant for an annotation on the shoot stating the initial API server class. + // It influences the size of the initial resource requests/limits. + // Possible values are [small, medium, large, xlarge, 2xlarge]. + // Note that this annotation is alpha and can be removed anytime without further notice. Only use it if you know + // what you do. + ShootAlphaScalingAPIServerClass = "alpha.kube-apiserver.scaling.shoot.gardener.cloud/class" + // ShootExperimentalAddonKyma is a constant for an annotation on the shoot stating that Kyma shall be installed. // TODO: Just a temporary solution. Remove this in a future version once Kyma is moved out again. ShootExperimentalAddonKyma = "experimental.addons.shoot.gardener.cloud/kyma" diff --git a/vendor/github.com/gardener/gardener/pkg/operation/garden/garden.go b/vendor/github.com/gardener/gardener/pkg/operation/garden/garden.go index e167afae6..169fd0fd8 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/garden/garden.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/garden/garden.go @@ -110,7 +110,7 @@ func constructDomainFromSecret(secret *corev1.Secret) (*Domain, error) { // DomainIsDefaultDomain identifies whether the given domain is a default domain. func DomainIsDefaultDomain(domain string, defaultDomains []*Domain) *Domain { for _, defaultDomain := range defaultDomains { - if strings.HasSuffix(domain, defaultDomain.Domain) { + if strings.HasSuffix(domain, "."+defaultDomain.Domain) { return defaultDomain } } diff --git a/vendor/github.com/gardener/gardener/pkg/operation/seed/seed.go b/vendor/github.com/gardener/gardener/pkg/operation/seed/seed.go index 195bc2c14..77f239df5 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/seed/seed.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/seed/seed.go @@ -300,7 +300,7 @@ func deployCertificates(seed *Seed, k8sSeedClient kubernetes.Interface, existing } // BootstrapCluster bootstraps a Seed cluster and deploys various required manifests. -func BootstrapCluster(k8sGardenClient kubernetes.Interface, seed *Seed, config *config.GardenletConfiguration, secrets map[string]*corev1.Secret, imageVector imagevector.ImageVector) error { +func BootstrapCluster(k8sGardenClient kubernetes.Interface, seed *Seed, config *config.GardenletConfiguration, secrets map[string]*corev1.Secret, imageVector imagevector.ImageVector, componentImageVectors imagevector.ComponentImageVectors) error { const chartName = "seed-bootstrap" k8sSeedClient, err := GetSeedClient(context.TODO(), k8sGardenClient.Client(), config.SeedClientConnection.ClientConnectionConfiguration, config.SeedSelector == nil, seed.Info.Name) @@ -575,10 +575,18 @@ func BootstrapCluster(k8sGardenClient kubernetes.Interface, seed *Seed, config * kibanaTLSOverride = wildcardCert.GetName() } + imageVectorOverwrites := map[string]interface{}{} + if componentImageVectors != nil { + for name, data := range componentImageVectors { + imageVectorOverwrites[name] = data + } + } + values := kubernetes.Values(map[string]interface{}{ "cloudProvider": seed.Info.Spec.Provider.Type, "global": map[string]interface{}{ - "images": chart.ImageMapToValues(images), + "images": chart.ImageMapToValues(images), + "imageVectorOverwrites": imageVectorOverwrites, }, "reserveExcessCapacity": seed.reserveExcessCapacity, "replicas": map[string]interface{}{ diff --git a/vendor/github.com/gardener/gardener/pkg/operation/shoot/shoot.go b/vendor/github.com/gardener/gardener/pkg/operation/shoot/shoot.go index f50872745..40533f003 100644 --- a/vendor/github.com/gardener/gardener/pkg/operation/shoot/shoot.go +++ b/vendor/github.com/gardener/gardener/pkg/operation/shoot/shoot.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/gardener/gardener/pkg/apis/core" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" @@ -33,8 +34,10 @@ import ( kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/Masterminds/semver" + dnsv1alpha1 "github.com/gardener/external-dns-management/pkg/apis/dns/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -349,11 +352,9 @@ func MergeExtensions(registrations []gardencorev1beta1.ControllerRegistration, e continue } - var timeout time.Duration + timeout := ExtensionDefaultTimeout if res.ReconcileTimeout != nil { timeout = res.ReconcileTimeout.Duration - } else { - timeout = ExtensionDefaultTimeout } typeToExtension[res.Type] = Extension{ @@ -431,3 +432,61 @@ func ToNetworks(s *gardencorev1beta1.Shoot) (*Networks, error) { APIServer: apiserver, }, nil } + +// ComputeRequiredExtensions compute the extension kind/type combinations that are required for the +// reconciliation flow. +func ComputeRequiredExtensions(shoot *gardencorev1beta1.Shoot, seed *gardencorev1beta1.Seed, controllerRegistrationList []*gardencorev1beta1.ControllerRegistration, internalDomain, externalDomain *garden.Domain) sets.String { + requiredExtensions := sets.NewString() + + if seed.Spec.Backup != nil { + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.BackupBucketResource, seed.Spec.Backup.Provider)) + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.BackupEntryResource, seed.Spec.Backup.Provider)) + } + // Hint: This is actually a temporary work-around to request the control plane extension of the seed provider type as + // it might come with webhooks that are configuring the exposure of shoot control planes. The ControllerRegistration resource + // does not reflect this today. + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.ControlPlaneResource, seed.Spec.Provider.Type)) + + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.ControlPlaneResource, shoot.Spec.Provider.Type)) + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.InfrastructureResource, shoot.Spec.Provider.Type)) + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.NetworkResource, shoot.Spec.Networking.Type)) + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.WorkerResource, shoot.Spec.Provider.Type)) + + for _, extension := range shoot.Spec.Extensions { + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.ExtensionResource, extension.Type)) + } + + for _, pool := range shoot.Spec.Provider.Workers { + if pool.Machine.Image != nil { + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.OperatingSystemConfigResource, pool.Machine.Image.Name)) + } + } + + if !gardencorev1beta1helper.TaintsHave(seed.Spec.Taints, gardencorev1beta1.SeedTaintDisableDNS) { + if shoot.Spec.DNS != nil { + for _, provider := range shoot.Spec.DNS.Providers { + if provider.Type != nil && *provider.Type != core.DNSUnmanaged { + requiredExtensions.Insert(fmt.Sprintf("%s/%s", dnsv1alpha1.DNSProviderKind, *provider.Type)) + } + } + } + + if internalDomain != nil && internalDomain.Provider != core.DNSUnmanaged { + requiredExtensions.Insert(fmt.Sprintf("%s/%s", dnsv1alpha1.DNSProviderKind, internalDomain.Provider)) + } + + if externalDomain != nil && externalDomain.Provider != core.DNSUnmanaged { + requiredExtensions.Insert(fmt.Sprintf("%s/%s", dnsv1alpha1.DNSProviderKind, externalDomain.Provider)) + } + } + + for _, controllerRegistration := range controllerRegistrationList { + for _, resource := range controllerRegistration.Spec.Resources { + if resource.Kind == extensionsv1alpha1.ExtensionResource && resource.GloballyEnabled != nil && *resource.GloballyEnabled { + requiredExtensions.Insert(fmt.Sprintf("%s/%s", extensionsv1alpha1.ExtensionResource, resource.Type)) + } + } + } + + return requiredExtensions +} diff --git a/vendor/github.com/gardener/gardener/pkg/utils/imagevector/imagevector_components.go b/vendor/github.com/gardener/gardener/pkg/utils/imagevector/imagevector_components.go new file mode 100644 index 000000000..729559760 --- /dev/null +++ b/vendor/github.com/gardener/gardener/pkg/utils/imagevector/imagevector_components.go @@ -0,0 +1,57 @@ +// Copyright (c) 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagevector + +import ( + "io" + "os" + + "gopkg.in/yaml.v2" +) + +const ( + // ComponentOverrideEnv is the name of the environment variable for image vector overrides of components deployed + // by Gardener. + ComponentOverrideEnv = "IMAGEVECTOR_OVERWRITE_COMPONENTS" +) + +// ReadComponentOverwrite reads an ComponentImageVector from the given io.Reader. +func ReadComponentOverwrite(r io.Reader) (ComponentImageVectors, error) { + data := struct { + Components []ComponentImageVector `json:"components" yaml:"components"` + }{} + + if err := yaml.NewDecoder(r).Decode(&data); err != nil { + return nil, err + } + + out := make(ComponentImageVectors, len(data.Components)) + for _, component := range data.Components { + out[component.Name] = component.ImageVectorOverwrite + } + + return out, nil +} + +// ReadComponentOverwriteFile reads an ComponentImageVector from the file with the given name. +func ReadComponentOverwriteFile(name string) (ComponentImageVectors, error) { + file, err := os.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + + return ReadComponentOverwrite(file) +} diff --git a/vendor/github.com/gardener/gardener/pkg/utils/imagevector/types.go b/vendor/github.com/gardener/gardener/pkg/utils/imagevector/types.go index 221214763..168fb8ed1 100644 --- a/vendor/github.com/gardener/gardener/pkg/utils/imagevector/types.go +++ b/vendor/github.com/gardener/gardener/pkg/utils/imagevector/types.go @@ -40,6 +40,15 @@ type Image struct { // ImageVector is a list of image sources. type ImageVector []*ImageSource +// ComponentImageVector contains an image vector overwrite for a component deployed by Gardener. +type ComponentImageVector struct { + Name string `json:"name" yaml:"name"` + ImageVectorOverwrite string `json:"imageVectorOverwrite" yaml:"imageVectorOverwrite"` +} + +// ComponentImageVectors maps a component with a given name (key) to the image vector overwrite content (value). +type ComponentImageVectors map[string]string + // FindOptions are options that can be supplied during either `FindImage` or `FindImages`. type FindOptions struct { RuntimeVersion *string diff --git a/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/controllerinstallation.go b/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/controllerinstallation.go index 24cb9905c..4246a5bbe 100644 --- a/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/controllerinstallation.go +++ b/vendor/github.com/gardener/gardener/pkg/utils/kubernetes/controllerinstallation.go @@ -20,9 +20,7 @@ import ( "github.com/gardener/gardener/pkg/logger" "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" ) @@ -108,36 +106,3 @@ func TryUpdateControllerInstallationStatus(g gardencore.Interface, backoff wait. return equality.Semantic.DeepEqual(cur.Status, updated.Status) }) } - -// CreateOrPatchControllerInstallation either creates the object or patches the existing one with the strategic merge patch type. -func CreateOrPatchControllerInstallation(g gardencore.Interface, meta metav1.ObjectMeta, transform func(*gardencorev1beta1.ControllerInstallation) *gardencorev1beta1.ControllerInstallation) (*gardencorev1beta1.ControllerInstallation, error) { - transformed := transform(&gardencorev1beta1.ControllerInstallation{ - TypeMeta: metav1.TypeMeta{ - APIVersion: gardencorev1beta1.SchemeGroupVersion.String(), - Kind: "ControllerInstallation", - }, - ObjectMeta: meta, - }) - - controllerInstallation, err := g.CoreV1beta1().ControllerInstallations().Get(meta.Name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return g.CoreV1beta1().ControllerInstallations().Create(transformed) - } - return nil, err - } - return patchControllerInstallation(g, controllerInstallation, transform(controllerInstallation.DeepCopy())) -} - -func patchControllerInstallation(g gardencore.Interface, oldObj, newObj *gardencorev1beta1.ControllerInstallation) (*gardencorev1beta1.ControllerInstallation, error) { - patch, err := CreateTwoWayMergePatch(oldObj, newObj) - if err != nil { - return nil, err - } - - if IsEmptyPatch(patch) { - return oldObj, nil - } - - return g.CoreV1beta1().ControllerInstallations().Patch(oldObj.Name, types.StrategicMergePatchType, patch) -} diff --git a/vendor/github.com/gardener/gardener/test/framework/k8s_utils.go b/vendor/github.com/gardener/gardener/test/framework/k8s_utils.go index 4918ba889..d56bce6fe 100644 --- a/vendor/github.com/gardener/gardener/test/framework/k8s_utils.go +++ b/vendor/github.com/gardener/gardener/test/framework/k8s_utils.go @@ -18,6 +18,7 @@ import ( "k8s.io/client-go/rest" k8sretry "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" + "time" ) // WaitUntilDaemonSetIsRunning waits until the daemon set with is running @@ -56,6 +57,27 @@ func (f *CommonFramework) WaitUntilStatefulSetIsRunning(ctx context.Context, sta }) } +// WaitUntilDeploymentIsReady waits until the given deployment is ready +func (f *CommonFramework) WaitUntilDeploymentIsReady(ctx context.Context, name string, namespace string, k8sClient kubernetes.Interface) error { + return retry.Until(ctx, defaultPollInterval, func(ctx context.Context) (done bool, err error) { + deployment := &appsv1.Deployment{} + if err := k8sClient.Client().Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, deployment); err != nil { + if apierrors.IsNotFound(err) { + f.Logger.Infof("Waiting for deployment %s/%s to be ready!", namespace, name) + return retry.MinorError(fmt.Errorf("deployment %q in namespace %q does not exist", name, namespace)) + } + return retry.SevereError(err) + } + + err = health.CheckDeployment(deployment) + if err != nil { + f.Logger.Infof("Waiting for deployment %s/%s to be ready!", namespace, name) + return retry.MinorError(fmt.Errorf("deployment %q in namespace %q is not healthy", name, namespace)) + } + return retry.Ok() + }) +} + // WaitUntilDeploymentsWithLabelsIsReady wait until pod with labels is running func (f *CommonFramework) WaitUntilDeploymentsWithLabelsIsReady(ctx context.Context, deploymentLabels labels.Selector, namespace string, k8sClient kubernetes.Interface) error { return retry.Until(ctx, defaultPollInterval, func(ctx context.Context) (done bool, err error) { @@ -92,6 +114,52 @@ func (f *CommonFramework) WaitUntilNamespaceIsDeleted(ctx context.Context, k8sCl }) } +// WaitForNNodesToBeHealthy waits for exactly Nodes to be healthy within a given timeout +func WaitForNNodesToBeHealthy(ctx context.Context, k8sClient kubernetes.Interface, n int, timeout time.Duration) error { + return WaitForNNodesToBeHealthyInWorkerPool(ctx, k8sClient, n, nil, timeout) +} + +// WaitForNNodesToBeHealthyInWorkerPool waits for exactly Nodes in a given worker pool to be healthy within a given timeout +func WaitForNNodesToBeHealthyInWorkerPool(ctx context.Context, k8sClient kubernetes.Interface, n int, workerGroup *string, timeout time.Duration) error { + return retry.UntilTimeout(ctx, defaultPollInterval, timeout, func(ctx context.Context) (done bool, err error) { + nodeList, err := GetAllNodesInWorkerPool(ctx, k8sClient, workerGroup) + if err != nil { + return retry.SevereError(err) + } + + nodeCount := len(nodeList.Items) + if nodeCount != n { + return retry.MinorError(fmt.Errorf("waiting for exactly %d nodes to be ready: only %d nodes registered in the cluster", n, nodeCount)) + } + + for _, node := range nodeList.Items { + if err := health.CheckNode(&node); err != nil { + return retry.MinorError(fmt.Errorf("waiting for exactly %d nodes to be ready: node %q is not healthy: %v", n, node.Name, err)) + } + } + + return retry.Ok() + }) +} + +// GetAllNodes fetches all nodes +func GetAllNodes(ctx context.Context, c kubernetes.Interface) (*corev1.NodeList, error) { + return GetAllNodesInWorkerPool(ctx, c, nil) +} + +// GetAllNodesInWorkerPool fetches all nodes of a specific worker group +func GetAllNodesInWorkerPool(ctx context.Context, c kubernetes.Interface, workerGroup *string) (*corev1.NodeList, error) { + nodeList := &corev1.NodeList{} + + selectorOption := &client.MatchingLabelsSelector{} + if workerGroup != nil && len(*workerGroup) > 0 { + selectorOption.Selector = labels.SelectorFromSet(labels.Set{"worker.gardener.cloud/pool": *workerGroup}) + } + + err := c.Client().List(ctx, nodeList, selectorOption) + return nodeList, err +} + // GetPodsByLabels fetches all pods with the desired set of labels func GetPodsByLabels(ctx context.Context, labelsSelector labels.Selector, c kubernetes.Interface, namespace string) (*corev1.PodList, error) { podList := &corev1.PodList{} diff --git a/vendor/modules.txt b/vendor/modules.txt index 25ac2423a..32e2c3cbf 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -52,7 +52,7 @@ github.com/gardener/etcd-druid/api/v1alpha1 github.com/gardener/external-dns-management/pkg/apis/dns github.com/gardener/external-dns-management/pkg/apis/dns/v1alpha1 github.com/gardener/external-dns-management/pkg/client/dns/clientset/versioned/scheme -# github.com/gardener/gardener v1.1.1-0.20200311075931-7f7e52b986e7 +# github.com/gardener/gardener v1.1.1-0.20200323102039-58593d8be86a github.com/gardener/gardener/pkg/api/extensions github.com/gardener/gardener/pkg/apis/core github.com/gardener/gardener/pkg/apis/core/helper @@ -119,7 +119,7 @@ github.com/gardener/gardener/test/framework/config github.com/gardener/gardener/test/framework/reporter github.com/gardener/gardener/test/integration/framework github.com/gardener/gardener/test/integration/shoots -# github.com/gardener/gardener-extensions v1.4.1-0.20200322183545-5329339c95e9 +# github.com/gardener/gardener-extensions v1.5.0 github.com/gardener/gardener-extensions/hack github.com/gardener/gardener-extensions/hack/.ci github.com/gardener/gardener-extensions/hack/api-reference/template