diff --git a/Makefile b/Makefile index 0fe21fcbc..186c4e87d 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ BUILD_DATE_PATH := github.com/kudobuilder/kudo/pkg/version.buildDate DATE_FMT := "%Y-%m-%dT%H:%M:%SZ" BUILD_DATE := $(shell date -u -d "@$SOURCE_DATE_EPOCH" "+${DATE_FMT}" 2>/dev/null || date -u -r "${SOURCE_DATE_EPOCH}" "+${DATE_FMT}" 2>/dev/null || date -u "+${DATE_FMT}") LDFLAGS := -X ${GIT_VERSION_PATH}=${GIT_VERSION} -X ${GIT_COMMIT_PATH}=${GIT_COMMIT} -X ${BUILD_DATE_PATH}=${BUILD_DATE} +ENABLE_WEBHOOKS ?= false export GO111MODULE=on @@ -67,7 +68,9 @@ manager-clean: .PHONY: run # Run against the configured Kubernetes cluster in ~/.kube/config run: - go run -ldflags "${LDFLAGS}" ./cmd/manager/main.go + # for local development, webhooks are disabled by default + # if you enable them, you have to take care of providing the TLS certs locally + ENABLE_WEBHOOKS=${ENABLE_WEBHOOKS} go run -ldflags "${LDFLAGS}" ./cmd/manager/main.go .PHONY: deploy # Install KUDO into a cluster via kubectl kudo init diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 41a3576dc..28c0be722 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -17,23 +17,33 @@ package main import ( "fmt" + "net/http" + "net/url" "os" + "strings" - "github.com/kudobuilder/kudo/pkg/apis" - "github.com/kudobuilder/kudo/pkg/controller/instance" - "github.com/kudobuilder/kudo/pkg/controller/operator" - "github.com/kudobuilder/kudo/pkg/controller/operatorversion" - "github.com/kudobuilder/kudo/pkg/version" + "github.com/go-logr/logr" apiextenstionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/kudobuilder/kudo/pkg/apis" + "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1" + "github.com/kudobuilder/kudo/pkg/controller/instance" + "github.com/kudobuilder/kudo/pkg/controller/operator" + "github.com/kudobuilder/kudo/pkg/controller/operatorversion" + "github.com/kudobuilder/kudo/pkg/util/kudo" + "github.com/kudobuilder/kudo/pkg/version" ) func main() { - logf.SetLogger(zap.Logger(false)) + logf.SetLogger(zap.New(zap.UseDevMode(false))) log := logf.Log.WithName("entrypoint") // Get version of KUDO @@ -41,7 +51,9 @@ func main() { // create new controller-runtime manager log.Info("setting up manager") - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + CertDir: "/tmp/cert", + }) if err != nil { log.Error(err, "unable to start manager") os.Exit(1) @@ -90,6 +102,14 @@ func main() { os.Exit(1) } + if strings.ToLower(os.Getenv("ENABLE_WEBHOOKS")) == "true" { + err = registerValidatingWebhook(&v1beta1.Instance{}, mgr, log) + if err != nil { + log.Error(err, "unable to create webhook") + os.Exit(1) + } + } + // Start the Cmd log.Info("Starting the Cmd.") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { @@ -97,3 +117,55 @@ func main() { os.Exit(1) } } + +// this is a fork of a code in controller-runtime to be able to pass in our own Validator interface +// see kudo.Validator docs for more details\ +// +// ideally in the future we should switch to just simply doing +// err = ctrl.NewWebhookManagedBy(mgr). +// For(&v1beta1.Instance{}). +// Complete() +// +// that internally calls this method but using their own internal Validator type +func registerValidatingWebhook(obj runtime.Object, mgr manager.Manager, log logr.Logger) error { + gvk, err := apiutil.GVKForObject(obj, mgr.GetScheme()) + if err != nil { + return err + } + validator, ok := obj.(kudo.Validator) + if !ok { + log.Info("skip registering a validating webhook, kudo.Validator interface is not implemented %v", gvk) + + return nil + } + vwh := kudo.ValidatingWebhookFor(validator) + if vwh != nil { + path := generateValidatePath(gvk) + + // Checking if the path is already registered. + // If so, just skip it. + if !isAlreadyHandled(path, mgr) { + log.Info("Registering a validating webhook for %v on path %s", gvk, path) + mgr.GetWebhookServer().Register(path, vwh) + } + } + return nil +} + +func isAlreadyHandled(path string, mgr manager.Manager) bool { + if mgr.GetWebhookServer().WebhookMux == nil { + return false + } + h, p := mgr.GetWebhookServer().WebhookMux.Handler(&http.Request{URL: &url.URL{Path: path}}) + if p == path && h != nil { + return true + } + return false +} + +// if the strategy to generate this path changes we should update init code and webhook setup +// right now this is in sync how controller-runtime generates these paths +func generateValidatePath(gvk schema.GroupVersionKind) string { + return "/validate-" + strings.Replace(gvk.Group, ".", "-", -1) + "-" + + gvk.Version + "-" + strings.ToLower(gvk.Kind) +} diff --git a/go.mod b/go.mod index 71ecb4e20..646abac18 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e + github.com/go-logr/logr v0.1.0 github.com/gogo/protobuf v1.3.1 // indirect github.com/golangci/golangci-lint v1.21.0 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf diff --git a/go.sum b/go.sum index 032bf2f4d..1df2bac5d 100644 --- a/go.sum +++ b/go.sum @@ -284,6 +284,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= diff --git a/pkg/apis/kudo/v1beta1/instance_validator.go b/pkg/apis/kudo/v1beta1/instance_validator.go new file mode 100644 index 000000000..8abf20eca --- /dev/null +++ b/pkg/apis/kudo/v1beta1/instance_validator.go @@ -0,0 +1,36 @@ +package v1beta1 + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/kudobuilder/kudo/pkg/util/kudo" +) + +// this forces the instance type to implement Validator interface, we'll get compile time error if it's not true anymore +var _ kudo.Validator = &Instance{} + +// ValidateCreate implements webhookutil.validator (from controller-runtime) +// we do not enforce any rules upon creation right now +func (i *Instance) ValidateCreate(req admission.Request) error { + return nil +} + +// ValidateUpdate hook called when UPDATE operation is triggered and our admission webhook is triggered +// ValidateUpdate implements webhookutil.validator (from controller-runtime) +func (i *Instance) ValidateUpdate(old runtime.Object, req admission.Request) error { + if i.Status.AggregatedStatus.Status.IsRunning() && req.RequestSubResource != "status" { + // when updating anything else than status, there shouldn't be a running plan + return fmt.Errorf("cannot update Instance %s/%s right now, there's plan %s in progress", i.Namespace, i.Name, i.Status.AggregatedStatus.ActivePlanName) + } + return nil +} + +// ValidateDelete hook called when DELETE operation is triggered and our admission webhook is triggered +// we don't enforce any validation on DELETE right now +// ValidateDelete implements webhookutil.validator (from controller-runtime) +func (i *Instance) ValidateDelete(req admission.Request) error { + return nil +} diff --git a/pkg/kudoctl/cmd/init.go b/pkg/kudoctl/cmd/init.go index 0e2230620..ae9b753e9 100644 --- a/pkg/kudoctl/cmd/init.go +++ b/pkg/kudoctl/cmd/init.go @@ -6,15 +6,15 @@ import ( "io" "strings" + "github.com/spf13/afero" + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" cmdInit "github.com/kudobuilder/kudo/pkg/kudoctl/cmd/init" "github.com/kudobuilder/kudo/pkg/kudoctl/kube" "github.com/kudobuilder/kudo/pkg/kudoctl/kudohome" "github.com/kudobuilder/kudo/pkg/kudoctl/util/repo" - - "github.com/spf13/afero" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" ) const ( @@ -68,6 +68,7 @@ type initCmd struct { crdOnly bool home kudohome.Home client *kube.Client + webhooks string } func newInitCmd(fs afero.Fs, out io.Writer) *cobra.Command { @@ -101,6 +102,7 @@ func newInitCmd(fs afero.Fs, out io.Writer) *cobra.Command { f.BoolVar(&i.crdOnly, "crd-only", false, "Add only KUDO CRDs to your cluster") f.BoolVarP(&i.wait, "wait", "w", false, "Block until KUDO manager is running and ready to receive requests") f.Int64Var(&i.timeout, "wait-timeout", 300, "Wait timeout to be used") + f.StringVar(&i.webhooks, "webhook", "", "List of webhooks to install separated by commas (One of: InstanceValidation)") f.StringVarP(&i.serviceAccount, "service-account", "", "", "Override for the default serviceAccount kudo-manager") return cmd @@ -122,13 +124,16 @@ func (initCmd *initCmd) validate(flags *flag.FlagSet) error { if flags.Changed("wait-timeout") && !initCmd.wait { return errors.New("wait-timeout is only useful when using the flag '--wait'") } + if initCmd.webhooks != "" && initCmd.webhooks != "InstanceValidation" { + return errors.New("webhooks can be only empty or contain a single string 'InstanceValidation'. No other webhooks supported") + } return nil } // run initializes local config and installs KUDO manager to Kubernetes cluster. func (initCmd *initCmd) run() error { - opts := cmdInit.NewOptions(initCmd.version, initCmd.ns, initCmd.serviceAccount) + opts := cmdInit.NewOptions(initCmd.version, initCmd.ns, initCmd.serviceAccount, webhooksArray(initCmd.webhooks)) // if image provided switch to it. if initCmd.image != "" { opts.Image = initCmd.image @@ -153,6 +158,14 @@ func (initCmd *initCmd) run() error { } mans = append(mans, prereq...) + if len(opts.Webhooks) != 0 { // right now there's only 0 or 1 webhook, so this is good enough + prereq, err := cmdInit.WebhookManifests(opts.Namespace) + if err != nil { + return err + } + mans = append(mans, prereq...) + } + deploy, err := cmdInit.ManagerManifests(opts) if err != nil { return err @@ -201,6 +214,13 @@ func (initCmd *initCmd) run() error { return nil } +func webhooksArray(webhooksAsStr string) []string { + if webhooksAsStr == "" { + return []string{} + } + return strings.Split(webhooksAsStr, ",") +} + // YAMLWriter writes yaml to writer. Looked into using https://godoc.org/gopkg.in/yaml.v2#NewEncoder which // looks like a better way, however the omitted JSON elements are encoded which results in a very verbose output. //TODO: Write a Encoder util which uses the "sigs.k8s.io/yaml" library for marshalling diff --git a/pkg/kudoctl/cmd/init/manager.go b/pkg/kudoctl/cmd/init/manager.go index 7ddd51750..72f3230b6 100644 --- a/pkg/kudoctl/cmd/init/manager.go +++ b/pkg/kudoctl/cmd/init/manager.go @@ -2,10 +2,7 @@ package init import ( "fmt" - - "github.com/kudobuilder/kudo/pkg/kudoctl/clog" - "github.com/kudobuilder/kudo/pkg/kudoctl/kube" - "github.com/kudobuilder/kudo/pkg/version" + "strconv" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -19,6 +16,10 @@ import ( appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/yaml" + + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" + "github.com/kudobuilder/kudo/pkg/kudoctl/kube" + "github.com/kudobuilder/kudo/pkg/version" ) //Defines the deployment of the KUDO manager and it's service definition. @@ -41,12 +42,17 @@ type Options struct { TerminationGracePeriodSeconds int64 // Image defines the image to be used Image string - // ServiceAccount defines the optional serviceaccount to be used to install KUDO + // Enable validation + Webhooks []string ServiceAccount string } +func (o Options) webhooksEnabled() bool { + return len(o.Webhooks) != 0 +} + // NewOptions provides an option struct with defaults -func NewOptions(v string, ns string, sa string) Options { +func NewOptions(v string, ns string, sa string, webhooks []string) Options { if v == "" { v = version.Get().GitVersion @@ -64,6 +70,7 @@ func NewOptions(v string, ns string, sa string) Options { Namespace: ns, TerminationGracePeriodSeconds: defaultGracePeriod, Image: fmt.Sprintf("kudobuilder/controller:v%v", v), + Webhooks: webhooks, ServiceAccount: sa, } } @@ -71,22 +78,29 @@ func NewOptions(v string, ns string, sa string) Options { // Install uses Kubernetes client to install KUDO. func Install(client *kube.Client, opts Options, crdOnly bool) error { - clog.Printf("✅ installing crds") if err := installCrds(client.ExtClient); err != nil { return err } + clog.Printf("✅ installed crds") if crdOnly { return nil } - clog.Printf("✅ preparing service accounts and other requirements for controller to run") if err := installPrereqs(client.KubeClient, opts); err != nil { return err } + clog.Printf("✅ installed service accounts and other requirements for controller to run") + + if opts.webhooksEnabled() { + if err := installInstanceValidatingWebhook(client.KubeClient, client.DynamicClient, opts.Namespace); err != nil { + return err + } + clog.Printf("✅ installed webhook") + } - clog.Printf("✅ installing kudo controller") if err := installManager(client.KubeClient, opts); err != nil { return err } + clog.Printf("✅ installed kudo controller") return nil } @@ -192,13 +206,14 @@ func generateDeployment(opts Options) *appsv1.StatefulSet { Env: []v1.EnvVar{ {Name: "POD_NAMESPACE", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, {Name: "SECRET_NAME", Value: "kudo-webhook-server-secret"}, + {Name: "ENABLE_WEBHOOKS", Value: strconv.FormatBool(opts.webhooksEnabled())}, }, Image: image, ImagePullPolicy: "Always", Name: "manager", Ports: []v1.ContainerPort{ // name matters for service - {ContainerPort: 9876, Name: "webhook-server", Protocol: "TCP"}, + {ContainerPort: 443, Name: "webhook-server", Protocol: "TCP"}, }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ diff --git a/pkg/kudoctl/cmd/init/prereqs.go b/pkg/kudoctl/cmd/init/prereqs.go index 9334461fe..a1924f6d6 100644 --- a/pkg/kudoctl/cmd/init/prereqs.go +++ b/pkg/kudoctl/cmd/init/prereqs.go @@ -3,8 +3,6 @@ package init import ( "fmt" - "github.com/kudobuilder/kudo/pkg/kudoctl/clog" - v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -13,6 +11,8 @@ import ( "k8s.io/client-go/kubernetes" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/yaml" + + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" ) //Defines the Prerequisites that need to be in place to run the KUDO manager. This includes setting up the kudo-system namespace and service account @@ -41,23 +41,9 @@ func installPrereqs(client kubernetes.Interface, opts Options) error { return err } } - - if err := installSecret(client.CoreV1(), opts); err != nil { - return err - } return nil } -func installSecret(client corev1.SecretsGetter, opts Options) error { - secret := generateWebHookSecret(opts) - _, err := client.Secrets(opts.Namespace).Create(secret) - if kerrors.IsAlreadyExists(err) { - clog.V(4).Printf("secret %v already exists", secret.Name) - return nil - } - return err -} - func installRoleBindings(client kubernetes.Interface, opts Options) error { rbac := generateRoleBinding(opts) _, err := client.RbacV1().ClusterRoleBindings().Create(rbac) @@ -147,19 +133,6 @@ func generateRoleBinding(opts Options) *rbacv1.ClusterRoleBinding { return sa } -// generateWebHookSecret builds the secret object used for webhooks -func generateWebHookSecret(opts Options) *v1.Secret { - secret := &v1.Secret{ - Data: make(map[string][]byte), - ObjectMeta: metav1.ObjectMeta{ - Name: "kudo-webhook-server-secret", - Namespace: opts.Namespace, - }, - } - - return secret -} - func generateLabels(labels map[string]string) map[string]string { labels["app"] = "kudo-manager" return labels @@ -193,7 +166,6 @@ func Prereq(opts Options) []runtime.Object { prereqs, serviceAccount(opts), roleBinding(opts), - webhookSecret(opts), ) } @@ -207,16 +179,6 @@ func roleBinding(opts Options) *rbacv1.ClusterRoleBinding { return rbac } -// webhookSecret provides the webhook secret manifest for printing -func webhookSecret(opts Options) *v1.Secret { - secret := generateWebHookSecret(opts) - secret.TypeMeta = metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - } - return secret -} - // serviceAccount provides the service account manifest for printing func serviceAccount(opts Options) *v1.ServiceAccount { sa := generateServiceAccount(opts) diff --git a/pkg/kudoctl/cmd/init/webhook.go b/pkg/kudoctl/cmd/init/webhook.go new file mode 100644 index 000000000..96728144f --- /dev/null +++ b/pkg/kudoctl/cmd/init/webhook.go @@ -0,0 +1,161 @@ +package init + +import ( + "fmt" + "strings" + + admissionv1beta1 "k8s.io/api/admissionregistration/v1beta1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + clientv1beta1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" + "sigs.k8s.io/yaml" + + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" + "github.com/kudobuilder/kudo/pkg/util/kudo" +) + +// installInstanceValidatingWebhook applies kubernetes resources related to the webhook to the cluster +func installInstanceValidatingWebhook(client kubernetes.Interface, dynamicClient dynamic.Interface, ns string) error { + if err := installUnstructured(dynamicClient, certificate(ns)); err != nil { + return err + } + if err := installAdmissionWebhook(client.AdmissionregistrationV1beta1(), instanceUpdateValidatingWebhook(ns)); err != nil { + return err + } + return nil +} + +// installUnstructured accepts kubernetes resource as unstructured.Unstructured and installs it into cluster +func installUnstructured(dynamicClient dynamic.Interface, items []unstructured.Unstructured) error { + for _, item := range items { + obj := item + gvk := item.GroupVersionKind() + _, err := dynamicClient.Resource(schema.GroupVersionResource{ + Group: gvk.Group, + Version: gvk.Version, + Resource: fmt.Sprintf("%ss", strings.ToLower(gvk.Kind)), // since we know what kinds are we dealing with here, this is OK + }).Namespace(obj.GetNamespace()).Create(&obj, metav1.CreateOptions{}) + if kerrors.IsAlreadyExists(err) { + clog.V(4).Printf("resource %s already registered", obj.GetName()) + } else if err != nil { + return fmt.Errorf("Error when creating resource %s/%s. %v", obj.GetName(), obj.GetNamespace(), err) + } + } + return nil +} + +func installAdmissionWebhook(client clientv1beta1.ValidatingWebhookConfigurationsGetter, webhook admissionv1beta1.ValidatingWebhookConfiguration) error { + _, err := client.ValidatingWebhookConfigurations().Create(&webhook) + if kerrors.IsAlreadyExists(err) { + clog.V(4).Printf("admission webhook %v already registered", webhook.Name) + return nil + } + return err +} + +func instanceUpdateValidatingWebhook(ns string) admissionv1beta1.ValidatingWebhookConfiguration { + namespacedScope := admissionv1beta1.NamespacedScope + failedType := admissionv1beta1.Fail + equivalentType := admissionv1beta1.Equivalent + noSideEffects := admissionv1beta1.SideEffectClassNone + return admissionv1beta1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kudo-manager-instance-validation-webhook-config", + Annotations: map[string]string{ + "cert-manager.io/inject-ca-from": fmt.Sprintf("%s/kudo-webhook-server-certificate", ns), + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ValidatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1beta1", + }, + Webhooks: []admissionv1beta1.ValidatingWebhook{ + { + Name: "instance-validation.kudo.dev", + Rules: []admissionv1beta1.RuleWithOperations{ + { + Operations: []admissionv1beta1.OperationType{"CREATE", "UPDATE"}, + Rule: admissionv1beta1.Rule{ + APIGroups: []string{"kudo.dev"}, + APIVersions: []string{"v1beta1"}, + Resources: []string{"instances"}, + Scope: &namespacedScope, + }, + }, + }, + FailurePolicy: &failedType, // this means that the request to update instance would fail, if webhook is not up + MatchPolicy: &equivalentType, + SideEffects: &noSideEffects, + ClientConfig: admissionv1beta1.WebhookClientConfig{ + Service: &admissionv1beta1.ServiceReference{ + Name: "kudo-controller-manager-service", + Namespace: ns, + Path: kudo.String("/validate-kudo-dev-v1beta1-instance"), + }, + }, + }, + }, + } +} + +func certificate(ns string) []unstructured.Unstructured { + return []unstructured.Unstructured{{ + Object: map[string]interface{}{ + "apiVersion": "cert-manager.io/v1alpha2", + "kind": "Issuer", + "metadata": map[string]interface{}{ + "name": "selfsigned-issuer", + "namespace": ns, + }, + "spec": map[string]interface{}{ + "selfSigned": map[string]interface{}{}, + }, + }, + }, + { + Object: map[string]interface{}{ + "apiVersion": "cert-manager.io/v1alpha2", + "kind": "Certificate", + "metadata": map[string]interface{}{ + "name": "kudo-webhook-server-certificate", + "namespace": ns, + }, + "spec": map[string]interface{}{ + "commonName": "kudo-controller-manager-service.kudo-system.svc", + "dnsNames": []string{"kudo-controller-manager-service.kudo-system.svc"}, + "issuerRef": map[string]interface{}{ + "kind": "Issuer", + "name": "selfsigned-issuer", + }, + "secretName": "kudo-webhook-server-secret", + }, + }, + }, + } +} + +// WebhookManifests provides webhook related resources as set of strings with serialized yaml +func WebhookManifests(ns string) ([]string, error) { + av := instanceUpdateValidatingWebhook(ns) + cert := certificate(ns) + objs := []runtime.Object{&av} + for _, c := range cert { + obj := c + objs = append(objs, &obj) + } + manifests := make([]string, len(objs)) + for i, obj := range objs { + o, err := yaml.Marshal(obj) + if err != nil { + return []string{}, err + } + manifests[i] = string(o) + } + + return manifests, nil +} diff --git a/pkg/kudoctl/cmd/init_integration_test.go b/pkg/kudoctl/cmd/init_integration_test.go index 6d6a8e6f7..48795e4ba 100644 --- a/pkg/kudoctl/cmd/init_integration_test.go +++ b/pkg/kudoctl/cmd/init_integration_test.go @@ -13,12 +13,6 @@ import ( "strings" "testing" - "github.com/kudobuilder/kudo/pkg/kudoctl/clog" - cmdinit "github.com/kudobuilder/kudo/pkg/kudoctl/cmd/init" - "github.com/kudobuilder/kudo/pkg/kudoctl/kube" - "sigs.k8s.io/yaml" - - testutils "github.com/kudobuilder/kudo/pkg/test/utils" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -28,6 +22,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" + cmdinit "github.com/kudobuilder/kudo/pkg/kudoctl/cmd/init" + "github.com/kudobuilder/kudo/pkg/kudoctl/kube" + testutils "github.com/kudobuilder/kudo/pkg/test/utils" ) var testenv testutils.TestEnvironment @@ -371,7 +371,7 @@ func TestNoErrorOnReInit(t *testing.T) { func deleteInitObjects(client *testutils.RetryClient) { crds := cmdinit.CRDs().AsArray() - prereqs := cmdinit.Prereq(cmdinit.NewOptions("", "", "")) + prereqs := cmdinit.Prereq(cmdinit.NewOptions("", "", "", []string{})) deleteCRDs(crds, client) deletePrereq(prereqs, client) } diff --git a/pkg/kudoctl/cmd/init_test.go b/pkg/kudoctl/cmd/init_test.go index d5a9a21dd..081777c6d 100644 --- a/pkg/kudoctl/cmd/init_test.go +++ b/pkg/kudoctl/cmd/init_test.go @@ -9,12 +9,6 @@ import ( "strings" "testing" - "github.com/kudobuilder/kudo/pkg/kudoctl/clog" - "github.com/kudobuilder/kudo/pkg/kudoctl/env" - "github.com/kudobuilder/kudo/pkg/kudoctl/kube" - "github.com/kudobuilder/kudo/pkg/kudoctl/kudohome" - "github.com/kudobuilder/kudo/pkg/kudoctl/util/repo" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -26,6 +20,11 @@ import ( yamlutil "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes/fake" testcore "k8s.io/client-go/testing" + + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" + "github.com/kudobuilder/kudo/pkg/kudoctl/kube" + "github.com/kudobuilder/kudo/pkg/kudoctl/kudohome" + "github.com/kudobuilder/kudo/pkg/kudoctl/util/repo" ) var updateGolden = flag.Bool("update", false, "update .golden files") @@ -118,51 +117,33 @@ func TestInitCmd_output(t *testing.T) { } } -func TestInitCmd_YAMLWriter(t *testing.T) { - testCases := []struct { - file string - settings *env.Settings - additionalFlags map[string]string +func TestInitCmd_yamlOutput(t *testing.T) { + tests := []struct { + name string + goldenFile string + flags map[string]string }{ - { - file: "deploy-kudo.yaml", - }, - { - file: "deploy-kudo-ns-default.yaml", - settings: env.DefaultSettings, // "default" namespace - }, + {"custom namespace", "deploy-kudo-ns.yaml", map[string]string{"dry-run": "true", "output": "yaml", "namespace": "foo"}}, + {"yaml output", "deploy-kudo.yaml", map[string]string{"dry-run": "true", "output": "yaml"}}, } - for _, tc := range testCases { - Settings = env.Settings{} - if tc.settings != nil { - Settings = *tc.settings - } - + for _, tt := range tests { fs := afero.NewMemMapFs() out := &bytes.Buffer{} initCmd := newInitCmd(fs, out) + Settings.AddFlags(initCmd.Flags()) - flags := map[string]string{ - "dry-run": "true", - "output": "yaml", - } - for name, value := range flags { - if err := initCmd.Flags().Set(name, value); err != nil { + for f, value := range tt.flags { + if err := initCmd.Flags().Set(f, value); err != nil { t.Fatal(err) } } - for name, value := range tc.additionalFlags { - if err := initCmd.Flags().Set(name, value); err != nil { - t.Fatal(err) - } - } if err := initCmd.RunE(initCmd, []string{}); err != nil { t.Fatal(err) } - gp := filepath.Join("testdata", tc.file+".golden") + gp := filepath.Join("testdata", tt.goldenFile+".golden") if *updateGolden { t.Log("update golden file") @@ -177,41 +158,7 @@ func TestInitCmd_YAMLWriter(t *testing.T) { assert.Equal(t, string(g), out.String(), "for golden file: %s", gp) } -} - -func TestInitCmd_CustomNamespace(t *testing.T) { - file := "deploy-kudo-ns.yaml" - fs := afero.NewMemMapFs() - out := &bytes.Buffer{} - initCmd := newInitCmd(fs, out) - Settings.AddFlags(initCmd.Flags()) - flags := map[string]string{"dry-run": "true", "output": "yaml", "namespace": "foo"} - for flag, value := range flags { - if err := initCmd.Flags().Set(flag, value); err != nil { - t.Fatal(err) - } - } - if err := initCmd.RunE(initCmd, []string{}); err != nil { - t.Fatal(err) - } - - gp := filepath.Join("testdata", file+".golden") - - if *updateGolden { - t.Log("update golden file") - if err := ioutil.WriteFile(gp, out.Bytes(), 0644); err != nil { - t.Fatalf("failed to update golden file: %s", err) - } - } - g, err := ioutil.ReadFile(gp) - if err != nil { - t.Fatalf("failed reading .golden: %s", err) - } - - if !bytes.Equal(out.Bytes(), g) { - t.Errorf("json does not match .golden file") - } } func TestInitCmd_ServiceAccount(t *testing.T) { diff --git a/pkg/kudoctl/cmd/testdata/deploy-kudo-ns-default.yaml.golden b/pkg/kudoctl/cmd/testdata/deploy-kudo-ns-default.yaml.golden deleted file mode 100644 index 80af0f18c..000000000 --- a/pkg/kudoctl/cmd/testdata/deploy-kudo-ns-default.yaml.golden +++ /dev/null @@ -1,339 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - app: kudo-manager - controller-tools.k8s.io: "1.0" - name: operators.kudo.dev -spec: - group: kudo.dev - names: - kind: Operator - plural: operators - singular: operator - scope: Namespaced - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - description: - type: string - kubernetesVersion: - type: string - kudoVersion: - type: string - maintainers: - items: - properties: - email: - type: string - name: - type: string - type: object - type: array - url: - type: string - type: object - status: - type: object - type: object - version: v1beta1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - app: kudo-manager - controller-tools.k8s.io: "1.0" - name: operatorversions.kudo.dev -spec: - group: kudo.dev - names: - kind: OperatorVersion - plural: operatorversions - singular: operatorversion - scope: Namespaced - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - connectionString: - description: ConnectionString defines a mustached string that can be - used to connect to an instance of the Operator - type: string - crdVersion: - type: string - operator: - type: object - parameters: - items: - properties: - default: - description: Default is a default value if no parameter is provided - by the instance - type: string - description: - description: Description captures a longer description of how - the variable will be used - type: string - displayName: - description: Human friendly crdVersion of the parameter name - type: string - name: - description: 'Name is the string that should be used in the template - file for example, if `name: COUNT` then using the variable `.Params.COUNT`' - type: string - required: - description: Required specifies if the parameter is required to - be provided by all instances, or whether a default can suffice - type: boolean - trigger: - description: Trigger identifies the plan that gets executed when - this parameter changes in the Instance object. Default is `update` - if present, or `deploy` if not present - type: string - type: object - type: array - plans: - description: Plans specify a map a plans that specify how to - type: object - tasks: - description: List of all tasks available in this OperatorVersions - items: - properties: - kind: - type: string - name: - type: string - spec: - type: object - type: object - type: array - templates: - description: List of go templates YAML files that define the application - operator instance - type: object - upgradableFrom: - description: UpgradableFrom lists all OperatorVersions that can upgrade - to this OperatorVersion - items: - type: object - type: array - type: object - status: - type: object - type: object - version: v1beta1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - app: kudo-manager - controller-tools.k8s.io: "1.0" - name: instances.kudo.dev -spec: - group: kudo.dev - names: - kind: Instance - plural: instances - singular: instance - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - OperatorVersion: - description: Operator specifies a reference to a specific Operator object - type: object - parameters: - type: object - type: object - status: - properties: - aggregatedStatus: - type: object - planStatus: - type: object - type: object - type: object - version: v1beta1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: v1 -kind: Namespace -metadata: - creationTimestamp: null - labels: - app: kudo-manager - controller-tools.k8s.io: "1.0" - name: kudo-system -spec: {} -status: {} - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - creationTimestamp: null - labels: - app: kudo-manager - name: kudo-manager - namespace: kudo-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - creationTimestamp: null - name: kudo-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: kudo-manager - namespace: kudo-system - ---- -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: kudo-webhook-server-secret - namespace: kudo-system - ---- -apiVersion: v1 -kind: Service -metadata: - creationTimestamp: null - labels: - app: kudo-manager - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: kudo-controller-manager-service - namespace: kudo-system -spec: - ports: - - name: kudo - port: 443 - targetPort: webhook-server - selector: - app: kudo-manager - control-plane: controller-manager - controller-tools.k8s.io: "1.0" -status: - loadBalancer: {} - ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - creationTimestamp: null - labels: - app: kudo-manager - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: kudo-controller-manager - namespace: kudo-system -spec: - selector: - matchLabels: - app: kudo-manager - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - serviceName: kudo-controller-manager-service - template: - metadata: - creationTimestamp: null - labels: - app: kudo-manager - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - spec: - containers: - - command: - - /root/manager - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SECRET_NAME - value: kudo-webhook-server-secret - image: kudobuilder/controller:vdev - imagePullPolicy: Always - name: manager - ports: - - containerPort: 9876 - name: webhook-server - protocol: TCP - resources: - requests: - cpu: 100m - memory: 50Mi - volumeMounts: - - mountPath: /tmp/cert - name: cert - readOnly: true - serviceAccountName: kudo-manager - terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: kudo-webhook-server-secret - updateStrategy: {} -status: - replicas: 0 - -... diff --git a/pkg/kudoctl/cmd/testdata/deploy-kudo-ns.yaml.golden b/pkg/kudoctl/cmd/testdata/deploy-kudo-ns.yaml.golden index d3f218ebd..be5e1492c 100644 --- a/pkg/kudoctl/cmd/testdata/deploy-kudo-ns.yaml.golden +++ b/pkg/kudoctl/cmd/testdata/deploy-kudo-ns.yaml.golden @@ -231,14 +231,6 @@ subjects: name: kudo-manager namespace: foo ---- -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: kudo-webhook-server-secret - namespace: foo - --- apiVersion: v1 kind: Service @@ -298,11 +290,13 @@ spec: fieldPath: metadata.namespace - name: SECRET_NAME value: kudo-webhook-server-secret + - name: ENABLE_WEBHOOKS + value: "false" image: kudobuilder/controller:vdev imagePullPolicy: Always name: manager ports: - - containerPort: 9876 + - containerPort: 443 name: webhook-server protocol: TCP resources: diff --git a/pkg/kudoctl/cmd/testdata/deploy-kudo-sa.yaml.golden b/pkg/kudoctl/cmd/testdata/deploy-kudo-sa.yaml.golden index 71d1b31a6..79b66acbd 100644 --- a/pkg/kudoctl/cmd/testdata/deploy-kudo-sa.yaml.golden +++ b/pkg/kudoctl/cmd/testdata/deploy-kudo-sa.yaml.golden @@ -231,14 +231,6 @@ subjects: name: safoo namespace: foo ---- -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: kudo-webhook-server-secret - namespace: foo - --- apiVersion: v1 kind: Service @@ -298,11 +290,13 @@ spec: fieldPath: metadata.namespace - name: SECRET_NAME value: kudo-webhook-server-secret + - name: ENABLE_WEBHOOKS + value: "false" image: kudobuilder/controller:vdev imagePullPolicy: Always name: manager ports: - - containerPort: 9876 + - containerPort: 443 name: webhook-server protocol: TCP resources: diff --git a/pkg/kudoctl/cmd/testdata/deploy-kudo.yaml.golden b/pkg/kudoctl/cmd/testdata/deploy-kudo.yaml.golden index 80af0f18c..455f84507 100644 --- a/pkg/kudoctl/cmd/testdata/deploy-kudo.yaml.golden +++ b/pkg/kudoctl/cmd/testdata/deploy-kudo.yaml.golden @@ -243,14 +243,6 @@ subjects: name: kudo-manager namespace: kudo-system ---- -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - name: kudo-webhook-server-secret - namespace: kudo-system - --- apiVersion: v1 kind: Service @@ -310,11 +302,13 @@ spec: fieldPath: metadata.namespace - name: SECRET_NAME value: kudo-webhook-server-secret + - name: ENABLE_WEBHOOKS + value: "false" image: kudobuilder/controller:vdev imagePullPolicy: Always name: manager ports: - - containerPort: 9876 + - containerPort: 443 name: webhook-server protocol: TCP resources: diff --git a/pkg/kudoctl/kube/config.go b/pkg/kudoctl/kube/config.go index 52722a7d6..db50ee832 100644 --- a/pkg/kudoctl/kube/config.go +++ b/pkg/kudoctl/kube/config.go @@ -3,22 +3,24 @@ package kube import ( "fmt" - "github.com/kudobuilder/kudo/pkg/kudoctl/clog" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + "github.com/kudobuilder/kudo/pkg/kudoctl/clog" ) // Client provides access different K8S clients type Client struct { - KubeClient kubernetes.Interface - ExtClient apiextensionsclient.Interface + KubeClient kubernetes.Interface + ExtClient apiextensionsclient.Interface + DynamicClient dynamic.Interface } -// GetConfig returns a Kubernetes client config for a given kubeconfig. -func GetConfig(kubeconfig string) clientcmd.ClientConfig { +// getConfig returns a Kubernetes client config for a given kubeconfig. +func getConfig(kubeconfig string) clientcmd.ClientConfig { rules := clientcmd.NewDefaultClientConfigLoadingRules() rules.DefaultClientConfig = &clientcmd.DefaultClientConfig @@ -32,7 +34,7 @@ func GetConfig(kubeconfig string) clientcmd.ClientConfig { } func getRestConfig(kubeconfig string) (*rest.Config, error) { - config, err := GetConfig(kubeconfig).ClientConfig() + config, err := getConfig(kubeconfig).ClientConfig() if err != nil { return nil, fmt.Errorf("could not get Kubernetes config using configuration %q: %s", kubeconfig, err) } @@ -54,6 +56,10 @@ func GetKubeClient(kubeconfig string) (*Client, error) { if err != nil { return nil, fmt.Errorf("could not get Kubernetes client: %s", err) } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("could not create Kubernetes dynamic client: %s", err) + } - return &Client{client, extClient}, nil + return &Client{client, extClient, dynamicClient}, nil } diff --git a/pkg/util/kudo/validator.go b/pkg/util/kudo/validator.go new file mode 100644 index 000000000..bbce9fa04 --- /dev/null +++ b/pkg/util/kudo/validator.go @@ -0,0 +1,95 @@ +package kudo + +import ( + "context" + "net/http" + + "k8s.io/api/admission/v1beta1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// Validator defines functions for validating an operation +// this is fork of Validator interface from controller-runtime enriched with the full request passed in +// if this issue get resolved, we can go back to using that interface https://github.com/kubernetes-sigs/controller-runtime/issues/688 +type Validator interface { + runtime.Object + ValidateCreate(req admission.Request) error + ValidateUpdate(old runtime.Object, req admission.Request) error + ValidateDelete(req admission.Request) error +} + +// ValidatingWebhookFor creates a new Webhook for validating the provided type. +func ValidatingWebhookFor(validator Validator) *admission.Webhook { + return &admission.Webhook{ + Handler: &validatingHandler{validator: validator}, + } +} + +type validatingHandler struct { + validator Validator + decoder *admission.Decoder +} + +var _ admission.DecoderInjector = &validatingHandler{} + +// InjectDecoder injects the decoder into a validatingHandler. +func (h *validatingHandler) InjectDecoder(d *admission.Decoder) error { + h.decoder = d + return nil +} + +// Handle handles admission requests. +func (h *validatingHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + if h.validator == nil { + panic("validator should never be nil") + } + + // Get the object in the request + obj := h.validator.DeepCopyObject().(Validator) + if req.Operation == v1beta1.Create { + err := h.decoder.Decode(req, obj) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + err = obj.ValidateCreate(req) + if err != nil { + return admission.Denied(err.Error()) + } + } + + if req.Operation == v1beta1.Update { + oldObj := obj.DeepCopyObject() + + err := h.decoder.DecodeRaw(req.Object, obj) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + err = h.decoder.DecodeRaw(req.OldObject, oldObj) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + err = obj.ValidateUpdate(oldObj, req) + if err != nil { + return admission.Denied(err.Error()) + } + } + + if req.Operation == v1beta1.Delete { + // In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346 + // OldObject contains the object being deleted + err := h.decoder.DecodeRaw(req.OldObject, obj) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + err = obj.ValidateDelete(req) + if err != nil { + return admission.Denied(err.Error()) + } + } + + return admission.Allowed("") +}