diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 95cf4dd9..43a5aaf6 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,5 +1,6 @@ resources: - manager.yaml +- webhook_service.yaml generatorOptions: disableNameSuffixHash: true diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 5f0effca..ae3c8f70 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -12,10 +12,12 @@ metadata: namespace: system labels: control-plane: controller-manager + app: ocs-client-operator spec: selector: matchLabels: control-plane: controller-manager + app: ocs-client-operator replicas: 1 template: metadata: @@ -23,6 +25,7 @@ spec: kubectl.kubernetes.io/default-container: manager labels: control-plane: controller-manager + app: ocs-client-operator spec: securityContext: runAsNonRoot: true @@ -36,6 +39,8 @@ spec: volumeMounts: - name: csi-images mountPath: /opt/config + - mountPath: /etc/tls/private + name: webhook-cert-secret env: - name: OPERATOR_NAMESPACE valueFrom: @@ -74,5 +79,8 @@ spec: - name: csi-images configMap: name: csi-images + - name: webhook-cert-secret + secret: + secretName: webhook-cert-secret serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/manager/webhook_service.yaml b/config/manager/webhook_service.yaml new file mode 100644 index 00000000..dd1f468b --- /dev/null +++ b/config/manager/webhook_service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.openshift.io/serving-cert-secret-name: webhook-cert-secret + name: webhook-server + namespace: system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 7443 + selector: + app: ocs-client-operator + type: ClusterIP diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a4e227c5..d2c036b0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -46,6 +46,15 @@ rules: verbs: - get - list +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - create + - get + - list + - update - watch - apiGroups: - apps @@ -204,6 +213,15 @@ rules: - get - list - watch +- apiGroups: + - operators.coreos.com + resources: + - subscriptions + verbs: + - get + - list + - update + - watch - apiGroups: - security.openshift.io resources: diff --git a/config/rbac/status-reporter-clusterrole.yaml b/config/rbac/status-reporter-clusterrole.yaml index 7de4c30a..ee19145e 100644 --- a/config/rbac/status-reporter-clusterrole.yaml +++ b/config/rbac/status-reporter-clusterrole.yaml @@ -10,6 +10,7 @@ rules: verbs: - get - list + - patch - apiGroups: - "" resources: diff --git a/controllers/clusterversion_controller.go b/controllers/clusterversion_controller.go index dd147ea3..076210aa 100644 --- a/controllers/clusterversion_controller.go +++ b/controllers/clusterversion_controller.go @@ -26,11 +26,14 @@ import ( "github.com/red-hat-storage/ocs-client-operator/pkg/console" "github.com/red-hat-storage/ocs-client-operator/pkg/csi" "github.com/red-hat-storage/ocs-client-operator/pkg/templates" + "github.com/red-hat-storage/ocs-client-operator/pkg/utils" "github.com/go-logr/logr" configv1 "github.com/openshift/api/config/v1" secv1 "github.com/openshift/api/security/v1" + opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + admrv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -57,9 +60,10 @@ const ( operatorConfigMapName = "ocs-client-operator-config" // ClusterVersionName is the name of the ClusterVersion object in the // openshift cluster. - clusterVersionName = "version" - - deployCSIKey = "DEPLOY_CSI" + clusterVersionName = "version" + deployCSIKey = "DEPLOY_CSI" + subscriptionLabelKey = "managed-by" + subscriptionLabelValue = "webhook.subscription.ocs.openshift.io" ) // ClusterVersionReconciler reconciles a ClusterVersion object @@ -106,10 +110,29 @@ func (c *ClusterVersionReconciler) SetupWithManager(mgr ctrl.Manager) error { }, ) + subscriptionPredicates := builder.WithPredicates( + predicate.NewPredicateFuncs( + func(client client.Object) bool { + return client.GetNamespace() == c.OperatorNamespace + }, + ), + predicate.LabelChangedPredicate{}, + ) + + webhookPredicates := builder.WithPredicates( + predicate.NewPredicateFuncs( + func(client client.Object) bool { + return client.GetName() == templates.SubscriptionWebhookName + }, + ), + ) + return ctrl.NewControllerManagedBy(mgr). For(&configv1.ClusterVersion{}, clusterVersionPredicates). Watches(&corev1.ConfigMap{}, enqueueClusterVersionRequest, configMapPredicates). Watches(&extv1.CustomResourceDefinition{}, enqueueClusterVersionRequest, builder.OnlyMetadata). + Watches(&opv1a1.Subscription{}, enqueueClusterVersionRequest, subscriptionPredicates). + Watches(&admrv1.ValidatingWebhookConfiguration{}, enqueueClusterVersionRequest, webhookPredicates). Complete(c) } @@ -127,6 +150,8 @@ func (c *ClusterVersionReconciler) SetupWithManager(mgr ctrl.Manager) error { //+kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=console.openshift.io,resources=consoleplugins,verbs=* +//+kubebuilder:rbac:groups=operators.coreos.com,resources=subscriptions,verbs=get;list;watch;update +//+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;update;create;watch // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile @@ -135,6 +160,16 @@ func (c *ClusterVersionReconciler) Reconcile(ctx context.Context, req ctrl.Reque c.log = log.FromContext(ctx, "ClusterVersion", req) c.log.Info("Reconciling ClusterVersion") + if err := c.reconcileSubscriptionValidatingWebhook(); err != nil { + c.log.Error(err, "unable to register subscription validating webhook") + return ctrl.Result{}, err + } + + if err := labelClientOperatorSubscription(c); err != nil { + c.log.Error(err, "unable to label ocs client operator subscription") + return ctrl.Result{}, err + } + if err := c.ensureConsolePlugin(); err != nil { c.log.Error(err, "unable to deploy client console") return ctrl.Result{}, err @@ -494,3 +529,81 @@ func (c *ClusterVersionReconciler) getDeployCSIConfig() (bool, error) { func (c *ClusterVersionReconciler) get(obj client.Object, opts ...client.GetOption) error { return c.Get(c.ctx, client.ObjectKeyFromObject(obj), obj, opts...) } + +func (c *ClusterVersionReconciler) reconcileSubscriptionValidatingWebhook() error { + whConfig := &admrv1.ValidatingWebhookConfiguration{} + whConfig.Name = templates.SubscriptionWebhookName + + // TODO (lgangava): after change to configmap controller, need to remove webhook during deletion + err := c.createOrUpdate(whConfig, func() error { + + // openshift fills in the ca on finding this annotation + whConfig.Annotations = map[string]string{ + "service.beta.openshift.io/inject-cabundle": "true", + } + + var caBundle []byte + if len(whConfig.Webhooks) == 0 { + whConfig.Webhooks = make([]admrv1.ValidatingWebhook, 1) + } else { + // do not mutate CA bundle that was injected by openshift + caBundle = whConfig.Webhooks[0].ClientConfig.CABundle + } + + // webhook desired state + var wh *admrv1.ValidatingWebhook = &whConfig.Webhooks[0] + templates.SubscriptionValidatingWebhook.DeepCopyInto(wh) + + wh.Name = whConfig.Name + // only send requests received from own namespace + wh.NamespaceSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": c.OperatorNamespace, + }, + } + // only send resources matching the label + wh.ObjectSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + subscriptionLabelKey: subscriptionLabelValue, + }, + } + // preserve the existing (injected) CA bundle if any + wh.ClientConfig.CABundle = caBundle + // send request to the service running in own namespace + wh.ClientConfig.Service.Namespace = c.OperatorNamespace + + return nil + }) + + if err != nil { + return err + } + + c.log.Info("successfully registered validating webhook") + return nil +} + +func labelClientOperatorSubscription(c *ClusterVersionReconciler) error { + subscriptionList := &opv1a1.SubscriptionList{} + err := c.List(c.ctx, subscriptionList, client.InNamespace(c.OperatorNamespace)) + if err != nil { + return fmt.Errorf("failed to list subscriptions") + } + + sub := utils.Find(subscriptionList.Items, func(sub *opv1a1.Subscription) bool { + return sub.Spec.Package == "ocs-client-operator" + }) + + if sub == nil { + return fmt.Errorf("failed to find subscription with ocs-client-operator package") + } + + if utils.AddLabel(sub, subscriptionLabelKey, subscriptionLabelValue) { + if err := c.Update(c.ctx, sub); err != nil { + return err + } + } + + c.log.Info("successfully labelled ocs-client-operator subscription") + return nil +} diff --git a/controllers/storageclient_controller.go b/controllers/storageclient_controller.go index 48e6f3b3..2305333a 100644 --- a/controllers/storageclient_controller.go +++ b/controllers/storageclient_controller.go @@ -38,7 +38,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" @@ -440,16 +439,6 @@ func getStatusReporterName(namespace, name string) string { return fmt.Sprintf("storageclient-%s-status-reporter", hex.EncodeToString(reporterName[:8])) } -// addLabel add a label to a resource metadata -func addLabel(obj metav1.Object, key string, value string) { - labels := obj.GetLabels() - if labels == nil { - labels = map[string]string{} - obj.SetLabels(labels) - } - labels[key] = value -} - func (s *StorageClientReconciler) delete(obj client.Object) error { if err := s.Client.Delete(s.ctx, obj); err != nil && !errors.IsNotFound(err) { return err @@ -462,8 +451,8 @@ func (s *StorageClientReconciler) reconcileClientStatusReporterJob(instance *v1a cronJob := &batchv1.CronJob{} cronJob.Name = getStatusReporterName(instance.Namespace, instance.Name) cronJob.Namespace = s.OperatorNamespace - addLabel(cronJob, storageClientNameLabel, instance.Name) - addLabel(cronJob, storageClientNamespaceLabel, instance.Namespace) + utils.AddLabel(cronJob, storageClientNameLabel, instance.Name) + utils.AddLabel(cronJob, storageClientNamespaceLabel, instance.Namespace) var podDeadLineSeconds int64 = 120 jobDeadLineSeconds := podDeadLineSeconds + 35 var keepJobResourceSeconds int32 = 600 diff --git a/main.go b/main.go index 63157141..0ede98a4 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,9 @@ import ( apiv1alpha1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" "github.com/red-hat-storage/ocs-client-operator/controllers" + "github.com/red-hat-storage/ocs-client-operator/pkg/templates" "github.com/red-hat-storage/ocs-client-operator/pkg/utils" + admwebhook "github.com/red-hat-storage/ocs-client-operator/pkg/webhook" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -33,6 +35,7 @@ import ( secv1 "github.com/openshift/api/security/v1" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + admrv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/fields" @@ -43,10 +46,11 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - apiclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metrics "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" //+kubebuilder:scaffold:imports ) @@ -74,8 +78,10 @@ func main() { var enableLeaderElection bool var probeAddr string var consolePort int + var webhookPort int flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.IntVar(&webhookPort, "webhook-port", 7443, "The port the webhook sever binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -89,6 +95,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) storageclustersSelector := fields.SelectorFromSet(fields.Set{"metadata.name": "storageclusters.ocs.openshift.io"}) + subscriptionwebhookSelector := fields.SelectorFromSet(fields.Set{"metadata.name": templates.SubscriptionWebhookName}) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metrics.Options{BindAddress: metricsAddr}, @@ -101,18 +108,26 @@ func main() { // only cache storagecluster crd Field: storageclustersSelector, }, + &admrv1.ValidatingWebhookConfiguration{}: { + // only cache our validation webhook + Field: subscriptionwebhookSelector, + }, }, }, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: webhookPort, + CertDir: "/etc/tls/private", + }), }) if err != nil { - setupLog.Error(err, "unable to start manager") + setupLog.Error(err, "unable to create manager") os.Exit(1) } // apiclient.New() returns a client without cache. // cache is not initialized before mgr.Start() // we need this because we need to interact with OperatorCondition - apiClient, err := apiclient.New(mgr.GetConfig(), apiclient.Options{ + apiClient, err := client.New(mgr.GetConfig(), client.Options{ Scheme: mgr.GetScheme(), }) if err != nil { @@ -133,6 +148,18 @@ func main() { os.Exit(1) } + setupLog.Info("setting up webhook server") + hookServer := mgr.GetWebhookServer() + + setupLog.Info("registering Subscription Channel validating webhook endpoint") + hookServer.Register("/validate-subscription", &webhook.Admission{ + Handler: &admwebhook.SubscriptionAdmission{ + Client: mgr.GetClient(), + Decoder: admission.NewDecoder(mgr.GetScheme()), + Log: mgr.GetLogger().WithName("webhook.subscription"), + }}, + ) + if err = (&controllers.StorageClientReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/pkg/templates/validatingwebhook.go b/pkg/templates/validatingwebhook.go new file mode 100644 index 00000000..7c22fc5c --- /dev/null +++ b/pkg/templates/validatingwebhook.go @@ -0,0 +1,36 @@ +package templates + +import ( + admrv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/utils/ptr" +) + +const ( + SubscriptionWebhookName = "subscription.ocs.openshift.io" +) + +var SubscriptionValidatingWebhook = admrv1.ValidatingWebhook{ + ClientConfig: admrv1.WebhookClientConfig{ + Service: &admrv1.ServiceReference{ + Name: "ocs-client-operator-webhook-server", + Path: ptr.To("/validate-subscription"), + Port: ptr.To(int32(443)), + }, + }, + Rules: []admrv1.RuleWithOperations{ + { + Rule: admrv1.Rule{ + APIGroups: []string{"operators.coreos.com"}, + APIVersions: []string{"v1alpha1"}, + Resources: []string{"subscriptions"}, + Scope: ptr.To(admrv1.NamespacedScope), + }, + Operations: []admrv1.OperationType{admrv1.Update}, + }, + }, + SideEffects: ptr.To(admrv1.SideEffectClassNone), + TimeoutSeconds: ptr.To(int32(30)), + AdmissionReviewVersions: []string{"v1"}, + // fail the validation if webhook can't be reached + FailurePolicy: ptr.To(admrv1.Fail), +} diff --git a/pkg/utils/k8sutils.go b/pkg/utils/k8sutils.go index 756faf32..5d47b2c2 100644 --- a/pkg/utils/k8sutils.go +++ b/pkg/utils/k8sutils.go @@ -39,6 +39,9 @@ const StorageClientNamespaceEnvVar = "STORAGE_CLIENT_NAMESPACE" const StatusReporterImageEnvVar = "STATUS_REPORTER_IMAGE" +// Value corresponding to annotation key has subscription channel +const DesiredSubscriptionChannelAnnotationKey = "ocs.openshift.io/subscription.channel" + const runCSIDaemonsetOnMaster = "RUN_CSI_DAEMONSET_ON_MASTER" // GetOperatorNamespace returns the namespace where the operator is deployed. @@ -74,6 +77,20 @@ func AddLabels(obj metav1.Object, newLabels map[string]string) { maps.Copy(labels, newLabels) } +// AddAnnotation adds label to a resource metadata, returns true if added else false +func AddLabel(obj metav1.Object, key string, value string) bool { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + obj.SetLabels(labels) + } + if oldValue, exist := labels[key]; !exist || oldValue != value { + labels[key] = value + return true + } + return false +} + // AddAnnotation adds an annotation to a resource metadata, returns true if added else false func AddAnnotation(obj metav1.Object, key string, value string) bool { annotations := obj.GetAnnotations() diff --git a/pkg/webhook/subscription.go b/pkg/webhook/subscription.go new file mode 100644 index 00000000..9d3e0afd --- /dev/null +++ b/pkg/webhook/subscription.go @@ -0,0 +1,61 @@ +package webhook + +import ( + "context" + "fmt" + "net/http" + + "github.com/go-logr/logr" + opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" + "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" + "github.com/red-hat-storage/ocs-client-operator/pkg/utils" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type SubscriptionAdmission struct { + Client client.Client + Decoder *admission.Decoder + Log logr.Logger +} + +func (s *SubscriptionAdmission) Handle(ctx context.Context, req admission.Request) admission.Response { + s.Log.Info("Request received for admission review") + + // review should be for a subscription + subscription := &opv1a1.Subscription{} + if err := s.Decoder.Decode(req, subscription); err != nil { + s.Log.Error(err, "failed to decode admission review as subscription") + return admission.Errored(http.StatusBadRequest, fmt.Errorf("only subscriptions admission reviews are supported: %v", err)) + } + + // review should be for ocs-client-operator subscription + if subscription.Spec.Package != "ocs-client-operator" { + s.Log.Info("subscription package is not 'ocs-client-operator'") + return admission.Errored(http.StatusBadRequest, fmt.Errorf("only ocs-client-operator subscription validation is supported")) + } + + storageClients := &v1alpha1.StorageClientList{} + if err := s.Client.List(ctx, storageClients); err != nil { + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("failed to list storageclients for validating subscription request: %v", err)) + } + + // this is the channel that user want to subscribe to + requestedSubcriptionChannel := subscription.Spec.Channel + for idx := range storageClients.Items { + storageClient := &storageClients.Items[idx] + namespacedName := client.ObjectKeyFromObject(storageClient) + + annotations := storageClient.GetAnnotations() + if annotations != nil { + allowedSubscriptionChannel, exist := annotations[utils.DesiredSubscriptionChannelAnnotationKey] + if exist && allowedSubscriptionChannel != requestedSubcriptionChannel { + s.Log.Info(fmt.Sprintf("Rejecting review as it doesn't conform to storageclient %q desired subscription channel", namespacedName)) + return admission.Denied(fmt.Sprintf("subscription channel %q not allowed as it'll violate storageclient %q requirements", requestedSubcriptionChannel, namespacedName)) + } + } + } + + s.Log.Info("Allowing review request as it doesn't violate any storageclients (if exist) desired subscription channel") + return admission.Allowed(fmt.Sprintf("valid subscription channel: %q", subscription.Spec.Channel)) +} diff --git a/service/status-report/main.go b/service/status-report/main.go index a4225eb5..d447c7ed 100644 --- a/service/status-report/main.go +++ b/service/status-report/main.go @@ -38,8 +38,7 @@ import ( ) const ( - desiredSubscriptionChannelAnnotationKey = "ocs.openshift.io/subscription.channel" - csvPrefix = "ocs-client-operator" + csvPrefix = "ocs-client-operator" ) func main() { @@ -146,7 +145,7 @@ func main() { storageClientCopy := &v1alpha1.StorageClient{} storageClient.DeepCopyInto(storageClientCopy) - if utils.AddAnnotation(storageClient, desiredSubscriptionChannelAnnotationKey, statusResponse.DesiredClientOperatorChannel) { + if utils.AddAnnotation(storageClient, utils.DesiredSubscriptionChannelAnnotationKey, statusResponse.DesiredClientOperatorChannel) { // patch is being used here as to not have any conflicts over storageclient cr changes as this annotation value doesn't depend on storageclient spec if err := cl.Patch(ctx, storageClient, client.MergeFrom(storageClientCopy)); err != nil { klog.Exitf("Failed to annotate storageclient %q: %v", storageClient.Name, err)