From 00962cc74a02962161d1fe0107bac621b6413a95 Mon Sep 17 00:00:00 2001 From: Sergio Arroutbi Date: Tue, 20 Feb 2024 17:10:17 +0100 Subject: [PATCH] WIP: Create CRD for component deployment Signed-off-by: Sergio Arroutbi --- PROJECT | 9 + api/attestation/v1alpha1/deployment_types.go | 99 +++ .../v1alpha1/zz_generated.deepcopy.go | 140 ++++ ...tation-operator.clusterserviceversion.yaml | 122 +++- .../attestation.keylime.dev_deployments.yaml | 117 ++++ cmd/attestation-operator/main.go | 132 +--- .../attestation.keylime.dev_deployments.yaml | 111 ++++ config/crd/kustomization.yaml | 3 + ...ainjection_in_attestation_deployments.yaml | 7 + .../webhook_in_attestation_deployments.yaml | 16 + config/manager/kustomization.yaml | 4 +- .../attestation_deployment_editor_role.yaml | 31 + .../attestation_deployment_viewer_role.yaml | 27 + config/rbac/role.yaml | 86 +++ .../attestation_v1alpha1_deployment.yaml | 12 + config/samples/kustomization.yaml | 1 + go.mod | 20 +- go.sum | 39 +- .../attestation/agent_controller.go | 132 +++- .../attestation/deployment_controller.go | 605 ++++++++++++++++++ .../deployment_controller_cluster_common.go | 82 +++ .../attestation/deployment_controller_info.go | 72 +++ .../attestation/deployment_controller_log.go | 38 ++ .../deployment_controller_parse_cert_files.go | 68 ++ .../deployment_controller_pod_command.go | 93 +++ ...deployment_controller_rand_password_gen.go | 39 ++ internal/controller/attestation/suite_test.go | 80 +++ .../keylime/registrar_synchronizer.go | 123 +++- .../attestation_v1alpha1_deployment.yaml | 8 + 29 files changed, 2159 insertions(+), 157 deletions(-) create mode 100644 api/attestation/v1alpha1/deployment_types.go create mode 100644 bundle/manifests/attestation.keylime.dev_deployments.yaml create mode 100644 config/crd/bases/attestation.keylime.dev_deployments.yaml create mode 100644 config/crd/patches/cainjection_in_attestation_deployments.yaml create mode 100644 config/crd/patches/webhook_in_attestation_deployments.yaml create mode 100644 config/rbac/attestation_deployment_editor_role.yaml create mode 100644 config/rbac/attestation_deployment_viewer_role.yaml create mode 100644 config/samples/attestation_v1alpha1_deployment.yaml create mode 100644 internal/controller/attestation/deployment_controller.go create mode 100644 internal/controller/attestation/deployment_controller_cluster_common.go create mode 100644 internal/controller/attestation/deployment_controller_info.go create mode 100644 internal/controller/attestation/deployment_controller_log.go create mode 100644 internal/controller/attestation/deployment_controller_parse_cert_files.go create mode 100644 internal/controller/attestation/deployment_controller_pod_command.go create mode 100644 internal/controller/attestation/deployment_controller_rand_password_gen.go create mode 100644 internal/controller/attestation/suite_test.go create mode 100644 operator_configs/attestation_v1alpha1_deployment.yaml diff --git a/PROJECT b/PROJECT index 3159c01..bbf2b1e 100644 --- a/PROJECT +++ b/PROJECT @@ -17,4 +17,13 @@ resources: kind: Agent path: github.com/keylime/attestation-operator/api/attestation/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: keylime.dev + group: attestation + kind: Deployment + path: github.com/keylime/attestation-operator/api/attestation/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/attestation/v1alpha1/deployment_types.go b/api/attestation/v1alpha1/deployment_types.go new file mode 100644 index 0000000..6ed7044 --- /dev/null +++ b/api/attestation/v1alpha1/deployment_types.go @@ -0,0 +1,99 @@ +/* +Copyright 2024 Keylime Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// ImageInfo defines global information for initial image in initial deployment task +type ImageInfo struct { + // ImageRepositoryPath is a string to specify where to download images + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Image Repository Path" + ImageRepositoryPath string `json:"imageRepositoryPath,omitempty"` + // ImageTag is a string to specify which image tag to download + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Image Tag" + ImageTag string `json:"imageTag,omitempty"` +} + +// InitGlobal defines global information for initial deployment task +type InitGlobal struct { + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Init Image" + InitialImage ImageInfo `json:"imageInfo,omitempty"` +} + +// NodeInfo defines global information for nodes to be deployed +type NodeInfo struct { + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Tenant Image Information" + TenantImageInfo ImageInfo `json:"tenantImageInfo,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Registrar Image Information" + RegistrarImageInfo ImageInfo `json:"registrarImageInfo,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Verifier Image Information" + VerifierImageInfo ImageInfo `json:"verifierImageInfo,omitempty"` +} + +// DeploymentSpec defines the desired state of Deployment +type DeploymentSpec struct { + // Enabled is a boolean that allows to specify if controller based deployment + // is enabled + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Enabled" + Enabled bool `json:"enabled,omitempty"` + // InitGlobal is a struct to define all information for the initial deployment + // is enabled + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Init Global" + InitGlobal InitGlobal `json:"initGlobal,omitempty"` + // NodeInfo is a struct to define all information for nodes deployed: + // - tenant + // - registrar + // - verifier + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Init Global" + NodeInfo NodeInfo `json:"nodeInfo,omitempty"` +} + +// DeploymentStatus defines the observed state of Deployment +type DeploymentStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// Deployment is the Schema for the deployments API +type Deployment struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DeploymentSpec `json:"spec,omitempty"` + Status DeploymentStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// DeploymentList contains a list of Deployment +type DeploymentList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Deployment `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Deployment{}, &DeploymentList{}) +} diff --git a/api/attestation/v1alpha1/zz_generated.deepcopy.go b/api/attestation/v1alpha1/zz_generated.deepcopy.go index e7db6a5..be889fa 100644 --- a/api/attestation/v1alpha1/zz_generated.deepcopy.go +++ b/api/attestation/v1alpha1/zz_generated.deepcopy.go @@ -131,6 +131,97 @@ func (in *AgentStatus) DeepCopy() *AgentStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Deployment) DeepCopyInto(out *Deployment) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Deployment. +func (in *Deployment) DeepCopy() *Deployment { + if in == nil { + return nil + } + out := new(Deployment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Deployment) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentList) DeepCopyInto(out *DeploymentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Deployment, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentList. +func (in *DeploymentList) DeepCopy() *DeploymentList { + if in == nil { + return nil + } + out := new(DeploymentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DeploymentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { + *out = *in + out.InitGlobal = in.InitGlobal + out.NodeInfo = in.NodeInfo +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec. +func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { + if in == nil { + return nil + } + out := new(DeploymentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatus. +func (in *DeploymentStatus) DeepCopy() *DeploymentStatus { + if in == nil { + return nil + } + out := new(DeploymentStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EKCertificate) DeepCopyInto(out *EKCertificate) { *out = *in @@ -177,6 +268,55 @@ func (in *EKCertificateStore) DeepCopy() *EKCertificateStore { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageInfo) DeepCopyInto(out *ImageInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageInfo. +func (in *ImageInfo) DeepCopy() *ImageInfo { + if in == nil { + return nil + } + out := new(ImageInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InitGlobal) DeepCopyInto(out *InitGlobal) { + *out = *in + out.InitialImage = in.InitialImage +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InitGlobal. +func (in *InitGlobal) DeepCopy() *InitGlobal { + if in == nil { + return nil + } + out := new(InitGlobal) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeInfo) DeepCopyInto(out *NodeInfo) { + *out = *in + out.TenantImageInfo = in.TenantImageInfo + out.RegistrarImageInfo = in.RegistrarImageInfo + out.VerifierImageInfo = in.VerifierImageInfo +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeInfo. +func (in *NodeInfo) DeepCopy() *NodeInfo { + if in == nil { + return nil + } + out := new(NodeInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RegistrarStatus) DeepCopyInto(out *RegistrarStatus) { *out = *in diff --git a/bundle/manifests/attestation-operator.clusterserviceversion.yaml b/bundle/manifests/attestation-operator.clusterserviceversion.yaml index 8c3db78..4fcf0c3 100644 --- a/bundle/manifests/attestation-operator.clusterserviceversion.yaml +++ b/bundle/manifests/attestation-operator.clusterserviceversion.yaml @@ -4,6 +4,21 @@ metadata: annotations: alm-examples: |- [ + { + "apiVersion": "attestation.keylime.dev/v1alpha1", + "kind": "Deployment", + "metadata": { + "labels": { + "app.kubernetes.io/created-by": "attestation-operator", + "app.kubernetes.io/instance": "deployment-sample", + "app.kubernetes.io/managed-by": "kustomize", + "app.kubernetes.io/name": "deployment", + "app.kubernetes.io/part-of": "attestation-operator" + }, + "name": "deployment-sample" + }, + "spec": null + }, { "apiVersion": "attestation.keylime.dev/v1alpha1", "kind": "Agent", @@ -21,7 +36,7 @@ metadata: } ] capabilities: Basic Install - createdAt: "2024-02-06T15:27:18Z" + createdAt: "2024-03-22T23:39:33Z" operators.operatorframework.io/builder: operator-sdk-v1.33.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 name: attestation-operator.v0.1.0 @@ -33,6 +48,9 @@ spec: - kind: Agent name: agents.attestation.keylime.dev version: v1alpha1 + - kind: Deployment + name: deployments.attestation.keylime.dev + version: v1alpha1 description: Operator SDK based Attestation Operator displayName: osdk-attestation-operator icon: @@ -42,6 +60,18 @@ spec: spec: clusterPermissions: - rules: + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - attestation.keylime.dev resources: @@ -68,22 +98,96 @@ spec: - get - patch - update + - apiGroups: + - attestation.keylime.dev + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - attestation.keylime.dev + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - attestation.keylime.dev + resources: + - deployments/status + verbs: + - get + - patch + - update + - apiGroups: + - attestation.keylime.dev + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: - pods verbs: + - create + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create - get - list + - update - watch - apiGroups: - "" resources: - secrets verbs: + - create + - delete - get - list + - patch + - update - watch + - apiGroups: + - security.openshift.io + resourceNames: + - anyuid + resources: + - securitycontextconstraints + verbs: + - use - apiGroups: - authentication.k8s.io resources: @@ -172,26 +276,10 @@ spec: capabilities: drop: - ALL - volumeMounts: - - mountPath: /var/lib/keylime/cv_ca/ - name: certs - readOnly: true - - mountPath: /var/lib/keylime/tpm_cert_store - name: tpm-cert-store - readOnly: true securityContext: runAsNonRoot: true serviceAccountName: attestation-operator-controller-manager terminationGracePeriodSeconds: 10 - volumes: - - name: certs - secret: - defaultMode: 420 - secretName: hhkl-keylime-certs - - name: tpm-cert-store - secret: - defaultMode: 420 - secretName: hhkl-keylime-tpm-cert-store permissions: - rules: - apiGroups: diff --git a/bundle/manifests/attestation.keylime.dev_deployments.yaml b/bundle/manifests/attestation.keylime.dev_deployments.yaml new file mode 100644 index 0000000..2763156 --- /dev/null +++ b/bundle/manifests/attestation.keylime.dev_deployments.yaml @@ -0,0 +1,117 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + creationTimestamp: null + name: deployments.attestation.keylime.dev +spec: + group: attestation.keylime.dev + names: + kind: Deployment + listKind: DeploymentList + plural: deployments + singular: deployment + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Deployment is the Schema for the deployments API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeploymentSpec defines the desired state of Deployment + properties: + enabled: + description: Enabled is a boolean that allows to specify if controller + based deployment is enabled + type: boolean + initGlobal: + description: InitGlobal is a struct to define all information for + the initial deployment is enabled + properties: + imageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + type: object + nodeInfo: + description: 'NodeInfo is a struct to define all information for nodes + deployed: - tenant - registrar - verifier' + properties: + registrarImageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + tenantImageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + verifierImageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + type: object + type: object + status: + description: DeploymentStatus defines the observed state of Deployment + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/cmd/attestation-operator/main.go b/cmd/attestation-operator/main.go index d9c816b..cddc131 100644 --- a/cmd/attestation-operator/main.go +++ b/cmd/attestation-operator/main.go @@ -18,9 +18,7 @@ package main import ( "flag" - "fmt" "os" - "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -36,8 +34,6 @@ import ( attestationv1alpha1 "github.com/keylime/attestation-operator/api/attestation/v1alpha1" attestationcontroller "github.com/keylime/attestation-operator/internal/controller/attestation" keylimecontroller "github.com/keylime/attestation-operator/internal/controller/keylime" - kclient "github.com/keylime/attestation-operator/pkg/client" - khttp "github.com/keylime/attestation-operator/pkg/client/http" "github.com/keylime/attestation-operator/pkg/version" //+kubebuilder:scaffold:imports ) @@ -71,110 +67,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - setupLog.Info("Attestation Operator", "info", version.Get()) - - var registrarURL string - var verifierURL string - if val, ok := os.LookupEnv("KEYLIME_REGISTRAR_URL"); ok { - if val == "" { - err := fmt.Errorf("environment variable KEYLIME_REGISTRAR_URL is empty") - setupLog.Error(err, "unable to determine URL for the keylime registrar") - os.Exit(1) - } - registrarURL = val - } else { - err := fmt.Errorf("environment variable KEYLIME_REGISTRAR_URL not set") - setupLog.Error(err, "unable to determine URL for the keylime registrar") - os.Exit(1) - } - - // TODO: we will actually need to detect and handle all verifiers - // Ideally we would detect scaling up/down at runtime, but let alone dealing with multiple would be good - if val, ok := os.LookupEnv("KEYLIME_VERIFIER_URL"); ok { - if val == "" { - err := fmt.Errorf("environment variable KEYLIME_VERIFIER_URL is empty") - setupLog.Error(err, "unable to determine URL for the keylime registrar") - os.Exit(1) - } - verifierURL = val - } else { - err := fmt.Errorf("environment variable KEYLIME_VERIFIER_URL not set") - setupLog.Error(err, "unable to determine URL for the keylime registrar") - os.Exit(1) - } - - var clientCertFile, clientKeyFile string - if val, ok := os.LookupEnv("KEYLIME_CLIENT_KEY"); ok { - if val == "" { - err := fmt.Errorf("environment variable KEYLIME_CLIENT_KEY is empty") - setupLog.Error(err, "unable to determine client key file for the keylime client") - os.Exit(1) - } - clientKeyFile = val - } else { - err := fmt.Errorf("environment variable KEYLIME_CLIENT_KEY not set") - setupLog.Error(err, "unable to determine client key file for the keylime client") - os.Exit(1) - } - if val, ok := os.LookupEnv("KEYLIME_CLIENT_CERT"); ok { - if val == "" { - err := fmt.Errorf("environment variable KEYLIME_CLIENT_CERT is empty") - setupLog.Error(err, "unable to determine client cert file for the keylime client") - os.Exit(1) - } - clientCertFile = val - } else { - err := fmt.Errorf("environment variable KEYLIME_CLIENT_CERT not set") - setupLog.Error(err, "unable to determine client cert file for the keylime client") - os.Exit(1) - } - - // if this is not set, we will have a baked in default - // compared to the URLs this is optional - var registrarSynchronizerInterval time.Duration - if val, ok := os.LookupEnv("KEYLIME_REGISTRAR_SYNCHRONIZER_INTERVAL_DURATION"); ok { - var err error - registrarSynchronizerInterval, err = time.ParseDuration(val) - if err != nil { - setupLog.Error(fmt.Errorf("environment variable KEYLIME_REGISTRAR_SYNCHRONIZER_INTERVAL_DURATION did not contain a duration string: %w", err), "unable to parse registrar synchronizer interval duration") - os.Exit(1) - } - } - - var agentReconcileInterval time.Duration - if val, ok := os.LookupEnv("KEYLIME_AGENT_RECONCILE_INTERVAL_DURATION"); ok { - var err error - agentReconcileInterval, err = time.ParseDuration(val) - if err != nil { - setupLog.Error(fmt.Errorf("environment variable KEYLIME_AGENT_RECONCILE_INTERVAL_DURATION did not contain a duration string: %w", err), "unable to parse agent reconcile interval duration") - os.Exit(1) - } - } - - tpmCertStore := os.Getenv("KEYLIME_TPM_CERT_STORE") - securePayloadDir := os.Getenv("KEYLIME_SECURE_PAYLOAD_DIR") - podNamespace := os.Getenv("POD_NAMESPACE") - - // we are going to reuse this context in several places - // so we'll create it already here - ctx := ctrl.SetupSignalHandler() - - hc, err := khttp.NewKeylimeHTTPClient( - khttp.ClientCertificate(clientCertFile, clientKeyFile), - // TODO: unfortunately currently our server certs don't have the correct SANs - // and for some reason that's not an issue for any of the other components - // However, golang is very picky when it comes to that, and one cannot disable SAN verification individually - khttp.InsecureSkipVerify(), - ) - if err != nil { - setupLog.Error(err, "unable to create HTTP client") - os.Exit(1) - } - keylimeClient, err := kclient.New(ctx, ctrl.Log.WithName("keylime"), hc, registrarURL, []string{verifierURL}, tpmCertStore) - if err != nil { - setupLog.Error(err, "failed to create keylime client") - os.Exit(1) - } + setupLog.Info("Initializing Attestation Operator", "info", version.Get()) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, @@ -200,13 +93,16 @@ func main() { os.Exit(1) } + if err = (&attestationcontroller.DeploymentReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Deployment") + os.Exit(1) + } if err = (&attestationcontroller.AgentReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Keylime: keylimeClient, - ReconcileInterval: agentReconcileInterval, - SecurePayloadDir: securePayloadDir, - PodNamespace: podNamespace, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Agent") os.Exit(1) @@ -215,10 +111,8 @@ func main() { // this is not a kubebuilder controller, so create it outside of the scaffold if err = (&keylimecontroller.RegistrarSynchronizer{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Keylime: keylimeClient, - LoopInterval: registrarSynchronizerInterval, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "RegistrarSynchronizer") } @@ -233,7 +127,7 @@ func main() { } setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } diff --git a/config/crd/bases/attestation.keylime.dev_deployments.yaml b/config/crd/bases/attestation.keylime.dev_deployments.yaml new file mode 100644 index 0000000..3903f90 --- /dev/null +++ b/config/crd/bases/attestation.keylime.dev_deployments.yaml @@ -0,0 +1,111 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: deployments.attestation.keylime.dev +spec: + group: attestation.keylime.dev + names: + kind: Deployment + listKind: DeploymentList + plural: deployments + singular: deployment + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Deployment is the Schema for the deployments API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeploymentSpec defines the desired state of Deployment + properties: + enabled: + description: Enabled is a boolean that allows to specify if controller + based deployment is enabled + type: boolean + initGlobal: + description: InitGlobal is a struct to define all information for + the initial deployment is enabled + properties: + imageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + type: object + nodeInfo: + description: 'NodeInfo is a struct to define all information for nodes + deployed: - tenant - registrar - verifier' + properties: + registrarImageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + tenantImageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + verifierImageInfo: + description: ImageInfo defines global information for initial + image in initial deployment task + properties: + imageRepositoryPath: + description: ImageRepositoryPath is a string to specify where + to download images + type: string + imageTag: + description: ImageTag is a string to specify which image tag + to download + type: string + type: object + type: object + type: object + status: + description: DeploymentStatus defines the observed state of Deployment + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 5028179..07b5ab4 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,17 +3,20 @@ # It should be run by config/default resources: - bases/attestation.keylime.dev_agents.yaml +- bases/attestation.keylime.dev_deployments.yaml #+kubebuilder:scaffold:crdkustomizeresource patches: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_agents.yaml +#- path: patches/webhook_in_deployments.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_agents.yaml +#- path: patches/cainjection_in_deployments.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_attestation_deployments.yaml b/config/crd/patches/cainjection_in_attestation_deployments.yaml new file mode 100644 index 0000000..2dd3c72 --- /dev/null +++ b/config/crd/patches/cainjection_in_attestation_deployments.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME + name: deployments.attestation.keylime.dev diff --git a/config/crd/patches/webhook_in_attestation_deployments.yaml b/config/crd/patches/webhook_in_attestation_deployments.yaml new file mode 100644 index 0000000..b99b364 --- /dev/null +++ b/config/crd/patches/webhook_in_attestation_deployments.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: deployments.attestation.keylime.dev +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 3ba9ba3..adeed37 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: mheese/attestation-operator - newTag: latest + newName: quay.io/sec-eng-special/attestation-operator + newTag: v0.1.0 diff --git a/config/rbac/attestation_deployment_editor_role.yaml b/config/rbac/attestation_deployment_editor_role.yaml new file mode 100644 index 0000000..12ea79c --- /dev/null +++ b/config/rbac/attestation_deployment_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit deployments. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: deployment-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: attestation-operator + app.kubernetes.io/part-of: attestation-operator + app.kubernetes.io/managed-by: kustomize + name: deployment-editor-role +rules: +- apiGroups: + - attestation.keylime.dev + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - attestation.keylime.dev + resources: + - deployments/status + verbs: + - get diff --git a/config/rbac/attestation_deployment_viewer_role.yaml b/config/rbac/attestation_deployment_viewer_role.yaml new file mode 100644 index 0000000..4c1ce9f --- /dev/null +++ b/config/rbac/attestation_deployment_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view deployments. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: deployment-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: attestation-operator + app.kubernetes.io/part-of: attestation-operator + app.kubernetes.io/managed-by: kustomize + name: deployment-viewer-role +rules: +- apiGroups: + - attestation.keylime.dev + resources: + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - attestation.keylime.dev + resources: + - deployments/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c33326e..4cdcacc 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - attestation.keylime.dev resources: @@ -30,19 +42,93 @@ rules: - get - patch - update +- apiGroups: + - attestation.keylime.dev + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - attestation.keylime.dev + resources: + - deployments/finalizers + verbs: + - update +- apiGroups: + - attestation.keylime.dev + resources: + - deployments/status + verbs: + - get + - patch + - update +- apiGroups: + - attestation.keylime.dev + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: - pods verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create - get - list + - update - watch - apiGroups: - "" resources: - secrets verbs: + - create + - delete - get - list + - patch + - update - watch +- apiGroups: + - security.openshift.io + resourceNames: + - anyuid + resources: + - securitycontextconstraints + verbs: + - use diff --git a/config/samples/attestation_v1alpha1_deployment.yaml b/config/samples/attestation_v1alpha1_deployment.yaml new file mode 100644 index 0000000..aec9d7e --- /dev/null +++ b/config/samples/attestation_v1alpha1_deployment.yaml @@ -0,0 +1,12 @@ +apiVersion: attestation.keylime.dev/v1alpha1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: deployment + app.kubernetes.io/instance: deployment-sample + app.kubernetes.io/part-of: attestation-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: attestation-operator + name: deployment-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index efae1b2..25050ad 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,5 @@ ## Append samples of your project ## resources: - attestation_v1alpha1_agent.yaml +- attestation_v1alpha1_deployment.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/go.mod b/go.mod index becf8dc..8162e23 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,22 @@ go 1.20 require ( github.com/go-logr/logr v1.2.4 github.com/google/go-tpm v0.9.0 + github.com/onsi/ginkgo/v2 v2.9.5 + github.com/onsi/gomega v1.27.7 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.2 k8s.io/client-go v0.27.2 sigs.k8s.io/controller-runtime v0.15.0 ) +require ( + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/moby/spdystream v0.2.0 // indirect + golang.org/x/mod v0.15.0 // indirect + golang.org/x/tools v0.18.0 // indirect +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -48,12 +58,12 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.11.0 - golang.org/x/net v0.10.0 // indirect + golang.org/x/crypto v0.19.0 + golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index d877bc0..7314cf3 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -22,7 +25,6 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -35,6 +37,7 @@ github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -69,17 +72,18 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -94,6 +98,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -101,9 +107,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -127,6 +134,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -148,6 +156,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -156,6 +166,9 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -168,6 +181,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= @@ -181,27 +196,28 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -214,6 +230,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/controller/attestation/agent_controller.go b/internal/controller/attestation/agent_controller.go index 8fcdd67..e5d4d2b 100644 --- a/internal/controller/attestation/agent_controller.go +++ b/internal/controller/attestation/agent_controller.go @@ -42,6 +42,7 @@ import ( attestationv1alpha1 "github.com/keylime/attestation-operator/api/attestation/v1alpha1" kclient "github.com/keylime/attestation-operator/pkg/client" "github.com/keylime/attestation-operator/pkg/client/http" + khttp "github.com/keylime/attestation-operator/pkg/client/http" "github.com/keylime/attestation-operator/pkg/client/registrar" "github.com/keylime/attestation-operator/pkg/client/verifier" ) @@ -81,11 +82,129 @@ type AgentReconciler struct { PodNamespace string } +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +func (r *AgentReconciler) InitializeKeylimeClient() error { + var registrarURL string + var verifierURL string + setupLog := ctrl.Log.WithName("setup") + if val, ok := os.LookupEnv("KEYLIME_REGISTRAR_URL"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_REGISTRAR_URL is empty") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + registrarURL = val + } else { + err := fmt.Errorf("environment variable KEYLIME_REGISTRAR_URL not set") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + + // TODO: we will actually need to detect and handle all verifiers + // Ideally we would detect scaling up/down at runtime, but let alone dealing with multiple would be good + if val, ok := os.LookupEnv("KEYLIME_VERIFIER_URL"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_VERIFIER_URL is empty") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + verifierURL = val + } else { + err := fmt.Errorf("environment variable KEYLIME_VERIFIER_URL not set") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + + var clientCertFile, clientKeyFile string + if val, ok := os.LookupEnv("KEYLIME_CLIENT_KEY"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_KEY is empty") + setupLog.Error(err, "unable to determine client key file for the keylime client") + os.Exit(1) + } + clientKeyFile = val + } else { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_KEY not set") + setupLog.Error(err, "unable to determine client key file for the keylime client") + os.Exit(1) + } + if val, ok := os.LookupEnv("KEYLIME_CLIENT_CERT"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_CERT is empty") + setupLog.Error(err, "unable to determine client cert file for the keylime client") + os.Exit(1) + } + clientCertFile = val + } else { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_CERT not set") + setupLog.Error(err, "unable to determine client cert file for the keylime client") + os.Exit(1) + } + + var agentReconcileInterval time.Duration + if val, ok := os.LookupEnv("KEYLIME_AGENT_RECONCILE_INTERVAL_DURATION"); ok { + var err error + agentReconcileInterval, err = time.ParseDuration(val) + if err != nil { + setupLog.Error(fmt.Errorf("environment variable KEYLIME_AGENT_RECONCILE_INTERVAL_DURATION did not contain a duration string: %w", err), "unable to parse agent reconcile interval duration") + os.Exit(1) + } + r.ReconcileInterval = agentReconcileInterval + } + + tpmCertStore := os.Getenv("KEYLIME_TPM_CERT_STORE") + r.SecurePayloadDir = os.Getenv("KEYLIME_SECURE_PAYLOAD_DIR") + r.PodNamespace = os.Getenv("POD_NAMESPACE") + + // we are going to reuse this context in several places + // so we'll create it already here + ctx := ctrl.SetupSignalHandler() + + // set this to info + setupLog.Info("AgentController: Certification Files Information", "CertFile", clientCertFile, "KeyFile", clientKeyFile) + + // if files don't exist, it is probable initialization of the secret is pending + // return with no error + if !fileExists(clientCertFile) { + setupLog.Info("Certificate Client file doesn't exist", "CertFile", clientCertFile) + return nil + } + if !fileExists(clientKeyFile) { + setupLog.Info("Certificate Key file doesn't exist", "KeyFile", clientKeyFile) + return nil + } + + hc, err := khttp.NewKeylimeHTTPClient( + khttp.ClientCertificate(clientCertFile, clientKeyFile), + // TODO: unfortunately currently our server certs don't have the correct SANs + // and for some reason that's not an issue for any of the other components + // However, golang is very picky when it comes to that, and one cannot disable SAN verification individually + khttp.InsecureSkipVerify(), + ) + if err != nil { + setupLog.Error(err, "unable to create Keylime HTTP client") + os.Exit(1) + } + keylimeClient, err := kclient.New(ctx, ctrl.Log.WithName("keylime"), hc, registrarURL, []string{verifierURL}, tpmCertStore) + if err != nil { + setupLog.Error(err, "failed to create keylime client") + os.Exit(1) + } + r.Keylime = keylimeClient + return nil +} + //+kubebuilder:rbac:groups=attestation.keylime.dev,resources=agents,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=attestation.keylime.dev,resources=agents/status,verbs=get;update;patch //+kubebuilder:rbac:groups=attestation.keylime.dev,resources=agents/finalizers,verbs=update //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch -//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -104,6 +223,17 @@ func (r *AgentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ct return ctrl.Result{}, client.IgnoreNotFound(err) } + // In case keylime client not created ... it might be waiting for secret creation + r.InitializeKeylimeClient() + if nil == r.Keylime { + l.Info("Waiting for keylime client to be initialized ... " + + "will attempt again in " + string(r.ReconcileInterval)) + return ctrl.Result{ + Requeue: true, + RequeueAfter: r.ReconcileInterval, + }, nil + } + // TODO: handle agent deletes: // - needs to delete it from the verifier // - needs to delete it from the registrar diff --git a/internal/controller/attestation/deployment_controller.go b/internal/controller/attestation/deployment_controller.go new file mode 100644 index 0000000..8bf384e --- /dev/null +++ b/internal/controller/attestation/deployment_controller.go @@ -0,0 +1,605 @@ +/* +Copyright 2024 Keylime Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "context" + "errors" + attestationv1alpha1 "github.com/keylime/attestation-operator/api/attestation/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "time" +) + +// Time in seconds to wait for init job +// TODO: set this configurable in CRD +const DEFAULT_WAIT_INIT_JOB = 30 + +// DeploymentReconciler reconciles a Deployment object +type DeploymentReconciler struct { + client.Client + Scheme *runtime.Scheme + InitJobCounter uint16 + InitJob *batchv1.Job + ContainerName string +} + +//+kubebuilder:rbac:groups=attestation.keylime.dev,resources=deployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=attestation.keylime.dev,resources=deployments/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=attestation.keylime.dev,resources=deployments/finalizers,verbs=update +//+kubebuilder:rbac:groups=attestation.keylime.dev,resources=jobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update +//+kubebuilder:rbac:groups=core,resources=pods/exec,verbs=get;list;watch;create;update +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,resourceNames=anyuid,verbs=use + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Deployment object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.15.0/pkg/reconcile +func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + l := log.FromContext(ctx) + SetLogInstance(l) + + // In case init job counter is enabled, reconcile after one second and decrease counter + if r.InitJobCounter > 0 { + l.Info("Waiting for initial job", "Pending count", r.InitJobCounter) + r.InitJobCounter -= 1 + return ctrl.Result{RequeueAfter: time.Duration(1) * time.Second}, nil + } + + deployment := &attestationv1alpha1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.NamespacedName.Namespace, + Name: req.NamespacedName.Name, + }, + } + err := r.Get(ctx, req.NamespacedName, deployment) + if err != nil { + l.Error(err, "Attestation Deployment resource not found") + return ctrl.Result{}, err + } + + err = r.deployComponents(deployment, req) + if err != nil { + l.Error(err, "Unable to deploy components") + } + return ctrl.Result{}, err +} + +// SetupWithManager sets up the controller with the Manager. +func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&attestationv1alpha1.Deployment{}). + Owns(&corev1.Secret{}). + Owns(&batchv1.Job{}). + Complete(r) +} + +// deployComponents will parse Attestation Deployment spec and will launch +// specified nodes in CRD +func (r *DeploymentReconciler) deployComponents(deployment *attestationv1alpha1.Deployment, req ctrl.Request) error { + l := GetLogInstance() + //TODO: perform deployment of the different components here + if deployment.Spec.Enabled == true { + l.Info("Deployment is enabled") + err := r.deployInitTasks(deployment, req) + if err != nil { + l.Error(err, "Unable to deploy initialization tasks") + return err + } + err = r.deployKeylimeNodes(deployment, req) + if err != nil { + l.Error(err, "Unable to deploy initialization tasks") + return err + } + } else { + l.Info("Deployment is not enabled") + } + return nil +} + +// createCAPasswordSecret +func (r *DeploymentReconciler) createCAPasswordSecret(deployment *attestationv1alpha1.Deployment, req ctrl.Request) error { + nameSpace := getCAPasswordSecretNamespace(req) + name := getCAPasswordSecretName(req) + search := types.NamespacedName{ + Namespace: nameSpace, + Name: name, + } + // CA Password Secret + secretCaPassword := &corev1.Secret{} + err := r.Get(context.Background(), search, secretCaPassword) + if err == nil { + GetLogInstance().Info("CA Password Secret is already there", "Namespace", secretCaPassword.Namespace, + "Name", secretCaPassword.Name, "Amount of Secrets", len(secretCaPassword.StringData)) + return nil + } + GetLogInstance().Info("Creating CA Password Secret", "Namespace", secretCaPassword.Namespace, "Name", secretCaPassword.Name) + secretCaPassword = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nameSpace, + Name: name, //TODO: set as configurable + }, + } + err = ctrl.SetControllerReference(deployment, secretCaPassword, r.Scheme) + if err != nil { + GetLogInstance().Error(err, "Unable to set CA password secret controller reference") + return err + } + err = r.Create(context.Background(), secretCaPassword) + if err != nil { + GetLogInstance().Error(err, "Unable to create CA password secret, maybe it already exists?") + } + secretCaPassword.StringData = make(map[string]string) + secretCaPassword.StringData["KEYLIME_CA_PASSWORD"] = NewRandPasswordGen().num(32) // TODO: set key and length as configurable + err = r.Update(context.Background(), secretCaPassword) + if err != nil { + GetLogInstance().Error(err, "Unable to update CA password secret") + return err + } + return nil +} + +// createCertsSecret will create certs secret if it does not exist +func (r *DeploymentReconciler) createCertsSecret(deployment *attestationv1alpha1.Deployment, req ctrl.Request) (*corev1.Secret, error) { + nameSpace := getCertsSecretNamespace(req) + name := getCertsSecretName(req) + search := types.NamespacedName{ + Namespace: nameSpace, + Name: name, + } + + // Certificates Secret + secretCerts := &corev1.Secret{} + err := r.Get(context.Background(), search, secretCerts) + if err == nil { + GetLogInstance().Info("Certificates Secret is already there", "Namespace", secretCerts.Namespace, + "Name", secretCerts.Name, "Amount of Secrets", len(secretCerts.StringData)) + return nil, nil + } + GetLogInstance().Info("Creating Certificates Secret", "Namespace", secretCerts.Namespace, "Name", secretCerts.Name) + secretCerts = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nameSpace, + Name: name, //TODO: set as configurable + }, + } + err = ctrl.SetControllerReference(deployment, secretCerts, r.Scheme) + if err != nil { + GetLogInstance().Error(err, "Unable to set certificates secret controller reference") + return nil, err + } + err = r.Create(context.Background(), secretCerts) + if err != nil { + GetLogInstance().Error(err, "Unable to create certificate secret, maybe it already exists?") + } + return secretCerts, err +} + +// getTPMCertsSecretName returns name of the TPM certificates secret +func getTPMCertsSecretName(req ctrl.Request) string { + return req.NamespacedName.Name + "-tpm-cert-store" +} + +// getTPMCertsSecretNamespace returns namespace of the TPM certificates secret +func getTPMCertsSecretNamespace(req ctrl.Request) string { + return req.NamespacedName.Namespace +} + +// getCertsSecretName returns name of the certificates secret +func getCertsSecretName(req ctrl.Request) string { + return req.NamespacedName.Name + "-certs" +} + +// getTPMCertsSecretNamespace returns namespace of the TPM certificates secret +func getCertsSecretNamespace(req ctrl.Request) string { + return req.NamespacedName.Namespace +} + +// getCAPasswordSecretName returns name of the CA password secret +func getCAPasswordSecretName(req ctrl.Request) string { + return req.NamespacedName.Name + "-ca-password" //TODO: set as configurable +} + +// getCAPasswordNamespace returns namespace of the CA password secret +func getCAPasswordSecretNamespace(req ctrl.Request) string { + return req.NamespacedName.Namespace +} + +// createTPMCertsSecret will create TPM certs secret if it does not exist +func (r *DeploymentReconciler) createTPMCertsSecret(deployment *attestationv1alpha1.Deployment, req ctrl.Request) (*corev1.Secret, error) { + nameSpace := getTPMCertsSecretNamespace(req) + name := getTPMCertsSecretName(req) + search := types.NamespacedName{ + Namespace: nameSpace, + Name: name, + } + // TPM Certificates Secret + secretTPMCerts := &corev1.Secret{} + err := r.Get(context.Background(), search, secretTPMCerts) + if err == nil { + GetLogInstance().Info("TPM Certificates Secret is already there", "Namespace", secretTPMCerts.Namespace, + "Name", secretTPMCerts.Name) + return nil, nil + } + GetLogInstance().Info("Creating TPM Secret", "Namespace", secretTPMCerts.Namespace, "Name", secretTPMCerts.Name) + secretTPMCerts = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nameSpace, + Name: name, //TODO: set as configurable + }, + } + err = ctrl.SetControllerReference(deployment, secretTPMCerts, r.Scheme) + if err != nil { + GetLogInstance().Error(err, "Unable to set TPM secret controller reference") + return nil, err + } + err = r.Create(context.Background(), secretTPMCerts) + if err != nil { + GetLogInstance().Error(err, "Unable to create TPM secret, maybe it already exists?") + } + return secretTPMCerts, err +} + +// createInitialJob will create initial job +func (r *DeploymentReconciler) createInitialJob(deployment *attestationv1alpha1.Deployment, req ctrl.Request) error { + // Initial Job + nameSpace := req.NamespacedName.Namespace + jobName := req.NamespacedName.Name + "-init-job" + containerName := req.NamespacedName.Name + "-init-job-container" + initJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nameSpace, + Name: jobName, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: &[]int32{1}[0], + Completions: &[]int32{1}[0], + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: getInitJobImageNameAndTag(deployment), + Name: containerName, + Ports: []corev1.ContainerPort{}, + LivenessProbe: nil, + ReadinessProbe: nil, + VolumeMounts: nil, + Resources: corev1.ResourceRequirements{}, + Command: getInitJobCommands(), + Env: getInitJobEnvVars(req), + }, + }, + RestartPolicy: "Never", + ServiceAccountName: "attestation-operator-controller-manager", + }, + }, + }, + } + search := types.NamespacedName{ + Namespace: nameSpace, + Name: jobName, + } + err := r.Get(context.Background(), search, initJob) + if err == nil { + GetLogInstance().Info("Init Job is already there", "Namespace", initJob.Namespace, "Name", initJob.Name) + return nil + } + GetLogInstance().Info("Creating Init Job", "Namespace", initJob.Namespace, "Name", initJob.Name) + err = ctrl.SetControllerReference(deployment, initJob, r.Scheme) + if err != nil { + GetLogInstance().Error(err, "Unable to set init job controller reference") + return err + } + err = r.Create(context.Background(), initJob) + if err != nil { + GetLogInstance().Error(err, "Unable to create initial job ... maybe it already exists?") + return err + } + r.InitJobCounter = DEFAULT_WAIT_INIT_JOB + r.InitJob = initJob + r.ContainerName = containerName + + return nil +} + +// deployInitTasks will deploy initial job +// specified nodes in CRD +func (r *DeploymentReconciler) deployInitTasks(deployment *attestationv1alpha1.Deployment, req ctrl.Request) error { + // Initial Job, the first thing + err := r.createInitialJob(deployment, req) + if err != nil { + GetLogInstance().Error(err, "Unable to create/read initial job") + return err + } + if r.InitJobCounter > 0 { + GetLogInstance().Info("Init job already exists ... (waiting for it)") + return nil + } + + err = r.createCAPasswordSecret(deployment, req) + if err != nil { + GetLogInstance().Error(err, "Unable to create/read CA Password secret") + return err + } + + // remove, if possible, this declaration: + nameSpace := req.NamespacedName.Namespace + + // Certificates Secret + secretCerts, err := r.createCertsSecret(deployment, req) + if err != nil { + GetLogInstance().Error(err, "Unable to create/read certs secret") + return err + } + + // Certificates TPM Secret + secretTPMCerts, err := r.createTPMCertsSecret(deployment, req) + if err != nil { + GetLogInstance().Error(err, "Unable to create/read TPM certs secret") + return err + } + + podList, err := PodList(nameSpace, context.Background(), r.InitJob.Name) + if len(podList) == 0 { + if err != nil { + return err + } + return errors.New("Unable to parse pod associated to job " + r.InitJob.Name) + } + initJobPod := podList[0] + + GetLogInstance().Info("InitJob information", "Namespace", r.InitJob.Namespace, + "Job Name", r.InitJob.Name, "Pod Name", initJobPod) + + // Once Initial Job is created, need to parse the certificates generated there (both keylime and tpm ones) only in case secrets are empty + if secretCerts != nil && len(secretCerts.StringData) == 0 { + certMap := parseCertificatesFromPod(initJobPod, r.ContainerName, "/tmp/certs", nameSpace) + secretCerts.StringData = make(map[string]string) + for k, v := range certMap { + secretCerts.StringData[k] = v + } + err = r.Update(context.Background(), secretCerts) + if err != nil { + GetLogInstance().Error(err, "Unable to update certificates secret") + return err + } + } + if secretTPMCerts != nil && len(secretTPMCerts.StringData) == 0 { + certMap := parseCertificatesFromPod(initJobPod, r.ContainerName, "/var/lib/keylime/tpm_cert_store/", nameSpace) + secretTPMCerts.StringData = make(map[string]string) + for k, v := range certMap { + secretTPMCerts.StringData[k] = v + } + err = r.Update(context.Background(), secretTPMCerts) + if err != nil { + GetLogInstance().Error(err, "Unable to update TPM certificates secret") + return err + } + } + return nil +} + +// getTenantPod function returns pod specification for tenant +func (r *DeploymentReconciler) getTenantPod(cr *attestationv1alpha1.Deployment, req ctrl.Request, + labels map[string]string) *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Command: getTenantPodCommands(), + Image: getTenantImageNameAndTag(cr), + Name: "keylime-tenant", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "certs", + MountPath: "/var/lib/keylime/cv_ca/", + ReadOnly: true, + }, + { + Name: "tpm-cert-store", + MountPath: "/var/lib/keylime/tpm_cert_store", + + ReadOnly: true, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "tpm-cert-store", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: getTPMCertsSecretName(req), + }, + }, + }, + { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: getCertsSecretName(req), + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyAlways, + ImagePullSecrets: []corev1.LocalObjectReference{}, + }, + } +} + +// deployKeylimeNodes will deploy +// specified nodes in CRD +func (r *DeploymentReconciler) deployKeylimeNodes(deployment *attestationv1alpha1.Deployment, req ctrl.Request) error { + // If InitJob is running, must wait for it + if r.InitJobCounter > 0 { + GetLogInstance().Info("Init job running ... (waiting for it)") + return nil + } + nameSpace := req.NamespacedName.Namespace + tenantName := req.NamespacedName.Name + "-tenant-deployment" + labels := map[string]string{ + "app": req.NamespacedName.Name, + } + tenantDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nameSpace, + Name: tenantName, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &[]int32{1}[0], // TODO: set configurable + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + Template: *r.getTenantPod(deployment, req, labels), + }, + } + search := types.NamespacedName{ + Namespace: nameSpace, + Name: tenantName, + } + err := r.Get(context.Background(), search, tenantDeployment) + if err == nil { + GetLogInstance().Info("Tenant is already there", "Namespace", nameSpace, + "Name", tenantName) + return nil + } + // Deploy Tenant + GetLogInstance().Info("Creating Tenant Deployment") + err = ctrl.SetControllerReference(deployment, tenantDeployment, r.Scheme) + if err != nil { + GetLogInstance().Error(err, "Unable to set tenant deployment controller reference") + return err + } + err = r.Create(context.Background(), tenantDeployment) + if err != nil { + GetLogInstance().Error(err, "Unable to create tenant") + return err + } + return nil +} + +// getTenantImageNameAndTag will return image path and tag according to CRD +func getTenantImageNameAndTag(deployment *attestationv1alpha1.Deployment) string { + path := deployment.Spec.NodeInfo.TenantImageInfo.ImageRepositoryPath + if path == "" { + GetLogInstance().Info("Did not find image path, using default one") + path = "quay.io/keylime/keylime_tenant" + } + tag := deployment.Spec.NodeInfo.TenantImageInfo.ImageTag + if tag == "" { + GetLogInstance().Info("Did not find image tag, using default one") + tag = "latest" + } + imagePathTag := path + ":" + tag + GetLogInstance().Info("Tenant Information", "Path", path, "Tag", tag, "Complete Name", imagePathTag) + return imagePathTag +} + +// getInitJobImageNameAndTag will return image path and tag according to CRD +func getInitJobImageNameAndTag(deployment *attestationv1alpha1.Deployment) string { + path := deployment.Spec.InitGlobal.InitialImage.ImageRepositoryPath + if path == "" { + GetLogInstance().Info("Did not find image path, using default one") + path = "quay.io/keylime/keylime_tenant" + } + tag := deployment.Spec.InitGlobal.InitialImage.ImageTag + if tag == "" { + GetLogInstance().Info("Did not find image tag, using default one") + tag = "latest" + } + imagePathTag := path + ":" + tag + GetLogInstance().Info("Init Job Information", "Path", path, "Tag", tag, "Complete Name", imagePathTag) + return imagePathTag +} + +// getInitJobCommands will return array of commands to execute at init job +func getInitJobCommands() []string { + commands := make([]string, 3) + commands[0] = "/bin/bash" + commands[1] = "-c" + commands[2] = ` + # generate the CV CA + mkdir -p /tmp/certs + cd /tmp + keylime_ca -d /tmp/certs --command init && keylime_ca -d /tmp/certs --command create --name server && keylime_ca -d /tmp/certs --command create --name client + if [[ $? -ne 0 ]] + then + echo "ERROR: unable to generate certificates" + exit 1 + fi + # TODO: set next sleep configurable + echo "GREAT: certificates generated! Will wait for a while until certs are parsed and stored in the certs secret (sleeping 40 secs)" + sleep 40 + exit 0 +` + return commands +} + +// getTenantPodCommands will return array of commands to execute in tenant +func getTenantPodCommands() []string { + commands := make([]string, 3) + commands[0] = "/bin/bash" + commands[1] = "-c" + commands[2] = ` + function on_exit() { + echo "Exiting..." + exit 0 + } + trap on_exit EXIT + echo "NOTE: This is not a service, but a simple exec-style Kubernetes pod. Access this pod through 'kubectl exec' and/or the 'keylime_tenant' script from the attestation operator repository (https://github.com/keylime/attestation-operator)." + while true; do sleep 30; done +` + return commands +} + +// getInitJobEnvVars will return array of commands to execute at init job +func getInitJobEnvVars(req ctrl.Request) []corev1.EnvVar { + init_job_env_vars := make([]corev1.EnvVar, 2) + init_job_env_vars[0] = corev1.EnvVar{ + Name: "KEYLIME_CA_PASSWORD", + Value: "PENDING-HOW-TO-READ-THIS-FROM-SECRET:hhkl-keylime-ca-password", + } + init_job_env_vars[1] = corev1.EnvVar{ + Name: "KEYLIME_SECRETS_CA_PW_NAME", + Value: getCAPasswordSecretName(req), + } + return init_job_env_vars +} diff --git a/internal/controller/attestation/deployment_controller_cluster_common.go b/internal/controller/attestation/deployment_controller_cluster_common.go new file mode 100644 index 0000000..3e91559 --- /dev/null +++ b/internal/controller/attestation/deployment_controller_cluster_common.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 Keylime Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// GetClusterClientConfig first tries to get a config object which uses the service account kubernetes gives to pods, +// if it is called from a process running in a kubernetes environment. +// Otherwise, it tries to build config from a default kubeconfig filepath if it fails, it fallback to the default config. +// Once it get the config, it returns the same. +func GetClusterClientConfig() (*rest.Config, error) { + config, err := rest.InClusterConfig() + if err != nil { + err1 := err + kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + err = fmt.Errorf("InClusterConfig as well as BuildConfigFromFlags Failed. Error in InClusterConfig: %+v\nError in BuildConfigFromFlags: %+v", err1, err) + return nil, err + } + } + return config, nil +} + +// GetClientsetFromClusterConfig takes REST config and Create a clientset based on that and return that clientset +func GetClientsetFromClusterConfig(config *rest.Config) (*kubernetes.Clientset, error) { + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + err = fmt.Errorf("failed creating clientset. Error: %+v", err) + return nil, err + } + + return clientset, nil +} + +// GetClusterClientset first tries to get a config object which uses the service account kubernetes gives to pods, +// if it is called from a process running in a kubernetes environment. +// Otherwise, it tries to build config from a default kubeconfig filepath if it fails, it fallback to the default config. +// Once it get the config, it creates a new Clientset for the given config and returns the clientset. +func GetClusterClientset() (*kubernetes.Clientset, error) { + config, err := GetClusterClientConfig() + if err != nil { + return nil, err + } + + return GetClientsetFromClusterConfig(config) +} + +// GetRESTClient first tries to get a config object which uses the service account kubernetes gives to pods, +// if it is called from a process running in a kubernetes environment. +// Otherwise, it tries to build config from a default kubeconfig filepath if it fails, it fallback to the default config. +// Once it get the config, it +func GetRESTClient() (*rest.RESTClient, error) { + config, err := GetClusterClientConfig() + if err != nil { + return &rest.RESTClient{}, err + } + + return rest.RESTClientFor(config) +} diff --git a/internal/controller/attestation/deployment_controller_info.go b/internal/controller/attestation/deployment_controller_info.go new file mode 100644 index 0000000..27ab5db --- /dev/null +++ b/internal/controller/attestation/deployment_controller_info.go @@ -0,0 +1,72 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "context" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PodList list the pods in a particular namespace +// :param string namespace: namespace of the Pod +// :param context: context of the controller +// :param startswith: string that indicates if pod names have to start with a certain string +// +// :return: +// +// string: Output of the command. (STDOUT) +// error: If any error has occurred otherwise `nil` +func PodList(namespace string, ctx context.Context, startsWith string) ([]string, error) { + config, err := GetClusterClientConfig() + if err != nil { + GetLogInstance().Info("Unable to get ClusterClientConfig") + return nil, err + } + if config == nil { + GetLogInstance().Info("Unable to get config") + err = fmt.Errorf("nil config") + return nil, err + } + + clientset, err := GetClientsetFromClusterConfig(config) + if err != nil { + GetLogInstance().Info("Unable to get ClientSetFromClusterConfig") + return nil, err + } + if clientset == nil { + GetLogInstance().Info("Clientset is null") + err = fmt.Errorf("nil clientset") + return nil, err + } + + pods, _ := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + lpods := make([]string, len(pods.Items)) + podIndex := 0 + for i, pod := range pods.Items { + GetLogInstance().Info("Execution information (Pod)", "i", i, "Pod", pod.GetName(), + "Pod Reason", pod.Status.Reason, "Pod Status", pod.Status) + if len(startsWith) == 0 || strings.HasPrefix(pod.GetName(), startsWith) { + GetLogInstance().Info("Pod added to list", "PodName", pod.GetName()) + lpods[podIndex] = pod.GetName() + podIndex += 1 + } + } + return lpods, nil +} diff --git a/internal/controller/attestation/deployment_controller_log.go b/internal/controller/attestation/deployment_controller_log.go new file mode 100644 index 0000000..29ae58d --- /dev/null +++ b/internal/controller/attestation/deployment_controller_log.go @@ -0,0 +1,38 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "github.com/go-logr/logr" + "sync" +) + +var lock = &sync.Mutex{} + +var logInstance logr.Logger + +func GetLogInstance() logr.Logger { + lock.Lock() + defer lock.Unlock() + return logInstance +} + +func SetLogInstance(l logr.Logger) { + lock.Lock() + defer lock.Unlock() + logInstance = l +} diff --git a/internal/controller/attestation/deployment_controller_parse_cert_files.go b/internal/controller/attestation/deployment_controller_parse_cert_files.go new file mode 100644 index 0000000..b300418 --- /dev/null +++ b/internal/controller/attestation/deployment_controller_parse_cert_files.go @@ -0,0 +1,68 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "strconv" + "strings" +) + +func getCertListFromPodStdout(stdout string) []string { + // stdout is a string of the type: + // a.pem\ncb.pem\nc.pem\n..z.pem\n + return strings.Split(stdout, "\n") +} + +func getCertContentsFromPod(podName, containerName, certDirectory, certFile, nameSpace string) string { + // stdout is a string of the type: + // a.pem\ncb.pem\nc.pem\n..z.pem\n + command := "cat " + certDirectory + "/" + certFile + stdout, _, err := podCommandExec(command, containerName, podName, nameSpace) + if err != nil { + GetLogInstance().Error(err, "Unable to execute command in pod", + "command", command, "podName", podName, "containerName", containerName, + "namespace", nameSpace) + return "" + } + return stdout +} + +func parseCertificatesFromPod(podName, containerName, certDirectory, nameSpace string) map[string]string { + command := "ls -1 " + certDirectory + // Connect to pod and read list of files + stdout, _, err := podCommandExec(command, containerName, podName, nameSpace) + if err != nil { + GetLogInstance().Error(err, "Unable to execute command in pod", + "command", command, "podName", podName, "containerName", containerName, + "namespace", nameSpace) + return nil + } + GetLogInstance().Info("Executed command in pod correctly", "stdout", stdout) + certList := getCertListFromPodStdout(stdout) + GetLogInstance().Info("Obtained a total of " + strconv.Itoa(len(certList)) + " certificates") + certMap := make(map[string]string) + var certContents string + for _, cert := range certList { + GetLogInstance().Info("Getting information for certificate [" + cert + "]") + if len(cert) > 0 { + certContents = getCertContentsFromPod(podName, containerName, certDirectory, + cert, nameSpace) + certMap[cert] = certContents + } + } + return certMap +} diff --git a/internal/controller/attestation/deployment_controller_pod_command.go b/internal/controller/attestation/deployment_controller_pod_command.go new file mode 100644 index 0000000..4c809e5 --- /dev/null +++ b/internal/controller/attestation/deployment_controller_pod_command.go @@ -0,0 +1,93 @@ +/* +Copyright 2024 Keylime Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "bytes" + "context" + "fmt" + "strings" + + core_v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/remotecommand" +) + +// podCommandExec uninterractively exec to the pod with the command specified. +// :param string command: command to execute +// :param string pod_name: Pod name +// :param string namespace: namespace of the Pod. +// :return: string: Output of the command. (STDOUT) +// +// string: Errors. (STDERR) +// error: If any error has occurred otherwise `nil` +func podCommandExec(command, containerName, podName, namespace string) (string, string, error) { + config, err := GetClusterClientConfig() + if err != nil { + return "", "", err + } + if config == nil { + err = fmt.Errorf("nil config") + return "", "", err + } + + clientset, err := GetClientsetFromClusterConfig(config) + if err != nil { + return "", "", err + } + if clientset == nil { + err = fmt.Errorf("nil clientset") + return "", "", err + } + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec") + scheme := runtime.NewScheme() + if err := core_v1.AddToScheme(scheme); err != nil { + return "", "", fmt.Errorf("error adding to scheme: %v", err) + } + + parameterCodec := runtime.NewParameterCodec(scheme) + req.VersionedParams(&core_v1.PodExecOptions{ + Command: []string{"/bin/bash", "-c", command}, + Container: containerName, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, parameterCodec) + + exec, spdyerr := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if spdyerr != nil { + return "", "", fmt.Errorf("error while creating Executor: %v, Command: %s", err, strings.Fields(command)) + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{ + Stdin: nil, + Stdout: &stdout, + Stderr: &stderr, + Tty: false, + }) + if err != nil { + return "", "", fmt.Errorf("error in Stream: %v, Command: %s", err, strings.Fields(command)) + } + return stdout.String(), stderr.String(), nil +} diff --git a/internal/controller/attestation/deployment_controller_rand_password_gen.go b/internal/controller/attestation/deployment_controller_rand_password_gen.go new file mode 100644 index 0000000..5275c24 --- /dev/null +++ b/internal/controller/attestation/deployment_controller_rand_password_gen.go @@ -0,0 +1,39 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "math/rand" + "time" +) + +var alphaNum = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + +type RandPasswordGen struct{} + +func NewRandPasswordGen() RandPasswordGen { + rand.Seed(time.Now().UnixNano()) + return RandPasswordGen{} +} + +func (r RandPasswordGen) num(passwordLen uint32) string { + b := make([]rune, passwordLen) + for i := range b { + b[i] = alphaNum[rand.Intn(len(alphaNum))] + } + return string(b) +} diff --git a/internal/controller/attestation/suite_test.go b/internal/controller/attestation/suite_test.go new file mode 100644 index 0000000..c240b08 --- /dev/null +++ b/internal/controller/attestation/suite_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2023 Keylime Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package attestation + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + attestationv1alpha1 "github.com/keylime/attestation-operator/api/attestation/v1alpha1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = attestationv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/internal/controller/keylime/registrar_synchronizer.go b/internal/controller/keylime/registrar_synchronizer.go index b0c9b08..641de1a 100644 --- a/internal/controller/keylime/registrar_synchronizer.go +++ b/internal/controller/keylime/registrar_synchronizer.go @@ -5,11 +5,15 @@ package keylime import ( "context" + "fmt" + "os" + "sync" "time" "github.com/go-logr/logr" kclient "github.com/keylime/attestation-operator/pkg/client" + khttp "github.com/keylime/attestation-operator/pkg/client/http" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -34,13 +38,120 @@ type RegistrarSynchronizer struct { log logr.Logger } -// Start implements manager.Runnable. -func (r *RegistrarSynchronizer) Start(ctx context.Context) error { +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +func (r *RegistrarSynchronizer) InitializeKeylimeClient(ctx context.Context) error { + var registrarURL string + var verifierURL string + setupLog := ctrl.Log.WithName("setup") + if val, ok := os.LookupEnv("KEYLIME_REGISTRAR_URL"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_REGISTRAR_URL is empty") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + registrarURL = val + } else { + err := fmt.Errorf("environment variable KEYLIME_REGISTRAR_URL not set") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + + // TODO: we will actually need to detect and handle all verifiers + // Ideally we would detect scaling up/down at runtime, but let alone dealing with multiple would be good + if val, ok := os.LookupEnv("KEYLIME_VERIFIER_URL"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_VERIFIER_URL is empty") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + verifierURL = val + } else { + err := fmt.Errorf("environment variable KEYLIME_VERIFIER_URL not set") + setupLog.Error(err, "unable to determine URL for the keylime registrar") + os.Exit(1) + } + + var clientCertFile, clientKeyFile string + if val, ok := os.LookupEnv("KEYLIME_CLIENT_KEY"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_KEY is empty") + setupLog.Error(err, "unable to determine client key file for the keylime client") + os.Exit(1) + } + clientKeyFile = val + } else { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_KEY not set") + setupLog.Error(err, "unable to determine client key file for the keylime client") + os.Exit(1) + } + if val, ok := os.LookupEnv("KEYLIME_CLIENT_CERT"); ok { + if val == "" { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_CERT is empty") + setupLog.Error(err, "unable to determine client cert file for the keylime client") + os.Exit(1) + } + clientCertFile = val + } else { + err := fmt.Errorf("environment variable KEYLIME_CLIENT_CERT not set") + setupLog.Error(err, "unable to determine client cert file for the keylime client") + os.Exit(1) + } + + tpmCertStore := os.Getenv("KEYLIME_TPM_CERT_STORE") + + // set this to info + setupLog.Info("RegistrationSynchronizer: Certification Files Information", "CertFile", clientCertFile, "KeyFile", clientKeyFile) + + // if files don't exist, it is probable initialization of the secret is pending + // return with no error + if !fileExists(clientCertFile) { + setupLog.Info("Certificate Client file doesn't exist", "CertFile", clientCertFile) + return nil + } + if !fileExists(clientKeyFile) { + setupLog.Info("Certificate Key file doesn't exist", "KeyFile", clientKeyFile) + return nil + } + + hc, err := khttp.NewKeylimeHTTPClient( + khttp.ClientCertificate(clientCertFile, clientKeyFile), + // TODO: unfortunately currently our server certs don't have the correct SANs + // and for some reason that's not an issue for any of the other components + // However, golang is very picky when it comes to that, and one cannot disable SAN verification individually + khttp.InsecureSkipVerify(), + ) + if err != nil { + setupLog.Error(err, "unable to create Keylime HTTP client") + os.Exit(1) + } + keylimeClient, err := kclient.New(ctx, ctrl.Log.WithName("keylime"), hc, registrarURL, []string{verifierURL}, tpmCertStore) + if err != nil { + setupLog.Error(err, "failed to create keylime client") + os.Exit(1) + } + r.Keylime = keylimeClient + return nil +} + +// getLoopInterval returns the interval to run reconciliation +func (r *RegistrarSynchronizer) getLoopInterval() time.Duration { loopInterval := r.LoopInterval if loopInterval == 0 { loopInterval = defaultLoopInterval } - t := time.NewTicker(loopInterval) + return loopInterval +} + +// Start implements manager.Runnable. +func (r *RegistrarSynchronizer) Start(ctx context.Context) error { + t := time.NewTicker(r.getLoopInterval()) defer t.Stop() loop: @@ -66,6 +177,12 @@ func (r *RegistrarSynchronizer) reconcile(ctx context.Context) { r.log.Error(err, "reconcile: failed to get list of agent CRs") return } + r.InitializeKeylimeClient(ctx) + if nil == r.Keylime { + r.log.Info("Waiting for keylime client to be initialized ... ", "Interval", r.getLoopInterval()) + return + } + k8smap := make(map[string]struct{}, len(k8sList.Items)) for _, cragent := range k8sList.Items { k8smap[cragent.Name] = struct{}{} diff --git a/operator_configs/attestation_v1alpha1_deployment.yaml b/operator_configs/attestation_v1alpha1_deployment.yaml new file mode 100644 index 0000000..9bacfcf --- /dev/null +++ b/operator_configs/attestation_v1alpha1_deployment.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: attestation.keylime.dev/v1alpha1 +kind: Deployment +metadata: + name: deployment-sample + namespace: keylime +spec: + enabled: true