diff --git a/.dockerignore b/.dockerignore index 2f65dbd017..c342702b73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ -.git -.github +.git/ +.github/ .vscode .dockerignore .gitignore @@ -7,12 +7,14 @@ .markdownlintrc *.out bin/ -build/ +hack/.bin/ +hack/tools/bin/ +hack/tools/haproxy/ out/ docs/ scripts/ **/.md -/*.test -/cluster-api-provider-vsphere -/examples/provider-components/provider-components*.yaml +*.test +cluster-api-provider-vsphere +examples/provider-components/provider-components*.yaml test/ diff --git a/PROJECT b/PROJECT index 0d1b60058e..25252e90fb 100644 --- a/PROJECT +++ b/PROJECT @@ -23,3 +23,7 @@ resources: - group: infrastructure version: v1alpha3 kind: VSphereVM +- group: infrastructure + version: v1alpha3 + kind: HAProxyLoadBalancer + diff --git a/api/v1alpha3/haproxyloadbalancer_types.go b/api/v1alpha3/haproxyloadbalancer_types.go new file mode 100644 index 0000000000..80a3d47f7b --- /dev/null +++ b/api/v1alpha3/haproxyloadbalancer_types.go @@ -0,0 +1,163 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // HAProxyLoadBalancerFinalizer allows a reconciler to clean up + // resources associated with an HAProxyLoadBalancer before removing + // it from the API server. + HAProxyLoadBalancerFinalizer = "haproxyloadbalancer.infrastructure.cluster.x-k8s.io" +) + +// HAProxyLoadBalancerSpec defines the desired state of HAProxyLoadBalancer. +type HAProxyLoadBalancerSpec struct { + // APIEndpoint is the address and port with which the load balancer API + // service may be accessed. + // If omitted then the VirtualMachineConfiguration field is required in + // order to deploy a new load balancer. + // +optional + APIEndpoint APIEndpoint `json:"apiEndpoint,omitempty"` + + // Ports is a list of one or more pairs of ports on which the load balancer + // listens for incoming traffic and the ports on the backend to which the + // traffic is transmitted. + Ports []LoadBalancerPort `json:"ports"` + + // Selector is used to identify the control-plane Machine resources that + // will be the backend servers for this load balancer. + Selector metav1.LabelSelector `json:"selector"` + + // CACertificateRef is a reference to a Secret resource that contains the + // following keys: + // * ca.crt - The PEM-encoded, public key for a CA certificate + // * ca.key - The PEM-encoded, private key for a CA certificate + // + // If unspecified, the Secret's Namespace defaults to + // HAProxyLoadBalancer.Namespace. + // + // If unspecified, the Secret's Name defaults to + // HAProxyLoadBalancer.Name+"-ca". + // + // When using an existing load balancer only the public key is required, + // however, if the private key is present as well and the + // ClientCertificateRef does not exist or contain a valid client + // certificate, then the public and private key in this Secret will be used + // to generate a valid, client certifiate for an existing load balancer. + // + // When provisioning a new load balancer this Secret must contain both + // a public *and* private key. If the Secret does not exist then a new + // Secret will be generated with a new CA key pair. If the Secret exists + // but does not contain a valid CA key pair then a new key pair will be + // generated and the Secret will be updated. + // + // If an existing load balancer is used then the Secret need only to contain + // the CA's public key. + CACertificateRef corev1.SecretReference `json:"caCertificateRef"` + + // ClientCredentialsRef is a reference to a Secret resource that contains + // the following keys: + // * client.crt - A PEM-encoded, public key for a client certificate + // used to access the load balancer's API server + // * client.key - A PEM-encoded, private key for a client certificate + // used to access the load balancer's API server + // * username - The username used to access the load balancer's API + // server + // * password - The password used to access the load balancer's API + // server + // + // If unspecified, the Secret's Namespace defaults to + // HAProxyLoadBalancer.Namespace. + // + // If unspecified, the Secret's Name defaults to + // HAProxyLoadBalancer.Name+"-client". + // + // This Secret must contain both a public *and* private key. If the Secret + // does not exist then a new Secret will be generated with a new client + // certificate key pair using the CA from CACertificateRef. + // + // If the Secret exists but does not contain a valid client certificate key + // pair, then a new client certificate key pair will be generated using the + // CA from CACertificateRef. + // + // When the username or password fields are empty, they both default to + // "guest". The HAProxy load balancer OVA built from the CAPV repository + // uses mutual certificate validation (client certificates) to control + // access to the load balancer's API server. However, a username and + // password are still required, even though they provide no actual access + // control. + ClientCredentialsRef corev1.SecretReference `json:"clientCredentialsRef"` + + // VirtualMachineConfiguration is optional information used to deploy a new + // load VM. + // If omitted then the APIEndpoint field is required to point to an existing + // load balancer. + // +omitempty + VirtualMachineConfiguration *VirtualMachineCloneSpec `json:"virtualMachineConfiguration,omitempty"` +} + +// HAProxyLoadBalancerStatus defines the observed state of HAProxyLoadBalancer. +type HAProxyLoadBalancerStatus struct { + // Ready indicates whether or not the load balancer is ready. + // + // This field is required as part of the Portable Load Balancer model and is + // inspected via an unstructured reader by other controllers to determine + // the status of the load balancer. + // + // +optional + Ready bool `json:"ready,omitempty"` + + // Address is the IP address or DNS name of the load balancer. + // + // This field is required as part of the Portable Load Balancer model and is + // inspected via an unstructured reader by other controllers to determine + // the status of the load balancer. + // + // +optional + Address string `json:"address,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=haproxyloadbalancers,scope=Namespaced +// +kubebuilder:storageversion +// +kubebuilder:subresource:status + +// HAProxyLoadBalancer is the Schema for the haproxyloadbalancers API +type HAProxyLoadBalancer struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec HAProxyLoadBalancerSpec `json:"spec,omitempty"` + Status HAProxyLoadBalancerStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// HAProxyLoadBalancerList contains a list of HAProxyLoadBalancer +type HAProxyLoadBalancerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []HAProxyLoadBalancer `json:"items"` +} + +func init() { + SchemeBuilder.Register(&HAProxyLoadBalancer{}, &HAProxyLoadBalancerList{}) +} diff --git a/api/v1alpha3/types.go b/api/v1alpha3/types.go index 80c28f2de6..c0c62a56f2 100644 --- a/api/v1alpha3/types.go +++ b/api/v1alpha3/types.go @@ -36,6 +36,63 @@ const ( ValueReady = "true" ) +// VirtualMachineCloneSpec is information used to clone a virtual machine. +type VirtualMachineCloneSpec struct { + // Template is the name or inventory path of the template used to clone + // the virtual machine. + Template string `json:"template"` + + // Server is the IP address or FQDN of the vSphere server on which + // the virtual machine is created/located. + // +optional + Server string `json:"server,omitempty"` + + // Datacenter is the name or inventory path of the datacenter in which the + // virtual machine is created/located. + // +optional + Datacenter string `json:"datacenter,omitempty"` + + // Folder is the name or inventory path of the folder in which the + // virtual machine is created/located. + // +optional + Folder string `json:"folder,omitempty"` + + // Datastore is the name or inventory path of the datastore in which the + // virtual machine is created/located. + // +optional + Datastore string `json:"datastore,omitempty"` + + // ResourcePool is the name or inventory path of the resource pool in which + // the virtual machine is created/located. + // +optional + ResourcePool string `json:"resourcePool,omitempty"` + + // Network is the network configuration for this machine's VM. + Network NetworkSpec `json:"network"` + + // NumCPUs is the number of virtual processors in a virtual machine. + // Defaults to the eponymous property value in the template from which the + // virtual machine is cloned. + // +optional + NumCPUs int32 `json:"numCPUs,omitempty"` + // NumCPUs is the number of cores among which to distribute CPUs in this + // virtual machine. + // Defaults to the eponymous property value in the template from which the + // virtual machine is cloned. + // +optional + NumCoresPerSocket int32 `json:"numCoresPerSocket,omitempty"` + // MemoryMiB is the size of a virtual machine's memory, in MiB. + // Defaults to the eponymous property value in the template from which the + // virtual machine is cloned. + // +optional + MemoryMiB int64 `json:"memoryMiB,omitempty"` + // DiskGiB is the size of a virtual machine's disk, in GiB. + // Defaults to the eponymous property value in the template from which the + // virtual machine is cloned. + // +optional + DiskGiB int32 `json:"diskGiB,omitempty"` +} + // VSphereMachineTemplateResource describes the data needed to create a VSphereMachine from a template type VSphereMachineTemplateResource struct { metav1.TypeMeta `json:",inline"` @@ -63,7 +120,7 @@ type APIEndpoint struct { Host string `json:"host"` // The port on which the API server is serving. - Port int `json:"port"` + Port int32 `json:"port"` } // IsZero returns true if either the host or the port are zero values. @@ -228,3 +285,15 @@ type VirtualMachine struct { // Network is the status of the VM's network devices. Network []NetworkStatus `json:"network"` } + +// LoadBalancerPort defines the pair of frontend and backend ports used by +// the load balancer. +type LoadBalancerPort struct { + // Ingress is the port on which a load balancer listens for incoming + // traffic. + Ingress int32 `json:"ingress"` + + // Egress is the port to which a load balancer transmits traffic to + // backend servers. + Egress int32 `json:"egress"` +} diff --git a/api/v1alpha3/vspherecluster_types.go b/api/v1alpha3/vspherecluster_types.go index 27ad2eb10c..f727a5d198 100644 --- a/api/v1alpha3/vspherecluster_types.go +++ b/api/v1alpha3/vspherecluster_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha3 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/cluster-api-provider-vsphere/api/v1alpha3/cloudprovider" @@ -46,6 +47,14 @@ type VSphereClusterSpec struct { // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. // +optional ControlPlaneEndpoint APIEndpoint `json:"controlPlaneEndpoint"` + + // LoadBalancerRef may be used to enable a control plane load balancer + // for this cluster. + // When a LoadBalancerRef is provided, the VSphereCluster.Status.Ready field + // will not be true until the referenced resource is Status.Ready and has a + // non-empty Status.Address value. + // +optional + LoadBalancerRef *corev1.ObjectReference `json:"loadBalancerRef,omitempty"` } // VSphereClusterStatus defines the observed state of VSphereClusterSpec diff --git a/api/v1alpha3/vspheremachine_types.go b/api/v1alpha3/vspheremachine_types.go index dfd3c8d58c..cdae2ca35b 100644 --- a/api/v1alpha3/vspheremachine_types.go +++ b/api/v1alpha3/vspheremachine_types.go @@ -31,43 +31,12 @@ const ( // VSphereMachineSpec defines the desired state of VSphereMachine type VSphereMachineSpec struct { + VirtualMachineCloneSpec `json:",inline"` + // ProviderID is the virtual machine's BIOS UUID formated as // vsphere://12345678-1234-1234-1234-123456789abc // +optional ProviderID *string `json:"providerID,omitempty"` - - // Template is the name, inventory path, or instance UUID of the template - // used to clone new machines. - Template string `json:"template"` - - // Datacenter is the name or inventory path of the datacenter where this - // machine's VM is created/located. - Datacenter string `json:"datacenter"` - - // Network is the network configuration for this machine's VM. - Network NetworkSpec `json:"network"` - - // NumCPUs is the number of virtual processors in a virtual machine. - // Defaults to the analogue property value in the template from which this - // machine is cloned. - // +optional - NumCPUs int32 `json:"numCPUs,omitempty"` - // NumCPUs is the number of cores among which to distribute CPUs in this - // virtual machine. - // Defaults to the analogue property value in the template from which this - // machine is cloned. - // +optional - NumCoresPerSocket int32 `json:"numCoresPerSocket,omitempty"` - // MemoryMiB is the size of a virtual machine's memory, in MiB. - // Defaults to the analogue property value in the template from which this - // machine is cloned. - // +optional - MemoryMiB int64 `json:"memoryMiB,omitempty"` - // DiskGiB is the size of a virtual machine's disk, in GiB. - // Defaults to the analogue property value in the template from which this - // machine is cloned. - // +optional - DiskGiB int32 `json:"diskGiB,omitempty"` } // VSphereMachineStatus defines the observed state of VSphereMachine @@ -79,17 +48,6 @@ type VSphereMachineStatus struct { // Addresses contains the VSphere instance associated addresses. Addresses []v1.NodeAddress `json:"addresses,omitempty"` - // TaskRef is a managed object reference to a Task related to the machine. - // This value is set automatically at runtime and should not be set or - // modified by users. - // +optional - TaskRef string `json:"taskRef,omitempty"` - - // Network returns the network status for each of the machine's configured - // network interfaces. - // +optional - Network []NetworkStatus `json:"networkStatus,omitempty"` - // ErrorReason will be set in the event that there is a terminal problem // reconciling the Machine and will contain a succinct value suitable // for machine interpretation. diff --git a/api/v1alpha3/vspherevm_types.go b/api/v1alpha3/vspherevm_types.go index dd0fe0a913..36a0535806 100644 --- a/api/v1alpha3/vspherevm_types.go +++ b/api/v1alpha3/vspherevm_types.go @@ -29,6 +29,8 @@ const ( // VSphereVMSpec defines the desired state of VSphereVM. type VSphereVMSpec struct { + VirtualMachineCloneSpec `json:",inline"` + // BootstrapRef is a reference to a bootstrap provider-specific resource // that holds configuration details. // This field is optional in case no bootstrap data is required to create @@ -40,55 +42,6 @@ type VSphereVMSpec struct { // the VM has been created. // +optional BiosUUID string `json:"biosUUID,omitempty"` - - // Template is the name, inventory path, or instance UUID of the template - // used to clone new VMs. - Template string `json:"template"` - - // Server is the name of the vSphere server on which this VM is - // created/located. - Server string `json:"server"` - - // Datacenter is the name or inventory path of the datacenter where this - // VM is created/located. - Datacenter string `json:"datacenter"` - - // Folder is the name of inventory path of the folder where this VM is - // located/created. - Folder string `json:"folder"` - - // Datastore is the name of inventory path of the datastore where this VM is - // located/created. - Datastore string `json:"datastore"` - - // ResourcePool is the name of inventory path of the resource pool where - // this VM is located/created. - ResourcePool string `json:"resourcePool"` - - // Network is the network configuration for this VM. - Network NetworkSpec `json:"network"` - - // NumCPUs is the number of virtual processors in a virtual machine. - // Defaults to the analogue property value in the template from which this - // VM is cloned. - // +optional - NumCPUs int32 `json:"numCPUs,omitempty"` - // NumCoresPerSocket is the number of cores among which to distribute CPUs - // in this virtual machine. - // Defaults to the analogue property value in the template from which this - // VM is cloned. - // +optional - NumCoresPerSocket int32 `json:"numCoresPerSocket,omitempty"` - // MemoryMiB is the size of a virtual machine's memory, in MiB. - // Defaults to the analogue property value in the template from which this - // VM is cloned. - // +optional - MemoryMiB int64 `json:"memoryMiB,omitempty"` - // DiskGiB is the size of a virtual machine's disk, in GiB. - // Defaults to the analogue property value in the template from which this - // VM is cloned. - // +optional - DiskGiB int32 `json:"diskGiB,omitempty"` } // VSphereVMStatus defines the observed state of VSphereVM diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index c3551ec4f1..2991c73984 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -41,6 +41,124 @@ func (in *APIEndpoint) DeepCopy() *APIEndpoint { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancer) DeepCopyInto(out *HAProxyLoadBalancer) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancer. +func (in *HAProxyLoadBalancer) DeepCopy() *HAProxyLoadBalancer { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HAProxyLoadBalancer) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancerList) DeepCopyInto(out *HAProxyLoadBalancerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]HAProxyLoadBalancer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancerList. +func (in *HAProxyLoadBalancerList) DeepCopy() *HAProxyLoadBalancerList { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *HAProxyLoadBalancerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancerSpec) DeepCopyInto(out *HAProxyLoadBalancerSpec) { + *out = *in + out.APIEndpoint = in.APIEndpoint + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]LoadBalancerPort, len(*in)) + copy(*out, *in) + } + in.Selector.DeepCopyInto(&out.Selector) + out.CACertificateRef = in.CACertificateRef + out.ClientCredentialsRef = in.ClientCredentialsRef + if in.VirtualMachineConfiguration != nil { + in, out := &in.VirtualMachineConfiguration, &out.VirtualMachineConfiguration + *out = new(VirtualMachineCloneSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancerSpec. +func (in *HAProxyLoadBalancerSpec) DeepCopy() *HAProxyLoadBalancerSpec { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HAProxyLoadBalancerStatus) DeepCopyInto(out *HAProxyLoadBalancerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAProxyLoadBalancerStatus. +func (in *HAProxyLoadBalancerStatus) DeepCopy() *HAProxyLoadBalancerStatus { + if in == nil { + return nil + } + out := new(HAProxyLoadBalancerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerPort) DeepCopyInto(out *LoadBalancerPort) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerPort. +func (in *LoadBalancerPort) DeepCopy() *LoadBalancerPort { + if in == nil { + return nil + } + out := new(LoadBalancerPort) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkDeviceSpec) DeepCopyInto(out *NetworkDeviceSpec) { *out = *in @@ -212,6 +330,11 @@ func (in *VSphereClusterSpec) DeepCopyInto(out *VSphereClusterSpec) { } in.CloudProviderConfiguration.DeepCopyInto(&out.CloudProviderConfiguration) out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + if in.LoadBalancerRef != nil { + in, out := &in.LoadBalancerRef, &out.LoadBalancerRef + *out = new(v1.ObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereClusterSpec. @@ -301,12 +424,12 @@ func (in *VSphereMachineList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VSphereMachineSpec) DeepCopyInto(out *VSphereMachineSpec) { *out = *in + in.VirtualMachineCloneSpec.DeepCopyInto(&out.VirtualMachineCloneSpec) if in.ProviderID != nil { in, out := &in.ProviderID, &out.ProviderID *out = new(string) **out = **in } - in.Network.DeepCopyInto(&out.Network) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereMachineSpec. @@ -327,13 +450,6 @@ func (in *VSphereMachineStatus) DeepCopyInto(out *VSphereMachineStatus) { *out = make([]v1.NodeAddress, len(*in)) copy(*out, *in) } - if in.Network != nil { - in, out := &in.Network, &out.Network - *out = make([]NetworkStatus, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } if in.ErrorReason != nil { in, out := &in.ErrorReason, &out.ErrorReason *out = new(errors.MachineStatusError) @@ -510,12 +626,12 @@ func (in *VSphereVMList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VSphereVMSpec) DeepCopyInto(out *VSphereVMSpec) { *out = *in + in.VirtualMachineCloneSpec.DeepCopyInto(&out.VirtualMachineCloneSpec) if in.BootstrapRef != nil { in, out := &in.BootstrapRef, &out.BootstrapRef *out = new(v1.ObjectReference) **out = **in } - in.Network.DeepCopyInto(&out.Network) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VSphereVMSpec. @@ -571,3 +687,19 @@ func (in *VirtualMachine) DeepCopy() *VirtualMachine { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachineCloneSpec) DeepCopyInto(out *VirtualMachineCloneSpec) { + *out = *in + in.Network.DeepCopyInto(&out.Network) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineCloneSpec. +func (in *VirtualMachineCloneSpec) DeepCopy() *VirtualMachineCloneSpec { + if in == nil { + return nil + } + out := new(VirtualMachineCloneSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_haproxyloadbalancers.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_haproxyloadbalancers.yaml new file mode 100644 index 0000000000..e59eac5a16 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_haproxyloadbalancers.yaml @@ -0,0 +1,396 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: haproxyloadbalancers.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + kind: HAProxyLoadBalancer + listKind: HAProxyLoadBalancerList + plural: haproxyloadbalancers + singular: haproxyloadbalancer + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: HAProxyLoadBalancer is the Schema for the haproxyloadbalancers + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HAProxyLoadBalancerSpec defines the desired state of HAProxyLoadBalancer. + properties: + apiEndpoint: + description: APIEndpoint is the address and port with which the load + balancer API service may be accessed. If omitted then the VirtualMachineConfiguration + field is required in order to deploy a new load balancer. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + caCertificateRef: + description: "CACertificateRef is a reference to a Secret resource that + contains the following keys: * ca.crt - The PEM-encoded, public + key for a CA certificate * ca.key - The PEM-encoded, private key + for a CA certificate \n If unspecified, the Secret's Namespace defaults + to HAProxyLoadBalancer.Namespace. \n If unspecified, the Secret's + Name defaults to HAProxyLoadBalancer.Name+\"-ca\". \n When using an + existing load balancer only the public key is required, however, if + the private key is present as well and the ClientCertificateRef does + not exist or contain a valid client certificate, then the public and + private key in this Secret will be used to generate a valid, client + certifiate for an existing load balancer. \n When provisioning a new + load balancer this Secret must contain both a public *and* private + key. If the Secret does not exist then a new Secret will be generated + with a new CA key pair. If the Secret exists but does not contain + a valid CA key pair then a new key pair will be generated and the + Secret will be updated. \n If an existing load balancer is used then + the Secret need only to contain the CA's public key." + properties: + name: + description: Name is unique within a namespace to reference a secret + resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + clientCredentialsRef: + description: "ClientCredentialsRef is a reference to a Secret resource + that contains the following keys: * client.crt - A PEM-encoded, + public key for a client certificate used to access + the load balancer's API server * client.key - A PEM-encoded, private + key for a client certificate used to access the load + balancer's API server * username - The username used to access + the load balancer's API server * password - The + password used to access the load balancer's API server + \n If unspecified, the Secret's Namespace defaults to HAProxyLoadBalancer.Namespace. + \n If unspecified, the Secret's Name defaults to HAProxyLoadBalancer.Name+\"-client\". + \n This Secret must contain both a public *and* private key. If the + Secret does not exist then a new Secret will be generated with a new + client certificate key pair using the CA from CACertificateRef. \n + If the Secret exists but does not contain a valid client certificate + key pair, then a new client certificate key pair will be generated + using the CA from CACertificateRef. \n When the username or password + fields are empty, they both default to \"guest\". The HAProxy load + balancer OVA built from the CAPV repository uses mutual certificate + validation (client certificates) to control access to the load balancer's + API server. However, a username and password are still required, even + though they provide no actual access control." + properties: + name: + description: Name is unique within a namespace to reference a secret + resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + ports: + description: Ports is a list of one or more pairs of ports on which + the load balancer listens for incoming traffic and the ports on the + backend to which the traffic is transmitted. + items: + description: LoadBalancerPort defines the pair of frontend and backend + ports used by the load balancer. + properties: + egress: + description: Egress is the port to which a load balancer transmits + traffic to backend servers. + format: int32 + type: integer + ingress: + description: Ingress is the port on which a load balancer listens + for incoming traffic. + format: int32 + type: integer + required: + - egress + - ingress + type: object + type: array + selector: + description: Selector is used to identify the control-plane Machine + resources that will be the backend servers for this load balancer. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + virtualMachineConfiguration: + description: VirtualMachineConfiguration is optional information used + to deploy a new load VM. If omitted then the APIEndpoint field is + required to point to an existing load balancer. + properties: + datacenter: + description: Datacenter is the name or inventory path of the datacenter + in which the virtual machine is created/located. + type: string + datastore: + description: Datastore is the name or inventory path of the datastore + in which the virtual machine is created/located. + type: string + diskGiB: + description: DiskGiB is the size of a virtual machine's disk, in + GiB. Defaults to the eponymous property value in the template + from which the virtual machine is cloned. + format: int32 + type: integer + folder: + description: Folder is the name or inventory path of the folder + in which the virtual machine is created/located. + type: string + memoryMiB: + description: MemoryMiB is the size of a virtual machine's memory, + in MiB. Defaults to the eponymous property value in the template + from which the virtual machine is cloned. + format: int64 + type: integer + network: + description: Network is the network configuration for this machine's + VM. + properties: + devices: + description: Devices is the list of network devices used by + the virtual machine. TODO(akutz) Make sure at least one network + matches the ClusterSpec.CloudProviderConfiguration.Network.Name + items: + description: NetworkDeviceSpec defines the network configuration + for a virtual machine's network device. + properties: + dhcp4: + description: DHCP4 is a flag that indicates whether or + not to use DHCP for IPv4 on this device. If true then + IPAddrs should not contain any IPv4 addresses. + type: boolean + dhcp6: + description: DHCP6 is a flag that indicates whether or + not to use DHCP for IPv6 on this device. If true then + IPAddrs should not contain any IPv6 addresses. + type: boolean + gateway4: + description: Gateway4 is the IPv4 gateway used by this + device. Required when DHCP4 is false. + type: string + gateway6: + description: Gateway4 is the IPv4 gateway used by this + device. Required when DHCP6 is false. + type: string + ipAddrs: + description: IPAddrs is a list of one or more IPv4 and/or + IPv6 addresses to assign to this device. Required when + DHCP4 and DHCP6 are both false. + items: + type: string + type: array + macAddr: + description: MACAddr is the MAC address used by this device. + It is generally a good idea to omit this field and allow + a MAC address to be generated. Please note that this + value must use the VMware OUI to work with the in-tree + vSphere cloud provider. + type: string + mtu: + description: MTU is the device’s Maximum Transmission + Unit size in bytes. + format: int64 + type: integer + nameservers: + description: Nameservers is a list of IPv4 and/or IPv6 + addresses used as DNS nameservers. Please note that + Linux allows only three nameservers (https://linux.die.net/man/5/resolv.conf). + items: + type: string + type: array + networkName: + description: NetworkName is the name of the vSphere network + to which the device will be connected. + type: string + routes: + description: Routes is a list of optional, static routes + applied to the device. + items: + description: NetworkRouteSpec defines a static network + route. + properties: + metric: + description: Metric is the weight/priority of the + route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + searchDomains: + description: SearchDomains is a list of search domains + used when resolving IP addresses with DNS. + items: + type: string + type: array + required: + - networkName + type: object + type: array + preferredAPIServerCidr: + description: PreferredAPIServeCIDR is the preferred CIDR for + the Kubernetes API server endpoint on this machine + type: string + routes: + description: Routes is a list of optional, static routes applied + to the virtual machine. + items: + description: NetworkRouteSpec defines a static network route. + properties: + metric: + description: Metric is the weight/priority of the route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + required: + - devices + type: object + numCPUs: + description: NumCPUs is the number of virtual processors in a virtual + machine. Defaults to the eponymous property value in the template + from which the virtual machine is cloned. + format: int32 + type: integer + numCoresPerSocket: + description: NumCPUs is the number of cores among which to distribute + CPUs in this virtual machine. Defaults to the eponymous property + value in the template from which the virtual machine is cloned. + format: int32 + type: integer + resourcePool: + description: ResourcePool is the name or inventory path of the resource + pool in which the virtual machine is created/located. + type: string + server: + description: Server is the IP address or FQDN of the vSphere server + on which the virtual machine is created/located. + type: string + template: + description: Template is the name or inventory path of the template + used to clone the virtual machine. + type: string + required: + - network + - template + type: object + required: + - caCertificateRef + - clientCredentialsRef + - ports + - selector + type: object + status: + description: HAProxyLoadBalancerStatus defines the observed state of HAProxyLoadBalancer. + properties: + address: + description: "Address is the IP address or DNS name of the load balancer. + \n This field is required as part of the Portable Load Balancer model + and is inspected via an unstructured reader by other controllers to + determine the status of the load balancer." + type: string + ready: + description: "Ready indicates whether or not the load balancer is ready. + \n This field is required as part of the Portable Load Balancer model + and is inspected via an unstructured reader by other controllers to + determine the status of the load balancer." + type: boolean + type: object + type: object + version: v1alpha3 + versions: + - name: v1alpha3 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_vsphereclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_vsphereclusters.yaml index 21bb05c8b5..7e2944568d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_vsphereclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_vsphereclusters.yaml @@ -468,6 +468,7 @@ spec: type: string port: description: The port on which the API server is serving. + format: int32 type: integer required: - host @@ -477,6 +478,46 @@ spec: description: Insecure is a flag that controls whether or not to validate the vSphere server's certificate. type: boolean + loadBalancerRef: + description: LoadBalancerRef may be used to enable a control plane + load balancer for this cluster. When a LoadBalancerRef is provided, + the VSphereCluster.Status.Ready field will not be true until the + referenced resource is Status.Ready and has a non-empty Status.Address + value. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object server: description: Server is the address of the vSphere endpoint. type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml index cb07a724a5..60a607b655 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml @@ -19,279 +19,535 @@ spec: scope: Namespaced subresources: status: {} - validation: - openAPIV3Schema: - description: VSphereMachine is the Schema for the vspheremachines API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VSphereMachineSpec defines the desired state of VSphereMachine - properties: - datacenter: - description: Datacenter is the name or inventory path of the datacenter - where this machine's VM is created/located. - type: string - diskGiB: - description: DiskGiB is the size of a virtual machine's disk, in GiB. - Defaults to the analogue property value in the template from which - this machine is cloned. - format: int32 - type: integer - memoryMiB: - description: MemoryMiB is the size of a virtual machine's memory, in - MiB. Defaults to the analogue property value in the template from - which this machine is cloned. - format: int64 - type: integer - network: - description: Network is the network configuration for this machine's - VM. - properties: - devices: - description: Devices is the list of network devices used by the - virtual machine. TODO(akutz) Make sure at least one network matches - the ClusterSpec.CloudProviderConfiguration.Network.Name - items: - description: NetworkDeviceSpec defines the network configuration - for a virtual machine's network device. - properties: - dhcp4: - description: DHCP4 is a flag that indicates whether or not - to use DHCP for IPv4 on this device. If true then IPAddrs - should not contain any IPv4 addresses. - type: boolean - dhcp6: - description: DHCP6 is a flag that indicates whether or not - to use DHCP for IPv6 on this device. If true then IPAddrs - should not contain any IPv6 addresses. - type: boolean - gateway4: - description: Gateway4 is the IPv4 gateway used by this device. - Required when DHCP4 is false. - type: string - gateway6: - description: Gateway4 is the IPv4 gateway used by this device. - Required when DHCP6 is false. - type: string - ipAddrs: - description: IPAddrs is a list of one or more IPv4 and/or - IPv6 addresses to assign to this device. Required when DHCP4 - and DHCP6 are both false. - items: + version: v1alpha2 + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: VSphereMachine is the Schema for the vspheremachines API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VSphereMachineSpec defines the desired state of VSphereMachine + properties: + datacenter: + description: Datacenter is the name or inventory path of the datacenter + where this machine's VM is created/located. + type: string + diskGiB: + description: DiskGiB is the size of a virtual machine's disk, in GiB. + Defaults to the analogue property value in the template from which + this machine is cloned. + format: int32 + type: integer + memoryMiB: + description: MemoryMiB is the size of a virtual machine's memory, + in MiB. Defaults to the analogue property value in the template + from which this machine is cloned. + format: int64 + type: integer + network: + description: Network is the network configuration for this machine's + VM. + properties: + devices: + description: Devices is the list of network devices used by the + virtual machine. TODO(akutz) Make sure at least one network + matches the ClusterSpec.CloudProviderConfiguration.Network.Name + items: + description: NetworkDeviceSpec defines the network configuration + for a virtual machine's network device. + properties: + dhcp4: + description: DHCP4 is a flag that indicates whether or not + to use DHCP for IPv4 on this device. If true then IPAddrs + should not contain any IPv4 addresses. + type: boolean + dhcp6: + description: DHCP6 is a flag that indicates whether or not + to use DHCP for IPv6 on this device. If true then IPAddrs + should not contain any IPv6 addresses. + type: boolean + gateway4: + description: Gateway4 is the IPv4 gateway used by this device. + Required when DHCP4 is false. type: string - type: array - macAddr: - description: MACAddr is the MAC address used by this device. - It is generally a good idea to omit this field and allow - a MAC address to be generated. Please note that this value - must use the VMware OUI to work with the in-tree vSphere - cloud provider. - type: string - mtu: - description: MTU is the device’s Maximum Transmission Unit - size in bytes. - format: int64 - type: integer - nameservers: - description: Nameservers is a list of IPv4 and/or IPv6 addresses - used as DNS nameservers. Please note that Linux allows only - three nameservers (https://linux.die.net/man/5/resolv.conf). - items: + gateway6: + description: Gateway4 is the IPv4 gateway used by this device. + Required when DHCP6 is false. type: string - type: array - networkName: - description: NetworkName is the name of the vSphere network - to which the device will be connected. - type: string - routes: - description: Routes is a list of optional, static routes applied - to the device. - items: - description: NetworkRouteSpec defines a static network route. - properties: - metric: - description: Metric is the weight/priority of the route. - format: int32 - type: integer - to: - description: To is an IPv4 or IPv6 address. - type: string - via: - description: Via is an IPv4 or IPv6 address. - type: string - required: - - metric - - to - - via - type: object - type: array - searchDomains: - description: SearchDomains is a list of search domains used - when resolving IP addresses with DNS. - items: + ipAddrs: + description: IPAddrs is a list of one or more IPv4 and/or + IPv6 addresses to assign to this device. Required when + DHCP4 and DHCP6 are both false. + items: + type: string + type: array + macAddr: + description: MACAddr is the MAC address used by this device. + It is generally a good idea to omit this field and allow + a MAC address to be generated. Please note that this value + must use the VMware OUI to work with the in-tree vSphere + cloud provider. type: string - type: array - required: - - networkName - type: object - type: array - preferredAPIServerCidr: - description: PreferredAPIServeCIDR is the preferred CIDR for the - Kubernetes API server endpoint on this machine - type: string - routes: - description: Routes is a list of optional, static routes applied - to the virtual machine. - items: - description: NetworkRouteSpec defines a static network route. - properties: - metric: - description: Metric is the weight/priority of the route. - format: int32 - type: integer - to: - description: To is an IPv4 or IPv6 address. - type: string - via: - description: Via is an IPv4 or IPv6 address. - type: string - required: - - metric - - to - - via - type: object - type: array - required: - - devices - type: object - numCPUs: - description: NumCPUs is the number of virtual processors in a virtual - machine. Defaults to the analogue property value in the template from - which this machine is cloned. - format: int32 - type: integer - numCoresPerSocket: - description: NumCPUs is the number of cores among which to distribute - CPUs in this virtual machine. Defaults to the analogue property value - in the template from which this machine is cloned. - format: int32 - type: integer - providerID: - description: ProviderID is the virtual machine's BIOS UUID formated - as vsphere://12345678-1234-1234-1234-123456789abc - type: string - template: - description: Template is the name, inventory path, or instance UUID - of the template used to clone new machines. - type: string - required: - - datacenter - - network - - template - type: object - status: - description: VSphereMachineStatus defines the observed state of VSphereMachine - properties: - addresses: - description: Addresses contains the VSphere instance associated addresses. - items: - description: NodeAddress contains information for the node's address. - properties: - address: - description: The node address. - type: string - type: - description: Node address type, one of Hostname, ExternalIP or - InternalIP. + mtu: + description: MTU is the device’s Maximum Transmission Unit + size in bytes. + format: int64 + type: integer + nameservers: + description: Nameservers is a list of IPv4 and/or IPv6 addresses + used as DNS nameservers. Please note that Linux allows + only three nameservers (https://linux.die.net/man/5/resolv.conf). + items: + type: string + type: array + networkName: + description: NetworkName is the name of the vSphere network + to which the device will be connected. + type: string + routes: + description: Routes is a list of optional, static routes + applied to the device. + items: + description: NetworkRouteSpec defines a static network + route. + properties: + metric: + description: Metric is the weight/priority of the + route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + searchDomains: + description: SearchDomains is a list of search domains used + when resolving IP addresses with DNS. + items: + type: string + type: array + required: + - networkName + type: object + type: array + preferredAPIServerCidr: + description: PreferredAPIServeCIDR is the preferred CIDR for the + Kubernetes API server endpoint on this machine type: string + routes: + description: Routes is a list of optional, static routes applied + to the virtual machine. + items: + description: NetworkRouteSpec defines a static network route. + properties: + metric: + description: Metric is the weight/priority of the route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array required: - - address - - type + - devices type: object - type: array - errorMessage: - description: "ErrorMessage will be set in the event that there is a - terminal problem reconciling the Machine and will contain a more verbose - string suitable for logging and human consumption. \n This field should - not be set for transitive errors that a controller faces that are - expected to be fixed automatically over time (like service outages), - but instead indicate that something is fundamentally wrong with the - Machine's spec or the configuration of the controller, and that manual - intervention is required. Examples of terminal errors would be invalid - combinations of settings in the spec, values that are unsupported - by the controller, or the responsible controller itself being critically - misconfigured. \n Any transient errors that occur during the reconciliation - of Machines can be added as events to the Machine object and/or logged - in the controller's output." - type: string - errorReason: - description: "ErrorReason will be set in the event that there is a terminal - problem reconciling the Machine and will contain a succinct value - suitable for machine interpretation. \n This field should not be set - for transitive errors that a controller faces that are expected to - be fixed automatically over time (like service outages), but instead - indicate that something is fundamentally wrong with the Machine's - spec or the configuration of the controller, and that manual intervention - is required. Examples of terminal errors would be invalid combinations - of settings in the spec, values that are unsupported by the controller, - or the responsible controller itself being critically misconfigured. - \n Any transient errors that occur during the reconciliation of Machines - can be added as events to the Machine object and/or logged in the - controller's output." - type: string - networkStatus: - description: Network returns the network status for each of the machine's - configured network interfaces. - items: - description: NetworkStatus provides information about one of a VM's - networks. + numCPUs: + description: NumCPUs is the number of virtual processors in a virtual + machine. Defaults to the analogue property value in the template + from which this machine is cloned. + format: int32 + type: integer + numCoresPerSocket: + description: NumCPUs is the number of cores among which to distribute + CPUs in this virtual machine. Defaults to the analogue property + value in the template from which this machine is cloned. + format: int32 + type: integer + providerID: + description: ProviderID is the virtual machine's BIOS UUID formated + as vsphere://12345678-1234-1234-1234-123456789abc + type: string + template: + description: Template is the name, inventory path, or instance UUID + of the template used to clone new machines. + type: string + required: + - datacenter + - network + - template + type: object + status: + description: VSphereMachineStatus defines the observed state of VSphereMachine + properties: + addresses: + description: Addresses contains the VSphere instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + errorMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + errorReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + networkStatus: + description: Network returns the network status for each of the machine's + configured network interfaces. + items: + description: NetworkStatus provides information about one of a VM's + networks. + properties: + connected: + description: Connected is a flag that indicates whether this + network is currently connected to the VM. + type: boolean + ipAddrs: + description: IPAddrs is one or more IP addresses reported by + vm-tools. + items: + type: string + type: array + macAddr: + description: MACAddr is the MAC address of the network device. + type: string + networkName: + description: NetworkName is the name of the network. + type: string + required: + - macAddr + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + taskRef: + description: TaskRef is a managed object reference to a Task related + to the machine. This value is set automatically at runtime and should + not be set or modified by users. + type: string + type: object + type: object + served: true + storage: true + - name: v1alpha3 + schema: + openAPIV3Schema: + description: VSphereMachine is the Schema for the vspheremachines API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VSphereMachineSpec defines the desired state of VSphereMachine + properties: + datacenter: + description: Datacenter is the name or inventory path of the datacenter + in which the virtual machine is created/located. + type: string + datastore: + description: Datastore is the name or inventory path of the datastore + in which the virtual machine is created/located. + type: string + diskGiB: + description: DiskGiB is the size of a virtual machine's disk, in GiB. + Defaults to the eponymous property value in the template from which + the virtual machine is cloned. + format: int32 + type: integer + folder: + description: Folder is the name or inventory path of the folder in + which the virtual machine is created/located. + type: string + memoryMiB: + description: MemoryMiB is the size of a virtual machine's memory, + in MiB. Defaults to the eponymous property value in the template + from which the virtual machine is cloned. + format: int64 + type: integer + network: + description: Network is the network configuration for this machine's + VM. properties: - connected: - description: Connected is a flag that indicates whether this network - is currently connected to the VM. - type: boolean - ipAddrs: - description: IPAddrs is one or more IP addresses reported by vm-tools. + devices: + description: Devices is the list of network devices used by the + virtual machine. TODO(akutz) Make sure at least one network + matches the ClusterSpec.CloudProviderConfiguration.Network.Name items: - type: string + description: NetworkDeviceSpec defines the network configuration + for a virtual machine's network device. + properties: + dhcp4: + description: DHCP4 is a flag that indicates whether or not + to use DHCP for IPv4 on this device. If true then IPAddrs + should not contain any IPv4 addresses. + type: boolean + dhcp6: + description: DHCP6 is a flag that indicates whether or not + to use DHCP for IPv6 on this device. If true then IPAddrs + should not contain any IPv6 addresses. + type: boolean + gateway4: + description: Gateway4 is the IPv4 gateway used by this device. + Required when DHCP4 is false. + type: string + gateway6: + description: Gateway4 is the IPv4 gateway used by this device. + Required when DHCP6 is false. + type: string + ipAddrs: + description: IPAddrs is a list of one or more IPv4 and/or + IPv6 addresses to assign to this device. Required when + DHCP4 and DHCP6 are both false. + items: + type: string + type: array + macAddr: + description: MACAddr is the MAC address used by this device. + It is generally a good idea to omit this field and allow + a MAC address to be generated. Please note that this value + must use the VMware OUI to work with the in-tree vSphere + cloud provider. + type: string + mtu: + description: MTU is the device’s Maximum Transmission Unit + size in bytes. + format: int64 + type: integer + nameservers: + description: Nameservers is a list of IPv4 and/or IPv6 addresses + used as DNS nameservers. Please note that Linux allows + only three nameservers (https://linux.die.net/man/5/resolv.conf). + items: + type: string + type: array + networkName: + description: NetworkName is the name of the vSphere network + to which the device will be connected. + type: string + routes: + description: Routes is a list of optional, static routes + applied to the device. + items: + description: NetworkRouteSpec defines a static network + route. + properties: + metric: + description: Metric is the weight/priority of the + route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + searchDomains: + description: SearchDomains is a list of search domains used + when resolving IP addresses with DNS. + items: + type: string + type: array + required: + - networkName + type: object type: array - macAddr: - description: MACAddr is the MAC address of the network device. - type: string - networkName: - description: NetworkName is the name of the network. + preferredAPIServerCidr: + description: PreferredAPIServeCIDR is the preferred CIDR for the + Kubernetes API server endpoint on this machine type: string + routes: + description: Routes is a list of optional, static routes applied + to the virtual machine. + items: + description: NetworkRouteSpec defines a static network route. + properties: + metric: + description: Metric is the weight/priority of the route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array required: - - macAddr + - devices type: object - type: array - ready: - description: Ready is true when the provider resource is ready. - type: boolean - taskRef: - description: TaskRef is a managed object reference to a Task related - to the machine. This value is set automatically at runtime and should - not be set or modified by users. - type: string - type: object - type: object - version: v1alpha2 - versions: - - name: v1alpha2 - served: true - storage: true - - name: v1alpha3 + numCPUs: + description: NumCPUs is the number of virtual processors in a virtual + machine. Defaults to the eponymous property value in the template + from which the virtual machine is cloned. + format: int32 + type: integer + numCoresPerSocket: + description: NumCPUs is the number of cores among which to distribute + CPUs in this virtual machine. Defaults to the eponymous property + value in the template from which the virtual machine is cloned. + format: int32 + type: integer + providerID: + description: ProviderID is the virtual machine's BIOS UUID formated + as vsphere://12345678-1234-1234-1234-123456789abc + type: string + resourcePool: + description: ResourcePool is the name or inventory path of the resource + pool in which the virtual machine is created/located. + type: string + server: + description: Server is the IP address or FQDN of the vSphere server + on which the virtual machine is created/located. + type: string + template: + description: Template is the name or inventory path of the template + used to clone the virtual machine. + type: string + required: + - network + - template + type: object + status: + description: VSphereMachineStatus defines the observed state of VSphereMachine + properties: + addresses: + description: Addresses contains the VSphere instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + errorMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + errorReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object served: true storage: false status: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml index 1b5be5dc94..6514712dbc 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml @@ -17,321 +17,659 @@ spec: plural: vspheremachinetemplates singular: vspheremachinetemplate scope: Namespaced - validation: - openAPIV3Schema: - description: VSphereMachineTemplate is the Schema for the vspheremachinetemplates - API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VSphereMachineTemplateSpec defines the desired state of VSphereMachineTemplate - properties: - template: - description: VSphereMachineTemplateResource describes the data needed - to create a VSphereMachine from a template - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the - latest internal value, and may reject unrecognized values. More - info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - description: Standard object's metadata. - properties: - annotations: - additionalProperties: + version: v1alpha2 + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: VSphereMachineTemplate is the Schema for the vspheremachinetemplates + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VSphereMachineTemplateSpec defines the desired state of VSphereMachineTemplate + properties: + template: + description: VSphereMachineTemplateResource describes the data needed + to create a VSphereMachine from a template + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: Standard object's metadata. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + generateName: + description: "GenerateName is an optional prefix, used by + the server, to generate a unique name ONLY IF the Name field + has not been provided. If this field is used, the name returned + to the client will be different than the name passed. This + value will also be combined with a unique suffix. The provided + value has the same validation rules as the Name field, and + may be truncated by the length of the suffix required to + make the value unique on the server. \n If this field is + specified and the generated name exists, the server will + NOT return a 409 - instead, it will either return 201 Created + or 500 with Reason ServerTimeout indicating a unique name + could not be found in the time allotted, and the client + should retry (optionally after the time indicated in the + Retry-After header). \n Applied only if Name is not specified. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency" type: string - description: 'Annotations is an unstructured key value map stored - with a resource that may be set by external tools to store - and retrieve arbitrary metadata. They are not queryable and - should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' - type: object - generateName: - description: "GenerateName is an optional prefix, used by the - server, to generate a unique name ONLY IF the Name field has - not been provided. If this field is used, the name returned - to the client will be different than the name passed. This - value will also be combined with a unique suffix. The provided - value has the same validation rules as the Name field, and - may be truncated by the length of the suffix required to make - the value unique on the server. \n If this field is specified - and the generated name exists, the server will NOT return - a 409 - instead, it will either return 201 Created or 500 - with Reason ServerTimeout indicating a unique name could not - be found in the time allotted, and the client should retry - (optionally after the time indicated in the Retry-After header). - \n Applied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency" - type: string - labels: - additionalProperties: + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + name: + description: 'Name must be unique within a namespace. Is required + when creating resources, although some resources may allow + a client to request the generation of an appropriate name + automatically. Name is primarily intended for creation idempotence + and configuration definition. Cannot be updated. More info: + http://kubernetes.io/docs/user-guide/identifiers#names' type: string - description: 'Map of string keys and values that can be used - to organize and categorize (scope and select) objects. May - match selectors of replication controllers and services. More - info: http://kubernetes.io/docs/user-guide/labels' - type: object - name: - description: 'Name must be unique within a namespace. Is required - when creating resources, although some resources may allow - a client to request the generation of an appropriate name - automatically. Name is primarily intended for creation idempotence - and configuration definition. Cannot be updated. More info: - http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - namespace: - description: "Namespace defines the space within each name must - be unique. An empty namespace is equivalent to the \"default\" - namespace, but \"default\" is the canonical representation. - Not all objects are required to be scoped to a namespace - - the value of this field for those objects will be empty. \n - Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces" - type: string - ownerReferences: - description: List of objects depended by this object. If ALL - objects in the list have been deleted, this object will be - garbage collected. If this object is managed by a controller, - then an entry in this list will point to this controller, - with the controller field set to true. There cannot be more - than one managing controller. - items: - description: OwnerReference contains enough information to - let you identify an owning object. An owning object must - be in the same namespace as the dependent, or be cluster-scoped, - so there is no namespace field. + namespace: + description: "Namespace defines the space within each name + must be unique. An empty namespace is equivalent to the + \"default\" namespace, but \"default\" is the canonical + representation. Not all objects are required to be scoped + to a namespace - the value of this field for those objects + will be empty. \n Must be a DNS_LABEL. Cannot be updated. + More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + ownerReferences: + description: List of objects depended by this object. If ALL + objects in the list have been deleted, this object will + be garbage collected. If this object is managed by a controller, + then an entry in this list will point to this controller, + with the controller field set to true. There cannot be more + than one managing controller. + items: + description: OwnerReference contains enough information + to let you identify an owning object. An owning object + must be in the same namespace as the dependent, or be + cluster-scoped, so there is no namespace field. + properties: + apiVersion: + description: API version of the referent. + type: string + blockOwnerDeletion: + description: If true, AND if the owner has the "foregroundDeletion" + finalizer, then the owner cannot be deleted from the + key-value store until this reference is removed. Defaults + to false. To set this field, a user needs "delete" + permission of the owner, otherwise 422 (Unprocessable + Entity) will be returned. + type: boolean + controller: + description: If true, this reference points to the managing + controller. + type: boolean + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + uid: + description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + type: array + type: object + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + datacenter: + description: Datacenter is the name or inventory path of the + datacenter where this machine's VM is created/located. + type: string + diskGiB: + description: DiskGiB is the size of a virtual machine's disk, + in GiB. Defaults to the analogue property value in the template + from which this machine is cloned. + format: int32 + type: integer + memoryMiB: + description: MemoryMiB is the size of a virtual machine's + memory, in MiB. Defaults to the analogue property value + in the template from which this machine is cloned. + format: int64 + type: integer + network: + description: Network is the network configuration for this + machine's VM. properties: - apiVersion: - description: API version of the referent. - type: string - blockOwnerDeletion: - description: If true, AND if the owner has the "foregroundDeletion" - finalizer, then the owner cannot be deleted from the - key-value store until this reference is removed. Defaults - to false. To set this field, a user needs "delete" permission - of the owner, otherwise 422 (Unprocessable Entity) will - be returned. - type: boolean - controller: - description: If true, this reference points to the managing - controller. - type: boolean - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' - type: string - required: - - apiVersion - - kind - - name - - uid - type: object - type: array - type: object - spec: - description: Spec is the specification of the desired behavior of - the machine. - properties: - datacenter: - description: Datacenter is the name or inventory path of the - datacenter where this machine's VM is created/located. - type: string - diskGiB: - description: DiskGiB is the size of a virtual machine's disk, - in GiB. Defaults to the analogue property value in the template - from which this machine is cloned. - format: int32 - type: integer - memoryMiB: - description: MemoryMiB is the size of a virtual machine's memory, - in MiB. Defaults to the analogue property value in the template - from which this machine is cloned. - format: int64 - type: integer - network: - description: Network is the network configuration for this machine's - VM. - properties: - devices: - description: Devices is the list of network devices used - by the virtual machine. TODO(akutz) Make sure at least - one network matches the ClusterSpec.CloudProviderConfiguration.Network.Name - items: - description: NetworkDeviceSpec defines the network configuration - for a virtual machine's network device. - properties: - dhcp4: - description: DHCP4 is a flag that indicates whether - or not to use DHCP for IPv4 on this device. If true - then IPAddrs should not contain any IPv4 addresses. - type: boolean - dhcp6: - description: DHCP6 is a flag that indicates whether - or not to use DHCP for IPv6 on this device. If true - then IPAddrs should not contain any IPv6 addresses. - type: boolean - gateway4: - description: Gateway4 is the IPv4 gateway used by - this device. Required when DHCP4 is false. - type: string - gateway6: - description: Gateway4 is the IPv4 gateway used by - this device. Required when DHCP6 is false. - type: string - ipAddrs: - description: IPAddrs is a list of one or more IPv4 - and/or IPv6 addresses to assign to this device. - Required when DHCP4 and DHCP6 are both false. - items: + devices: + description: Devices is the list of network devices used + by the virtual machine. TODO(akutz) Make sure at least + one network matches the ClusterSpec.CloudProviderConfiguration.Network.Name + items: + description: NetworkDeviceSpec defines the network configuration + for a virtual machine's network device. + properties: + dhcp4: + description: DHCP4 is a flag that indicates whether + or not to use DHCP for IPv4 on this device. If + true then IPAddrs should not contain any IPv4 + addresses. + type: boolean + dhcp6: + description: DHCP6 is a flag that indicates whether + or not to use DHCP for IPv6 on this device. If + true then IPAddrs should not contain any IPv6 + addresses. + type: boolean + gateway4: + description: Gateway4 is the IPv4 gateway used by + this device. Required when DHCP4 is false. + type: string + gateway6: + description: Gateway4 is the IPv4 gateway used by + this device. Required when DHCP6 is false. + type: string + ipAddrs: + description: IPAddrs is a list of one or more IPv4 + and/or IPv6 addresses to assign to this device. + Required when DHCP4 and DHCP6 are both false. + items: + type: string + type: array + macAddr: + description: MACAddr is the MAC address used by + this device. It is generally a good idea to omit + this field and allow a MAC address to be generated. + Please note that this value must use the VMware + OUI to work with the in-tree vSphere cloud provider. type: string - type: array - macAddr: - description: MACAddr is the MAC address used by this - device. It is generally a good idea to omit this - field and allow a MAC address to be generated. Please - note that this value must use the VMware OUI to - work with the in-tree vSphere cloud provider. - type: string - mtu: - description: MTU is the device’s Maximum Transmission - Unit size in bytes. - format: int64 - type: integer - nameservers: - description: Nameservers is a list of IPv4 and/or - IPv6 addresses used as DNS nameservers. Please note - that Linux allows only three nameservers (https://linux.die.net/man/5/resolv.conf). - items: + mtu: + description: MTU is the device’s Maximum Transmission + Unit size in bytes. + format: int64 + type: integer + nameservers: + description: Nameservers is a list of IPv4 and/or + IPv6 addresses used as DNS nameservers. Please + note that Linux allows only three nameservers + (https://linux.die.net/man/5/resolv.conf). + items: + type: string + type: array + networkName: + description: NetworkName is the name of the vSphere + network to which the device will be connected. type: string - type: array - networkName: - description: NetworkName is the name of the vSphere - network to which the device will be connected. - type: string - routes: - description: Routes is a list of optional, static - routes applied to the device. - items: - description: NetworkRouteSpec defines a static network + routes: + description: Routes is a list of optional, static + routes applied to the device. + items: + description: NetworkRouteSpec defines a static + network route. + properties: + metric: + description: Metric is the weight/priority + of the route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + searchDomains: + description: SearchDomains is a list of search domains + used when resolving IP addresses with DNS. + items: + type: string + type: array + required: + - networkName + type: object + type: array + preferredAPIServerCidr: + description: PreferredAPIServeCIDR is the preferred CIDR + for the Kubernetes API server endpoint on this machine + type: string + routes: + description: Routes is a list of optional, static routes + applied to the virtual machine. + items: + description: NetworkRouteSpec defines a static network + route. + properties: + metric: + description: Metric is the weight/priority of the route. - properties: - metric: - description: Metric is the weight/priority of - the route. - format: int32 - type: integer - to: - description: To is an IPv4 or IPv6 address. - type: string - via: - description: Via is an IPv4 or IPv6 address. - type: string - required: - - metric - - to - - via - type: object - type: array - searchDomains: - description: SearchDomains is a list of search domains - used when resolving IP addresses with DNS. - items: + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. type: string - type: array - required: - - networkName - type: object - type: array - preferredAPIServerCidr: - description: PreferredAPIServeCIDR is the preferred CIDR - for the Kubernetes API server endpoint on this machine - type: string - routes: - description: Routes is a list of optional, static routes - applied to the virtual machine. - items: - description: NetworkRouteSpec defines a static network - route. - properties: - metric: - description: Metric is the weight/priority of the - route. - format: int32 - type: integer - to: - description: To is an IPv4 or IPv6 address. - type: string - via: - description: Via is an IPv4 or IPv6 address. - type: string - required: - - metric - - to - - via - type: object - type: array - required: - - devices - type: object - numCPUs: - description: NumCPUs is the number of virtual processors in - a virtual machine. Defaults to the analogue property value - in the template from which this machine is cloned. - format: int32 - type: integer - numCoresPerSocket: - description: NumCPUs is the number of cores among which to distribute - CPUs in this virtual machine. Defaults to the analogue property - value in the template from which this machine is cloned. - format: int32 - type: integer - providerID: - description: ProviderID is the virtual machine's BIOS UUID formated - as vsphere://12345678-1234-1234-1234-123456789abc - type: string - template: - description: Template is the name, inventory path, or instance - UUID of the template used to clone new machines. - type: string - required: - - datacenter - - network - - template - type: object - required: - - spec - type: object - required: - - template - type: object - type: object - version: v1alpha2 - versions: - - name: v1alpha2 + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + required: + - devices + type: object + numCPUs: + description: NumCPUs is the number of virtual processors in + a virtual machine. Defaults to the analogue property value + in the template from which this machine is cloned. + format: int32 + type: integer + numCoresPerSocket: + description: NumCPUs is the number of cores among which to + distribute CPUs in this virtual machine. Defaults to the + analogue property value in the template from which this + machine is cloned. + format: int32 + type: integer + providerID: + description: ProviderID is the virtual machine's BIOS UUID + formated as vsphere://12345678-1234-1234-1234-123456789abc + type: string + template: + description: Template is the name, inventory path, or instance + UUID of the template used to clone new machines. + type: string + required: + - datacenter + - network + - template + type: object + required: + - spec + type: object + required: + - template + type: object + type: object served: true storage: true - name: v1alpha3 + schema: + openAPIV3Schema: + description: VSphereMachineTemplate is the Schema for the vspheremachinetemplates + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VSphereMachineTemplateSpec defines the desired state of VSphereMachineTemplate + properties: + template: + description: VSphereMachineTemplateResource describes the data needed + to create a VSphereMachine from a template + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: Standard object's metadata. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + generateName: + description: "GenerateName is an optional prefix, used by + the server, to generate a unique name ONLY IF the Name field + has not been provided. If this field is used, the name returned + to the client will be different than the name passed. This + value will also be combined with a unique suffix. The provided + value has the same validation rules as the Name field, and + may be truncated by the length of the suffix required to + make the value unique on the server. \n If this field is + specified and the generated name exists, the server will + NOT return a 409 - instead, it will either return 201 Created + or 500 with Reason ServerTimeout indicating a unique name + could not be found in the time allotted, and the client + should retry (optionally after the time indicated in the + Retry-After header). \n Applied only if Name is not specified. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency" + type: string + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + name: + description: 'Name must be unique within a namespace. Is required + when creating resources, although some resources may allow + a client to request the generation of an appropriate name + automatically. Name is primarily intended for creation idempotence + and configuration definition. Cannot be updated. More info: + http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within each name + must be unique. An empty namespace is equivalent to the + \"default\" namespace, but \"default\" is the canonical + representation. Not all objects are required to be scoped + to a namespace - the value of this field for those objects + will be empty. \n Must be a DNS_LABEL. Cannot be updated. + More info: http://kubernetes.io/docs/user-guide/namespaces" + type: string + ownerReferences: + description: List of objects depended by this object. If ALL + objects in the list have been deleted, this object will + be garbage collected. If this object is managed by a controller, + then an entry in this list will point to this controller, + with the controller field set to true. There cannot be more + than one managing controller. + items: + description: OwnerReference contains enough information + to let you identify an owning object. An owning object + must be in the same namespace as the dependent, or be + cluster-scoped, so there is no namespace field. + properties: + apiVersion: + description: API version of the referent. + type: string + blockOwnerDeletion: + description: If true, AND if the owner has the "foregroundDeletion" + finalizer, then the owner cannot be deleted from the + key-value store until this reference is removed. Defaults + to false. To set this field, a user needs "delete" + permission of the owner, otherwise 422 (Unprocessable + Entity) will be returned. + type: boolean + controller: + description: If true, this reference points to the managing + controller. + type: boolean + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + uid: + description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + type: array + type: object + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + datacenter: + description: Datacenter is the name or inventory path of the + datacenter in which the virtual machine is created/located. + type: string + datastore: + description: Datastore is the name or inventory path of the + datastore in which the virtual machine is created/located. + type: string + diskGiB: + description: DiskGiB is the size of a virtual machine's disk, + in GiB. Defaults to the eponymous property value in the + template from which the virtual machine is cloned. + format: int32 + type: integer + folder: + description: Folder is the name or inventory path of the folder + in which the virtual machine is created/located. + type: string + memoryMiB: + description: MemoryMiB is the size of a virtual machine's + memory, in MiB. Defaults to the eponymous property value + in the template from which the virtual machine is cloned. + format: int64 + type: integer + network: + description: Network is the network configuration for this + machine's VM. + properties: + devices: + description: Devices is the list of network devices used + by the virtual machine. TODO(akutz) Make sure at least + one network matches the ClusterSpec.CloudProviderConfiguration.Network.Name + items: + description: NetworkDeviceSpec defines the network configuration + for a virtual machine's network device. + properties: + dhcp4: + description: DHCP4 is a flag that indicates whether + or not to use DHCP for IPv4 on this device. If + true then IPAddrs should not contain any IPv4 + addresses. + type: boolean + dhcp6: + description: DHCP6 is a flag that indicates whether + or not to use DHCP for IPv6 on this device. If + true then IPAddrs should not contain any IPv6 + addresses. + type: boolean + gateway4: + description: Gateway4 is the IPv4 gateway used by + this device. Required when DHCP4 is false. + type: string + gateway6: + description: Gateway4 is the IPv4 gateway used by + this device. Required when DHCP6 is false. + type: string + ipAddrs: + description: IPAddrs is a list of one or more IPv4 + and/or IPv6 addresses to assign to this device. + Required when DHCP4 and DHCP6 are both false. + items: + type: string + type: array + macAddr: + description: MACAddr is the MAC address used by + this device. It is generally a good idea to omit + this field and allow a MAC address to be generated. + Please note that this value must use the VMware + OUI to work with the in-tree vSphere cloud provider. + type: string + mtu: + description: MTU is the device’s Maximum Transmission + Unit size in bytes. + format: int64 + type: integer + nameservers: + description: Nameservers is a list of IPv4 and/or + IPv6 addresses used as DNS nameservers. Please + note that Linux allows only three nameservers + (https://linux.die.net/man/5/resolv.conf). + items: + type: string + type: array + networkName: + description: NetworkName is the name of the vSphere + network to which the device will be connected. + type: string + routes: + description: Routes is a list of optional, static + routes applied to the device. + items: + description: NetworkRouteSpec defines a static + network route. + properties: + metric: + description: Metric is the weight/priority + of the route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + searchDomains: + description: SearchDomains is a list of search domains + used when resolving IP addresses with DNS. + items: + type: string + type: array + required: + - networkName + type: object + type: array + preferredAPIServerCidr: + description: PreferredAPIServeCIDR is the preferred CIDR + for the Kubernetes API server endpoint on this machine + type: string + routes: + description: Routes is a list of optional, static routes + applied to the virtual machine. + items: + description: NetworkRouteSpec defines a static network + route. + properties: + metric: + description: Metric is the weight/priority of the + route. + format: int32 + type: integer + to: + description: To is an IPv4 or IPv6 address. + type: string + via: + description: Via is an IPv4 or IPv6 address. + type: string + required: + - metric + - to + - via + type: object + type: array + required: + - devices + type: object + numCPUs: + description: NumCPUs is the number of virtual processors in + a virtual machine. Defaults to the eponymous property value + in the template from which the virtual machine is cloned. + format: int32 + type: integer + numCoresPerSocket: + description: NumCPUs is the number of cores among which to + distribute CPUs in this virtual machine. Defaults to the + eponymous property value in the template from which the + virtual machine is cloned. + format: int32 + type: integer + providerID: + description: ProviderID is the virtual machine's BIOS UUID + formated as vsphere://12345678-1234-1234-1234-123456789abc + type: string + resourcePool: + description: ResourcePool is the name or inventory path of + the resource pool in which the virtual machine is created/located. + type: string + server: + description: Server is the IP address or FQDN of the vSphere + server on which the virtual machine is created/located. + type: string + template: + description: Template is the name or inventory path of the + template used to clone the virtual machine. + type: string + required: + - network + - template + type: object + required: + - spec + type: object + required: + - template + type: object + type: object served: true storage: false status: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml index 6a857bc153..73e69125ef 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml @@ -80,30 +80,31 @@ spec: type: object datacenter: description: Datacenter is the name or inventory path of the datacenter - where this VM is created/located. + in which the virtual machine is created/located. type: string datastore: - description: Datastore is the name of inventory path of the datastore - where this VM is located/created. + description: Datastore is the name or inventory path of the datastore + in which the virtual machine is created/located. type: string diskGiB: description: DiskGiB is the size of a virtual machine's disk, in GiB. - Defaults to the analogue property value in the template from which - this VM is cloned. + Defaults to the eponymous property value in the template from which + the virtual machine is cloned. format: int32 type: integer folder: - description: Folder is the name of inventory path of the folder where - this VM is located/created. + description: Folder is the name or inventory path of the folder in which + the virtual machine is created/located. type: string memoryMiB: description: MemoryMiB is the size of a virtual machine's memory, in - MiB. Defaults to the analogue property value in the template from - which this VM is cloned. + MiB. Defaults to the eponymous property value in the template from + which the virtual machine is cloned. format: int64 type: integer network: - description: Network is the network configuration for this VM. + description: Network is the network configuration for this machine's + VM. properties: devices: description: Devices is the list of network devices used by the @@ -224,35 +225,30 @@ spec: type: object numCPUs: description: NumCPUs is the number of virtual processors in a virtual - machine. Defaults to the analogue property value in the template from - which this VM is cloned. + machine. Defaults to the eponymous property value in the template + from which the virtual machine is cloned. format: int32 type: integer numCoresPerSocket: - description: NumCoresPerSocket is the number of cores among which to - distribute CPUs in this virtual machine. Defaults to the analogue - property value in the template from which this VM is cloned. + description: NumCPUs is the number of cores among which to distribute + CPUs in this virtual machine. Defaults to the eponymous property value + in the template from which the virtual machine is cloned. format: int32 type: integer resourcePool: - description: ResourcePool is the name of inventory path of the resource - pool where this VM is located/created. + description: ResourcePool is the name or inventory path of the resource + pool in which the virtual machine is created/located. type: string server: - description: Server is the name of the vSphere server on which this - VM is created/located. + description: Server is the IP address or FQDN of the vSphere server + on which the virtual machine is created/located. type: string template: - description: Template is the name, inventory path, or instance UUID - of the template used to clone new VMs. + description: Template is the name or inventory path of the template + used to clone the virtual machine. type: string required: - - datacenter - - datastore - - folder - network - - resourcePool - - server - template type: object status: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 539d9786ba..f8628ccb90 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,8 @@ resources: - bases/infrastructure.cluster.x-k8s.io_vspheremachines.yaml - bases/infrastructure.cluster.x-k8s.io_vsphereclusters.yaml - bases/infrastructure.cluster.x-k8s.io_vspheremachinetemplates.yaml +- bases/infrastructure.cluster.x-k8s.io_vspherevms.yaml +- bases/infrastructure.cluster.x-k8s.io_haproxyloadbalancers.yaml # +kubebuilder:scaffold:crdkustomizeresource #patches: diff --git a/controllers/vspherecluster_controller.go b/controllers/vspherecluster_controller.go index 047845ddd5..d5916a7e6e 100644 --- a/controllers/vspherecluster_controller.go +++ b/controllers/vspherecluster_controller.go @@ -26,6 +26,7 @@ import ( apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" @@ -33,6 +34,7 @@ import ( "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -181,7 +183,7 @@ func (r clusterReconciler) reconcileDelete(ctx *context.ClusterContext) (reconci ctx.Logger.Info("Reconciling VSphereCluster delete") // Cluster is deleted so remove the finalizer. - ctx.VSphereCluster.Finalizers = clusterutilv1.Filter(ctx.VSphereCluster.Finalizers, infrav1.ClusterFinalizer) + ctrlutil.RemoveFinalizer(ctx.VSphereCluster, infrav1.ClusterFinalizer) return reconcile.Result{}, nil } @@ -189,25 +191,22 @@ func (r clusterReconciler) reconcileDelete(ctx *context.ClusterContext) (reconci func (r clusterReconciler) reconcileNormal(ctx *context.ClusterContext) (reconcile.Result, error) { ctx.Logger.Info("Reconciling VSphereCluster") - // TODO(akutz) Update this logic to include infrastructure prep such as: - // * Downloading OVAs into the content library for any machines that - // use them. - // * Create any load balancers for VMC on AWS, etc. - ctx.VSphereCluster.Status.Ready = true - ctx.Logger.V(6).Info("VSphereCluster is infrastructure-ready") - // If the VSphereCluster doesn't have our finalizer, add it. - if !clusterutilv1.Contains(ctx.VSphereCluster.Finalizers, infrav1.ClusterFinalizer) { - ctx.VSphereCluster.Finalizers = append(ctx.VSphereCluster.Finalizers, infrav1.ClusterFinalizer) - ctx.Logger.V(6).Info( - "adding finalizer for VSphereCluster", - "cluster-namespace", ctx.VSphereCluster.Namespace, - "cluster-name", ctx.VSphereCluster.Name) + ctrlutil.AddFinalizer(ctx.VSphereCluster, infrav1.ClusterFinalizer) + + // Reconcile the VSphereCluster resource's ready state. + if err := r.reconcileReadyState(ctx); err != nil { + if err == errNotReady { + ctx.Logger.Info("VSphereCluster is not ready") + return reconcile.Result{}, nil + } + return reconcile.Result{}, errors.Wrapf(err, + "failed to reconcile ready state for %s", ctx) } - // Update the VSphereCluster resource with a control plane endpoint. + // Reconcile the VSphereCluster resource's control plane endpoint. if err := r.reconcileControlPlaneEndpoint(ctx); err != nil { - if err == infrautilv1.ErrNoMachineIPAddr { + if err == errNotReady { ctx.Logger.Info("Waiting on an control plane endpoint") return reconcile.Result{}, nil } @@ -244,18 +243,118 @@ func (r clusterReconciler) reconcileNormal(ctx *context.ClusterContext) (reconci return reconcile.Result{}, nil } +func (r clusterReconciler) reconcileReadyState(ctx *context.ClusterContext) error { + if ctx.VSphereCluster.Spec.LoadBalancerRef == nil { + ctx.VSphereCluster.Status.Ready = true + return nil + } + if err := r.reconcileLoadBalancer(ctx); err != nil { + return err + } + ctx.VSphereCluster.Status.Ready = true + return nil +} + +var errNotReady = errors.New("not ready") + +func (r clusterReconciler) reconcileLoadBalancer(ctx *context.ClusterContext) error { + loadBalancerRef := ctx.VSphereCluster.Spec.LoadBalancerRef + loadBalancer := &unstructured.Unstructured{} + loadBalancerKey := types.NamespacedName{ + Namespace: loadBalancerRef.Namespace, + Name: loadBalancerRef.Name, + } + if err := ctx.Client.Get(ctx, loadBalancerKey, loadBalancer); err != nil { + if apierrors.IsNotFound(err) { + ctx.Logger.Info("Resource specified by LoadBalancerRef not found", + "load-balancer-gvk", loadBalancerRef.GroupVersionKind().String(), + "load-balancer-namespace", loadBalancerRef.Namespace, + "load-balancer-name", loadBalancerRef.Name) + return errNotReady + } + return err + } + + loadBalancerStatus, _ := loadBalancer.Object["status"].(map[string]interface{}) + if loadBalancerStatus == nil { + return errors.Errorf("load balancer status is nil %s %s/%s", + loadBalancerRef.GroupVersionKind().String(), + loadBalancerRef.Namespace, + loadBalancerRef.Name) + } + + if ready, _ := loadBalancerStatus["ready"].(bool); !ready { + ctx.Logger.Info("LoadBalancerRef.Status.Ready is false", + "load-balancer-gvk", loadBalancerRef.GroupVersionKind().String(), + "load-balancer-namespace", loadBalancerRef.Namespace, + "load-balancer-name", loadBalancerRef.Name) + return errNotReady + } + + address := loadBalancerStatus["address"].(string) + if address == "" { + ctx.Logger.Info("LoadBalancerRef.Status.Address is empty", + "load-balancer-gvk", loadBalancerRef.GroupVersionKind().String(), + "load-balancer-namespace", loadBalancerRef.Namespace, + "load-balancer-name", loadBalancerRef.Name) + return errNotReady + } + + // Update the VSphereCluster.Spec.ControlPlaneEndpoint with the address + // from the load balancer. + // The control plane endpoint also requires a port, but assigning the + // default port is handled in reconcileControlPlaneEndpoint. + ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Host = address + + return nil +} + func (r clusterReconciler) reconcileControlPlaneEndpoint(ctx *context.ClusterContext) error { // If the cluster already has a control plane endpoint set then there // is nothing to do. if !ctx.Cluster.Spec.ControlPlaneEndpoint.IsZero() { + ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Host = ctx.Cluster.Spec.ControlPlaneEndpoint.Host + ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Port = ctx.Cluster.Spec.ControlPlaneEndpoint.Port ctx.Logger.V(4).Info("ControlPlaneEndpoint already set on Cluster") return nil } + + // Enqueue a reconcile request for the cluster when the target API server is + // online. + go func() { + // Block until the target API server is online. + wait.PollImmediateInfinite(time.Second*1, func() (bool, error) { return r.isAPIServerOnline(ctx), nil }) + ctx.Logger.Info("triggering GenericEvent", "reason", "api-server-online") + eventChannel := ctx.GetGenericEventChannelFor(ctx.VSphereCluster.GetObjectKind().GroupVersionKind()) + eventChannel <- event.GenericEvent{ + Meta: ctx.VSphereCluster, + Object: ctx.VSphereCluster, + } + }() + if !ctx.VSphereCluster.Spec.ControlPlaneEndpoint.IsZero() { - ctx.Logger.V(4).Info("ControlPlaneEndpoint already set on VSphereCluster") + ctx.Logger.Info( + "ControlPlaneEndpoint already set on VSphereCluster", + "host", ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Host, + "port", ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Port) return nil } + // If the VSphereCluster only has the host-part of a control plane endpoint, + // then it is likely the control plane endpoint was provided via a load + // balancer. In this case, when the port is zero, assign the default API + // port to the control plane endpoint. + if ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Host != "" { + if ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Port == 0 { + ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Port = apiEndpointPort + ctx.Logger.Info( + "ControlPlaneEndpoint.Host already set on VSphereCluster", + "host", ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Host, + "port", ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Port) + return nil + } + } + // Get the CAPI Machine resources for the cluster. machines, err := infrautilv1.GetMachinesInCluster(ctx, ctx.Client, ctx.VSphereCluster.Namespace, ctx.VSphereCluster.Name) if err != nil { @@ -300,27 +399,13 @@ func (r clusterReconciler) reconcileControlPlaneEndpoint(ctx *context.ClusterCon ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Host = ipAddr ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Port = apiEndpointPort - // Enqueue a reconcile request for the cluster when the target API - // server is online. - go func() { - // Block until the target API server is online. - wait.PollImmediateInfinite(time.Second*1, func() (bool, error) { return r.isAPIServerOnline(ctx), nil }) - ctx.Logger.Info("triggering GenericEvent", "reason", "api-server-online") - eventChannel := ctx.GetGenericEventChannelFor(ctx.VSphereCluster.GetObjectKind().GroupVersionKind()) - eventChannel <- event.GenericEvent{ - Meta: ctx.VSphereCluster, - Object: ctx.VSphereCluster, - } - }() - - ctx.Logger.V(4).Info( - "found API endpoint via control plane machine", + ctx.Logger.Info( + "ControlPlaneEndpoin discovered via control plane machine", "host", ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Host, "port", ctx.VSphereCluster.Spec.ControlPlaneEndpoint.Port) - return nil } - return infrautilv1.ErrNoMachineIPAddr + return errNotReady } func (r clusterReconciler) isAPIServerOnline(ctx *context.ClusterContext) bool { @@ -556,7 +641,7 @@ func (r clusterReconciler) controlPlaneMachineToCluster(o handler.MapObject) []c if !infrautilv1.IsControlPlaneMachine(vsphereMachine) { return nil } - if len(vsphereMachine.Status.Network) == 0 { + if len(vsphereMachine.Status.Addresses) == 0 { return nil } // Get the VSphereMachine's preferred IP address. diff --git a/controllers/vspheremachine_controller.go b/controllers/vspheremachine_controller.go index 0567ce1907..4db7fe380a 100644 --- a/controllers/vspheremachine_controller.go +++ b/controllers/vspheremachine_controller.go @@ -254,17 +254,41 @@ func (r machineReconciler) reconcileNormal(ctx *context.MachineContext) (reconci vm.Namespace, vm.Name) } vm.Spec.BootstrapRef = ctx.Machine.Spec.Bootstrap.ConfigRef - vm.Spec.Server = ctx.VSphereCluster.Spec.Server - vm.Spec.Datacenter = ctx.VSphereMachine.Spec.Datacenter - vm.Spec.Datastore = ctx.VSphereCluster.Spec.CloudProviderConfiguration.Workspace.Datastore - vm.Spec.Folder = ctx.VSphereCluster.Spec.CloudProviderConfiguration.Workspace.Folder - vm.Spec.ResourcePool = ctx.VSphereCluster.Spec.CloudProviderConfiguration.Workspace.ResourcePool + + // Several of the VSphereVM's clone spec properties can be derived + // from multiple places. The order is: + // + // 1. From the VSphereMachine.Spec + // 2. From the VSphereCluster.Spec.CloudProviderConfiguration.Workspace + // 3. From the VSphereCluster.Spec + vsphereMachineSpec := ctx.VSphereMachine.Spec + vsphereClusterSpec := ctx.VSphereCluster.Spec + vsphereCloudConfig := ctx.VSphereCluster.Spec.CloudProviderConfiguration.Workspace + if vm.Spec.Server = vsphereMachineSpec.Server; vm.Spec.Server == "" { + if vm.Spec.Server = vsphereCloudConfig.Server; vm.Spec.Server == "" { + vm.Spec.Server = vsphereClusterSpec.Server + } + } + if vm.Spec.Datacenter = vsphereMachineSpec.Datacenter; vm.Spec.Datacenter == "" { + vm.Spec.Datacenter = vsphereCloudConfig.Datacenter + } + if vm.Spec.Datastore = vsphereMachineSpec.Datastore; vm.Spec.Datastore == "" { + vm.Spec.Datastore = vsphereCloudConfig.Datastore + } + if vm.Spec.Folder = vsphereMachineSpec.Folder; vm.Spec.Folder == "" { + vm.Spec.Folder = vsphereCloudConfig.Folder + } + if vm.Spec.ResourcePool = vsphereMachineSpec.ResourcePool; vm.Spec.ResourcePool == "" { + vm.Spec.ResourcePool = vsphereCloudConfig.ResourcePool + } + vm.Spec.Network = ctx.VSphereMachine.Spec.Network vm.Spec.NumCPUs = ctx.VSphereMachine.Spec.NumCPUs vm.Spec.NumCoresPerSocket = ctx.VSphereMachine.Spec.NumCoresPerSocket vm.Spec.MemoryMiB = ctx.VSphereMachine.Spec.MemoryMiB vm.Spec.DiskGiB = ctx.VSphereMachine.Spec.DiskGiB vm.Spec.Template = ctx.VSphereMachine.Spec.Template + return nil } if _, err := ctrlutil.CreateOrUpdate(ctx, ctx.Client, vm, mutateFn); err != nil { @@ -306,13 +330,11 @@ func (r machineReconciler) reconcileNetwork(ctx *context.MachineContext, vm *inf if expNetCount != actNetCount { return false, errors.Errorf("invalid network count for %q: exp=%d act=%d", ctx, expNetCount, actNetCount) } - ctx.VSphereMachine.Status.Network = vm.Status.Network - // If the VM is powered on then issue requeues until all of the VM's - // networks have IP addresses. + // Translate the VSphereVM.Status.Network field into a list of NodeAddress + // resources and store them in the VSphereMachine.Status.Addresses field. var ipAddrs []corev1.NodeAddress - - for _, netStatus := range ctx.VSphereMachine.Status.Network { + for _, netStatus := range vm.Status.Network { for _, ip := range netStatus.IPAddrs { ipAddrs = append(ipAddrs, corev1.NodeAddress{ Type: corev1.NodeInternalIP, @@ -326,7 +348,7 @@ func (r machineReconciler) reconcileNetwork(ctx *context.MachineContext, vm *inf return false, nil } - // Use the collected IP addresses to assign the Machine's addresses. + // Use the collected IP addresses to assign the VSphereMachine's addresses. ctx.VSphereMachine.Status.Addresses = ipAddrs return true, nil diff --git a/main.go b/main.go index 817d1cf236..76e8da7524 100644 --- a/main.go +++ b/main.go @@ -144,6 +144,9 @@ func main() { if err := controllers.AddMachineControllerToManager(ctx, mgr); err != nil { return err } + if err := controllers.AddVMControllerToManager(ctx, mgr); err != nil { + return err + } return nil } diff --git a/pkg/context/fake/fake_machine_context.go b/pkg/context/fake/fake_machine_context.go index cf7467f981..8e56a0ebac 100644 --- a/pkg/context/fake/fake_machine_context.go +++ b/pkg/context/fake/fake_machine_context.go @@ -78,19 +78,21 @@ func newVSphereMachine(owner clusterv1a2.Machine) infrav1.VSphereMachine { }, }, Spec: infrav1.VSphereMachineSpec{ - Datacenter: "dc0", - Network: infrav1.NetworkSpec{ - Devices: []infrav1.NetworkDeviceSpec{ - { - NetworkName: "VM Network", - DHCP4: true, - DHCP6: true, + VirtualMachineCloneSpec: infrav1.VirtualMachineCloneSpec{ + Datacenter: "dc0", + Network: infrav1.NetworkSpec{ + Devices: []infrav1.NetworkDeviceSpec{ + { + NetworkName: "VM Network", + DHCP4: true, + DHCP6: true, + }, }, }, + NumCPUs: 2, + MemoryMiB: 2048, + DiskGiB: 20, }, - NumCPUs: 2, - MemoryMiB: 2048, - DiskGiB: 20, }, } } diff --git a/pkg/context/fake/fake_vm_context.go b/pkg/context/fake/fake_vm_context.go index 9470c392d9..913bd2585d 100644 --- a/pkg/context/fake/fake_vm_context.go +++ b/pkg/context/fake/fake_vm_context.go @@ -51,20 +51,22 @@ func newVSphereVM() infrav1.VSphereVM { UID: types.UID(VSphereVMUUID), }, Spec: infrav1.VSphereVMSpec{ - Server: "10.10.10.10", - Datacenter: "dc0", - Network: infrav1.NetworkSpec{ - Devices: []infrav1.NetworkDeviceSpec{ - { - NetworkName: "VM Network", - DHCP4: true, - DHCP6: true, + VirtualMachineCloneSpec: infrav1.VirtualMachineCloneSpec{ + Server: "10.10.10.10", + Datacenter: "dc0", + Network: infrav1.NetworkSpec{ + Devices: []infrav1.NetworkDeviceSpec{ + { + NetworkName: "VM Network", + DHCP4: true, + DHCP6: true, + }, }, }, + NumCPUs: 2, + MemoryMiB: 2048, + DiskGiB: 20, }, - NumCPUs: 2, - MemoryMiB: 2048, - DiskGiB: 20, }, } } diff --git a/pkg/util/kubeadm.go b/pkg/util/kubeadm.go index b459eb4555..fa3bd21927 100644 --- a/pkg/util/kubeadm.go +++ b/pkg/util/kubeadm.go @@ -85,7 +85,7 @@ func GetAPIEndpointForControlPlaneEndpoint(controlPlaneEndpoint string) (*infrav } if szPort := u.Port(); szPort != "" { - port, err := strconv.Atoi(szPort) + port, err := strconv.ParseInt(szPort, 10, 32) if err != nil { return nil, errors.Wrapf( err, @@ -93,7 +93,7 @@ func GetAPIEndpointForControlPlaneEndpoint(controlPlaneEndpoint string) (*infrav szPort, controlPlaneEndpoint) } - apiEndpoint.Port = port + apiEndpoint.Port = int32(port) } return apiEndpoint, nil diff --git a/pkg/util/machines_test.go b/pkg/util/machines_test.go index 9b8168b015..4dd20f3895 100644 --- a/pkg/util/machines_test.go +++ b/pkg/util/machines_test.go @@ -90,8 +90,10 @@ func Test_GetMachinePreferredIPAddress(t *testing.T) { name: "multiple IPv4 addresses, preferred CIDR set to v4", machine: &v1alpha3.VSphereMachine{ Spec: v1alpha3.VSphereMachineSpec{ - Network: v1alpha3.NetworkSpec{ - PreferredAPIServerCIDR: "192.168.0.0/16", + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + PreferredAPIServerCIDR: "192.168.0.0/16", + }, }, }, Status: v1alpha3.VSphereMachineStatus{ @@ -114,8 +116,10 @@ func Test_GetMachinePreferredIPAddress(t *testing.T) { name: "multiple IPv4 and IPv6 addresses, preferred CIDR set to v4", machine: &v1alpha3.VSphereMachine{ Spec: v1alpha3.VSphereMachineSpec{ - Network: v1alpha3.NetworkSpec{ - PreferredAPIServerCIDR: "192.168.0.0/16", + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + PreferredAPIServerCIDR: "192.168.0.0/16", + }, }, }, Status: v1alpha3.VSphereMachineStatus{ @@ -138,8 +142,10 @@ func Test_GetMachinePreferredIPAddress(t *testing.T) { name: "multiple IPv4 and IPv6 addresses, preferred CIDR set to v6", machine: &v1alpha3.VSphereMachine{ Spec: v1alpha3.VSphereMachineSpec{ - Network: v1alpha3.NetworkSpec{ - PreferredAPIServerCIDR: "fdf3:35b5:9dad:6e09::/64", + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + PreferredAPIServerCIDR: "fdf3:35b5:9dad:6e09::/64", + }, }, }, Status: v1alpha3.VSphereMachineStatus{ @@ -163,8 +169,10 @@ func Test_GetMachinePreferredIPAddress(t *testing.T) { name: "no addresses found", machine: &v1alpha3.VSphereMachine{ Spec: v1alpha3.VSphereMachineSpec{ - Network: v1alpha3.NetworkSpec{ - PreferredAPIServerCIDR: "fdf3:35b5:9dad:6e09::/64", + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + PreferredAPIServerCIDR: "fdf3:35b5:9dad:6e09::/64", + }, }, }, Status: v1alpha3.VSphereMachineStatus{ @@ -178,8 +186,10 @@ func Test_GetMachinePreferredIPAddress(t *testing.T) { name: "no addresses found with preferred CIDR", machine: &v1alpha3.VSphereMachine{ Spec: v1alpha3.VSphereMachineSpec{ - Network: v1alpha3.NetworkSpec{ - PreferredAPIServerCIDR: "192.168.0.0/16", + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + PreferredAPIServerCIDR: "192.168.0.0/16", + }, }, }, Status: v1alpha3.VSphereMachineStatus{ @@ -225,12 +235,14 @@ func Test_GetMachineMetadata(t *testing.T) { name: "dhcp4", machine: &v1alpha3.VSphereVM{ Spec: v1alpha3.VSphereVMSpec{ - Network: v1alpha3.NetworkSpec{ - Devices: []v1alpha3.NetworkDeviceSpec{ - { - NetworkName: "network1", - MACAddr: "00:00:00:00:00", - DHCP4: true, + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + Devices: []v1alpha3.NetworkDeviceSpec{ + { + NetworkName: "network1", + MACAddr: "00:00:00:00:00", + DHCP4: true, + }, }, }, }, @@ -254,12 +266,14 @@ network: name: "dhcp6", machine: &v1alpha3.VSphereVM{ Spec: v1alpha3.VSphereVMSpec{ - Network: v1alpha3.NetworkSpec{ - Devices: []v1alpha3.NetworkDeviceSpec{ - { - NetworkName: "network1", - MACAddr: "00:00:00:00:00", - DHCP6: true, + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + Devices: []v1alpha3.NetworkDeviceSpec{ + { + NetworkName: "network1", + MACAddr: "00:00:00:00:00", + DHCP6: true, + }, }, }, }, @@ -283,13 +297,15 @@ network: name: "dhcp4+dhcp6", machine: &v1alpha3.VSphereVM{ Spec: v1alpha3.VSphereVMSpec{ - Network: v1alpha3.NetworkSpec{ - Devices: []v1alpha3.NetworkDeviceSpec{ - { - NetworkName: "network1", - MACAddr: "00:00:00:00:00", - DHCP4: true, - DHCP6: true, + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + Devices: []v1alpha3.NetworkDeviceSpec{ + { + NetworkName: "network1", + MACAddr: "00:00:00:00:00", + DHCP4: true, + DHCP6: true, + }, }, }, }, @@ -313,14 +329,16 @@ network: name: "static4+dhcp6", machine: &v1alpha3.VSphereVM{ Spec: v1alpha3.VSphereVMSpec{ - Network: v1alpha3.NetworkSpec{ - Devices: []v1alpha3.NetworkDeviceSpec{ - { - NetworkName: "network1", - MACAddr: "00:00:00:00:00", - DHCP6: true, - IPAddrs: []string{"192.168.4.21"}, - Gateway4: "192.168.4.1", + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + Devices: []v1alpha3.NetworkDeviceSpec{ + { + NetworkName: "network1", + MACAddr: "00:00:00:00:00", + DHCP6: true, + IPAddrs: []string{"192.168.4.21"}, + Gateway4: "192.168.4.1", + }, }, }, }, @@ -347,21 +365,23 @@ network: name: "static4+dhcp6+static-routes", machine: &v1alpha3.VSphereVM{ Spec: v1alpha3.VSphereVMSpec{ - Network: v1alpha3.NetworkSpec{ - Devices: []v1alpha3.NetworkDeviceSpec{ - { - NetworkName: "network1", - MACAddr: "00:00:00:00:00", - DHCP6: true, - IPAddrs: []string{"192.168.4.21"}, - Gateway4: "192.168.4.1", + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + Devices: []v1alpha3.NetworkDeviceSpec{ + { + NetworkName: "network1", + MACAddr: "00:00:00:00:00", + DHCP6: true, + IPAddrs: []string{"192.168.4.21"}, + Gateway4: "192.168.4.1", + }, }, - }, - Routes: []v1alpha3.NetworkRouteSpec{ - { - To: "192.168.5.1/24", - Via: "192.168.4.254", - Metric: 3, + Routes: []v1alpha3.NetworkRouteSpec{ + { + To: "192.168.5.1/24", + Via: "192.168.4.254", + Metric: 3, + }, }, }, }, @@ -392,25 +412,27 @@ network: name: "2nets", machine: &v1alpha3.VSphereVM{ Spec: v1alpha3.VSphereVMSpec{ - Network: v1alpha3.NetworkSpec{ - Devices: []v1alpha3.NetworkDeviceSpec{ - { - NetworkName: "network1", - MACAddr: "00:00:00:00:00", - DHCP4: true, - Routes: []v1alpha3.NetworkRouteSpec{ - { - To: "192.168.5.1/24", - Via: "192.168.4.254", - Metric: 3, + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + Devices: []v1alpha3.NetworkDeviceSpec{ + { + NetworkName: "network1", + MACAddr: "00:00:00:00:00", + DHCP4: true, + Routes: []v1alpha3.NetworkRouteSpec{ + { + To: "192.168.5.1/24", + Via: "192.168.4.254", + Metric: 3, + }, }, }, - }, - { - NetworkName: "network12", - MACAddr: "00:00:00:00:01", - DHCP6: true, - MTU: mtu(100), + { + NetworkName: "network12", + MACAddr: "00:00:00:00:01", + DHCP6: true, + MTU: mtu(100), + }, }, }, }, @@ -445,22 +467,24 @@ network: name: "2nets-static+dhcp", machine: &v1alpha3.VSphereVM{ Spec: v1alpha3.VSphereVMSpec{ - Network: v1alpha3.NetworkSpec{ - Devices: []v1alpha3.NetworkDeviceSpec{ - { - NetworkName: "network1", - MACAddr: "00:00:00:00:00", - IPAddrs: []string{"192.168.4.21"}, - Gateway4: "192.168.4.1", - MTU: mtu(0), - Nameservers: []string{"1.1.1.1"}, - SearchDomains: []string{"vmware.ci"}, - }, - { - NetworkName: "network12", - MACAddr: "00:00:00:00:01", - DHCP6: true, - SearchDomains: []string{"vmware6.ci"}, + VirtualMachineCloneSpec: v1alpha3.VirtualMachineCloneSpec{ + Network: v1alpha3.NetworkSpec{ + Devices: []v1alpha3.NetworkDeviceSpec{ + { + NetworkName: "network1", + MACAddr: "00:00:00:00:00", + IPAddrs: []string{"192.168.4.21"}, + Gateway4: "192.168.4.1", + MTU: mtu(0), + Nameservers: []string{"1.1.1.1"}, + SearchDomains: []string{"vmware.ci"}, + }, + { + NetworkName: "network12", + MACAddr: "00:00:00:00:01", + DHCP6: true, + SearchDomains: []string{"vmware6.ci"}, + }, }, }, }, diff --git a/test/e2e/capv_resource_generators.go b/test/e2e/capv_resource_generators.go index 3d12c1f570..da544fc11b 100644 --- a/test/e2e/capv_resource_generators.go +++ b/test/e2e/capv_resource_generators.go @@ -144,19 +144,21 @@ func (n *NodeGenerator) Generate(clusterNamespace, clusterName string) framework }, }, Spec: infrav1.VSphereMachineSpec{ - Datacenter: vsphereDatacenter, - DiskGiB: 50, - MemoryMiB: 2048, - Network: infrav1.NetworkSpec{ - Devices: []infrav1.NetworkDeviceSpec{ - { - NetworkName: vsphereNetwork, - DHCP4: true, + VirtualMachineCloneSpec: infrav1.VirtualMachineCloneSpec{ + Datacenter: vsphereDatacenter, + DiskGiB: 50, + MemoryMiB: 2048, + Network: infrav1.NetworkSpec{ + Devices: []infrav1.NetworkDeviceSpec{ + { + NetworkName: vsphereNetwork, + DHCP4: true, + }, }, }, + NumCPUs: 2, + Template: vsphereTemplate, }, - NumCPUs: 2, - Template: vsphereTemplate, }, } @@ -264,19 +266,21 @@ func (n *MachineDeploymentGenerator) Generate(clusterNamespace, clusterName stri Spec: infrav1.VSphereMachineTemplateSpec{ Template: infrav1.VSphereMachineTemplateResource{ Spec: infrav1.VSphereMachineSpec{ - Datacenter: vsphereDatacenter, - DiskGiB: 50, - MemoryMiB: 2048, - Network: infrav1.NetworkSpec{ - Devices: []infrav1.NetworkDeviceSpec{ - { - NetworkName: vsphereNetwork, - DHCP4: true, + VirtualMachineCloneSpec: infrav1.VirtualMachineCloneSpec{ + Datacenter: vsphereDatacenter, + DiskGiB: 50, + MemoryMiB: 2048, + Network: infrav1.NetworkSpec{ + Devices: []infrav1.NetworkDeviceSpec{ + { + NetworkName: vsphereNetwork, + DHCP4: true, + }, }, }, + NumCPUs: 2, + Template: vsphereTemplate, }, - NumCPUs: 2, - Template: vsphereTemplate, }, }, }, diff --git a/test/e2e/framework/single_node_control_plane.go b/test/e2e/framework/single_node_control_plane.go index 553ed69700..5b329d0277 100644 --- a/test/e2e/framework/single_node_control_plane.go +++ b/test/e2e/framework/single_node_control_plane.go @@ -107,6 +107,7 @@ func SingleNodeControlPlane(input *SingleNodeControlPlaneInput) { } // Wait for the CAPI Cluster resource to enter the Provisioned phase. + By("waiting for cluster resources to enter the provisioned phase") Eventually(func() (string, error) { cluster := &clusterv1.Cluster{} key := client.ObjectKey{