Fix api service reg (#1337)

* fix: allow custom k3s token * aux metrics proxy svc Signed-off-by: Ishan Khare <[email protected]> * sync endpoints and detect single binary distros Signed-off-by: Ishan Khare <[email protected]> * fix values test Signed-off-by: Ishan Khare <[email protected]> * fix lint errors Signed-off-by: Ishan Khare <[email protected]> * only add SANs when not in single binary distro Signed-off-by: Ishan Khare <[email protected]> * remove unwanted configmap Signed-off-by: Ishan Khare <[email protected]> * modify eks chart for api registration fix Signed-off-by: Ishan Khare <[email protected]> * move service creation from chart to code address other minor review comments Signed-off-by: Ishan Khare <[email protected]> * add deletion of aux services in case of toggle Signed-off-by: Ishan Khare <[email protected]> * add e2e for metrics server api registration check Signed-off-by: Ishan Khare <[email protected]> * fix metrics server helm chart install Signed-off-by: Ishan Khare <[email protected]> * modify special service default setup and remove env var Signed-off-by: Ishan Khare <[email protected]> * add tests for metrics proxy - node and pod metrics enable hostNetwork for metrics server Signed-off-by: Ishan Khare <[email protected]> * fix metrics server chart installation Signed-off-by: Ishan Khare <[email protected]> * remove single bin flag, utilize existing distro detection mechanism Signed-off-by: Ishan Khare <[email protected]> * refactor: use cache correctly --------- Signed-off-by: Ishan Khare <[email protected]> Co-authored-by: Fabian Kramm <[email protected]>
loft-sh · Nov 15, 2023 · e61c61b · e61c61b
1 parent f369daa
commit e61c61b
Show file tree

Hide file tree

Showing 36 changed files with 3,420 additions and 42 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -209,6 +209,13 @@ jobs:
           -f ${{ matrix.test-suite-path }}/values.yaml \
           "${extraArgs[@]}"
         continue-on-error: true
+
+      - name: Install metrics server in host cluster
+        id: install-metrics-server
+        run: |-
+          set -x
+          helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
+          helm upgrade --install metrics-server metrics-server/metrics-server --set args={--kubelet-insecure-tls} --set containerPort=4443 -n kube-system
 
       - name: Wait until vcluster is ready
         id: wait-until-vcluster-is-ready

diff --git a/charts/eks/templates/syncer-deployment.yaml b/charts/eks/templates/syncer-deployment.yaml
@@ -118,7 +118,7 @@ spec:
         {{- if not .Values.syncer.noArgs }}
         args:
           - --name={{ .Release.Name }}
-          - --request-header-ca-cert=/pki/ca.crt
+          - --request-header-ca-cert=/pki/front-proxy-ca.crt
           - --client-ca-cert=/pki/ca.crt
           - --server-ca-cert=/pki/ca.crt
           - --server-ca-key=/pki/ca.key

diff --git a/charts/k8s/templates/syncer-deployment.yaml b/charts/k8s/templates/syncer-deployment.yaml
@@ -148,7 +148,7 @@ spec:
         {{- if not .Values.syncer.noArgs }}
         args:
           - --name={{ .Release.Name }}
-          - --request-header-ca-cert=/pki/ca.crt
+          - --request-header-ca-cert=/pki/front-proxy-ca.crt
           - --client-ca-cert=/pki/ca.crt
           - --server-ca-cert=/pki/ca.crt
           - --server-ca-key=/pki/ca.key

diff --git a/devspace.yaml b/devspace.yaml
@@ -52,6 +52,12 @@ deployments:
           disabled: "false"
           instanceCreatorType: "devspace"
           instanceCreatorUID: "devspace"
+        proxy:
+          metricsServer:
+            nodes:
+              enabled: true
+            pods:
+              enabled: true
         sync:
           generic:
             clusterRole:

diff --git a/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go b/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go
@@ -9,6 +9,7 @@ import (
 	"github.com/loft-sh/vcluster/pkg/setup/options"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
+	"github.com/loft-sh/vcluster/pkg/specialservices"
 	"github.com/loft-sh/vcluster/pkg/util/loghelper"
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
@@ -42,6 +43,8 @@ type EndpointController struct {
 	Log loghelper.Logger
 
 	provider provider
+
+	k8sDistro bool
 }
 
 func NewEndpointController(ctx *options.ControllerContext, provider provider) *EndpointController {
@@ -53,6 +56,7 @@ func NewEndpointController(ctx *options.ControllerContext, provider provider) *E
 		VirtualManagerCache: ctx.VirtualManager.GetCache(),
 		Log:                 loghelper.New("kubernetes-default-endpoint-controller"),
 		provider:            provider,
+		k8sDistro:           ctx.Options.IsK8sDistro,
 	}
 }
 
@@ -69,6 +73,13 @@ func (e *EndpointController) Reconcile(ctx context.Context, _ ctrl.Request) (ctr
 	if err != nil {
 		return ctrl.Result{RequeueAfter: time.Second}, err
 	}
+
+	if e.k8sDistro {
+		err = e.syncMetricsServerEndpoints(ctx, e.VirtualClient, e.LocalClient, e.ServiceName, e.ServiceNamespace)
+		if err != nil {
+			return ctrl.Result{RequeueAfter: time.Second}, err
+		}
+	}
 	return ctrl.Result{}, nil
 }
 
@@ -81,7 +92,18 @@ func (e *EndpointController) SetupWithManager(mgr ctrl.Manager) error {
 	pfuncs := predicate.NewPredicateFuncs(pp)
 
 	vp := func(object client.Object) bool {
-		return object.GetNamespace() == "default" && object.GetName() == "kubernetes"
+		if object.GetNamespace() == specialservices.DefaultKubernetesSvcKey.Namespace && object.GetName() == specialservices.DefaultKubernetesSvcKey.Name {
+			return true
+		}
+
+		if e.k8sDistro {
+			if object.GetNamespace() == specialservices.VclusterProxyMetricsSvcKey.Namespace &&
+				object.GetName() == specialservices.VclusterProxyMetricsSvcKey.Name {
+				return true
+			}
+		}
+
+		return false
 	}
 	vfuncs := predicate.NewPredicateFuncs(vp)
 
@@ -99,6 +121,82 @@ func (e *EndpointController) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(e)
 }
 
+func (e *EndpointController) syncMetricsServerEndpoints(ctx context.Context, virtualClient, localClient client.Client, serviceName, serviceNamespace string) error {
+	// get physical service endpoints
+	pEndpoints := &corev1.Endpoints{}
+	err := localClient.Get(ctx, types.NamespacedName{
+		Namespace: serviceNamespace,
+		Name:      serviceName,
+	}, pEndpoints)
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			return nil
+		}
+
+		return err
+	}
+
+	vEndpoints := &corev1.Endpoints{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: specialservices.VclusterProxyMetricsSvcKey.Namespace,
+			Name:      specialservices.VclusterProxyMetricsSvcKey.Name,
+		},
+	}
+
+	result, err := controllerutil.CreateOrPatch(ctx, virtualClient, vEndpoints, func() error {
+		if vEndpoints.Labels == nil {
+			vEndpoints.Labels = map[string]string{}
+		}
+		// vEndpoints.Labels[discoveryv1.LabelSkipMirror] = "true"
+
+		// build new subsets
+		newSubsets := []corev1.EndpointSubset{}
+		for _, subset := range pEndpoints.Subsets {
+			newPorts := []corev1.EndpointPort{}
+			for _, p := range subset.Ports {
+				if p.Name != "https" {
+					continue
+				}
+
+				newPorts = append(newPorts, p)
+			}
+
+			newAddresses := []corev1.EndpointAddress{}
+			for _, address := range subset.Addresses {
+				address.Hostname = ""
+				address.NodeName = nil
+				address.TargetRef = nil
+				newAddresses = append(newAddresses, address)
+			}
+			newNotReadyAddresses := []corev1.EndpointAddress{}
+			for _, address := range subset.NotReadyAddresses {
+				address.Hostname = ""
+				address.NodeName = nil
+				address.TargetRef = nil
+				newNotReadyAddresses = append(newNotReadyAddresses, address)
+			}
+
+			newSubsets = append(newSubsets, corev1.EndpointSubset{
+				Addresses:         newAddresses,
+				NotReadyAddresses: newNotReadyAddresses,
+				Ports:             newPorts,
+			})
+		}
+
+		vEndpoints.Subsets = newSubsets
+		return nil
+	})
+	if err != nil {
+		return nil
+	}
+
+	if result == controllerutil.OperationResultCreated || result == controllerutil.OperationResultUpdated {
+		return e.provider.createOrPatch(ctx, virtualClient, vEndpoints)
+	}
+
+	return err
+}
+
 func (e *EndpointController) syncKubernetesServiceEndpoints(ctx context.Context, virtualClient client.Client, localClient client.Client, serviceName, serviceNamespace string) error {
 	// get physical service endpoints
 	pEndpoints := &corev1.Endpoints{}

diff --git a/pkg/controllers/resources/endpoints/syncer.go b/pkg/controllers/resources/endpoints/syncer.go
@@ -40,7 +40,7 @@ func (s *endpointsSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object,
 var _ syncer.Starter = &endpointsSyncer{}
 
 func (s *endpointsSyncer) ReconcileStart(ctx *synccontext.SyncContext, req ctrl.Request) (bool, error) {
-	if req.Namespace == "default" && req.Name == "kubernetes" {
+	if req.NamespacedName == specialservices.DefaultKubernetesSvcKey {
 		return true, nil
 	} else if _, ok := specialservices.Default.SpecialServicesToSync()[req.NamespacedName]; ok {
 		return true, nil

diff --git a/pkg/controllers/resources/pods/syncer_test.go b/pkg/controllers/resources/pods/syncer_test.go
@@ -7,6 +7,8 @@ import (
 	podtranslate "github.com/loft-sh/vcluster/pkg/controllers/resources/pods/translate"
 	synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context"
 	generictesting "github.com/loft-sh/vcluster/pkg/controllers/syncer/testing"
+	"github.com/loft-sh/vcluster/pkg/setup/options"
+	"github.com/loft-sh/vcluster/pkg/specialservices"
 	"github.com/loft-sh/vcluster/pkg/util/maps"
 	"github.com/loft-sh/vcluster/pkg/util/translate"
 	"gotest.tools/assert"
@@ -21,6 +23,8 @@ import (
 func TestSync(t *testing.T) {
 	translate.Default = translate.NewSingleNamespaceTranslator(generictesting.DefaultTestTargetNamespace)
 
+	specialservices.SetDefault(&options.VirtualClusterOptions{})
+
 	PodLogsVolumeName := "pod-logs"
 	LogsVolumeName := "logs"
 	KubeletPodVolumeName := "kubelet-pods"