diff --git a/Dockerfile b/Dockerfile index e8eb869..3c14177 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN go mod download COPY main.go main.go COPY apis/ apis/ COPY controllers/ controllers/ +COPY pkg/ pkg/ # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=$arch go build -a -ldflags '-s -w' -o manager main.go diff --git a/controllers/cloudinit/cloudinit.go b/controllers/cloudinit/cloudinit.go index 681e252..5319133 100644 --- a/controllers/cloudinit/cloudinit.go +++ b/controllers/cloudinit/cloudinit.go @@ -19,9 +19,14 @@ package cloudinit import ( "bytes" "fmt" + "path/filepath" "text/template" ) +var ( + CAPIAuthTokenPath = filepath.Join("/capi", "etc", "token") +) + // File is a file that cloud-init will create. type File struct { // Content of the file to create. diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index cfb5515..9dde04b 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -27,6 +27,8 @@ import ( // ControlPlaneInitInput defines the context needed to generate a controlplane instance to init a cluster. type ControlPlaneInitInput struct { + // AuthToken will be used for authenticating CAPI-only requests to the cluster-agent. + AuthToken string // CAKey is the PEM-encoded key of the cluster CA certificate. CAKey string // CACert is the PEM-encoded cert of the cluster CA certificate. @@ -131,6 +133,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { cloudConfig.WriteFiles, File{Content: input.CAKey, Path: filepath.Join("/var", "tmp", "ca.key"), Permissions: "0600", Owner: "root:root"}, File{Content: input.CACert, Path: filepath.Join("/var", "tmp", "ca.crt"), Permissions: "0600", Owner: "root:root"}, + File{Content: input.AuthToken, Path: CAPIAuthTokenPath, Permissions: "0600", Owner: "root:root"}, ) cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...) diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go index 36e7849..9031db2 100644 --- a/controllers/cloudinit/controlplane_init_test.go +++ b/controllers/cloudinit/controlplane_init_test.go @@ -28,7 +28,9 @@ func TestControlPlaneInit(t *testing.T) { t.Run("Simple", func(t *testing.T) { g := NewWithT(t) + authToken := "capi-auth-token" cloudConfig, err := cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{ + AuthToken: authToken, CAKey: `CA KEY DATA`, CACert: `CA CERT DATA`, ControlPlaneEndpoint: "k8s.my-domain.com", @@ -75,6 +77,12 @@ func TestControlPlaneInit(t *testing.T) { Permissions: "0600", Owner: "root:root", }, + cloudinit.File{ + Content: authToken, + Path: cloudinit.CAPIAuthTokenPath, + Permissions: "0600", + Owner: "root:root", + }, )) _, err = cloudinit.GenerateCloudConfig(cloudConfig) diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index 56362ea..0ac4492 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -27,6 +27,8 @@ import ( // ControlPlaneJoinInput defines the context needed to generate a controlplane instance to join a cluster. type ControlPlaneJoinInput struct { + // AuthToken will be used for authenticating CAPI-only requests to the cluster-agent. + AuthToken string // ControlPlaneEndpoint is the control plane endpoint of the cluster. ControlPlaneEndpoint string // Token is the token that will be used for joining other nodes to the cluster. @@ -109,6 +111,12 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { } cloudConfig := NewBaseCloudConfig() + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{ + Content: input.AuthToken, + Path: CAPIAuthTokenPath, + Permissions: "0600", + Owner: "root:root", + }) cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...) if args := input.ExtraKubeletArgs; len(args) > 0 { cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{ diff --git a/controllers/cloudinit/controlplane_join_test.go b/controllers/cloudinit/controlplane_join_test.go index aa31202..0d98a57 100644 --- a/controllers/cloudinit/controlplane_join_test.go +++ b/controllers/cloudinit/controlplane_join_test.go @@ -28,7 +28,9 @@ func TestControlPlaneJoin(t *testing.T) { t.Run("Simple", func(t *testing.T) { g := NewWithT(t) + authToken := "capi-auth-token" cloudConfig, err := cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ + AuthToken: authToken, ControlPlaneEndpoint: "k8s.my-domain.com", KubernetesVersion: "v1.25.2", ClusterAgentPort: "30000", @@ -60,6 +62,15 @@ func TestControlPlaneJoin(t *testing.T) { `microk8s add-node --token-ttl 10000 --token "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, })) + g.Expect(cloudConfig.WriteFiles).To(ContainElements( + cloudinit.File{ + Content: authToken, + Path: cloudinit.CAPIAuthTokenPath, + Permissions: "0600", + Owner: "root:root", + }, + )) + _, err = cloudinit.GenerateCloudConfig(cloudConfig) g.Expect(err).ToNot(HaveOccurred()) }) diff --git a/controllers/microk8sconfig_controller.go b/controllers/microk8sconfig_controller.go index c4fb761..9acb506 100644 --- a/controllers/microk8sconfig_controller.go +++ b/controllers/microk8sconfig_controller.go @@ -59,6 +59,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/source" + + tokenpkg "github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token" ) type InitLocker interface { @@ -213,6 +215,10 @@ func (r *MicroK8sConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, nil } + if err = tokenpkg.Reconcile(ctx, r.Client, util.ObjectKey(scope.Cluster)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile token: %w", err) + } + // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { log.Info("Cluster control plane is not initialized, waiting") @@ -296,7 +302,13 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte portOfDqlite = remappedDqlitePort } + authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster)) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err) + } + controlPlaneInput := &cloudinit.ControlPlaneInitInput{ + AuthToken: authToken, CACert: *cert, CAKey: *key, ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host, @@ -403,7 +415,13 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } + authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster)) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err) + } + controlPlaneInput := &cloudinit.ControlPlaneJoinInput{ + AuthToken: authToken, ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host, Token: token, TokenTTL: microk8sConfig.Spec.InitConfiguration.JoinTokenTTLInSecs, diff --git a/pkg/token/token.go b/pkg/token/token.go new file mode 100644 index 0000000..4feee74 --- /dev/null +++ b/pkg/token/token.go @@ -0,0 +1,101 @@ +package token + +import ( + "context" + cryptorand "crypto/rand" + "encoding/base64" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + AuthTokenNameSuffix = "capi-auth-token" +) + +// Reconcile ensures that a token secret exists for the given cluster. +func Reconcile(ctx context.Context, c client.Client, clusterKey client.ObjectKey) error { + if _, err := getSecret(ctx, c, clusterKey); err != nil { + if apierrors.IsNotFound(err) { + if _, err := generateAndStore(ctx, c, clusterKey); err != nil { + return fmt.Errorf("failed to generate and store token: %w", err) + } + return nil + } + } + + return nil +} + +// Lookup retrieves the token for the given cluster. +func Lookup(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (string, error) { + secret, err := getSecret(ctx, c, clusterKey) + if err != nil { + return "", fmt.Errorf("failed to get secret: %w", err) + } + + v, ok := secret.Data["token"] + if !ok { + return "", fmt.Errorf("token not found in secret") + } + + return string(v), nil +} + +// authTokenName returns the name of the auth-token secret, computed by convention using the name of the cluster. +func authTokenName(clusterName string) string { + return fmt.Sprintf("%s-%s", clusterName, AuthTokenNameSuffix) +} + +// getSecret retrieves the token secret for the given cluster. +func getSecret(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) { + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: authTokenName(clusterKey.Name), + Namespace: clusterKey.Namespace, + } + if err := c.Get(ctx, key, s); err != nil { + return nil, fmt.Errorf("failed to get secret: %w", err) + } + + return s, nil +} + +// generateAndStore generates a new token and stores it in a secret. +func generateAndStore(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) { + token, err := randomB64(16) + if err != nil { + return nil, fmt.Errorf("failed to generate token: %w", err) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterKey.Namespace, + Name: authTokenName(clusterKey.Name), + }, + Data: map[string][]byte{ + "token": []byte(token), + }, + Type: clusterv1.ClusterSecretType, + } + + if err := c.Create(ctx, secret); err != nil { + return nil, fmt.Errorf("failed to create secret: %w", err) + } + + return secret, nil +} + +// randomB64 generates a random base64 string of n bytes. +func randomB64(n int) (string, error) { + b := make([]byte, n) + _, err := cryptorand.Read(b) + if err != nil { + return "", fmt.Errorf("failed to read random bytes: %w", err) + } + return base64.StdEncoding.EncodeToString(b), nil +} diff --git a/pkg/token/token_test.go b/pkg/token/token_test.go new file mode 100644 index 0000000..419b05f --- /dev/null +++ b/pkg/token/token_test.go @@ -0,0 +1,86 @@ +package token_test + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token" +) + +func TestReconcile(t *testing.T) { + t.Run("SecretAvailableSucceeds", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix), + Namespace: namespace, + }, + } + c := fake.NewClientBuilder().WithObjects(secret).Build() + + g := NewWithT(t) + + g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed()) + }) + + t.Run("SecretNotFoundGenerates", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + c := fake.NewClientBuilder().Build() + + g := NewWithT(t) + + g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed()) + + s := &corev1.Secret{} + key := client.ObjectKey{ + Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix), + Namespace: namespace, + } + g.Expect(c.Get(context.Background(), key, s)).To(Succeed()) + g.Expect(s.ObjectMeta.Name).To(Equal(fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix))) + g.Expect(s.ObjectMeta.Namespace).To(Equal(namespace)) + g.Expect(string(s.Data["token"])).ToNot(BeEmpty()) + }) + + t.Run("LookupFailsIfNoSecret", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + c := fake.NewClientBuilder().Build() + + g := NewWithT(t) + + _, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace}) + g.Expect(err).To(HaveOccurred()) + }) + + t.Run("LookupSucceedsIfSecretExists", func(t *testing.T) { + namespace := "test-namespace" + clusterName := "test-cluster" + expToken := "test-token" + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix), + Namespace: namespace, + }, + Data: map[string][]byte{ + "token": []byte(expToken), + }, + } + c := fake.NewClientBuilder().WithObjects(secret).Build() + + g := NewWithT(t) + + token, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(token).To(Equal(expToken)) + }) +}