Skip to content

Commit

Permalink
Add capi-auth-token file to control plane machines (#115)
Browse files Browse the repository at this point in the history
  • Loading branch information
HomayoonAlimohammadi authored Oct 14, 2024
1 parent eab4b8b commit f60ac2e
Show file tree
Hide file tree
Showing 9 changed files with 241 additions and 0 deletions.
1 change: 1 addition & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ RUN go mod download
COPY main.go main.go
COPY apis/ apis/
COPY controllers/ controllers/
COPY pkg/ pkg/

# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=$arch go build -a -ldflags '-s -w' -o manager main.go
Expand Down
5 changes: 5 additions & 0 deletions controllers/cloudinit/cloudinit.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,14 @@ package cloudinit
import (
"bytes"
"fmt"
"path/filepath"
"text/template"
)

var (
CAPIAuthTokenPath = filepath.Join("/capi", "etc", "token")
)

// File is a file that cloud-init will create.
type File struct {
// Content of the file to create.
Expand Down
3 changes: 3 additions & 0 deletions controllers/cloudinit/controlplane_init.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import (

// ControlPlaneInitInput defines the context needed to generate a controlplane instance to init a cluster.
type ControlPlaneInitInput struct {
// AuthToken will be used for authenticating CAPI-only requests to the cluster-agent.
AuthToken string
// CAKey is the PEM-encoded key of the cluster CA certificate.
CAKey string
// CACert is the PEM-encoded cert of the cluster CA certificate.
Expand Down Expand Up @@ -131,6 +133,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
cloudConfig.WriteFiles,
File{Content: input.CAKey, Path: filepath.Join("/var", "tmp", "ca.key"), Permissions: "0600", Owner: "root:root"},
File{Content: input.CACert, Path: filepath.Join("/var", "tmp", "ca.crt"), Permissions: "0600", Owner: "root:root"},
File{Content: input.AuthToken, Path: CAPIAuthTokenPath, Permissions: "0600", Owner: "root:root"},
)
cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...)

Expand Down
8 changes: 8 additions & 0 deletions controllers/cloudinit/controlplane_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ func TestControlPlaneInit(t *testing.T) {
t.Run("Simple", func(t *testing.T) {
g := NewWithT(t)

authToken := "capi-auth-token"
cloudConfig, err := cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{
AuthToken: authToken,
CAKey: `CA KEY DATA`,
CACert: `CA CERT DATA`,
ControlPlaneEndpoint: "k8s.my-domain.com",
Expand Down Expand Up @@ -75,6 +77,12 @@ func TestControlPlaneInit(t *testing.T) {
Permissions: "0600",
Owner: "root:root",
},
cloudinit.File{
Content: authToken,
Path: cloudinit.CAPIAuthTokenPath,
Permissions: "0600",
Owner: "root:root",
},
))

_, err = cloudinit.GenerateCloudConfig(cloudConfig)
Expand Down
8 changes: 8 additions & 0 deletions controllers/cloudinit/controlplane_join.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import (

// ControlPlaneJoinInput defines the context needed to generate a controlplane instance to join a cluster.
type ControlPlaneJoinInput struct {
// AuthToken will be used for authenticating CAPI-only requests to the cluster-agent.
AuthToken string
// ControlPlaneEndpoint is the control plane endpoint of the cluster.
ControlPlaneEndpoint string
// Token is the token that will be used for joining other nodes to the cluster.
Expand Down Expand Up @@ -109,6 +111,12 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
}

cloudConfig := NewBaseCloudConfig()
cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{
Content: input.AuthToken,
Path: CAPIAuthTokenPath,
Permissions: "0600",
Owner: "root:root",
})
cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...)
if args := input.ExtraKubeletArgs; len(args) > 0 {
cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{
Expand Down
11 changes: 11 additions & 0 deletions controllers/cloudinit/controlplane_join_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ func TestControlPlaneJoin(t *testing.T) {
t.Run("Simple", func(t *testing.T) {
g := NewWithT(t)

authToken := "capi-auth-token"
cloudConfig, err := cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
AuthToken: authToken,
ControlPlaneEndpoint: "k8s.my-domain.com",
KubernetesVersion: "v1.25.2",
ClusterAgentPort: "30000",
Expand Down Expand Up @@ -60,6 +62,15 @@ func TestControlPlaneJoin(t *testing.T) {
`microk8s add-node --token-ttl 10000 --token "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
}))

g.Expect(cloudConfig.WriteFiles).To(ContainElements(
cloudinit.File{
Content: authToken,
Path: cloudinit.CAPIAuthTokenPath,
Permissions: "0600",
Owner: "root:root",
},
))

_, err = cloudinit.GenerateCloudConfig(cloudConfig)
g.Expect(err).ToNot(HaveOccurred())
})
Expand Down
18 changes: 18 additions & 0 deletions controllers/microk8sconfig_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/source"

tokenpkg "github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token"
)

type InitLocker interface {
Expand Down Expand Up @@ -213,6 +215,10 @@ func (r *MicroK8sConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reque
return ctrl.Result{}, nil
}

if err = tokenpkg.Reconcile(ctx, r.Client, util.ObjectKey(scope.Cluster)); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to reconcile token: %w", err)
}

// Note: can't use IsFalse here because we need to handle the absence of the condition as well as false.
if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) {
log.Info("Cluster control plane is not initialized, waiting")
Expand Down Expand Up @@ -296,7 +302,13 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte
portOfDqlite = remappedDqlitePort
}

authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster))
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err)
}

controlPlaneInput := &cloudinit.ControlPlaneInitInput{
AuthToken: authToken,
CACert: *cert,
CAKey: *key,
ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host,
Expand Down Expand Up @@ -403,7 +415,13 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster))
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err)
}

controlPlaneInput := &cloudinit.ControlPlaneJoinInput{
AuthToken: authToken,
ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host,
Token: token,
TokenTTL: microk8sConfig.Spec.InitConfiguration.JoinTokenTTLInSecs,
Expand Down
101 changes: 101 additions & 0 deletions pkg/token/token.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package token

import (
"context"
cryptorand "crypto/rand"
"encoding/base64"
"fmt"

corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
AuthTokenNameSuffix = "capi-auth-token"
)

// Reconcile ensures that a token secret exists for the given cluster.
func Reconcile(ctx context.Context, c client.Client, clusterKey client.ObjectKey) error {
if _, err := getSecret(ctx, c, clusterKey); err != nil {
if apierrors.IsNotFound(err) {
if _, err := generateAndStore(ctx, c, clusterKey); err != nil {
return fmt.Errorf("failed to generate and store token: %w", err)
}
return nil
}
}

return nil
}

// Lookup retrieves the token for the given cluster.
func Lookup(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (string, error) {
secret, err := getSecret(ctx, c, clusterKey)
if err != nil {
return "", fmt.Errorf("failed to get secret: %w", err)
}

v, ok := secret.Data["token"]
if !ok {
return "", fmt.Errorf("token not found in secret")
}

return string(v), nil
}

// authTokenName returns the name of the auth-token secret, computed by convention using the name of the cluster.
func authTokenName(clusterName string) string {
return fmt.Sprintf("%s-%s", clusterName, AuthTokenNameSuffix)
}

// getSecret retrieves the token secret for the given cluster.
func getSecret(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) {
s := &corev1.Secret{}
key := client.ObjectKey{
Name: authTokenName(clusterKey.Name),
Namespace: clusterKey.Namespace,
}
if err := c.Get(ctx, key, s); err != nil {
return nil, fmt.Errorf("failed to get secret: %w", err)
}

return s, nil
}

// generateAndStore generates a new token and stores it in a secret.
func generateAndStore(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) {
token, err := randomB64(16)
if err != nil {
return nil, fmt.Errorf("failed to generate token: %w", err)
}

secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: clusterKey.Namespace,
Name: authTokenName(clusterKey.Name),
},
Data: map[string][]byte{
"token": []byte(token),
},
Type: clusterv1.ClusterSecretType,
}

if err := c.Create(ctx, secret); err != nil {
return nil, fmt.Errorf("failed to create secret: %w", err)
}

return secret, nil
}

// randomB64 generates a random base64 string of n bytes.
func randomB64(n int) (string, error) {
b := make([]byte, n)
_, err := cryptorand.Read(b)
if err != nil {
return "", fmt.Errorf("failed to read random bytes: %w", err)
}
return base64.StdEncoding.EncodeToString(b), nil
}
86 changes: 86 additions & 0 deletions pkg/token/token_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package token_test

import (
"context"
"fmt"
"testing"

. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

"github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token"
)

func TestReconcile(t *testing.T) {
t.Run("SecretAvailableSucceeds", func(t *testing.T) {
namespace := "test-namespace"
clusterName := "test-cluster"
secret := &corev1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix),
Namespace: namespace,
},
}
c := fake.NewClientBuilder().WithObjects(secret).Build()

g := NewWithT(t)

g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed())
})

t.Run("SecretNotFoundGenerates", func(t *testing.T) {
namespace := "test-namespace"
clusterName := "test-cluster"
c := fake.NewClientBuilder().Build()

g := NewWithT(t)

g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed())

s := &corev1.Secret{}
key := client.ObjectKey{
Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix),
Namespace: namespace,
}
g.Expect(c.Get(context.Background(), key, s)).To(Succeed())
g.Expect(s.ObjectMeta.Name).To(Equal(fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix)))
g.Expect(s.ObjectMeta.Namespace).To(Equal(namespace))
g.Expect(string(s.Data["token"])).ToNot(BeEmpty())
})

t.Run("LookupFailsIfNoSecret", func(t *testing.T) {
namespace := "test-namespace"
clusterName := "test-cluster"
c := fake.NewClientBuilder().Build()

g := NewWithT(t)

_, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})
g.Expect(err).To(HaveOccurred())
})

t.Run("LookupSucceedsIfSecretExists", func(t *testing.T) {
namespace := "test-namespace"
clusterName := "test-cluster"
expToken := "test-token"
secret := &corev1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix),
Namespace: namespace,
},
Data: map[string][]byte{
"token": []byte(expToken),
},
}
c := fake.NewClientBuilder().WithObjects(secret).Build()

g := NewWithT(t)

token, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(token).To(Equal(expToken))
})
}

0 comments on commit f60ac2e

Please sign in to comment.