Skip to content

Commit

Permalink
feat: add bootstrap secrets (#969)
Browse files Browse the repository at this point in the history
* feat: add bootstrap secrets

Signed-off-by: jacky-xbb <[email protected]>
  • Loading branch information
jacky-xbb authored Nov 8, 2023
1 parent 7ebf8c9 commit f8421c7
Show file tree
Hide file tree
Showing 11 changed files with 528 additions and 18 deletions.
18 changes: 15 additions & 3 deletions apis/apps/v2beta1/emqx_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,22 @@ type EMQXSpec struct {

type BootstrapAPIKey struct {
// +kubebuilder:validation:Pattern:=`^[a-zA-Z\d-_]+$`
Key string `json:"key"`
Key string `json:"key,omitempty"`
// +kubebuilder:validation:MinLength:=3
// +kubebuilder:validation:MaxLength:=32
Secret string `json:"secret"`
// +kubebuilder:validation:MaxLength:=128
Secret string `json:"secret,omitempty"`
SecretRef *SecretRef `json:"secretRef,omitempty"`
}

type SecretRef struct {
Key KeyRef `json:"key"`
Secret KeyRef `json:"secret"`
}

type KeyRef struct {
SecretName string `json:"secretName"`
// +kubebuilder:validation:Pattern:=`^[a-zA-Z\d-_]+$`
SecretKey string `json:"secretKey"`
}

type Config struct {
Expand Down
41 changes: 40 additions & 1 deletion apis/apps/v2beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 29 additions & 4 deletions config/crd/bases/apps.emqx.io_emqxes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6489,12 +6489,37 @@ spec:
pattern: ^[a-zA-Z\d-_]+$
type: string
secret:
maxLength: 32
maxLength: 128
minLength: 3
type: string
required:
- key
- secret
secretRef:
properties:
key:
properties:
secretKey:
pattern: ^[a-zA-Z\d-_]+$
type: string
secretName:
type: string
required:
- secretKey
- secretName
type: object
secret:
properties:
secretKey:
pattern: ^[a-zA-Z\d-_]+$
type: string
secretName:
type: string
required:
- secretKey
- secretName
type: object
required:
- key
- secret
type: object
type: object
type: array
clusterDomain:
Expand Down
50 changes: 46 additions & 4 deletions controllers/apps/v2beta1/add_bootstrap_resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ import (
corev1 "k8s.io/api/core/v1"
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"

appsv2beta1 "github.com/emqx/emqx-operator/apis/apps/v2beta1"
innerReq "github.com/emqx/emqx-operator/internal/requester"
Expand All @@ -23,7 +25,7 @@ type addBootstrap struct {
func (a *addBootstrap) reconcile(ctx context.Context, instance *appsv2beta1.EMQX, _ innerReq.RequesterInterface) subResult {
for _, resource := range []client.Object{
generateNodeCookieSecret(instance),
generateBootstrapAPIKeySecret(instance),
generateBootstrapAPIKeySecret(a.Client, ctx, instance),
} {
if err := ctrl.SetControllerReference(instance, resource, a.Scheme); err != nil {
return subResult{err: emperror.Wrap(err, "failed to set controller reference")}
Expand Down Expand Up @@ -63,12 +65,52 @@ func generateNodeCookieSecret(instance *appsv2beta1.EMQX) *corev1.Secret {
}
}

func generateBootstrapAPIKeySecret(instance *appsv2beta1.EMQX) *corev1.Secret {
// ReadSecret reads a secret from the Kubernetes cluster.
func ReadSecret(k8sClient client.Client, ctx context.Context, namespace string, name string, key string) (string, error) {
// Define a new Secret object
secret := &corev1.Secret{}

// Define the Secret Name and Namespace
secretName := types.NamespacedName{
Namespace: namespace,
Name: name,
}

// Use the client to fetch the Secret
if err := k8sClient.Get(ctx, secretName, secret); err != nil {
return "", err
}

// secret.Data is a map[string][]byte
secretValue := string(secret.Data[key])

return secretValue, nil
}

func generateBootstrapAPIKeySecret(k8sClient client.Client, ctx context.Context, instance *appsv2beta1.EMQX) *corev1.Secret {
logger := log.FromContext(ctx)
bootstrapAPIKeys := ""

for _, apiKey := range instance.Spec.BootstrapAPIKeys {
bootstrapAPIKeys += apiKey.Key + ":" + apiKey.Secret + "\n"
}
if apiKey.SecretRef != nil {
logger.V(1).Info("Read SecretRef")

// Read key and secret values from the refenced secrets
keyValue, err := ReadSecret(k8sClient, ctx, instance.Namespace, apiKey.SecretRef.Key.SecretName, apiKey.SecretRef.Key.SecretKey)
if err != nil {
logger.V(1).Error(err, "read secretRef", "key")
continue
}
secretValue, err := ReadSecret(k8sClient, ctx, instance.Namespace, apiKey.SecretRef.Secret.SecretName, apiKey.SecretRef.Secret.SecretKey)
if err != nil {
logger.V(1).Error(err, "read secretRef", "secret")
continue
}
bootstrapAPIKeys += keyValue + ":" + secretValue + "\n"
} else {
bootstrapAPIKeys += apiKey.Key + ":" + apiKey.Secret + "\n"
}
}
defPassword, _ := password.Generate(64, 10, 0, true, true)
bootstrapAPIKeys += appsv2beta1.DefaultBootstrapAPIKey + ":" + defPassword

Expand Down
169 changes: 169 additions & 0 deletions controllers/apps/v2beta1/add_bootstrap_resource_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
package v2beta1

import (
"context"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

appsv2beta1 "github.com/emqx/emqx-operator/apis/apps/v2beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var _ = Describe("AddBootstrap", Ordered, Label("bootstrap"), func() {
var (
instance *appsv2beta1.EMQX
a *addBootstrap
ns *corev1.Namespace
)
instance = new(appsv2beta1.EMQX)
ns = &corev1.Namespace{}

BeforeEach(func() {
a = &addBootstrap{emqxReconciler}
ns = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "controller-v2beta1-add-emqx-bootstrap-test",
Labels: map[string]string{
"test": "e2e",
},
},
}
instance = emqx.DeepCopy()
instance.Namespace = ns.Name
})

AfterEach(func() {
// Clean up bootstrap_api_key secret
bootstrapSecret := &corev1.Secret{}
err := k8sClient.Get(context.TODO(), client.ObjectKey{
Namespace: ns.Name,
Name: instance.BootstrapAPIKeyNamespacedName().Name,
}, bootstrapSecret)
if err == nil {
// If the secret exists, delete it
Expect(k8sClient.Delete(context.TODO(), bootstrapSecret)).Should(Succeed())
} else if !errors.IsNotFound(err) {
// If the error is not a NotFound error, fail the test
Expect(err).NotTo(HaveOccurred())
}
})

It("create namespace", func() {
Expect(k8sClient.Create(context.TODO(), ns)).Should(Succeed())
})

It("should create bootstrap secrets", func() {
// Wait until the bootstrap secrets are created
// Call the reconciler.
result := a.reconcile(ctx, instance, nil)

// Make sure there were no errors.
Expect(result.err).NotTo(HaveOccurred())
// Check the created secrets.
cookieSecret := &corev1.Secret{}
err := k8sClient.Get(context.Background(), client.ObjectKey{
Namespace: ns.Name,
Name: instance.NodeCookieNamespacedName().Name,
}, cookieSecret)
Expect(err).NotTo(HaveOccurred())
Expect(cookieSecret.Data["node_cookie"]).ShouldNot(BeEmpty())

bootstrapSecret := &corev1.Secret{}
err = k8sClient.Get(context.Background(), client.ObjectKey{
Namespace: ns.Name,
Name: instance.BootstrapAPIKeyNamespacedName().Name,
}, bootstrapSecret)
Expect(err).NotTo(HaveOccurred())
Expect(bootstrapSecret.Data["bootstrap_api_key"]).ShouldNot(BeEmpty())
})

It("should contain key and secret in bootstrap secret given initial values", func() {
// Given
instance.Spec.BootstrapAPIKeys = []appsv2beta1.BootstrapAPIKey{
{
Key: "test_key",
Secret: "test_secret",
},
}

// Call the reconciler.
result := a.reconcile(ctx, instance, nil)

// Make sure there were no errors.
Expect(result.err).NotTo(HaveOccurred())

// Check the created secrets.
bootstrapSecret := &corev1.Secret{}
err := k8sClient.Get(context.Background(), client.ObjectKey{
Namespace: ns.Name,
Name: instance.BootstrapAPIKeyNamespacedName().Name,
}, bootstrapSecret)
Expect(err).NotTo(HaveOccurred())

// Verify that the bootstrap API key contains the initial key and secret.
Expect(string(bootstrapSecret.Data["bootstrap_api_key"])).Should(ContainSubstring("test_key:test_secret"))
})

It("should contain key and secret in bootstrap secret given SecretRef values", func() {
// Given
instance.Spec.BootstrapAPIKeys = []appsv2beta1.BootstrapAPIKey{
{
SecretRef: &appsv2beta1.SecretRef{
Key: appsv2beta1.KeyRef{
// Note: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters
SecretName: "test-key-secret",
SecretKey: "key",
},
Secret: appsv2beta1.KeyRef{
SecretName: "test-value-secret",
SecretKey: "secret",
},
},
},
}

// Create referenced secrets
keySecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: ns.Name,
Name: "test-key-secret",
},
StringData: map[string]string{
"key": "test_key",
},
}
Expect(k8sClient.Create(context.TODO(), keySecret)).Should(Succeed())

secretSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: ns.Name,
Name: "test-value-secret",
},
StringData: map[string]string{
"secret": "test_secret",
},
}
Expect(k8sClient.Create(context.TODO(), secretSecret)).Should(Succeed())

// Call the reconciler.
result := a.reconcile(ctx, instance, nil)

// Make sure there were no errors.
Expect(result.err).NotTo(HaveOccurred())

// Check the created secrets.
bootstrapSecret := &corev1.Secret{}
err := k8sClient.Get(context.Background(), client.ObjectKey{
Namespace: ns.Name,
Name: instance.BootstrapAPIKeyNamespacedName().Name,
}, bootstrapSecret)
Expect(err).NotTo(HaveOccurred())

// Verify that the bootstrap API key contains the initial key and secret.
Expect(string(bootstrapSecret.Data["bootstrap_api_key"])).Should(ContainSubstring("test_key:test_secret"))
})
})
Loading

0 comments on commit f8421c7

Please sign in to comment.