Build Time Improvement (#1067) (#1069)

* 🚧 build(VSecM): parallelize building images * minor * minor * move vendor * removed proto generation from bundle.sh * even faster * typo * code cleanup --------- Signed-off-by: Volkan Özçelik <[email protected]>
vmware-tanzu · Jul 13, 2024 · 6f29299 · 6f29299
1 parent f96536d
commit 6f29299
Show file tree

Hide file tree

Showing 26 changed files with 215 additions and 193 deletions.
diff --git a/app/safe/internal/server/route/delete/delete.go b/app/safe/internal/server/route/delete/delete.go
@@ -67,6 +67,8 @@ func Delete(
 
 	// Only sentinel can execute delete requests.
 	if ok, respond := validation.IsSentinel(j, cid, spiffeid); !ok {
+		j.Event = audit.NotSentinel
+		journal.Log(j)
 		respond(w)
 		return
 	}

diff --git a/app/safe/internal/server/route/keystone/keystone.go b/app/safe/internal/server/route/keystone/keystone.go
@@ -87,11 +87,9 @@ func Status(
 
 	// Only sentinel can get the status.
 	if ok, respond := validation.IsSentinel(j, cid, spiffeid); !ok {
-		respond(w)
-
-		j.Event = audit.BadSpiffeId
+		j.Event = audit.NotSentinel
 		journal.Log(j)
-
+		respond(w)
 		return
 	}
 

diff --git a/app/safe/internal/server/route/list/impl.go b/app/safe/internal/server/route/list/impl.go
@@ -58,6 +58,8 @@ func doList(
 
 	// Only sentinel can list.
 	if ok, respond := validation.IsSentinel(j, cid, spiffeid); !ok {
+		j.Event = audit.NotSentinel
+		journal.Log(j)
 		respond(w)
 		return
 	}

diff --git a/app/safe/internal/server/route/receive/receive.go b/app/safe/internal/server/route/receive/receive.go
@@ -56,10 +56,8 @@ func Keys(cid string, r *http.Request, w http.ResponseWriter) {
 	// Only sentinel can set keys.
 	if ok, respond := validation.IsSentinel(j, cid, spiffeid); !ok {
 		respond(w)
-
-		j.Event = audit.BadSpiffeId
+		j.Event = audit.NotSentinel
 		journal.Log(j)
-
 		return
 	}
 
@@ -105,8 +103,12 @@ func Keys(cid string, r *http.Request, w http.ResponseWriter) {
 		return
 	}
 
-	keysCombined := agePrivateKey + "\n" + agePublicKey + "\n" + aesCipherKey
-	crypto.SetRootKeyInMemory(keysCombined)
+	rkt := data.RootKeyCollection{
+		PrivateKey: agePrivateKey,
+		PublicKey:  agePublicKey,
+		AesSeed:    aesCipherKey,
+	}
+	crypto.SetRootKeyInMemory(rkt.Combine())
 
 	if err := bootstrap.PersistRootKeysToRootKeyBackingStore(
 		data.RootKeyCollection{

diff --git a/app/safe/internal/server/route/secret/secret.go b/app/safe/internal/server/route/secret/secret.go
@@ -21,6 +21,7 @@ import (
 	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/server/route/base/validation"
 	"github.com/vmware-tanzu/secrets-manager/core/audit/journal"
 	"github.com/vmware-tanzu/secrets-manager/core/constants/audit"
+	"github.com/vmware-tanzu/secrets-manager/core/constants/val"
 	"github.com/vmware-tanzu/secrets-manager/core/crypto"
 	entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
 	log "github.com/vmware-tanzu/secrets-manager/core/log/std"
@@ -43,7 +44,7 @@ func Secret(cid string, r *http.Request, w http.ResponseWriter) {
 	spiffeid := s.IdAsString(r)
 	if spiffeid == "" {
 		w.WriteHeader(http.StatusBadRequest)
-		_, err := io.WriteString(w, "NOK!")
+		_, err := io.WriteString(w, val.NotOk)
 		if err != nil {
 			log.ErrorLn(&cid, "error writing response", err.Error())
 		}
@@ -53,7 +54,7 @@ func Secret(cid string, r *http.Request, w http.ResponseWriter) {
 
 	if !crypto.RootKeySetInMemory() {
 		w.WriteHeader(http.StatusBadRequest)
-		_, err := io.WriteString(w, "NOK!")
+		_, err := io.WriteString(w, val.NotOk)
 		if err != nil {
 			log.ErrorLn(&cid, "error writing response", err.Error())
 		}
@@ -67,8 +68,9 @@ func Secret(cid string, r *http.Request, w http.ResponseWriter) {
 
 	// Only sentinel can do this.
 	if ok, respond := validation.IsSentinel(j, cid, spiffeid); !ok {
+		j.Event = audit.NotSentinel
+		journal.Log(j)
 		respond(w)
-
 		return
 	}
 

diff --git a/app/safe/internal/state/secret/collection/io.go b/app/safe/internal/state/secret/collection/io.go
@@ -0,0 +1,56 @@
+package collection
+
+import (
+	"errors"
+	"os"
+	"strings"
+
+	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/io"
+	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/stats"
+	f "github.com/vmware-tanzu/secrets-manager/core/constants/file"
+	"github.com/vmware-tanzu/secrets-manager/core/env"
+	log "github.com/vmware-tanzu/secrets-manager/core/log/std"
+)
+
+func populateSecretsFromFileStore(cid string) error {
+	root := env.DataPathForSafe()
+	files, err := os.ReadDir(root)
+	if err != nil {
+		return errors.Join(
+			err,
+			errors.New("populateSecrets: problem reading secrets directory"),
+		)
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+
+		fn := file.Name()
+		if strings.HasSuffix(fn, f.AgeBackupExtension) {
+			continue
+		}
+
+		key := strings.Replace(fn, f.AgeExtension, "", 1)
+
+		_, exists := Secrets.Load(key)
+		if exists {
+			continue
+		}
+
+		secretOnDisk, err := io.ReadFromDisk(key)
+		if err != nil {
+			log.ErrorLn(&cid,
+				"populateSecrets: problem reading secret from disk:",
+				err.Error())
+			continue
+		}
+		if secretOnDisk != nil {
+			stats.CurrentState.Increment(key, Secrets.Load)
+			Secrets.Store(key, *secretOnDisk)
+		}
+	}
+
+	return nil
+}
diff --git a/app/safe/internal/state/secret/collection/populate.go b/app/safe/internal/state/secret/collection/populate.go
@@ -11,14 +11,8 @@
 package collection
 
 import (
-	"errors"
-	"os"
-	"strings"
 	"sync"
 
-	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/io"
-	"github.com/vmware-tanzu/secrets-manager/app/safe/internal/state/stats"
-	f "github.com/vmware-tanzu/secrets-manager/core/constants/file"
 	"github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
 	"github.com/vmware-tanzu/secrets-manager/core/env"
 	log "github.com/vmware-tanzu/secrets-manager/core/log/std"
@@ -39,49 +33,6 @@ func SecretsPopulated() bool {
 	return secretsPopulated
 }
 
-func populateSecretsFromFileStore(cid string) error {
-	root := env.DataPathForSafe()
-	files, err := os.ReadDir(root)
-	if err != nil {
-		return errors.Join(
-			err,
-			errors.New("populateSecrets: problem reading secrets directory"),
-		)
-	}
-
-	for _, file := range files {
-		if file.IsDir() {
-			continue
-		}
-
-		fn := file.Name()
-		if strings.HasSuffix(fn, f.AgeBackupExtension) {
-			continue
-		}
-
-		key := strings.Replace(fn, f.AgeExtension, "", 1)
-
-		_, exists := Secrets.Load(key)
-		if exists {
-			continue
-		}
-
-		secretOnDisk, err := io.ReadFromDisk(key)
-		if err != nil {
-			log.ErrorLn(&cid,
-				"populateSecrets: problem reading secret from disk:",
-				err.Error())
-			continue
-		}
-		if secretOnDisk != nil {
-			stats.CurrentState.Increment(key, Secrets.Load)
-			Secrets.Store(key, *secretOnDisk)
-		}
-	}
-
-	return nil
-}
-
 // PopulateSecrets scans the designated secrets storage directory on disk,
 // reading each secret file that is not marked as a backup, and loads the
 // secrets into a global store if they have not already been loaded. This

diff --git a/app/sentinel/internal/oidc/engine/adapter.go b/app/sentinel/internal/oidc/engine/adapter.go
@@ -20,24 +20,9 @@ import (
 	"github.com/vmware-tanzu/secrets-manager/core/constants/key"
 	"github.com/vmware-tanzu/secrets-manager/core/crypto"
 	entity "github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
+	"github.com/vmware-tanzu/secrets-manager/core/entity/v1/reqres/sentinel"
 )
 
-// SecretRequest encapsulates a VSecM Safe REST command payload.
-type SecretRequest struct {
-	Workloads          []string `json:"workload"`
-	Secret             string   `json:"secret"`
-	Namespaces         []string `json:"namespaces,omitempty"`
-	Encrypt            bool     `json:"encrypt,omitempty"`
-	Delete             bool     `json:"delete,omitempty"`
-	Append             bool     `json:"append,omitempty"`
-	List               bool     `json:"list,omitempty"`
-	Template           string   `json:"template,omitempty"`
-	Format             string   `json:"format,omitempty"`
-	SerializedRootKeys string   `json:"root-keys,omitempty"`
-	NotBefore          string   `json:"nbf,omitempty"`
-	Expires            string   `json:"exp,omitempty"`
-}
-
 // HandleCommandSecrets processes HTTP requests related to secret management.
 //
 // This function handles both listing and modifying secrets based on the
@@ -81,7 +66,7 @@ type SecretRequest struct {
 //   - If there is an error during secret retrieval or modification, it returns
 //     a 500 Internal Server Error status with the error message.
 func HandleCommandSecrets(
-	w http.ResponseWriter, r *http.Request, req *SecretRequest,
+	w http.ResponseWriter, r *http.Request, req *sentinel.SecretRequest,
 ) {
 	id := crypto.Id()
 

diff --git a/app/sentinel/internal/oidc/engine/handle.go b/app/sentinel/internal/oidc/engine/handle.go
@@ -13,6 +13,8 @@ package engine
 import (
 	"encoding/json"
 	"net/http"
+
+	"github.com/vmware-tanzu/secrets-manager/core/entity/v1/reqres/sentinel"
 )
 
 // HandleSecrets processes incoming HTTP requests related to secrets management.
@@ -39,7 +41,7 @@ func HandleSecrets(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var req SecretRequest
+	var req sentinel.SecretRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return

diff --git a/app/sentinel/internal/oidc/safe/post.go b/app/sentinel/internal/oidc/safe/post.go
@@ -135,7 +135,8 @@ func Post(
 				},
 			}
 
-			parts := strings.Split(sc.SerializedRootKeys, "\n")
+			parts := sc.SplitRootKeys()
+
 			if len(parts) != 3 {
 				return "", printPayloadError(
 					cid, errors.New("post: Bad data! Very bad data"))

diff --git a/app/sentinel/internal/safe/post.go b/app/sentinel/internal/safe/post.go
@@ -159,7 +159,7 @@ func Post(parentContext context.Context,
 				},
 			}
 
-			parts := strings.Split(sc.SerializedRootKeys, "\n")
+			parts := sc.SplitRootKeys()
 			if len(parts) != 3 {
 				return errors.New("post: Bad data! Very bad data")
 			}

diff --git a/core/audit/journal/log.go b/core/audit/journal/log.go
@@ -11,9 +11,9 @@
 package journal
 
 import (
-	"github.com/vmware-tanzu/secrets-manager/core/constants/audit"
 	"net/http"
 
+	"github.com/vmware-tanzu/secrets-manager/core/constants/audit"
 	"github.com/vmware-tanzu/secrets-manager/core/entity/v1/data"
 )
 

diff --git a/core/constants/audit/event.go b/core/constants/audit/event.go
@@ -12,16 +12,18 @@ package audit
 
 type Event string
 
-const Enter Event = "vsecm-enter"
+const BadPayload Event = "vsecm-bad-payload"
+const BadPeerSvid Event = "vsecm-bad-peer-spiffeid"
 const BadSpiffeId Event = "vsecm-bad-spiffeid"
 const BrokenBody Event = "vsecm-broken-body"
-const RequestTypeMismatch Event = "vsecm-request-type-mismatch"
-const BadPeerSvid Event = "vsecm-bad-peer-spiffeid"
+const DecryptionFailed Event = "vsecm-decryption-failed"
+const EncryptionFailed Event = "vsecm-encryption-failed"
+const Enter Event = "vsecm-enter"
 const NoSecret Event = "vsecm-no-secret"
-const Ok Event = "vsecm-ok"
-const NoWorkloadId Event = "vsecm-no-wl-id"
 const NoValue Event = "vsecm-no-value"
-const EncryptionFailed Event = "vsecm-encryption-failed"
-const DecryptionFailed Event = "vsecm-decryption-failed"
-const BadPayload Event = "vsecm-bad-payload"
+const NotWorkload = "vsecm-not-workload"
+const NoWorkloadId Event = "vsecm-no-wl-id"
+const NotSentinel = "vsecm-not-sentinel"
+const Ok Event = "vsecm-ok"
+const RequestTypeMismatch Event = "vsecm-request-type-mismatch"
 const RootKeyNotSet = "vsecm-root-key-not-set"
diff --git a/core/constants/val/val.go b/core/constants/val/val.go
@@ -18,6 +18,7 @@ const TimeNever = "never"
 const BlankRootKey = "{}"
 
 const Ok = "OK"
+const NotOk = "NOK!"
 
 // JsonEmpty is a constant string representing an empty value.
 // This value is generated by the go templating engine

diff --git a/core/entity/v1/data/command.go b/core/entity/v1/data/command.go
@@ -10,6 +10,8 @@
 
 package data
 
+import "strings"
+
 // VSecMInternalCommand is the command that VSecM uses to perform
 // internal operations.
 type VSecMInternalCommand struct {
@@ -33,3 +35,18 @@ type SentinelCommand struct {
 	ShouldSleep        bool
 	SleepIntervalMs    int
 }
+
+// SplitRootKeys splits the SerializedRootKeys of the SentinelCommand
+// into a slice of strings based on newline characters.
+//
+// It returns a slice of strings, where each string represents a root key.
+// If there are no newline characters in SerializedRootKeys, the returned
+// slice will contain a single element.
+//
+// Example:
+//
+//	sc := SentinelCommand{SerializedRootKeys: "key1\nkey2\nkey3"}
+//	keys := sc.SplitRootKeys() // returns []string{"key1", "key2", "key3"}
+func (sc SentinelCommand) SplitRootKeys() []string {
+	return strings.Split(sc.SerializedRootKeys, "\n")
+}
diff --git a/core/entity/v1/reqres/sentinel/sentinel.go b/core/entity/v1/reqres/sentinel/sentinel.go
@@ -0,0 +1,27 @@
+/*
+|    Protect your secrets, protect your sensitive data.
+:    Explore VMware Secrets Manager docs at https://vsecm.com/
+</
+<>/  keep your secrets… secret
+>/
+<>/' Copyright 2023–present VMware Secrets Manager contributors.
+>/'  SPDX-License-Identifier: BSD-2-Clause
+*/
+
+package sentinel
+
+// SecretRequest encapsulates a VSecM Safe REST command payload.
+type SecretRequest struct {
+	Workloads          []string `json:"workload"`
+	Secret             string   `json:"secret"`
+	Namespaces         []string `json:"namespaces,omitempty"`
+	Encrypt            bool     `json:"encrypt,omitempty"`
+	Delete             bool     `json:"delete,omitempty"`
+	Append             bool     `json:"append,omitempty"`
+	List               bool     `json:"list,omitempty"`
+	Template           string   `json:"template,omitempty"`
+	Format             string   `json:"format,omitempty"`
+	SerializedRootKeys string   `json:"root-keys,omitempty"`
+	NotBefore          string   `json:"nbf,omitempty"`
+	Expires            string   `json:"exp,omitempty"`
+}