Collect block storage type during setup (when started using run) (#1430)

treeverse · Feb 9, 2021 · a72168d · a72168d
1 parent 1998b8a
commit a72168d
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 41 deletions.
diff --git a/api/controller.go b/api/controller.go
@@ -32,6 +32,7 @@ import (
 	"github.com/treeverse/lakefs/auth/model"
 	"github.com/treeverse/lakefs/block"
 	"github.com/treeverse/lakefs/catalog"
+	"github.com/treeverse/lakefs/cloud"
 	"github.com/treeverse/lakefs/db"
 	"github.com/treeverse/lakefs/graveler"
 	"github.com/treeverse/lakefs/httputil"
@@ -52,14 +53,15 @@ const (
 )
 
 type Dependencies struct {
-	ctx             context.Context
-	Cataloger       catalog.Cataloger
-	Auth            auth.Service
-	BlockAdapter    block.Adapter
-	MetadataManager auth.MetadataManager
-	Migrator        db.Migrator
-	Collector       stats.Collector
-	Logger          logging.Logger
+	ctx                   context.Context
+	Cataloger             catalog.Cataloger
+	Auth                  auth.Service
+	BlockAdapter          block.Adapter
+	MetadataManager       auth.MetadataManager
+	Migrator              db.Migrator
+	Collector             stats.Collector
+	CloudMetadataProvider cloud.MetadataProvider
+	Logger                logging.Logger
 }
 
 func (d *Dependencies) WithContext(ctx context.Context) *Dependencies {
@@ -239,14 +241,12 @@ func (c *Controller) SetupLakeFSHandler() setupop.SetupLakeFSHandler {
 			return setupop.NewSetupLakeFSDefault(http.StatusInternalServerError).
 				WithPayload(&models.Error{Message: err.Error()})
 		}
-		metadata, err := c.deps.MetadataManager.Write()
-		if err != nil {
-			c.deps.Logger.Error("failed to write metadata after setup")
-		} else {
-			c.deps.Collector.SetInstallationID(metadata[auth.InstallationIDKeyName])
-			c.deps.Collector.CollectMetadata(convertMetadata(metadata))
-		}
+
+		metadata := stats.NewMetadata(c.deps.Logger, c.deps.BlockAdapter.BlockstoreType(), c.deps.MetadataManager, c.deps.CloudMetadataProvider)
+		c.deps.Collector.SetInstallationID(metadata.InstallationID)
+		c.deps.Collector.CollectMetadata(metadata)
 		c.deps.Collector.CollectEvent("global", "init")
+
 		return setupop.NewSetupLakeFSOK().WithPayload(&models.CredentialsWithSecret{
 			AccessKeyID:     cred.AccessKeyID,
 			AccessSecretKey: cred.AccessSecretKey,
@@ -255,20 +255,6 @@ func (c *Controller) SetupLakeFSHandler() setupop.SetupLakeFSHandler {
 	})
 }
 
-func convertMetadata(metadata map[string]string) *stats.Metadata {
-	res := &stats.Metadata{
-		InstallationID: metadata[auth.InstallationIDKeyName],
-	}
-
-	for k, v := range metadata {
-		res.Entries = append(res.Entries, stats.MetadataEntry{
-			Name:  k,
-			Value: v,
-		})
-	}
-	return res
-}
-
 func (c *Controller) GetCurrentUserHandler() authop.GetCurrentUserHandler {
 	return authop.GetCurrentUserHandlerFunc(func(params authop.GetCurrentUserParams, user *models.User) middleware.Responder {
 		return authop.NewGetCurrentUserOK().WithPayload(&authop.GetCurrentUserOKBody{

diff --git a/api/controller_test.go b/api/controller_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/treeverse/lakefs/catalog"
 	"github.com/treeverse/lakefs/db"
 	"github.com/treeverse/lakefs/httputil"
+	"github.com/treeverse/lakefs/stats"
 	"github.com/treeverse/lakefs/testutil"
 	"github.com/treeverse/lakefs/upload"
 )
@@ -1431,6 +1432,20 @@ func TestController_SetupLakeFSHandler(t *testing.T) {
 				if len(deps.collector.metadata[0].Entries) < 5 {
 					t.Fatalf("There should be at least 5 metadata entries: %s", deps.collector.metadata[0].Entries)
 				}
+
+				hasBlockStoreType := false
+				for _, ent := range deps.collector.metadata[0].Entries {
+					if ent.Name == stats.BlockstoreTypeKey {
+						hasBlockStoreType = true
+						if ent.Value == "" {
+							t.Fatalf("Blockstorage key exists but with empty value: %s", deps.collector.metadata[0].Entries)
+						}
+						break
+					}
+				}
+				if !hasBlockStoreType {
+					t.Fatalf("missing blockstorage key: %s", deps.collector.metadata[0].Entries)
+				}
 			})
 
 			if c.errorDefaultCode == 0 {

diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go
@@ -85,7 +85,7 @@ var runCmd = &cobra.Command{
 			cfg.GetAuthCacheConfig())
 		authMetadataManager := auth.NewDBMetadataManager(config.Version, dbPool)
 		cloudMetadataProvider := stats.BuildMetadataProvider(logger, cfg)
-		metadata := stats.NewMetadata(logger, cfg, authMetadataManager, cloudMetadataProvider)
+		metadata := stats.NewMetadata(logger, cfg.GetBlockstoreType(), authMetadataManager, cloudMetadataProvider)
 		bufferedCollector := stats.NewBufferedCollector(metadata.InstallationID, cfg)
 
 		// send metadata
@@ -104,13 +104,14 @@ var runCmd = &cobra.Command{
 		signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
 
 		apiHandler := api.Serve(api.Dependencies{
-			Cataloger:       cataloger,
-			Auth:            authService,
-			BlockAdapter:    blockStore,
-			MetadataManager: authMetadataManager,
-			Migrator:        migrator,
-			Collector:       bufferedCollector,
-			Logger:          logger.WithField("service", "api_gateway"),
+			Cataloger:             cataloger,
+			Auth:                  authService,
+			BlockAdapter:          blockStore,
+			MetadataManager:       authMetadataManager,
+			CloudMetadataProvider: cloudMetadataProvider,
+			Migrator:              migrator,
+			Collector:             bufferedCollector,
+			Logger:                logger.WithField("service", "api_gateway"),
 		})
 
 		// init gateway server

diff --git a/cmd/lakefs/cmd/setup.go b/cmd/lakefs/cmd/setup.go
@@ -54,7 +54,7 @@ var setupCmd = &cobra.Command{
 			cfg.GetAuthCacheConfig())
 		metadataManager := auth.NewDBMetadataManager(config.Version, dbPool)
 		cloudMetadataProvider := stats.BuildMetadataProvider(logging.Default(), cfg)
-		metadata := stats.NewMetadata(logging.Default(), cfg, metadataManager, cloudMetadataProvider)
+		metadata := stats.NewMetadata(logging.Default(), cfg.GetBlockstoreType(), metadataManager, cloudMetadataProvider)
 
 		credentials, err := auth.CreateInitialAdminUserWithKeys(authService, metadataManager, userName, &accessKeyID, &secretAccessKey)
 		if err != nil {

diff --git a/cmd/lakefs/cmd/superuser.go b/cmd/lakefs/cmd/superuser.go
@@ -46,7 +46,7 @@ var superuserCmd = &cobra.Command{
 			cfg.GetAuthCacheConfig())
 		authMetadataManager := auth.NewDBMetadataManager(config.Version, dbPool)
 		metadataProvider := stats.BuildMetadataProvider(logging.Default(), cfg)
-		metadata := stats.NewMetadata(logging.Default(), cfg, authMetadataManager, metadataProvider)
+		metadata := stats.NewMetadata(logging.Default(), cfg.GetBlockstoreType(), authMetadataManager, metadataProvider)
 		credentials, err := auth.AddAdminUser(authService, &model.SuperuserConfiguration{
 			User: model.User{
 				CreatedAt: time.Now(),

diff --git a/stats/metadata.go b/stats/metadata.go
@@ -23,7 +23,7 @@ type Metadata struct {
 	Entries        []MetadataEntry `json:"entries"`
 }
 
-func NewMetadata(logger logging.Logger, c *config.Config, authMetadataManager auth.MetadataManager, cloudMetadataProvider cloud.MetadataProvider) *Metadata {
+func NewMetadata(logger logging.Logger, blockstoreType string, authMetadataManager auth.MetadataManager, cloudMetadataProvider cloud.MetadataProvider) *Metadata {
 	res := &Metadata{}
 	authMetadata, err := authMetadataManager.Write()
 	if err != nil {
@@ -41,7 +41,6 @@ func NewMetadata(logger logging.Logger, c *config.Config, authMetadataManager au
 			res.Entries = append(res.Entries, MetadataEntry{Name: k, Value: v})
 		}
 	}
-	blockstoreType := c.GetBlockstoreType()
 	res.Entries = append(res.Entries, MetadataEntry{Name: BlockstoreTypeKey, Value: blockstoreType})
 	return res
 }