Merge pull request #1207 from tanbirali/feature/1206-Resource-fetchin…

…g-Improvement-MongoDB fix(1206): Resource fetching performance improvement for MongoDb
tailwarden · Nov 15, 2023 · 3e31650 · 3e31650
2 parents a35c1d0 + ef4c890
commit 3e31650
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 18 deletions.
diff --git a/internal/internal.go b/internal/internal.go
@@ -252,7 +252,7 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien
 	case "Scaleway":
 		scaleway.FetchResources(ctx, client, db, telemetry, analytics)
 	case "MongoDBAtlas":
-		mongodbatlas.FetchResources(ctx, client, db, telemetry, analytics)
+		mongodbatlas.FetchResources(ctx, client, db, telemetry, analytics, wp)
 	case "GCP":
 		gcp.FetchResources(ctx, client, db, telemetry, analytics, wp)
 	case "OVH":

diff --git a/providers/mongodbatlas/mongodbatlas.go b/providers/mongodbatlas/mongodbatlas.go
@@ -17,25 +17,27 @@ func listOfSupportedServices() []providers.FetchDataFunction {
 	}
 }
 
-func FetchResources(ctx context.Context, client providers.ProviderClient, db *bun.DB, telemetry bool, analytics utils.Analytics) {
+func FetchResources(ctx context.Context, client providers.ProviderClient, db *bun.DB, telemetry bool, analytics utils.Analytics, wp *providers.WorkerPool) {
 	for _, fetchResources := range listOfSupportedServices() {
-		resources, err := fetchResources(ctx, client)
-		if err != nil {
-			log.Printf("[%s][MongoDBAtlas] %s", client.Name, err)
-		} else {
-			for _, resource := range resources {
-				_, err := db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
-				if err != nil {
-					logrus.WithError(err).Errorf("db trigger failed")
+		wp.SubmitTask(func() {
+			resources, err := fetchResources(ctx, client)
+			if err != nil {
+				log.Printf("[%s][MongoDBAtlas] %s", client.Name, err)
+			} else {
+				for _, resource := range resources {
+					_, err := db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
+					if err != nil {
+						logrus.WithError(err).Errorf("db trigger failed")
+					}
+				}
+				if telemetry {
+					analytics.TrackEvent("discovered_resources", map[string]interface{}{
+						"provider":  "MongoDBAtlas",
+						"resources": len(resources),
+					})
 				}
-
-			}
-			if telemetry {
-				analytics.TrackEvent("discovered_resources", map[string]interface{}{
-					"provider":  "MongoDBAtlas",
-					"resources": len(resources),
-				})
 			}
-		}
+		})
+
 	}
 }