fix: Resource fetching performance improvement for Scaleway (tailward…

…en#1212) Co-authored-by: Azanul Haque <[email protected]>
Azanul · Dec 6, 2023 · cad8aff · cad8aff
1 parent d344687
commit cad8aff
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 18 deletions.
diff --git a/internal/internal.go b/internal/internal.go
@@ -250,7 +250,7 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien
 	case "Azure":
 		azure.FetchResources(ctx, client, db, telemetry, analytics, wp)
 	case "Scaleway":
-		scaleway.FetchResources(ctx, client, db, telemetry, analytics)
+		scaleway.FetchResources(ctx, client, db, telemetry, analytics, wp)
 	case "MongoDBAtlas":
 		mongodbatlas.FetchResources(ctx, client, db, telemetry, analytics, wp)
 	case "GCP":

diff --git a/providers/scaleway/scaleway.go b/providers/scaleway/scaleway.go
@@ -28,24 +28,26 @@ func listOfSupportedServices() []providers.FetchDataFunction {
 	}
 }
 
-func FetchResources(ctx context.Context, client providers.ProviderClient, db *bun.DB, telemetry bool, analytics utils.Analytics) {
-	for _, fetchResources := range listOfSupportedServices() {
-		resources, err := fetchResources(ctx, client)
-		if err != nil {
-			log.Printf("[%s][Scaleway] %s", client.Name, err)
-		} else {
-			for _, resource := range resources {
-				_, err := db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
-				if err != nil {
-					logrus.WithError(err).Errorf("db trigger failed")
+func FetchResources(ctx context.Context, client providers.ProviderClient, db *bun.DB, telemetry bool, analytics utils.Analytics, wp *providers.WorkerPool) {
+	wp.SubmitTask(func() {
+		for _, fetchResources := range listOfSupportedServices() {
+			resources, err := fetchResources(ctx, client)
+			if err != nil {
+				log.Printf("[%s][Scaleway] %s", client.Name, err)
+			} else {
+				for _, resource := range resources {
+					_, err := db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost").Exec(context.Background())
+					if err != nil {
+						logrus.WithError(err).Errorf("db trigger failed")
+					}
+				}
+				if telemetry {
+					analytics.TrackEvent("discovered_resources", map[string]interface{}{
+						"provider":  "Scaleway",
+						"resources": len(resources),
+					})
 				}
-			}
-			if telemetry {
-				analytics.TrackEvent("discovered_resources", map[string]interface{}{
-					"provider":  "Scaleway",
-					"resources": len(resources),
-				})
 			}
 		}
-	}
+	})
 }