Merge branch 'develop' of github.com:tailwarden/komiser into improve-…

…dashboard-readme

.devcontainer/devcontainer.json

@@ -0,0 +1,34 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/go
+{
+	"name": "Go",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/go:1-1.21-bullseye",
+	"features": {
+		"ghcr.io/devcontainers-contrib/features/node-asdf:0": {},
+		"ghcr.io/devcontainers-contrib/features/typescript:2": {}
+	},
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"golang.go-nightly",
+				"bradlc.vscode-tailwindcss"
+			]
+		}
+	}
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "go version",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

dashboard/README.md

@@ -4,16 +4,18 @@ Komiser dashboard is a [Next.js](https://nextjs.org/) project bootstrapped with
 
 **Full frontend stack:**
 
-- 🖥 `Next.js`
-- 📜 `Typescript`
-- 🎨 `Tailwind`
-- 📖 `Storybook`
-- 🧪 `Jest`
-- 📚 `React Testing Library`
+- 🖥 [`Next.js`](https://nextjs.org/)
+- 📜 [`Typescript`](https://www.typescriptlang.org/)
+- 🎨 [`Tailwind`](https://tailwindcss.com/)
+- 📖 [`Storybook`](https://storybook.js.org/)
+- 🧪 [`Jest`](https://jestjs.io/)
+- 📚 [`React Testing Library`](https://testing-library.com/docs/react-testing-library/intro)
 
 ## 🚀 Getting Started
 
-First, run the development server:
+Follow the [Contribution Guide](https://github.com/tailwarden/komiser/blob/develop/CONTRIBUTING.md#contributing-to-komiser-dashboard-ui) first if you haven't done so already. Then come back here and follow the next steps:
+
+1. Run the development server:
 
 ```shell
 
@@ -36,6 +38,7 @@ Open [http://localhost:3002/](http://localhost:3002). If you see the dashboard,
 ❗ If you get an error page such as this, please refer to the logs and our [docs](https://docs.komiser.io/docs/introduction/getting-started).
 ![Error Image](https://user-images.githubusercontent.com/13384559/224320642-0bf6814b-d97a-4ad9-95a0-ca82e353c5d0.png)
 
+
 ## 🧩 Components
 
 Komiser components are documented under `/components`
@@ -52,6 +55,11 @@ Komiser components are documented under `/components`
 > }
 > ```
 
+You can find all the shared Components also inside [Storybook](https://storybook.komiser.io/). If you're implementing a new Story, please check for existing or new components with Storybook.
+We will require a story for new shared components like icons, inputs or similar.
+
+Component convention:
+
 **Component convention:**
 
 - 📁 Component folder: component name in `kebab-case`
@@ -60,19 +68,25 @@ Komiser components are documented under `/components`
 - 🎭 Component story mock (if needed): component name in `UpperCamelCase.mocks.*`
 - 🧪 Component unit test: component name in `UpperCamelCase.test.*`
 - 🧐 Check `Card` example for more details:
+
   ![Component Example](https://user-images.githubusercontent.com/13384559/224307211-2ce62245-de24-4ee7-a156-fb54d8d34b4f.png)
 
 **Additional instructions:**
 
 - 📖 To view this component on Storybook, run: `npm run storybook`, then pick `Card`
   ![Storybook Image](https://user-images.githubusercontent.com/13384559/224320112-e21d2ed4-1e22-4a33-adb3-6c236c4d4208.png)
 - 🧪 To run the unit tests, run: `npm run test:watch`, hit `p`, then `card`
+
   ![Unit Test Image](https://user-images.githubusercontent.com/13384559/224320260-19b1359e-1bfb-4db5-8379-918dacd7da44.png)
 
 ## 🧪 Testing
 
 We use Jest & React Testing Library for our unit tests.
 
+- To run the unit tests, run: `npm run test:watch`, hit `p`, then `card`
+  <img width="668" alt="image" src="https://user-images.githubusercontent.com/13384559/224320260-19b1359e-1bfb-4db5-8379-918dacd7da44.png">
+
+
 **Testing convention:**
 
 - ✅ All new Utils need to be tested. Existing ones when being changed
@@ -151,6 +165,90 @@ it('opens the dropdown when clicked', () => {
 });
 ```
 
+## 🎨 Adding to Storybook
+
+[**Storybook**](https://storybook.komiser.io/) is a tool for UI development. It makes development faster by isolating components. This allows you to work on one component at a time. If you create a new shared component or want to visualize variations of an existing one, follow these steps:
+
+- To view this component on Storybook locally, run: `npm run storybook`, then pick an example (`Card`) or your new component story
+  <img width="1411" alt="image" src="https://user-images.githubusercontent.com/13384559/224320112-e21d2ed4-1e22-4a33-adb3-6c236c4d4208.png">
+
+### 1. **Create the Story**:
+
+In the same directory as your component, create a Storybook story:
+
+- Create a story file: component name in `UpperCamelCase.stories.*`.
+
+Here's a basic story format:
+
+```typescript
+import React from 'react';
+import { ComponentStory, ComponentMeta } from '@storybook/react';
+
+import YourComponent from './YourComponent';
+
+export default {
+  title: 'Path/To/YourComponent',
+  component: YourComponent
+} as ComponentMeta<typeof YourComponent>;
+
+const Template: ComponentStory<typeof YourComponent> = args => (
+  <YourComponent {...args} />
+);
+
+export const Default = Template.bind({});
+Default.args = {
+  // default props here...
+};
+```
+
+### 2. **Add Variations**:
+
+You can create multiple variations of your component by replicating the `Default` pattern. For example, if your component has a variation for a "disabled" state:
+
+```typescript
+export const Disabled = Template.bind({});
+Disabled.args = {
+  // props to set the component to its disabled state...
+};
+```
+
+### 3. **Mock Data**:
+
+If your component requires mock data, create a mock file: component name in `UpperCamelCase.mocks.*`. Import this data into your story file to use with your component variations.
+
+### 4. **Visual Check**:
+
+Run Storybook:
+
+```bash
+npm run storybook
+```
+
+Your component should now appear in the Storybook UI. Navigate to it, and verify all the variations display correctly.
+
+### 5. **Documentation**:
+
+Add a brief description and any notes on your component's functionality within the Storybook UI. Use the `parameters` object in your default export:
+
+```typescript
+export default {
+  title: 'Path/To/YourComponent',
+  component: YourComponent,
+  parameters: {
+    docs: {
+      description: {
+        component: 'Your description here...'
+      }
+    }
+  }
+} as ComponentMeta<typeof YourComponent>;
+```
+
+---
+
+> Remember: Storybook is not just a tool but also a way to document components. Ensure you provide meaningful names, descriptions, and use cases to help other developers understand the use and purpose of each component.
+
+
 ## 🤝 Contributing
 
 We welcome all contributors to join us on the mission of improving Komiser, especially when it comes to writing tests and adding documentation.

internal/internal.go

@@ -11,6 +11,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/getsentry/sentry-go"
@@ -54,7 +55,7 @@ var Arch = runtime.GOARCH
 var db *bun.DB
 var analytics utils.Analytics
 
-func Exec(address string, port int, configPath string, telemetry bool, a utils.Analytics, regions []string, cmd *cobra.Command) error {
+func Exec(address string, port int, configPath string, telemetry bool, a utils.Analytics, regions []string, _ *cobra.Command) error {
 	analytics = a
 
 	ctx := context.Background()
@@ -79,10 +80,8 @@ func Exec(address string, port int, configPath string, telemetry bool, a utils.A
 
 		_, err = cron.Every(1).Hours().Do(func() {
 			log.Info("Fetching resources workflow has started")
-			err = fetchResources(ctx, clients, regions, telemetry)
-			if err != nil {
-				log.Fatal(err)
-			}
+
+			fetchResources(ctx, clients, regions, telemetry)
 		})
 
 		if err != nil {
@@ -209,7 +208,7 @@ func setupDBConnection(c *models.Config) error {
 	return nil
 }
 
-func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string) {
+func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string, wp *providers.WorkerPool) {
 	localHub := sentry.CurrentHub().Clone()
 
 	defer func() {
@@ -233,7 +232,7 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien
 
 	switch provider {
 	case "AWS":
-		aws.FetchResources(ctx, client, regions, db, telemetry, analytics)
+		aws.FetchResources(ctx, client, regions, db, telemetry, analytics, wp)
 	case "DigitalOcean":
 		do.FetchResources(ctx, client, db, telemetry, analytics)
 	case "OCI":
@@ -257,33 +256,48 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien
 	}
 }
 
-func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool) error {
+func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool) {
+	numWorkers := 64
+	wp := providers.NewWorkerPool(numWorkers)
+	wp.Start()
+
+	var wwg sync.WaitGroup
+	workflowTrigger := func(client providers.ProviderClient, provider string) {
+		wwg.Add(1)
+		go func() {
+			defer wwg.Done()
+			triggerFetchingWorfklow(ctx, client, provider, telemetry, regions, wp)
+		}()
+	}
+
 	for _, client := range clients {
 		if client.AWSClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "AWS", telemetry, regions)
+			workflowTrigger(client, "AWS")
 		} else if client.DigitalOceanClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "DigitalOcean", telemetry, regions)
+			workflowTrigger(client, "DigitalOcean")
 		} else if client.OciClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "OCI", telemetry, regions)
+			workflowTrigger(client, "OCI")
 		} else if client.CivoClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "Civo", telemetry, regions)
+			workflowTrigger(client, "Civo")
 		} else if client.K8sClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "Kubernetes", telemetry, regions)
+			workflowTrigger(client, "Kubernetes")
 		} else if client.LinodeClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "Linode", telemetry, regions)
+			workflowTrigger(client, "Linode")
 		} else if client.TencentClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "Tencent", telemetry, regions)
+			workflowTrigger(client, "Tencent")
 		} else if client.AzureClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "Azure", telemetry, regions)
+			workflowTrigger(client, "Azure")
 		} else if client.ScalewayClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "Scaleway", telemetry, regions)
+			workflowTrigger(client, "Scaleway")
 		} else if client.MongoDBAtlasClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "MongoDBAtlas", telemetry, regions)
+			workflowTrigger(client, "MongoDBAtlas")
 		} else if client.GCPClient != nil {
-			go triggerFetchingWorfklow(ctx, client, "GCP", telemetry, regions)
+			workflowTrigger(client, "GCP")
 		}
 	}
-	return nil
+
+	wwg.Wait()
+	wp.Wait()
 }
 
 func checkUpgrade() {

internal/internal_test.go

@@ -0,0 +1,37 @@
+package internal
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/tailwarden/komiser/internal/config"
+	"github.com/tailwarden/komiser/utils"
+)
+
+// BenchmarkFactorial benchmarks the Factorial function.
+func BenchmarkFetchResources(b *testing.B) {
+	// Setup
+	ctx := context.TODO()
+	log.SetOutput(io.Discard)
+	analytics.Init()
+	cfg, clients, accounts, err := config.Load("/workspaces/komiser/config.toml", false, analytics, db)
+	if err != nil {
+		b.Fatalf("Error during config setup: %v", err)
+	}
+	err = setupDBConnection(cfg)
+	if err != nil {
+		b.Fatalf("Error during DB setup: %v", err)
+	}
+	err = utils.SetupSchema(db, cfg, accounts)
+	if err != nil {
+		b.Fatalf("Error during schema setup: %v", err)
+	}
+
+	// The benchmark function will run b.N times
+	for i := 0; i < b.N; i++ {
+		fetchResources(ctx, clients, []string{}, false)
+	}
+}

providers/aws/aws.go

@@ -98,7 +98,7 @@ func listOfSupportedServices() []providers.FetchDataFunction {
 	}
 }
 
-func FetchResources(ctx context.Context, client providers.ProviderClient, regions []string, db *bun.DB, telemetry bool, analytics utils.Analytics) {
+func FetchResources(ctx context.Context, client providers.ProviderClient, regions []string, db *bun.DB, telemetry bool, analytics utils.Analytics, wp *providers.WorkerPool) {
 	listOfSupportedRegions := getRegions()
 	if len(regions) > 0 {
 		log.Infof("Komiser will fetch resources from the following regions: %s", strings.Join(regions, ","))
@@ -108,23 +108,25 @@ func FetchResources(ctx context.Context, client providers.ProviderClient, region
 	for _, region := range listOfSupportedRegions {
 		client.AWSClient.Region = region
 		for _, fetchResources := range listOfSupportedServices() {
-			resources, err := fetchResources(ctx, client)
-			if err != nil {
-				log.Warnf("[%s][AWS] %s", client.Name, err)
-			} else {
-				for _, resource := range resources {
-				_, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost, relations=EXCLUDED.relations").Exec(context.Background())
-					if err != nil {
-						log.WithError(err).Errorf("db trigger failed")
+			wp.SubmitTask(func() {
+				resources, err := fetchResources(ctx, client)
+				if err != nil {
+					log.Warnf("[%s][AWS] %s", client.Name, err)
+				} else {
+					for _, resource := range resources {
+						_, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost, relations=EXCLUDED.relations").Exec(context.Background())
+						if err != nil {
+							log.WithError(err).Errorf("db trigger failed")
+						}
+					}
+					if telemetry {
+						analytics.TrackEvent("discovered_resources", map[string]interface{}{
+							"provider":  "AWS",
+							"resources": len(resources),
+						})
 					}
 				}
-				if telemetry {
-					analytics.TrackEvent("discovered_resources", map[string]interface{}{
-						"provider":  "AWS",
-						"resources": len(resources),
-					})
-				}
-			}
+			})
 		}
 	}
 }