From 90963c45953edfd323361817b7f63e8e0f613cdd Mon Sep 17 00:00:00 2001 From: titanventura Date: Mon, 9 Oct 2023 21:22:12 +0530 Subject: [PATCH 01/19] add. concurrent resource fetching code --- internal/internal.go | 34 +++++++++++++++++++--------------- providers/aws/aws.go | 34 ++++++++++++++++++---------------- providers/providers.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 31 deletions(-) diff --git a/internal/internal.go b/internal/internal.go index 1765a1171..48b966938 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -79,10 +79,14 @@ func Exec(address string, port int, configPath string, telemetry bool, a utils.A _, err = cron.Every(1).Hours().Do(func() { log.Info("Fetching resources workflow has started") - err = fetchResources(ctx, clients, regions, telemetry) + + numWorkers := 64 + wp := providers.NewWorkerPool(numWorkers) + err = fetchResources(ctx, clients, regions, telemetry, wp) if err != nil { log.Fatal(err) } + wp.Wait() }) if err != nil { @@ -209,7 +213,7 @@ func setupDBConnection(c *models.Config) error { return nil } -func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string) { +func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string, wp *providers.WorkerPool) { localHub := sentry.CurrentHub().Clone() defer func() { @@ -233,7 +237,7 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien switch provider { case "AWS": - aws.FetchResources(ctx, client, regions, db, telemetry, analytics) + aws.FetchResources(ctx, client, regions, db, telemetry, analytics, wp) case "DigitalOcean": do.FetchResources(ctx, client, db, telemetry, analytics) case "OCI": @@ -257,30 +261,30 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien } } -func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool) error { +func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool, wp *providers.WorkerPool) error { for _, client := range clients { if client.AWSClient != nil { - go triggerFetchingWorfklow(ctx, client, "AWS", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "AWS", telemetry, regions, wp) } else if client.DigitalOceanClient != nil { - go triggerFetchingWorfklow(ctx, client, "DigitalOcean", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "DigitalOcean", telemetry, regions, wp) } else if client.OciClient != nil { - go triggerFetchingWorfklow(ctx, client, "OCI", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "OCI", telemetry, regions, wp) } else if client.CivoClient != nil { - go triggerFetchingWorfklow(ctx, client, "Civo", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "Civo", telemetry, regions, wp) } else if client.K8sClient != nil { - go triggerFetchingWorfklow(ctx, client, "Kubernetes", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "Kubernetes", telemetry, regions, wp) } else if client.LinodeClient != nil { - go triggerFetchingWorfklow(ctx, client, "Linode", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "Linode", telemetry, regions, wp) } else if client.TencentClient != nil { - go triggerFetchingWorfklow(ctx, client, "Tencent", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "Tencent", telemetry, regions, wp) } else if client.AzureClient != nil { - go triggerFetchingWorfklow(ctx, client, "Azure", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "Azure", telemetry, regions, wp) } else if client.ScalewayClient != nil { - go triggerFetchingWorfklow(ctx, client, "Scaleway", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "Scaleway", telemetry, regions, wp) } else if client.MongoDBAtlasClient != nil { - go triggerFetchingWorfklow(ctx, client, "MongoDBAtlas", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "MongoDBAtlas", telemetry, regions, wp) } else if client.GCPClient != nil { - go triggerFetchingWorfklow(ctx, client, "GCP", telemetry, regions) + go triggerFetchingWorfklow(ctx, client, "GCP", telemetry, regions, wp) } } return nil diff --git a/providers/aws/aws.go b/providers/aws/aws.go index ecf1e862e..7b1cc8f4d 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -98,7 +98,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { } } -func FetchResources(ctx context.Context, client providers.ProviderClient, regions []string, db *bun.DB, telemetry bool, analytics utils.Analytics) { +func FetchResources(ctx context.Context, client providers.ProviderClient, regions []string, db *bun.DB, telemetry bool, analytics utils.Analytics, wp *providers.WorkerPool) { listOfSupportedRegions := getRegions() if len(regions) > 0 { log.Infof("Komiser will fetch resources from the following regions: %s", strings.Join(regions, ",")) @@ -108,23 +108,25 @@ func FetchResources(ctx context.Context, client providers.ProviderClient, region for _, region := range listOfSupportedRegions { client.AWSClient.Region = region for _, fetchResources := range listOfSupportedServices() { - resources, err := fetchResources(ctx, client) - if err != nil { - log.Warnf("[%s][AWS] %s", client.Name, err) - } else { - for _, resource := range resources { - _, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost, relations=EXCLUDED.relations").Exec(context.Background()) - if err != nil { - log.WithError(err).Errorf("db trigger failed") + wp.SubmitTask(func() { + resources, err := fetchResources(ctx, client) + if err != nil { + log.Warnf("[%s][AWS] %s", client.Name, err) + } else { + for _, resource := range resources { + _, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost, relations=EXCLUDED.relations").Exec(context.Background()) + if err != nil { + log.WithError(err).Errorf("db trigger failed") + } + } + if telemetry { + analytics.TrackEvent("discovered_resources", map[string]interface{}{ + "provider": "AWS", + "resources": len(resources), + }) } } - if telemetry { - analytics.TrackEvent("discovered_resources", map[string]interface{}{ - "provider": "AWS", - "resources": len(resources), - }) - } - } + }) } } } diff --git a/providers/providers.go b/providers/providers.go index 138fa34da..9e2b1dc22 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -2,6 +2,7 @@ package providers import ( "context" + "sync" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/aws/aws-sdk-go-v2/aws" @@ -47,3 +48,39 @@ type K8sClient struct { Client *kubernetes.Clientset OpencostBaseUrl string } + +type WorkerPool struct { + numWorkers int + tasks chan func() + wg sync.WaitGroup +} + +func NewWorkerPool(numWorkers int) *WorkerPool { + return &WorkerPool{ + numWorkers: numWorkers, + tasks: make(chan func()), + } +} + +func (wp *WorkerPool) Start() { + for i := 0; i < wp.numWorkers; i++ { + wp.wg.Add(1) + go wp.worker() + } +} + +func (wp *WorkerPool) SubmitTask(task func()) { + wp.tasks <- task +} + +func (wp *WorkerPool) Wait() { + close(wp.tasks) + wp.wg.Wait() +} + +func (wp *WorkerPool) worker() { + defer wp.wg.Done() + for task := range wp.tasks { + task() + } +} From 3599f1f22e0dd30e5edd499c4752ea775b4ddc5a Mon Sep 17 00:00:00 2001 From: titanventura Date: Tue, 10 Oct 2023 00:35:10 +0530 Subject: [PATCH 02/19] feat(concurrency): fix - waiting and waitgroup logic --- internal/internal.go | 32 +++++++++++++++++++++----------- providers/providers.go | 8 ++++---- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/internal/internal.go b/internal/internal.go index 48b966938..2e3288c09 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -82,6 +82,7 @@ func Exec(address string, port int, configPath string, telemetry bool, a utils.A numWorkers := 64 wp := providers.NewWorkerPool(numWorkers) + wp.Start() err = fetchResources(ctx, clients, regions, telemetry, wp) if err != nil { log.Fatal(err) @@ -262,29 +263,38 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien } func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool, wp *providers.WorkerPool) error { + + workflowTrigger := func(client providers.ProviderClient, provider string) { + wp.Wg.Add(1) + go func() { + defer wp.Wg.Done() + triggerFetchingWorfklow(ctx, client, provider, telemetry, regions, wp) + }() + } + for _, client := range clients { if client.AWSClient != nil { - go triggerFetchingWorfklow(ctx, client, "AWS", telemetry, regions, wp) + workflowTrigger(client, "AWS") } else if client.DigitalOceanClient != nil { - go triggerFetchingWorfklow(ctx, client, "DigitalOcean", telemetry, regions, wp) + workflowTrigger(client, "DigitalOcean") } else if client.OciClient != nil { - go triggerFetchingWorfklow(ctx, client, "OCI", telemetry, regions, wp) + workflowTrigger(client, "OCI") } else if client.CivoClient != nil { - go triggerFetchingWorfklow(ctx, client, "Civo", telemetry, regions, wp) + workflowTrigger(client, "Civo") } else if client.K8sClient != nil { - go triggerFetchingWorfklow(ctx, client, "Kubernetes", telemetry, regions, wp) + workflowTrigger(client, "Kubernetes") } else if client.LinodeClient != nil { - go triggerFetchingWorfklow(ctx, client, "Linode", telemetry, regions, wp) + workflowTrigger(client, "Linode") } else if client.TencentClient != nil { - go triggerFetchingWorfklow(ctx, client, "Tencent", telemetry, regions, wp) + workflowTrigger(client, "Tencent") } else if client.AzureClient != nil { - go triggerFetchingWorfklow(ctx, client, "Azure", telemetry, regions, wp) + workflowTrigger(client, "Azure") } else if client.ScalewayClient != nil { - go triggerFetchingWorfklow(ctx, client, "Scaleway", telemetry, regions, wp) + workflowTrigger(client, "Scaleway") } else if client.MongoDBAtlasClient != nil { - go triggerFetchingWorfklow(ctx, client, "MongoDBAtlas", telemetry, regions, wp) + workflowTrigger(client, "MongoDBAtlas") } else if client.GCPClient != nil { - go triggerFetchingWorfklow(ctx, client, "GCP", telemetry, regions, wp) + workflowTrigger(client, "GCP") } } return nil diff --git a/providers/providers.go b/providers/providers.go index 9e2b1dc22..81afb7b62 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -52,7 +52,7 @@ type K8sClient struct { type WorkerPool struct { numWorkers int tasks chan func() - wg sync.WaitGroup + Wg sync.WaitGroup } func NewWorkerPool(numWorkers int) *WorkerPool { @@ -64,23 +64,23 @@ func NewWorkerPool(numWorkers int) *WorkerPool { func (wp *WorkerPool) Start() { for i := 0; i < wp.numWorkers; i++ { - wp.wg.Add(1) go wp.worker() } } func (wp *WorkerPool) SubmitTask(task func()) { + wp.Wg.Add(1) wp.tasks <- task } func (wp *WorkerPool) Wait() { + wp.Wg.Wait() close(wp.tasks) - wp.wg.Wait() } func (wp *WorkerPool) worker() { - defer wp.wg.Done() for task := range wp.tasks { task() + wp.Wg.Done() } } From 493845ddeac6eab63f3a7f15dd7876937a725a16 Mon Sep 17 00:00:00 2001 From: Azanul Haque <42029519+Azanul@users.noreply.github.com> Date: Tue, 10 Oct 2023 06:30:56 +0000 Subject: [PATCH 03/19] using a separate WaitGroup Signed-off-by: Azanul Haque <42029519+Azanul@users.noreply.github.com> --- internal/internal.go | 10 +++++++--- providers/providers.go | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/internal.go b/internal/internal.go index 2e3288c09..1c55b6de5 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -11,6 +11,7 @@ import ( "runtime" "strconv" "strings" + "sync" "time" "github.com/getsentry/sentry-go" @@ -54,7 +55,7 @@ var Arch = runtime.GOARCH var db *bun.DB var analytics utils.Analytics -func Exec(address string, port int, configPath string, telemetry bool, a utils.Analytics, regions []string, cmd *cobra.Command) error { +func Exec(address string, port int, configPath string, telemetry bool, a utils.Analytics, regions []string, _ *cobra.Command) error { analytics = a ctx := context.Background() @@ -264,10 +265,11 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool, wp *providers.WorkerPool) error { + var wwg sync.WaitGroup workflowTrigger := func(client providers.ProviderClient, provider string) { - wp.Wg.Add(1) + wwg.Add(1) go func() { - defer wp.Wg.Done() + defer wwg.Done() triggerFetchingWorfklow(ctx, client, provider, telemetry, regions, wp) }() } @@ -297,6 +299,8 @@ func fetchResources(ctx context.Context, clients []providers.ProviderClient, reg workflowTrigger(client, "GCP") } } + + wwg.Wait() return nil } diff --git a/providers/providers.go b/providers/providers.go index 81afb7b62..ef0198714 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -52,7 +52,7 @@ type K8sClient struct { type WorkerPool struct { numWorkers int tasks chan func() - Wg sync.WaitGroup + wg sync.WaitGroup } func NewWorkerPool(numWorkers int) *WorkerPool { @@ -69,18 +69,18 @@ func (wp *WorkerPool) Start() { } func (wp *WorkerPool) SubmitTask(task func()) { - wp.Wg.Add(1) + wp.wg.Add(1) wp.tasks <- task } func (wp *WorkerPool) Wait() { - wp.Wg.Wait() + wp.wg.Wait() close(wp.tasks) } func (wp *WorkerPool) worker() { for task := range wp.tasks { task() - wp.Wg.Done() + wp.wg.Done() } } From 352d7d91624c7b58edc2382f41d13bc9e2fb95e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:07:30 +0200 Subject: [PATCH 04/19] fix: improve contribution guidelines for setup and add proper links to references --- CONTRIBUTING.md | 58 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 591bd8dca..d1c554359 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -302,10 +302,10 @@ Komiser Dashboard is built on **Typescript** and **Next.js**. The entire fronten - **React Testing Library** Following are the pre-requisites needed to setup a Dev environment of Komiser dashboard: -- In nearly all cases, while contributing to Komiser UI, you will need to build and run the Komiser Server as well, using the CLI. Make sure to follow the steps mentioned in the **"Komiser Installation"** section above. +- In nearly all cases, while contributing to Komiser UI, you will need to build and run the Komiser Server as well, using the CLI. Make sure to follow the steps mentioned in the [**"Komiser Installation"**](#komiser-installation) section above. - Make sure to have all the **latest versions** of the frontend stack listed above. -### Setup a local Developement Server +### Setup a local Development Server Here are the steps to setup and access the Komiser dashboard: @@ -317,9 +317,43 @@ go mod download ``` **Step 1:** -From the root folder, start the Komiser backend server using the following command: +Generate a `config.toml` file in the root folder if you haven't done so already. +Paste in the following example content: + +```toml +[[aws]] + name="sandbox" + source="CREDENTIALS_FILE" + path="./path/to/credentials/file" + profile="default" + +[[aws]] + name="staging" + source="CREDENTIALS_FILE" + path="./path/to/credentials/file" + profile="staging-account" + +[[gcp]] + name="production" + source="ENVIRONMENT_VARIABLES" + # path="./path/to/credentials/file" specify if CREDENTIALS_FILE is used + profile="production" + +[sqlite] + file="komiser.db" ``` -go run *.go start --config /path/to/config.toml + +You can find more options in our [Quickstart Guide](https://docs.komiser.io/getting-started/quickstart). It shows you how to connect to PostreSQL and other things. + +**Edit the config** + +Then you also need to create your credentials file for any of your providers. Head over [here](https://docs.komiser.io/getting-started/quickstart#self-hosted) to have a list of guides to chose from. +Update the above config following one of the guides and remove the integrations you don't need. + +**Step 2:** +From the root folder, start the Komiser backend server using the following command: +```bash +go run *.go start --config ./config.toml ``` > As soon as you run this, you'll be able to access the dashboard at `http://localhost:3000`. @@ -330,26 +364,28 @@ go run *.go start --config /path/to/config.toml > Follow the steps given below to do so 👇 > -**Step 2:** +**Step 3:** Head over to the **`dashboard`** directory: -``` +```bash cd dashboard ``` -**Step 3:** +**Step 4:** Create a new environment variable in the **`.env`** file: ``` -EXT_PUBLIC_API_URL=http://localhost:3000 +NEXT_PUBLIC_API_URL=http://localhost:3000 ``` -**Step 4:** +**Step 5:** Install the necessary **`npm`** dependencies and start the dev server using the following command: -``` +```bash npm install npm run dev ``` +The official supported NodeJS version is the latest `18.x.x` LTS release. + You'll be able to access the dashboard at **`http://localhost:3002/`** ![](https://hackmd.io/_uploads/ryvOPmFla.png) @@ -408,4 +444,4 @@ Feel free to reach out to us on our [Discord Server](https://discord.tailwarden. ## License -Komiser is an open-source software released under the [Elastic License 2.0 (ELv2)](https://github.com/tailwarden/komiser/blob/develop/LICENSE). \ No newline at end of file +Komiser is an open-source software released under the [Elastic License 2.0 (ELv2)](https://github.com/tailwarden/komiser/blob/develop/LICENSE). From 75ffd4984acd01e2370999a6d1f7fb3e42d937db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:09:09 +0200 Subject: [PATCH 05/19] fix: fix port in dashboard URLs --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1c554359..48ac8d827 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,7 +115,7 @@ In order to deploy a **self-hosted (local) instance** of Komiser, the next step **Step 3: Accessing the Komiser UI** -Once the local Komiser instance is running, you can access the dashboard UI on **`http://localhost:3000`** +Once the local Komiser instance is running, you can access the dashboard UI on **`http://localhost:3002`** ![komiser-dashboard](https://hackmd.io/_uploads/Syo0bMtgT.png) @@ -356,7 +356,7 @@ From the root folder, start the Komiser backend server using the following comma go run *.go start --config ./config.toml ``` -> As soon as you run this, you'll be able to access the dashboard at `http://localhost:3000`. +> As soon as you run this, you'll be able to access the dashboard at `http://localhost:3002`. > > An important point to note here is, this dashboard only reflects the changes from the **`master`** branch. > From ef2f5358a6516a5af1574bd2eabde6e19c237b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:43:08 +0200 Subject: [PATCH 06/19] feat: add Storybook section for frontend contribution guide --- dashboard/README.md | 78 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/dashboard/README.md b/dashboard/README.md index 514b43c94..aa80c4fd0 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -27,6 +27,9 @@ If you get an error page such as this, please refer to the logs and our [docs](h Komiser components are documented under `/components` +You can find all the shared Components also inside [Storybook](https://storybook.komiser.io/). If you're implementing a new Story, please check for existing or new components with Storybook. +We will require a story for new shared components like icons, inputs or similar. + Component convention: - Component folder: component name in `kebab-case` @@ -40,12 +43,85 @@ Component convention: Additional instructions: -- To view this component on Storybook, run: `npm run storybook`, then pick `Card` +- To view this component on Storybook locally, run: `npm run storybook`, then pick `Card` image - To run the unit tests, run: `npm run test:watch`, hit `p`, then `card` image +## Adding to Storybook + +[**Storybook**](https://storybook.komiser.io/) is a tool for UI development. It makes development faster by isolating components. This allows you to work on one component at a time. If you create a new shared component or want to visualize variations of an existing one, follow these steps: + +### 1. **Create the Story**: + +In the same directory as your component, create a Storybook story: + +- Create a story file: component name in `UpperCamelCase.stories.*`. + +Here's a basic story format: +\```typescript +import React from 'react'; +import { ComponentStory, ComponentMeta } from '@storybook/react'; + +import YourComponent from './YourComponent'; + +export default { +title: 'Path/To/YourComponent', +component: YourComponent, +} as ComponentMeta; + +const Template: ComponentStory = (args) => ; + +export const Default = Template.bind({}); +Default.args = { +// default props here... +}; +\``` + +### 2. **Add Variations**: + +You can create multiple variations of your component by replicating the `Default` pattern. For example, if your component has a variation for a "disabled" state: +\```typescript +export const Disabled = Template.bind({}); +Disabled.args = { +// props to set the component to its disabled state... +}; +\``` + +### 3. **Mock Data**: + +If your component requires mock data, create a mock file: component name in `UpperCamelCase.mocks.*`. Import this data into your story file to use with your component variations. + +### 4. **Visual Check**: + +Run Storybook: +\```bash +npm run storybook +\``` +Your component should now appear in the Storybook UI. Navigate to it, and verify all the variations display correctly. + +### 5. **Documentation**: + +Add a brief description and any notes on your component's functionality within the Storybook UI. Use the `parameters` object in your default export: +\```typescript +export default { +title: 'Path/To/YourComponent', +component: YourComponent, +parameters: { +docs: { +description: { +component: 'Your description here...', +}, +}, +}, +} as ComponentMeta; +\``` + +--- + +Remember: Storybook is not just a tool but also a way to document components. Ensure you provide meaningful names, descriptions, and use cases to help other developers understand the use and purpose of each component. + ## Testing We use Jest & React Testing Library for our unit tests. From 261488f251b7911d413ea2cec432d693485c15e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:46:40 +0200 Subject: [PATCH 07/19] fix: unify line breaks and remove escapes --- dashboard/README.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/dashboard/README.md b/dashboard/README.md index aa80c4fd0..46bfbdb68 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -60,7 +60,7 @@ In the same directory as your component, create a Storybook story: - Create a story file: component name in `UpperCamelCase.stories.*`. Here's a basic story format: -\```typescript +```typescript import React from 'react'; import { ComponentStory, ComponentMeta } from '@storybook/react'; @@ -77,17 +77,17 @@ export const Default = Template.bind({}); Default.args = { // default props here... }; -\``` +``` ### 2. **Add Variations**: You can create multiple variations of your component by replicating the `Default` pattern. For example, if your component has a variation for a "disabled" state: -\```typescript +```typescript export const Disabled = Template.bind({}); Disabled.args = { // props to set the component to its disabled state... }; -\``` +``` ### 3. **Mock Data**: @@ -96,15 +96,15 @@ If your component requires mock data, create a mock file: component name in `Upp ### 4. **Visual Check**: Run Storybook: -\```bash +```bash npm run storybook -\``` +``` Your component should now appear in the Storybook UI. Navigate to it, and verify all the variations display correctly. ### 5. **Documentation**: Add a brief description and any notes on your component's functionality within the Storybook UI. Use the `parameters` object in your default export: -\```typescript +```typescript export default { title: 'Path/To/YourComponent', component: YourComponent, @@ -116,7 +116,7 @@ component: 'Your description here...', }, }, } as ComponentMeta; -\``` +``` --- @@ -136,8 +136,7 @@ Testing convention: Testing examples: - Simple Jest unit test example (snippet from `/utils/formatNumber.test.ts`): - -```Typescript +```typescript import formatNumber from './formatNumber'; describe('formatNumber outputs', () => { @@ -152,8 +151,7 @@ describe('formatNumber outputs', () => { ``` - Jest & Testing library example (snippet from `/components/card/Card.test.tsx`): - -```Typescript +```typescript import { render, screen } from '@testing-library/react'; import RefreshIcon from '../icons/RefreshIcon'; import Card from './Card'; From 096740ef4e460ec176001398d19c197f600c6732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:56:26 +0200 Subject: [PATCH 08/19] feat: cross reference frontend readme and contribution guidelines --- CONTRIBUTING.md | 4 +++- dashboard/README.md | 47 ++++++++++++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48ac8d827..6930cbc32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -410,7 +410,9 @@ go-bindata-assetfs -o template.go dist/ dist/assets/images/ ## Contributing Best Practices -Here are some best practices to follow during the development process to make your changes more structured and making it easier for us to review: +> For frontend you can also follow the [Getting Started Guide](https://github.com/tailwarden/komiser/tree/develop/dashboard#getting-started) in the `/dashboard` README. + +Here are some best practices to follow during the backend development process to make your changes more structured and making it easier for us to review: 1. **Write Comprehensive Unit Tests:** diff --git a/dashboard/README.md b/dashboard/README.md index 46bfbdb68..4e6c05d05 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -4,7 +4,9 @@ Full frontend stack: `Next.js`, `Typescript`, `Tailwind`, `Storybook`, `Jest` & ## Getting Started -First, run the development server: +Follow the [Contribution Guide](https://github.com/tailwarden/komiser/blob/develop/CONTRIBUTING.md#contributing-to-komiser-dashboard-ui) first if you haven't done so already. Then come back here and follow the next steps: + +1. run the development server: ```bash # From the Komiser root folder start the Komiser server, run: @@ -17,11 +19,11 @@ NEXT_PUBLIC_API_URL=http://localhost:3000 npm run dev NEXT_PUBLIC_API_URL=http://localhost:3000 ``` -Open [http://localhost:3002/](http://localhost:3002). If you see the dashboard, congrats! It's all up and running correctly. +2. open [http://localhost:3002/](http://localhost:3002). If you see the dashboard, congrats! It's all up and running correctly. image -If you get an error page such as this, please refer to the logs and our [docs](https://docs.komiser.io/docs/introduction/getting-started). -image +> If you get an error page such as this, please refer to the logs and our [docs](https://docs.komiser.io/docs/introduction/getting-started). +> image ## Components @@ -60,6 +62,7 @@ In the same directory as your component, create a Storybook story: - Create a story file: component name in `UpperCamelCase.stories.*`. Here's a basic story format: + ```typescript import React from 'react'; import { ComponentStory, ComponentMeta } from '@storybook/react'; @@ -67,25 +70,28 @@ import { ComponentStory, ComponentMeta } from '@storybook/react'; import YourComponent from './YourComponent'; export default { -title: 'Path/To/YourComponent', -component: YourComponent, + title: 'Path/To/YourComponent', + component: YourComponent } as ComponentMeta; -const Template: ComponentStory = (args) => ; +const Template: ComponentStory = args => ( + +); export const Default = Template.bind({}); Default.args = { -// default props here... + // default props here... }; ``` ### 2. **Add Variations**: You can create multiple variations of your component by replicating the `Default` pattern. For example, if your component has a variation for a "disabled" state: + ```typescript export const Disabled = Template.bind({}); Disabled.args = { -// props to set the component to its disabled state... + // props to set the component to its disabled state... }; ``` @@ -96,25 +102,28 @@ If your component requires mock data, create a mock file: component name in `Upp ### 4. **Visual Check**: Run Storybook: + ```bash npm run storybook ``` + Your component should now appear in the Storybook UI. Navigate to it, and verify all the variations display correctly. ### 5. **Documentation**: Add a brief description and any notes on your component's functionality within the Storybook UI. Use the `parameters` object in your default export: + ```typescript export default { -title: 'Path/To/YourComponent', -component: YourComponent, -parameters: { -docs: { -description: { -component: 'Your description here...', -}, -}, -}, + title: 'Path/To/YourComponent', + component: YourComponent, + parameters: { + docs: { + description: { + component: 'Your description here...' + } + } + } } as ComponentMeta; ``` @@ -136,6 +145,7 @@ Testing convention: Testing examples: - Simple Jest unit test example (snippet from `/utils/formatNumber.test.ts`): + ```typescript import formatNumber from './formatNumber'; @@ -151,6 +161,7 @@ describe('formatNumber outputs', () => { ``` - Jest & Testing library example (snippet from `/components/card/Card.test.tsx`): + ```typescript import { render, screen } from '@testing-library/react'; import RefreshIcon from '../icons/RefreshIcon'; From 49e3335f89ea6dac9e9a650c3c62db82492ec42e Mon Sep 17 00:00:00 2001 From: Azanul Date: Thu, 12 Oct 2023 18:36:31 +0530 Subject: [PATCH 09/19] feat(devcontainer): devcontainer setup Signed-off-by: Azanul --- .devcontainer/devcontainer.json | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..94503ce34 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/go +{ + "name": "Go", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/go:1-1.21-bullseye", + "features": { + "ghcr.io/devcontainers-contrib/features/node-asdf:0": {}, + "ghcr.io/devcontainers-contrib/features/typescript:2": {} + }, + "customizations": { + "vscode": { + "extensions": [ + "golang.go-nightly", + "bradlc.vscode-tailwindcss" + ] + } + } + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "go version", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} From 0138660ff1b61f85694e2e5ee336e3be4cacbad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:08:16 +0200 Subject: [PATCH 10/19] feat: add more links and improve content formatting unity --- CONTRIBUTING.md | 74 ++++++++++++++++++++------------------------- dashboard/README.md | 2 +- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6930cbc32..344b2e1d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,32 +69,29 @@ Following these steps will ensure that your contributions are well-received, rev ``` 6. While submitting Pull Request, **make sure to change the base branch from**: [master](https://github.com/tailwarden/komiser/tree/master) to [develop](v). Making sure to avoid any possible merge conflicts -> ### Keeping your Fork Up-to-Date -> -> If you plan on doing anything more than just a tiny quick fix, you’ll want to **make sure you keep your fork up to date** by tracking the original [“upstream” repo](https://github.com/tailwarden/komiser) that you forked. -> -> Follow the steps given below to do so: -> -> 1. Add the 'upstream' repo to list of remotes: -> -> ```bash -> git remote add upstream https://github.com/tailwarden/komiser.git -> ``` -> -> 2. Fetch upstream repo’s branches and latest commits: -> -> ```bash -> git fetch upstream -> ``` -> -> 3. Checkout to the **`develop`** branch and merge the upstream: -> -> ```bash -> git checkout develop -> git merge upstream/develop -> ``` -> -> **Now, your local 'develop' branch is up-to-date with everything modified upstream!** +### Keeping your Fork Up-to-Date + +If you plan on doing anything more than just a tiny quick fix, you’ll want to **make sure you keep your fork up to date** by tracking the original [“upstream” repo](https://github.com/tailwarden/komiser) that you forked. + +Follow the steps given below to do so: + +1. Add the 'upstream' repo to list of remotes: + ```bash + git remote add upstream https://github.com/tailwarden/komiser.git +``` + +2. Fetch upstream repo’s branches and latest commits: + ```bash + git fetch upstream + ``` + +3. Checkout to the **`develop`** branch and merge the upstream: + ```bash + git checkout develop + git merge upstream/develop + ``` + +**Now, your local 'develop' branch is up-to-date with everything modified upstream!** ## Contributing to Komiser Engine @@ -137,7 +134,6 @@ Create a new **`provider_name.go`** file under **`providers/provider_name`** dir **Step 2:** Add the following boilerplate code, which defines the structure of any new provider to be added: - ```go package PROVIDER_NAME @@ -171,7 +167,6 @@ Then, the main task is writing the code to fetch all the resources/services usin **Step 3:** Add the information about the appropriate provider's SDK client in [**`providers/provider.go`**](https://github.com/tailwarden/komiser/blob/develop/providers/providers.go) file: - ```go type ProviderClient struct { @@ -200,7 +195,7 @@ type AzureClient struct { Add the provider configuration in TOML format in your **`config.toml`** file, which will be used by Komiser to configure your account with the CLI. An example configuration entry for configuring a Google Cloud account in the **`config.toml`** file would look like this: -``` +```toml [[gcp]] name="production" source="ENVIRONMENT_VARIABLES" @@ -210,15 +205,13 @@ profile="production" **Step 5:** Build a new Komiser binary with the latest code changes by running: - -``` +```bash go build ``` **Step 6:** Start a new Komiser development server using this new binary: - -``` +```bash ./komiser start ``` @@ -233,7 +226,6 @@ Create a new file **`servicename.go`** under the path **`providers/provider_name **Step 2:** Add the following boilerplate code, which defines the structure of any new service/resource to be added for a cloud provider: - ```go package service @@ -312,7 +304,7 @@ Here are the steps to setup and access the Komiser dashboard: **Step 0:** Install the necessary Go dependencies using the following command: -``` +```bash go mod download ``` @@ -343,7 +335,8 @@ Paste in the following example content: file="komiser.db" ``` -You can find more options in our [Quickstart Guide](https://docs.komiser.io/getting-started/quickstart). It shows you how to connect to PostreSQL and other things. +> You can find more options in our [Quickstart Guide](https://docs.komiser.io/getting-started/quickstart). +> For example it shows you how to connect to PostreSQL and other things. **Edit the config** @@ -356,13 +349,10 @@ From the root folder, start the Komiser backend server using the following comma go run *.go start --config ./config.toml ``` -> As soon as you run this, you'll be able to access the dashboard at `http://localhost:3002`. -> +> As soon as you run this, you'll be able to access the dashboard at [`http://localhost:3002/`](http://localhost:3002). > An important point to note here is, this dashboard only reflects the changes from the **`master`** branch. -> -> For our purpose, we certainly need changes to be reflected from our development branch! +> For our purpose, we certainly need changes to be reflected from our development branch! > Follow the steps given below to do so 👇 -> **Step 3:** Head over to the **`dashboard`** directory: @@ -386,7 +376,7 @@ npm run dev The official supported NodeJS version is the latest `18.x.x` LTS release. -You'll be able to access the dashboard at **`http://localhost:3002/`** +You'll be able to access the dashboard at [**`http://localhost:3002/`**](http://localhost:3002) ![](https://hackmd.io/_uploads/ryvOPmFla.png) diff --git a/dashboard/README.md b/dashboard/README.md index 4e6c05d05..9a53d1d0e 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -20,7 +20,7 @@ NEXT_PUBLIC_API_URL=http://localhost:3000 ``` 2. open [http://localhost:3002/](http://localhost:3002). If you see the dashboard, congrats! It's all up and running correctly. -image + image > If you get an error page such as this, please refer to the logs and our [docs](https://docs.komiser.io/docs/introduction/getting-started). > image From 083e4c6df7e509a120aceb86e61d8a97efa80336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:19:27 +0200 Subject: [PATCH 11/19] fix: adress all formatting issues and add links to tools mentioned --- CONTRIBUTING.md | 77 +++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 344b2e1d0..e98509746 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,31 +42,34 @@ Following these steps will ensure that your contributions are well-received, rev ### Fork and Pull Request Flow 1. Head over to the [Komiser GitHub repo](https://github.com/tailwarden/komiser) and "fork it" into your own GitHub account. + 2. Clone your fork to your local machine, using the following command: - ```bash - git clone git@github.com:USERNAME/FORKED-PROJECT.git - ``` + ```bash + git clone git@github.com:USERNAME/FORKED-PROJECT.git + ``` + 3. Create a new branch based-off **`develop`** branch: - ```bash - git checkout develop - git checkout -b fix/XXX-something develop - ``` - Make sure to follow the following branch naming conventions: - - For feature/enchancements, use: **`feature/xxxx-name_of_feature`** - - For bug fixes, use: **`fix/xxxx-name_of_bug`**

- - > Here, **`xxxx`** is the issue number associated with the bug/feature! - - For example: - ```bash - git checkout -b feature/1022-kubecost-integration develop - ``` + ```bash + git checkout develop + git checkout -b fix/XXX-something develop + ``` + Make sure to follow the following branch naming conventions: + - For feature/enchancements, use: **`feature/xxxx-name_of_feature`** + - For bug fixes, use: **`fix/xxxx-name_of_bug`**

+ > Here, **`xxxx`** is the issue number associated with the bug/feature! + > For example: + > ```bash + > git checkout -b feature/1022-kubecost-integration develop + > ``` + 4. Implement the changes or additions you intend to contribute. Whether it's **bug fixes**, **new features**, or **enhancements**, this is where you put your coding skills to use. + 5. Once your changes are ready, you may then commit and push the changes from your working branch: - ```bash - git commit -m "nice_commit_description" - git push origin feature/1022-kubecost-integration - ``` + ```bash + git commit -m "nice_commit_description" + git push origin feature/1022-kubecost-integration + ``` + 6. While submitting Pull Request, **make sure to change the base branch from**: [master](https://github.com/tailwarden/komiser/tree/master) to [develop](v). Making sure to avoid any possible merge conflicts ### Keeping your Fork Up-to-Date @@ -76,20 +79,20 @@ If you plan on doing anything more than just a tiny quick fix, you’ll want to Follow the steps given below to do so: 1. Add the 'upstream' repo to list of remotes: - ```bash - git remote add upstream https://github.com/tailwarden/komiser.git -``` + ```bash + git remote add upstream https://github.com/tailwarden/komiser.git + ``` 2. Fetch upstream repo’s branches and latest commits: - ```bash - git fetch upstream - ``` + ```bash + git fetch upstream + ``` 3. Checkout to the **`develop`** branch and merge the upstream: - ```bash - git checkout develop - git merge upstream/develop - ``` + ```bash + git checkout develop + git merge upstream/develop + ``` **Now, your local 'develop' branch is up-to-date with everything modified upstream!** @@ -286,12 +289,12 @@ Additionally, [here](https://youtu.be/Vn5uc2elcVg?feature=shared) is a video tut ## Contributing to Komiser Dashboard UI Komiser Dashboard is built on **Typescript** and **Next.js**. The entire frontend stack used is as follows: -- **Next.js** -- **Typescript** -- **Tailwind** -- **Storybook** -- **Jest** -- **React Testing Library** +- [**Next.js**](https://nextjs.org/) +- [**Typescript**](https://www.typescriptlang.org/) +- [**Tailwind**](https://tailwindcss.com/) +- [**Storybook**](https://storybook.js.org/docs/react/get-started/why-storybook) +- [**Jest** ](https://jestjs.io/) +- [**React Testing Library**](https://testing-library.com/docs/react-testing-library/intro/) Following are the pre-requisites needed to setup a Dev environment of Komiser dashboard: - In nearly all cases, while contributing to Komiser UI, you will need to build and run the Komiser Server as well, using the CLI. Make sure to follow the steps mentioned in the [**"Komiser Installation"**](#komiser-installation) section above. From 02fe761c7ae04d2c1c7c8c600cd94df66401a012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 16:12:11 +0200 Subject: [PATCH 12/19] feat: reformat Contribution Guidelines to habe niver formatting --- CONTRIBUTING.md | 328 +++++++++++++++++++++++++----------------------- 1 file changed, 168 insertions(+), 160 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e98509746..54daed77f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,11 +2,11 @@ ## Welcome -**Want to contribute to Komiser, but not sure where to start?** We've got you covered! +🎉 **Want to contribute to Komiser, but not sure where to start?** We've got you covered! **Welcome to the Komiser Contributing Guide** where you'll find everything you need to get started with contributing to Komiser. -We are thrilled to have you as part of our community and looking forward to your valuable contributions! +We are thrilled to have you as part of our community and looking forward to your valuable contributions! 🌟 ## Before You Get Started! @@ -14,9 +14,9 @@ Before getting started with your first contribution, here are a few things to ke ### For Major Feature Enhancements -**Planning to work on adding a major feature enhancement?** That's amazing we always encourage ambitious contributions. **Keep in mind though that we always recommend discussing your plans with the team first.** +🚀 **Planning to work on adding a major feature enhancement?** That's amazing we always encourage ambitious contributions. **Keep in mind though that we always recommend discussing your plans with the team first.** -This provides an opportunity for fellow contributors to - **guide you in the right direction**, **offer feedback on your design**, and **potentially identify if another contributor is already working on a similar task**. +This provides an opportunity for fellow contributors to - **guide you in the right direction**, **offer feedback on your design**, and **potentially identify if another contributor is already working on a similar task**. Here's how to reach out: @@ -25,52 +25,52 @@ Here's how to reach out: ### For Bug Reports and Minor Fixes -**Found a bug and wish to fix it?** Each and every contribution matters regardless of its size! +🐞 **Found a bug and wish to fix it?** Each and every contribution matters regardless of its size! Here are a few things to keep in mind in this case: - Before creating a new issue, please make sure to check for an already [existing issue](https://github.com/tailwarden/komiser/issues) for that bug. -- If an issue doesn't exist, [create a new issue](https://github.com/tailwarden/komiser/issues/new/choose) of **`type: Bug Report`** and make sure to provide as much detail as possible. +- If an issue doesn't exist, [create a new issue](https://github.com/tailwarden/komiser/issues/new/choose) of **\`type: Bug Report\`** and make sure to provide as much detail as possible. - Feel free to reach out to us on **#contributors** or **#feedback** channel on our Discord Server, if you need any assistance or have questions. ## General Contribution Flow This section covers the **general contribution flow** when contributing to any area of **Komiser (Engine or the UI)** and some best practices to follow along the way. -Following these steps will ensure that your contributions are well-received, reviewed, and integrated effectively into Komiser's codebase. +Following these steps will ensure that your contributions are well-received, reviewed, and integrated effectively into Komiser's codebase. ### Fork and Pull Request Flow -1. Head over to the [Komiser GitHub repo](https://github.com/tailwarden/komiser) and "fork it" into your own GitHub account. - -2. Clone your fork to your local machine, using the following command: - ```bash - git clone git@github.com:USERNAME/FORKED-PROJECT.git - ``` - -3. Create a new branch based-off **`develop`** branch: - ```bash - git checkout develop - git checkout -b fix/XXX-something develop - ``` - Make sure to follow the following branch naming conventions: - - For feature/enchancements, use: **`feature/xxxx-name_of_feature`** - - For bug fixes, use: **`fix/xxxx-name_of_bug`**

- > Here, **`xxxx`** is the issue number associated with the bug/feature! - > For example: - > ```bash - > git checkout -b feature/1022-kubecost-integration develop - > ``` - -4. Implement the changes or additions you intend to contribute. Whether it's **bug fixes**, **new features**, or **enhancements**, this is where you put your coding skills to use. - -5. Once your changes are ready, you may then commit and push the changes from your working branch: - ```bash - git commit -m "nice_commit_description" - git push origin feature/1022-kubecost-integration - ``` - -6. While submitting Pull Request, **make sure to change the base branch from**: [master](https://github.com/tailwarden/komiser/tree/master) to [develop](v). Making sure to avoid any possible merge conflicts +1️⃣ Head over to the [Komiser GitHub repo](https://github.com/tailwarden/komiser) and "fork it" into your own GitHub account. + +2️⃣ Clone your fork to your local machine, using the following command: +```bash +git clone git@github.com:USERNAME/FORKED-PROJECT.git +``` + +3️⃣ Create a new branch based-off **\`develop\`** branch: +```bash +git checkout develop +git checkout -b fix/XXX-something develop +``` +Make sure to follow the following branch naming conventions: +- For feature/enchancements, use: **\`feature/xxxx-name_of_feature\`** +- For bug fixes, use: **\`fix/xxxx-name_of_bug\`** +> Here, **\`xxxx\`** is the issue number associated with the bug/feature! +> For example: +> ```bash +> git checkout -b feature/1022-kubecost-integration develop +> ``` + +4️⃣ Implement the changes or additions you intend to contribute. Whether it's **bug fixes**, **new features**, or **enhancements**, this is where you put your coding skills to use. + +5️⃣ Once your changes are ready, you may then commit and push the changes from your working branch: +``` +git commit -m "nice_commit_description" +git push origin feature/1022-kubecost-integration +``` + +6️⃣ While submitting Pull Request, **make sure to change the base branch from**: [master](https://github.com/tailwarden/komiser/tree/master) to [develop](v). Making sure to avoid any possible merge conflicts ### Keeping your Fork Up-to-Date @@ -78,65 +78,91 @@ If you plan on doing anything more than just a tiny quick fix, you’ll want to Follow the steps given below to do so: -1. Add the 'upstream' repo to list of remotes: - ```bash - git remote add upstream https://github.com/tailwarden/komiser.git - ``` +1️⃣ Add the 'upstream' repo to list of remotes: +```bash +git remote add upstream https://github.com/tailwarden/komiser.git +``` -2. Fetch upstream repo’s branches and latest commits: - ```bash - git fetch upstream - ``` +2️⃣ Fetch upstream repo’s branches and latest commits: +```bash +git fetch upstream +``` -3. Checkout to the **`develop`** branch and merge the upstream: - ```bash - git checkout develop - git merge upstream/develop - ``` +3️⃣ Checkout to the **\`develop\`** branch and merge the upstream: +```bash +git checkout develop +git merge upstream/develop +``` **Now, your local 'develop' branch is up-to-date with everything modified upstream!** -## Contributing to Komiser Engine -The core Komiser Engine is written in Go (Golang) and leverages Go Modules. Following are the pre-requisistes needed to run Komiser on your local machine: -1. Latest Go version (currently its **`1.19`**) must be installed if you want to build and/or make changes to the existing code. The binary **`go1.19`** should be available in your path.

- > If you wish to keep multiple versions of Go in your system and don't want to disturb the existing ones, refer [the guide](https://go.dev/doc/manage-install). -2. Make sure that the **`GOPATH`** environment variable is configured appropriately. +# 🚀 Contributing to Komiser Engine + +The core Komiser Engine is written in Go (Golang) and leverages Go Modules. Here are the prerequisites to run Komiser on your local machine: -### Komiser Installation +1. 🌐 **Go Version**: + - Latest Go version (currently its **`1.19`**) must be installed if you want to build and/or make changes to the existing code. The binary **`go1.19`** should be available in your path. + > 💡 If you wish to keep multiple versions of Go in your system and don't want to disturb the existing ones, refer [the guide](https://go.dev/doc/manage-install). -**Step 1: Installing Komiser CLI** +2. 🔧 **GOPATH**: + - Ensure that the **`GOPATH`** environment variable is configured appropriately. -Follow the instructions given in the [documentation](https://docs.komiser.io/getting-started/installation) to install the **Komiser CLI**, according to your operating system. +--- -**Step 2: Connect to a Cloud Account** +## 🛠️ Komiser Installation -In order to deploy a **self-hosted (local) instance** of Komiser, the next step would be to connect your Komiser CLI to a cloud account of your choice. You may refer the documentation of the [supported cloud providers](https://docs.komiser.io/configuration/cloud-providers/aws) and follow the instructions using any one (Let's say AWS). +### **Step 1: Installing Komiser CLI** +Follow the instructions in the [documentation](https://docs.komiser.io/getting-started/installation) to install the **Komiser CLI** for your operating system. -**Step 3: Accessing the Komiser UI** +### **Step 2: Connect to a Cloud Account** +To deploy a **self-hosted (local) instance** of Komiser, connect your Komiser CLI to a cloud account of your choice. Refer to the documentation of the [supported cloud providers](https://docs.komiser.io/configuration/cloud-providers/aws). -Once the local Komiser instance is running, you can access the dashboard UI on **`http://localhost:3002`** +### **Step 3: Accessing the Komiser UI** +Access the dashboard UI at **`http://localhost:3002`** once the local Komiser instance is running. ![komiser-dashboard](https://hackmd.io/_uploads/Syo0bMtgT.png) -### Ways to Contribute in Komiser Engine +--- + +# 🚀 Contributing to Komiser Engine + +The core Komiser Engine is written in Go (Golang) and leverages Go Modules. Here are the prerequisites to run Komiser on your local machine: + +1. 🌐 **Go Version**: + - Latest Go version (currently its **`1.19`**) must be installed if you want to build and/or make changes to the existing code. The binary **`go1.19`** should be available in your path. + > 💡 If you wish to keep multiple versions of Go in your system and don't want to disturb the existing ones, refer [the guide](https://go.dev/doc/manage-install). + +2. 🔧 **GOPATH**: + - Ensure that the **`GOPATH`** environment variable is configured appropriately. + +--- +## 🛠️ Komiser Installation -Komiser is an open source cloud-agnostic resource manager which helps you break down your cloud resources cost at the resource level. +### **Step 1: Installing Komiser CLI** +Follow the instructions in the [documentation](https://docs.komiser.io/getting-started/installation) to install the **Komiser CLI** for your operating system. -Due to the nature of Komiser, a cloud-agnostic cloud management tool, our work is never really done! There are always more providers and cloud services that can be added, updated and cost calculated. +### **Step 2: Connect to a Cloud Account** +To deploy a **self-hosted (local) instance** of Komiser, connect your Komiser CLI to a cloud account of your choice. Refer to the documentation of the [supported cloud providers](https://docs.komiser.io/configuration/cloud-providers/aws). -Therefore, there are mainly three ways you can contribute to the Komiser Engine: +### **Step 3: Accessing the Komiser UI** +Access the dashboard UI at **`http://localhost:3002`** once the local Komiser instance is running. -### 1. Adding a new Cloud Provider +![komiser-dashboard](https://hackmd.io/_uploads/Syo0bMtgT.png) + +--- + +## 🌟 Ways to Contribute to Komiser Engine -Here are the general steps to integrate a new cloud provider in Komiser: +Komiser is an open-source cloud-agnostic resource manager. It helps you break down cloud resource costs at the resource level. As a cloud-agnostic cloud management tool, we always have more providers and cloud services to add, update, and cost-calculate. -**Step 1:** -Create a new **`provider_name.go`** file under **`providers/provider_name`** directory. +### 1️⃣ Adding a new Cloud Provider + +- **Step 1:** Create **`provider_name.go`** in **`providers/provider_name`** directory. + +- **Step 2:** Add the following boilerplate: -**Step 2:** -Add the following boilerplate code, which defines the structure of any new provider to be added: ```go package PROVIDER_NAME @@ -166,12 +192,9 @@ func FetchProviderData(ctx context.Context, client ProviderClient, db *bun.DB) { } ``` -Then, the main task is writing the code to fetch all the resources/services using the provider's Go client SDK. You may refer any [existing examples](https://github.com/tailwarden/komiser/tree/develop/providers) to understand better. +- **Step 3:** Add SDK client details in [**`providers/provider.go`**](https://github.com/tailwarden/komiser/blob/develop/providers/providers.go): -**Step 3:** -Add the information about the appropriate provider's SDK client in [**`providers/provider.go`**](https://github.com/tailwarden/komiser/blob/develop/providers/providers.go) file: ```go - type ProviderClient struct { AWSClient *aws.Config DigitalOceanClient *godo.Client @@ -191,13 +214,10 @@ type AzureClient struct { Credentials *azidentity.ClientSecretCredential SubscriptionId string } - ``` -**Step 4:** -Add the provider configuration in TOML format in your **`config.toml`** file, which will be used by Komiser to configure your account with the CLI. +- **Step 4:** Add provider configuration in TOML format in **`config.toml`**: -An example configuration entry for configuring a Google Cloud account in the **`config.toml`** file would look like this: ```toml [[gcp]] name="production" @@ -206,21 +226,19 @@ source="ENVIRONMENT_VARIABLES" profile="production" ``` -**Step 5:** -Build a new Komiser binary with the latest code changes by running: +- **Step 5:** Compile a new Komiser binary: + ```bash go build ``` -**Step 6:** -Start a new Komiser development server using this new binary: +- **Step 6:** Start a new Komiser development server: + ```bash ./komiser start ``` -**If everything goes well, you'll see a new cloud provider added in the Komiser Dashboard!** - -### 2. Adding a new Cloud Service/Resource +### 2️⃣ Adding a new Cloud Service/Resource Here are the general steps to add a new service/resource for a cloud provider in Komiser: @@ -282,38 +300,39 @@ Repeat steps **`4,5,6`** accordingly and you'll see a new resource/service added Additionally, [here](https://youtu.be/Vn5uc2elcVg?feature=shared) is a video tutorial of the entire process for your reference. -#### 3. Enhance existing Cloud service/resource +### 3️⃣ Enhance existing Cloud service/resource **So, you wish to improve the code quality of an existing cloud service/resource?** Feel free to discuss your ideas with us on our [Discord Server](https://discord.tailwarden.com) and [open a new issue](https://github.com/tailwarden/komiser/issues). -## Contributing to Komiser Dashboard UI +# 🚀 Contributing to Komiser Dashboard UI + +Komiser Dashboard utilizes a modern tech stack. Here's a brief about it: -Komiser Dashboard is built on **Typescript** and **Next.js**. The entire frontend stack used is as follows: -- [**Next.js**](https://nextjs.org/) -- [**Typescript**](https://www.typescriptlang.org/) -- [**Tailwind**](https://tailwindcss.com/) -- [**Storybook**](https://storybook.js.org/docs/react/get-started/why-storybook) -- [**Jest** ](https://jestjs.io/) -- [**React Testing Library**](https://testing-library.com/docs/react-testing-library/intro/) +- **Framework**: [**Next.js**](https://nextjs.org/) +- **Language**: [**Typescript**](https://www.typescriptlang.org/) +- **CSS**: [**Tailwind**](https://tailwindcss.com/) +- **Component Library**: [**Storybook**](https://storybook.js.org/docs/react/get-started/why-storybook) +- **Testing**: + - [**Jest**](https://jestjs.io/) + - [**React Testing Library**](https://testing-library.com/docs/react-testing-library/intro/) -Following are the pre-requisites needed to setup a Dev environment of Komiser dashboard: -- In nearly all cases, while contributing to Komiser UI, you will need to build and run the Komiser Server as well, using the CLI. Make sure to follow the steps mentioned in the [**"Komiser Installation"**](#komiser-installation) section above. -- Make sure to have all the **latest versions** of the frontend stack listed above. +## 🧩 Prerequisites -### Setup a local Development Server +1. While working on Komiser UI, you'd typically need the Komiser Server as well. Follow the instructions in the [**Komiser Installation**](#komiser-installation) section above. +2. Ensure you're using the **latest versions** of the tech stack mentioned above. -Here are the steps to setup and access the Komiser dashboard: +## 🛠 Setup a Local Development Server -**Step 0:** +Let's get your hands dirty by setting up the Komiser dashboard: -Install the necessary Go dependencies using the following command: +### Step 0: Grab the Go Dependencies ```bash go mod download ``` -**Step 1:** -Generate a `config.toml` file in the root folder if you haven't done so already. -Paste in the following example content: +### Step 1: Configure `config.toml` + +If it doesn't exist, create `config.toml` in the root and use this template: ```toml [[aws]] @@ -338,105 +357,94 @@ Paste in the following example content: file="komiser.db" ``` -> You can find more options in our [Quickstart Guide](https://docs.komiser.io/getting-started/quickstart). -> For example it shows you how to connect to PostreSQL and other things. - -**Edit the config** +> 📘 Dive deeper in our [Quickstart Guide](https://docs.komiser.io/getting-started/quickstart) for more configurations like connecting to PostgreSQL. -Then you also need to create your credentials file for any of your providers. Head over [here](https://docs.komiser.io/getting-started/quickstart#self-hosted) to have a list of guides to chose from. -Update the above config following one of the guides and remove the integrations you don't need. +Now, craft your credentials. Check out the guides [here](https://docs.komiser.io/getting-started/quickstart#self-hosted) to tailor the config for your needs. -**Step 2:** -From the root folder, start the Komiser backend server using the following command: +### Step 2: Boot Up the Komiser Backend ```bash go run *.go start --config ./config.toml ``` -> As soon as you run this, you'll be able to access the dashboard at [`http://localhost:3002/`](http://localhost:3002). -> An important point to note here is, this dashboard only reflects the changes from the **`master`** branch. -> For our purpose, we certainly need changes to be reflected from our development branch! -> Follow the steps given below to do so 👇 - -**Step 3:** -Head over to the **`dashboard`** directory: +> 🖥️ This fires up the dashboard at [`http://localhost:3002/`](http://localhost:3002). However, it mirrors the **`master`** branch. Let's make it reflect your development branch! +### Step 3: Navigate to the Dashboard Directory ```bash cd dashboard ``` -**Step 4:** -Create a new environment variable in the **`.env`** file: +### Step 4: Set up Environment Variables +Create or update the **`.env`** file: ``` NEXT_PUBLIC_API_URL=http://localhost:3000 ``` -**Step 5:** -Install the necessary **`npm`** dependencies and start the dev server using the following command: +### Step 5: Spin up the Dev Server +Install dependencies and fire up the dev server: ```bash npm install npm run dev ``` -The official supported NodeJS version is the latest `18.x.x` LTS release. +> 🟢 NodeJS: Ensure you're on the `18.x.x` LTS release. -You'll be able to access the dashboard at [**`http://localhost:3002/`**](http://localhost:3002) +Once done, open up [**`http://localhost:3002/`**](http://localhost:3002) -![](https://hackmd.io/_uploads/ryvOPmFla.png) +![Komiser Dashboard](https://hackmd.io/_uploads/ryvOPmFla.png) -To understand the installation process in a bit more detail, you may refer the [video walkthrough](https://youtu.be/uwxj11-eRt8?feature=shared). +📺 For a more detailed walkthrough, check the [video tutorial](https://youtu.be/uwxj11-eRt8?feature=shared). -### Understanding the UI components +## 🎨 Understanding the UI Components -The Komiser UI components are being handled and organised using [Storybook](https://storybook.js.org/). -Refer the [components section](https://github.com/tailwarden/komiser/tree/develop/dashboard#components) of the dashboard README to understand the component conventions used. +Komiser's UI elegance is sculpted using [Storybook](https://storybook.js.org/). Dive deep into the [components section](https://github.com/tailwarden/komiser/tree/develop/dashboard#components) to grasp our conventions. -### Testing Your Changes +## 🧪 Testing Your Changes -The Komiser dashboard uses **Jest** and **React Testing Library** for unit tests. Refer the [testing section](https://github.com/tailwarden/komiser/tree/develop/dashboard#testing) of the dashboard README to understand how you can write simple unit tests, to validate your changes. +We leverage **Jest** and **React Testing Library** for unit tests. Navigate to the [testing section](https://github.com/tailwarden/komiser/tree/develop/dashboard#testing) for insights on writing tests. -### Building a Go Artifact +## 📦 Building a Go Artifact -Once you have implemented the necessary frontend changes, make sure to build a new Go artifact using the following command: +After refining the frontend, craft a new Go artifact: ``` go-bindata-assetfs -o template.go dist/ dist/assets/images/ ``` -## Contributing Best Practices - -> For frontend you can also follow the [Getting Started Guide](https://github.com/tailwarden/komiser/tree/develop/dashboard#getting-started) in the `/dashboard` README. -Here are some best practices to follow during the backend development process to make your changes more structured and making it easier for us to review: +# 📚 Contributing Best Practices -1. **Write Comprehensive Unit Tests:** - - - When making code changes, be sure to include well-structured unit tests. - - Utilize [Go's built-in testing framework](https://pkg.go.dev/testing) for this purpose. - - Take inspiration from existing tests in the project. - - Before submitting your pull request, run the full test suite on your development branch to ensure your changes are thoroughly validated. +> 📘 For frontend endeavors, do explore the [Getting Started Guide](https://github.com/tailwarden/komiser/tree/develop/dashboard#getting-started) in the `/dashboard` README. -2. **Keep Documentation Updated:** +Embarking on backend development? Here are golden practices to streamline your contributions and facilitate our review process: - - Ensure relevant documentation is updated or added, according to new features being added or modified. +### 1. 🧪 Write Comprehensive Unit Tests +- Every code change craves well-thought-out unit tests. +- Leverage [Go's built-in testing framework](https://pkg.go.dev/testing) for this. +- Seek inspiration from tests already enhancing the project. +- 🚀 Before firing that pull request, run the entire test suite on your branch ensuring robust validation of your changes. -3. **Prioritize Clean Code:** +### 2. 📖 Keep Documentation Updated +- Newly added or tinkered features? Ensure the documentation mirrors your magic. - - Make sure to use **`go fmt`** to ensure uniform code formatting before committing your changes. - - Using IDEs and code editors like **VSCode** makes this easy, as they offer plugins that automate the formatting process. +### 3. 🎨 Prioritize Clean Code +- Seal your changes with **`go fmt`** for a consistent code style. +- Tools like **VSCode** come handy with plugins automating this formatting quest. -4. **Mindful Code Comments:** +### 4. 🖊 Mindful Code Comments +- Complex logic? Esoteric algorithms? Or just proud of your intricate code snippet? Comment them. +- Thoughtful comments pave way for an insightful review, letting others decipher your masterpiece with ease. - - Use comments to explain complex logic, algorithms, or any non-obvious parts of your code. - - Well-placed comments will make your code more accessible to others and will ultimately help in a smoother review process of your changes. +--- +# 🌟 Ending Note -## Ending Note +We trust this guide lights up your contribution path. Diving into Komiser's codebase should now be thrilling and engaging. -We hope this guide proves to be helpful and makes contributing to Komiser an exciting and fun process for you all. +In wrapping up, a **MASSIVE THANK YOU** echoes for your invaluable time and efforts. You're making Komiser even more radiant and welcoming for the community. -At the end, we wanna give you a **HUGE THANK YOU** for taking out your time in contributing and making Komiser better and more accessible to the community! +> 📞 Need a chat? Have queries buzzing? Buzz us on our [Discord Server](https://discord.tailwarden.com). -Feel free to reach out to us on our [Discord Server](https://discord.tailwarden.com) if you need any assistance or have any questions. +--- -## License +# 📜 License -Komiser is an open-source software released under the [Elastic License 2.0 (ELv2)](https://github.com/tailwarden/komiser/blob/develop/LICENSE). +Komiser, an emblem of open-source, basks under the [Elastic License 2.0 (ELv2)](https://github.com/tailwarden/komiser/blob/develop/LICENSE). From 757f39d8bd3a8f6c252e803d0eecb433f2f21fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 17:04:20 +0200 Subject: [PATCH 13/19] fix: adress review comments and remove duplicated content, also cleanup --- CONTRIBUTING.md | 50 +++++++++++-------------------------------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54daed77f..06b73a145 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,8 +65,8 @@ Make sure to follow the following branch naming conventions: 4️⃣ Implement the changes or additions you intend to contribute. Whether it's **bug fixes**, **new features**, or **enhancements**, this is where you put your coding skills to use. 5️⃣ Once your changes are ready, you may then commit and push the changes from your working branch: -``` -git commit -m "nice_commit_description" +```bash +git commit -m "fix(xxxx-name_of_bug): nice commit description" git push origin feature/1022-kubecost-integration ``` @@ -97,34 +97,6 @@ git merge upstream/develop **Now, your local 'develop' branch is up-to-date with everything modified upstream!** -# 🚀 Contributing to Komiser Engine - -The core Komiser Engine is written in Go (Golang) and leverages Go Modules. Here are the prerequisites to run Komiser on your local machine: - -1. 🌐 **Go Version**: - - Latest Go version (currently its **`1.19`**) must be installed if you want to build and/or make changes to the existing code. The binary **`go1.19`** should be available in your path. - > 💡 If you wish to keep multiple versions of Go in your system and don't want to disturb the existing ones, refer [the guide](https://go.dev/doc/manage-install). - -2. 🔧 **GOPATH**: - - Ensure that the **`GOPATH`** environment variable is configured appropriately. - ---- - -## 🛠️ Komiser Installation - -### **Step 1: Installing Komiser CLI** -Follow the instructions in the [documentation](https://docs.komiser.io/getting-started/installation) to install the **Komiser CLI** for your operating system. - -### **Step 2: Connect to a Cloud Account** -To deploy a **self-hosted (local) instance** of Komiser, connect your Komiser CLI to a cloud account of your choice. Refer to the documentation of the [supported cloud providers](https://docs.komiser.io/configuration/cloud-providers/aws). - -### **Step 3: Accessing the Komiser UI** -Access the dashboard UI at **`http://localhost:3002`** once the local Komiser instance is running. - -![komiser-dashboard](https://hackmd.io/_uploads/Syo0bMtgT.png) - ---- - # 🚀 Contributing to Komiser Engine The core Komiser Engine is written in Go (Golang) and leverages Go Modules. Here are the prerequisites to run Komiser on your local machine: @@ -159,9 +131,9 @@ Komiser is an open-source cloud-agnostic resource manager. It helps you break do ### 1️⃣ Adding a new Cloud Provider -- **Step 1:** Create **`provider_name.go`** in **`providers/provider_name`** directory. +- Step 1: Create **`provider_name.go`** in **`providers/provider_name`** directory. -- **Step 2:** Add the following boilerplate: +- Step 2: Add the following boilerplate: ```go package PROVIDER_NAME @@ -192,7 +164,7 @@ func FetchProviderData(ctx context.Context, client ProviderClient, db *bun.DB) { } ``` -- **Step 3:** Add SDK client details in [**`providers/provider.go`**](https://github.com/tailwarden/komiser/blob/develop/providers/providers.go): +- Step 3: Add SDK client details in [**`providers/provider.go`**](https://github.com/tailwarden/komiser/blob/develop/providers/providers.go): ```go type ProviderClient struct { @@ -325,12 +297,12 @@ Komiser Dashboard utilizes a modern tech stack. Here's a brief about it: Let's get your hands dirty by setting up the Komiser dashboard: -### Step 0: Grab the Go Dependencies +### 1️⃣ Grab the Go Dependencies ```bash go mod download ``` -### Step 1: Configure `config.toml` +### 2️⃣ Configure `config.toml` If it doesn't exist, create `config.toml` in the root and use this template: @@ -361,25 +333,25 @@ If it doesn't exist, create `config.toml` in the root and use this template: Now, craft your credentials. Check out the guides [here](https://docs.komiser.io/getting-started/quickstart#self-hosted) to tailor the config for your needs. -### Step 2: Boot Up the Komiser Backend +### 3️⃣ Boot Up the Komiser Backend ```bash go run *.go start --config ./config.toml ``` > 🖥️ This fires up the dashboard at [`http://localhost:3002/`](http://localhost:3002). However, it mirrors the **`master`** branch. Let's make it reflect your development branch! -### Step 3: Navigate to the Dashboard Directory +### 4️⃣ Navigate to the Dashboard Directory ```bash cd dashboard ``` -### Step 4: Set up Environment Variables +### 5️⃣ Set up Environment Variables Create or update the **`.env`** file: ``` NEXT_PUBLIC_API_URL=http://localhost:3000 ``` -### Step 5: Spin up the Dev Server +### 6️⃣ Spin up the Dev Server Install dependencies and fire up the dev server: ```bash npm install From d0d5f73f8aba8d31f92553674fbd5faad49493f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20R=C3=B6sel?= <320272+Traxmaxx@users.noreply.github.com> Date: Thu, 12 Oct 2023 17:06:28 +0200 Subject: [PATCH 14/19] fix: adress review comments --- dashboard/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/README.md b/dashboard/README.md index 9a53d1d0e..ef3fbb7d9 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -6,7 +6,7 @@ Full frontend stack: `Next.js`, `Typescript`, `Tailwind`, `Storybook`, `Jest` & Follow the [Contribution Guide](https://github.com/tailwarden/komiser/blob/develop/CONTRIBUTING.md#contributing-to-komiser-dashboard-ui) first if you haven't done so already. Then come back here and follow the next steps: -1. run the development server: +1. Run the development server: ```bash # From the Komiser root folder start the Komiser server, run: @@ -19,7 +19,7 @@ NEXT_PUBLIC_API_URL=http://localhost:3000 npm run dev NEXT_PUBLIC_API_URL=http://localhost:3000 ``` -2. open [http://localhost:3002/](http://localhost:3002). If you see the dashboard, congrats! It's all up and running correctly. +2. Open [http://localhost:3002/](http://localhost:3002). If you see the dashboard, congrats! It's all up and running correctly. image > If you get an error page such as this, please refer to the logs and our [docs](https://docs.komiser.io/docs/introduction/getting-started). From afab697af02a94a3fa1dca1475b9bca6a5df9370 Mon Sep 17 00:00:00 2001 From: Azanul Date: Fri, 13 Oct 2023 23:01:34 +0530 Subject: [PATCH 15/19] feat(aws utils): util func extension Signed-off-by: Azanul --- providers/aws/lambda/functions.go | 2 +- providers/aws/utils/utils.go | 29 +++++++++++++++++++---- providers/aws/utils/utils_test.go | 38 ++++++++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/providers/aws/lambda/functions.go b/providers/aws/lambda/functions.go index b7a345c6e..02a41e387 100644 --- a/providers/aws/lambda/functions.go +++ b/providers/aws/lambda/functions.go @@ -48,7 +48,7 @@ func Functions(ctx context.Context, client providers.ProviderClient) ([]models.R return resources, err } - priceMap, err := awsUtils.GetPriceMap(pricingOutput) + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) return resources, err diff --git a/providers/aws/utils/utils.go b/providers/aws/utils/utils.go index bc1708972..529c00bcb 100644 --- a/providers/aws/utils/utils.go +++ b/providers/aws/utils/utils.go @@ -11,8 +11,12 @@ import ( type ProductEntry struct { Product struct { Attributes struct { - Group string `json:"group"` - Operation string `json:"operation"` + Group string `json:"group"` + Operation string `json:"operation"` + GroupDescription string `json:"groupDescription"` + RequestDescription string `json:"requestDescription"` + InstanceType string `json:"instanceType"` + InstanceTypeFamily string `json:"instanceTypeFamily"` } `json:"attributes"` } `json:"product"` Terms struct { @@ -47,7 +51,7 @@ func GetCost(pds []PriceDimensions, v float64) float64 { return total } -func GetPriceMap(pricingOutput *pricing.GetProductsOutput) (map[string][]PriceDimensions, error) { +func GetPriceMap(pricingOutput *pricing.GetProductsOutput, field string) (map[string][]PriceDimensions, error) { priceMap := make(map[string][]PriceDimensions) if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { @@ -58,7 +62,22 @@ func GetPriceMap(pricingOutput *pricing.GetProductsOutput) (map[string][]PriceDi return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) } - group := price.Product.Attributes.Group + var key string + switch field { + case "group": + key = price.Product.Attributes.Group + case "operation": + key = price.Product.Attributes.Operation + case "groupDescription": + key = price.Product.Attributes.GroupDescription + case "requestDescription": + key = price.Product.Attributes.RequestDescription + case "instanceType": + key = price.Product.Attributes.InstanceType + case "instanceTypeFamily": + key = price.Product.Attributes.InstanceTypeFamily + } + unitPrices := []PriceDimensions{} for _, pd := range price.Terms.OnDemand { for _, p := range pd.PriceDimensions { @@ -66,7 +85,7 @@ func GetPriceMap(pricingOutput *pricing.GetProductsOutput) (map[string][]PriceDi } } - priceMap[group] = unitPrices + priceMap[key] = unitPrices } } diff --git a/providers/aws/utils/utils_test.go b/providers/aws/utils/utils_test.go index 1602fb242..bf2f6fb39 100644 --- a/providers/aws/utils/utils_test.go +++ b/providers/aws/utils/utils_test.go @@ -49,6 +49,7 @@ func TestGetCost(t *testing.T) { func TestGetPriceMap(t *testing.T) { testCases := []struct { inputPriceList []string + field string expectedNumProducts int expectedNumPriceDims map[string]int }{ @@ -77,6 +78,7 @@ func TestGetPriceMap(t *testing.T) { } } }`}, + field: "group", expectedNumProducts: 1, expectedNumPriceDims: map[string]int{"TestGroup": 1}, }, @@ -153,16 +155,46 @@ func TestGetPriceMap(t *testing.T) { } }`, }, + field: "group", expectedNumProducts: 2, expectedNumPriceDims: map[string]int{"TestGroup1": 2, "TestGroup2": 3}, }, + // Minimal valid JSON input with a single product, one price dimension & "instanceType" attribute + { + inputPriceList: []string{` + { + "product": { + "attributes": { + "instanceType": "TestInstanceType" + } + }, + "terms": { + "OnDemand": { + "test_term": { + "priceDimensions": { + "test_price_dimension": { + "beginRange": "0", + "endRange": "Inf", + "pricePerUnit": { + "USD": "0.1" + } + } + } + } + } + } + }`}, + field: "instanceType", + expectedNumProducts: 1, + expectedNumPriceDims: map[string]int{"TestInstanceType": 1}, + }, } for i, testCase := range testCases { t.Run(fmt.Sprintf("Test case %d", i+1), func(t *testing.T) { output := pricing.GetProductsOutput{ PriceList: testCase.inputPriceList, } - priceMap, err := GetPriceMap(&output) + priceMap, err := GetPriceMap(&output, "group") if err != nil { t.Errorf("Expected no error, but got: %v", err) } @@ -186,7 +218,7 @@ func TestGetPriceMap_InvalidJSON(t *testing.T) { output := pricing.GetProductsOutput{ PriceList: []string{invalidJSON}, } - _, err := GetPriceMap(&output) + _, err := GetPriceMap(&output, "group") if err == nil { t.Error("Expected an error, but got nil") } @@ -194,7 +226,7 @@ func TestGetPriceMap_InvalidJSON(t *testing.T) { func TestGetPriceMap_NoPricingOutput(t *testing.T) { // PricingOutput is nil - priceMap, err := GetPriceMap(nil) + priceMap, err := GetPriceMap(nil, "group") if err != nil { t.Errorf("Expected no error, but got: %v", err) } From a5a51a6fa8c8efb383d8da4f64d50fd9ebc85093 Mon Sep 17 00:00:00 2001 From: Azanul Date: Fri, 13 Oct 2023 23:02:48 +0530 Subject: [PATCH 16/19] feat(aws utils): util func extension s3 buckets Signed-off-by: Azanul --- providers/aws/s3/buckets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/aws/s3/buckets.go b/providers/aws/s3/buckets.go index de60e6ed6..005f6104e 100644 --- a/providers/aws/s3/buckets.go +++ b/providers/aws/s3/buckets.go @@ -45,7 +45,7 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { return resources, err } - priceMap, err := awsUtils.GetPriceMap(pricingOutput) + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) return resources, err From ce53d680bcce1708376ec475e6ae1992b6a83fb3 Mon Sep 17 00:00:00 2001 From: Azanul Date: Sun, 15 Oct 2023 22:55:34 +0530 Subject: [PATCH 17/19] refac(fetchResources): encapsulate Signed-off-by: Azanul --- internal/internal.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/internal/internal.go b/internal/internal.go index 1c55b6de5..29d8ea202 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -81,14 +81,7 @@ func Exec(address string, port int, configPath string, telemetry bool, a utils.A _, err = cron.Every(1).Hours().Do(func() { log.Info("Fetching resources workflow has started") - numWorkers := 64 - wp := providers.NewWorkerPool(numWorkers) - wp.Start() - err = fetchResources(ctx, clients, regions, telemetry, wp) - if err != nil { - log.Fatal(err) - } - wp.Wait() + fetchResources(ctx, clients, regions, telemetry) }) if err != nil { @@ -263,7 +256,10 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien } } -func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool, wp *providers.WorkerPool) error { +func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool) { + numWorkers := 64 + wp := providers.NewWorkerPool(numWorkers) + wp.Start() var wwg sync.WaitGroup workflowTrigger := func(client providers.ProviderClient, provider string) { @@ -301,7 +297,7 @@ func fetchResources(ctx context.Context, clients []providers.ProviderClient, reg } wwg.Wait() - return nil + wp.Wait() } func checkUpgrade() { From 1d46d1dcaf166170532959644982b2dd03c49e47 Mon Sep 17 00:00:00 2001 From: Azanul Date: Sun, 15 Oct 2023 22:56:09 +0530 Subject: [PATCH 18/19] fear(fetchResources): benchmark test Signed-off-by: Azanul --- internal/internal_test.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 internal/internal_test.go diff --git a/internal/internal_test.go b/internal/internal_test.go new file mode 100644 index 000000000..f8bfc9ee9 --- /dev/null +++ b/internal/internal_test.go @@ -0,0 +1,37 @@ +package internal + +import ( + "context" + "io" + "testing" + + log "github.com/sirupsen/logrus" + + "github.com/tailwarden/komiser/internal/config" + "github.com/tailwarden/komiser/utils" +) + +// BenchmarkFactorial benchmarks the Factorial function. +func BenchmarkFetchResources(b *testing.B) { + // Setup + ctx := context.TODO() + log.SetOutput(io.Discard) + analytics.Init() + cfg, clients, accounts, err := config.Load("/workspaces/komiser/config.toml", false, analytics, db) + if err != nil { + b.Fatalf("Error during config setup: %v", err) + } + err = setupDBConnection(cfg) + if err != nil { + b.Fatalf("Error during DB setup: %v", err) + } + err = utils.SetupSchema(db, cfg, accounts) + if err != nil { + b.Fatalf("Error during schema setup: %v", err) + } + + // The benchmark function will run b.N times + for i := 0; i < b.N; i++ { + fetchResources(ctx, clients, []string{}, false) + } +} From 96dba3af9825fa421d65517ed2fee2233e754279 Mon Sep 17 00:00:00 2001 From: Kavibritto Chalaman Date: Mon, 16 Oct 2023 11:24:58 +0000 Subject: [PATCH 19/19] #1089 added nil check on associated KeyPair --- providers/aws/ec2/instances.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/providers/aws/ec2/instances.go b/providers/aws/ec2/instances.go index b4997579a..32eb93c08 100644 --- a/providers/aws/ec2/instances.go +++ b/providers/aws/ec2/instances.go @@ -212,12 +212,14 @@ func getEC2Relations(inst *etype.Instance, resourceArn string) (rel []models.Lin }) // Get associated Keypair - rel = append(rel, models.Link{ - ResourceID: *inst.KeyName, - Name: *inst.KeyName, - Type: "Key Pair", - Relation: "USES", - }) + if inst.KeyName != nil { + rel = append(rel, models.Link{ + ResourceID: *inst.KeyName, + Name: *inst.KeyName, + Type: "Key Pair", + Relation: "USES", + }) + } // Get associated IAM roles if inst.IamInstanceProfile != nil {