diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..94503ce34 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/go +{ + "name": "Go", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/go:1-1.21-bullseye", + "features": { + "ghcr.io/devcontainers-contrib/features/node-asdf:0": {}, + "ghcr.io/devcontainers-contrib/features/typescript:2": {} + }, + "customizations": { + "vscode": { + "extensions": [ + "golang.go-nightly", + "bradlc.vscode-tailwindcss" + ] + } + } + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "go version", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 591bd8dca..06b73a145 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,11 +2,11 @@ ## Welcome -**Want to contribute to Komiser, but not sure where to start?** We've got you covered! +πŸŽ‰ **Want to contribute to Komiser, but not sure where to start?** We've got you covered! **Welcome to the Komiser Contributing Guide** where you'll find everything you need to get started with contributing to Komiser. -We are thrilled to have you as part of our community and looking forward to your valuable contributions! +We are thrilled to have you as part of our community and looking forward to your valuable contributions! 🌟 ## Before You Get Started! @@ -14,9 +14,9 @@ Before getting started with your first contribution, here are a few things to ke ### For Major Feature Enhancements -**Planning to work on adding a major feature enhancement?** That's amazing we always encourage ambitious contributions. **Keep in mind though that we always recommend discussing your plans with the team first.** +πŸš€ **Planning to work on adding a major feature enhancement?** That's amazing we always encourage ambitious contributions. **Keep in mind though that we always recommend discussing your plans with the team first.** -This provides an opportunity for fellow contributors to - **guide you in the right direction**, **offer feedback on your design**, and **potentially identify if another contributor is already working on a similar task**. +This provides an opportunity for fellow contributors to - **guide you in the right direction**, **offer feedback on your design**, and **potentially identify if another contributor is already working on a similar task**. Here's how to reach out: @@ -25,118 +25,115 @@ Here's how to reach out: ### For Bug Reports and Minor Fixes -**Found a bug and wish to fix it?** Each and every contribution matters regardless of its size! +🐞 **Found a bug and wish to fix it?** Each and every contribution matters regardless of its size! Here are a few things to keep in mind in this case: - Before creating a new issue, please make sure to check for an already [existing issue](https://github.com/tailwarden/komiser/issues) for that bug. -- If an issue doesn't exist, [create a new issue](https://github.com/tailwarden/komiser/issues/new/choose) of **`type: Bug Report`** and make sure to provide as much detail as possible. +- If an issue doesn't exist, [create a new issue](https://github.com/tailwarden/komiser/issues/new/choose) of **\`type: Bug Report\`** and make sure to provide as much detail as possible. - Feel free to reach out to us on **#contributors** or **#feedback** channel on our Discord Server, if you need any assistance or have questions. ## General Contribution Flow This section covers the **general contribution flow** when contributing to any area of **Komiser (Engine or the UI)** and some best practices to follow along the way. -Following these steps will ensure that your contributions are well-received, reviewed, and integrated effectively into Komiser's codebase. +Following these steps will ensure that your contributions are well-received, reviewed, and integrated effectively into Komiser's codebase. ### Fork and Pull Request Flow -1. Head over to the [Komiser GitHub repo](https://github.com/tailwarden/komiser) and "fork it" into your own GitHub account. -2. Clone your fork to your local machine, using the following command: - ```bash - git clone git@github.com:USERNAME/FORKED-PROJECT.git - ``` -3. Create a new branch based-off **`develop`** branch: - ```bash - git checkout develop - git checkout -b fix/XXX-something develop - ``` - Make sure to follow the following branch naming conventions: - - For feature/enchancements, use: **`feature/xxxx-name_of_feature`** - - For bug fixes, use: **`fix/xxxx-name_of_bug`**

- - > Here, **`xxxx`** is the issue number associated with the bug/feature! - - For example: - ```bash - git checkout -b feature/1022-kubecost-integration develop - ``` -4. Implement the changes or additions you intend to contribute. Whether it's **bug fixes**, **new features**, or **enhancements**, this is where you put your coding skills to use. -5. Once your changes are ready, you may then commit and push the changes from your working branch: - ```bash - git commit -m "nice_commit_description" - git push origin feature/1022-kubecost-integration - ``` -6. While submitting Pull Request, **make sure to change the base branch from**: [master](https://github.com/tailwarden/komiser/tree/master) to [develop](v). Making sure to avoid any possible merge conflicts - -> ### Keeping your Fork Up-to-Date -> -> If you plan on doing anything more than just a tiny quick fix, you’ll want to **make sure you keep your fork up to date** by tracking the original [β€œupstream” repo](https://github.com/tailwarden/komiser) that you forked. -> -> Follow the steps given below to do so: -> -> 1. Add the 'upstream' repo to list of remotes: -> -> ```bash -> git remote add upstream https://github.com/tailwarden/komiser.git -> ``` -> -> 2. Fetch upstream repo’s branches and latest commits: -> -> ```bash -> git fetch upstream -> ``` -> -> 3. Checkout to the **`develop`** branch and merge the upstream: -> +1️⃣ Head over to the [Komiser GitHub repo](https://github.com/tailwarden/komiser) and "fork it" into your own GitHub account. + +2️⃣ Clone your fork to your local machine, using the following command: +```bash +git clone git@github.com:USERNAME/FORKED-PROJECT.git +``` + +3️⃣ Create a new branch based-off **\`develop\`** branch: +```bash +git checkout develop +git checkout -b fix/XXX-something develop +``` +Make sure to follow the following branch naming conventions: +- For feature/enchancements, use: **\`feature/xxxx-name_of_feature\`** +- For bug fixes, use: **\`fix/xxxx-name_of_bug\`** +> Here, **\`xxxx\`** is the issue number associated with the bug/feature! +> For example: > ```bash -> git checkout develop -> git merge upstream/develop +> git checkout -b feature/1022-kubecost-integration develop > ``` -> -> **Now, your local 'develop' branch is up-to-date with everything modified upstream!** -## Contributing to Komiser Engine +4️⃣ Implement the changes or additions you intend to contribute. Whether it's **bug fixes**, **new features**, or **enhancements**, this is where you put your coding skills to use. -The core Komiser Engine is written in Go (Golang) and leverages Go Modules. Following are the pre-requisistes needed to run Komiser on your local machine: -1. Latest Go version (currently its **`1.19`**) must be installed if you want to build and/or make changes to the existing code. The binary **`go1.19`** should be available in your path.

- > If you wish to keep multiple versions of Go in your system and don't want to disturb the existing ones, refer [the guide](https://go.dev/doc/manage-install). -2. Make sure that the **`GOPATH`** environment variable is configured appropriately. +5️⃣ Once your changes are ready, you may then commit and push the changes from your working branch: +```bash +git commit -m "fix(xxxx-name_of_bug): nice commit description" +git push origin feature/1022-kubecost-integration +``` -### Komiser Installation +6️⃣ While submitting Pull Request, **make sure to change the base branch from**: [master](https://github.com/tailwarden/komiser/tree/master) to [develop](v). Making sure to avoid any possible merge conflicts -**Step 1: Installing Komiser CLI** +### Keeping your Fork Up-to-Date -Follow the instructions given in the [documentation](https://docs.komiser.io/getting-started/installation) to install the **Komiser CLI**, according to your operating system. +If you plan on doing anything more than just a tiny quick fix, you’ll want to **make sure you keep your fork up to date** by tracking the original [β€œupstream” repo](https://github.com/tailwarden/komiser) that you forked. -**Step 2: Connect to a Cloud Account** +Follow the steps given below to do so: -In order to deploy a **self-hosted (local) instance** of Komiser, the next step would be to connect your Komiser CLI to a cloud account of your choice. You may refer the documentation of the [supported cloud providers](https://docs.komiser.io/configuration/cloud-providers/aws) and follow the instructions using any one (Let's say AWS). +1️⃣ Add the 'upstream' repo to list of remotes: +```bash +git remote add upstream https://github.com/tailwarden/komiser.git +``` -**Step 3: Accessing the Komiser UI** +2️⃣ Fetch upstream repo’s branches and latest commits: +```bash +git fetch upstream +``` -Once the local Komiser instance is running, you can access the dashboard UI on **`http://localhost:3000`** +3️⃣ Checkout to the **\`develop\`** branch and merge the upstream: +```bash +git checkout develop +git merge upstream/develop +``` -![komiser-dashboard](https://hackmd.io/_uploads/Syo0bMtgT.png) +**Now, your local 'develop' branch is up-to-date with everything modified upstream!** -### Ways to Contribute in Komiser Engine +# πŸš€ Contributing to Komiser Engine -Komiser is an open source cloud-agnostic resource manager which helps you break down your cloud resources cost at the resource level. +The core Komiser Engine is written in Go (Golang) and leverages Go Modules. Here are the prerequisites to run Komiser on your local machine: -Due to the nature of Komiser, a cloud-agnostic cloud management tool, our work is never really done! There are always more providers and cloud services that can be added, updated and cost calculated. +1. 🌐 **Go Version**: + - Latest Go version (currently its **`1.19`**) must be installed if you want to build and/or make changes to the existing code. The binary **`go1.19`** should be available in your path. + > πŸ’‘ If you wish to keep multiple versions of Go in your system and don't want to disturb the existing ones, refer [the guide](https://go.dev/doc/manage-install). -Therefore, there are mainly three ways you can contribute to the Komiser Engine: +2. πŸ”§ **GOPATH**: + - Ensure that the **`GOPATH`** environment variable is configured appropriately. -### 1. Adding a new Cloud Provider +--- -Here are the general steps to integrate a new cloud provider in Komiser: +## πŸ› οΈ Komiser Installation -**Step 1:** -Create a new **`provider_name.go`** file under **`providers/provider_name`** directory. +### **Step 1: Installing Komiser CLI** +Follow the instructions in the [documentation](https://docs.komiser.io/getting-started/installation) to install the **Komiser CLI** for your operating system. -**Step 2:** -Add the following boilerplate code, which defines the structure of any new provider to be added: +### **Step 2: Connect to a Cloud Account** +To deploy a **self-hosted (local) instance** of Komiser, connect your Komiser CLI to a cloud account of your choice. Refer to the documentation of the [supported cloud providers](https://docs.komiser.io/configuration/cloud-providers/aws). + +### **Step 3: Accessing the Komiser UI** +Access the dashboard UI at **`http://localhost:3002`** once the local Komiser instance is running. + +![komiser-dashboard](https://hackmd.io/_uploads/Syo0bMtgT.png) + +--- + +## 🌟 Ways to Contribute to Komiser Engine + +Komiser is an open-source cloud-agnostic resource manager. It helps you break down cloud resource costs at the resource level. As a cloud-agnostic cloud management tool, we always have more providers and cloud services to add, update, and cost-calculate. + +### 1️⃣ Adding a new Cloud Provider + +- Step 1: Create **`provider_name.go`** in **`providers/provider_name`** directory. + +- Step 2: Add the following boilerplate: ```go package PROVIDER_NAME @@ -167,13 +164,9 @@ func FetchProviderData(ctx context.Context, client ProviderClient, db *bun.DB) { } ``` -Then, the main task is writing the code to fetch all the resources/services using the provider's Go client SDK. You may refer any [existing examples](https://github.com/tailwarden/komiser/tree/develop/providers) to understand better. - -**Step 3:** -Add the information about the appropriate provider's SDK client in [**`providers/provider.go`**](https://github.com/tailwarden/komiser/blob/develop/providers/providers.go) file: +- Step 3: Add SDK client details in [**`providers/provider.go`**](https://github.com/tailwarden/komiser/blob/develop/providers/providers.go): ```go - type ProviderClient struct { AWSClient *aws.Config DigitalOceanClient *godo.Client @@ -193,14 +186,11 @@ type AzureClient struct { Credentials *azidentity.ClientSecretCredential SubscriptionId string } - ``` -**Step 4:** -Add the provider configuration in TOML format in your **`config.toml`** file, which will be used by Komiser to configure your account with the CLI. +- **Step 4:** Add provider configuration in TOML format in **`config.toml`**: -An example configuration entry for configuring a Google Cloud account in the **`config.toml`** file would look like this: -``` +```toml [[gcp]] name="production" source="ENVIRONMENT_VARIABLES" @@ -208,23 +198,19 @@ source="ENVIRONMENT_VARIABLES" profile="production" ``` -**Step 5:** -Build a new Komiser binary with the latest code changes by running: +- **Step 5:** Compile a new Komiser binary: -``` +```bash go build ``` -**Step 6:** -Start a new Komiser development server using this new binary: +- **Step 6:** Start a new Komiser development server: -``` +```bash ./komiser start ``` -**If everything goes well, you'll see a new cloud provider added in the Komiser Dashboard!** - -### 2. Adding a new Cloud Service/Resource +### 2️⃣ Adding a new Cloud Service/Resource Here are the general steps to add a new service/resource for a cloud provider in Komiser: @@ -233,7 +219,6 @@ Create a new file **`servicename.go`** under the path **`providers/provider_name **Step 2:** Add the following boilerplate code, which defines the structure of any new service/resource to be added for a cloud provider: - ```go package service @@ -287,125 +272,151 @@ Repeat steps **`4,5,6`** accordingly and you'll see a new resource/service added Additionally, [here](https://youtu.be/Vn5uc2elcVg?feature=shared) is a video tutorial of the entire process for your reference. -#### 3. Enhance existing Cloud service/resource +### 3️⃣ Enhance existing Cloud service/resource **So, you wish to improve the code quality of an existing cloud service/resource?** Feel free to discuss your ideas with us on our [Discord Server](https://discord.tailwarden.com) and [open a new issue](https://github.com/tailwarden/komiser/issues). -## Contributing to Komiser Dashboard UI +# πŸš€ Contributing to Komiser Dashboard UI -Komiser Dashboard is built on **Typescript** and **Next.js**. The entire frontend stack used is as follows: -- **Next.js** -- **Typescript** -- **Tailwind** -- **Storybook** -- **Jest** -- **React Testing Library** +Komiser Dashboard utilizes a modern tech stack. Here's a brief about it: -Following are the pre-requisites needed to setup a Dev environment of Komiser dashboard: -- In nearly all cases, while contributing to Komiser UI, you will need to build and run the Komiser Server as well, using the CLI. Make sure to follow the steps mentioned in the **"Komiser Installation"** section above. -- Make sure to have all the **latest versions** of the frontend stack listed above. +- **Framework**: [**Next.js**](https://nextjs.org/) +- **Language**: [**Typescript**](https://www.typescriptlang.org/) +- **CSS**: [**Tailwind**](https://tailwindcss.com/) +- **Component Library**: [**Storybook**](https://storybook.js.org/docs/react/get-started/why-storybook) +- **Testing**: + - [**Jest**](https://jestjs.io/) + - [**React Testing Library**](https://testing-library.com/docs/react-testing-library/intro/) -### Setup a local Developement Server +## 🧩 Prerequisites -Here are the steps to setup and access the Komiser dashboard: +1. While working on Komiser UI, you'd typically need the Komiser Server as well. Follow the instructions in the [**Komiser Installation**](#komiser-installation) section above. +2. Ensure you're using the **latest versions** of the tech stack mentioned above. -**Step 0:** +## πŸ›  Setup a Local Development Server -Install the necessary Go dependencies using the following command: -``` +Let's get your hands dirty by setting up the Komiser dashboard: + +### 1️⃣ Grab the Go Dependencies +```bash go mod download ``` -**Step 1:** -From the root folder, start the Komiser backend server using the following command: -``` -go run *.go start --config /path/to/config.toml +### 2️⃣ Configure `config.toml` + +If it doesn't exist, create `config.toml` in the root and use this template: + +```toml +[[aws]] + name="sandbox" + source="CREDENTIALS_FILE" + path="./path/to/credentials/file" + profile="default" + +[[aws]] + name="staging" + source="CREDENTIALS_FILE" + path="./path/to/credentials/file" + profile="staging-account" + +[[gcp]] + name="production" + source="ENVIRONMENT_VARIABLES" + # path="./path/to/credentials/file" specify if CREDENTIALS_FILE is used + profile="production" + +[sqlite] + file="komiser.db" ``` -> As soon as you run this, you'll be able to access the dashboard at `http://localhost:3000`. -> -> An important point to note here is, this dashboard only reflects the changes from the **`master`** branch. -> -> For our purpose, we certainly need changes to be reflected from our development branch! -> Follow the steps given below to do so πŸ‘‡ -> +> πŸ“˜ Dive deeper in our [Quickstart Guide](https://docs.komiser.io/getting-started/quickstart) for more configurations like connecting to PostgreSQL. -**Step 2:** -Head over to the **`dashboard`** directory: +Now, craft your credentials. Check out the guides [here](https://docs.komiser.io/getting-started/quickstart#self-hosted) to tailor the config for your needs. +### 3️⃣ Boot Up the Komiser Backend +```bash +go run *.go start --config ./config.toml ``` + +> πŸ–₯️ This fires up the dashboard at [`http://localhost:3002/`](http://localhost:3002). However, it mirrors the **`master`** branch. Let's make it reflect your development branch! + +### 4️⃣ Navigate to the Dashboard Directory +```bash cd dashboard ``` -**Step 3:** -Create a new environment variable in the **`.env`** file: +### 5️⃣ Set up Environment Variables +Create or update the **`.env`** file: ``` -EXT_PUBLIC_API_URL=http://localhost:3000 +NEXT_PUBLIC_API_URL=http://localhost:3000 ``` -**Step 4:** -Install the necessary **`npm`** dependencies and start the dev server using the following command: -``` +### 6️⃣ Spin up the Dev Server +Install dependencies and fire up the dev server: +```bash npm install npm run dev ``` -You'll be able to access the dashboard at **`http://localhost:3002/`** +> 🟒 NodeJS: Ensure you're on the `18.x.x` LTS release. + +Once done, open up [**`http://localhost:3002/`**](http://localhost:3002) -![](https://hackmd.io/_uploads/ryvOPmFla.png) +![Komiser Dashboard](https://hackmd.io/_uploads/ryvOPmFla.png) -To understand the installation process in a bit more detail, you may refer the [video walkthrough](https://youtu.be/uwxj11-eRt8?feature=shared). +πŸ“Ί For a more detailed walkthrough, check the [video tutorial](https://youtu.be/uwxj11-eRt8?feature=shared). -### Understanding the UI components +## 🎨 Understanding the UI Components -The Komiser UI components are being handled and organised using [Storybook](https://storybook.js.org/). -Refer the [components section](https://github.com/tailwarden/komiser/tree/develop/dashboard#components) of the dashboard README to understand the component conventions used. +Komiser's UI elegance is sculpted using [Storybook](https://storybook.js.org/). Dive deep into the [components section](https://github.com/tailwarden/komiser/tree/develop/dashboard#components) to grasp our conventions. -### Testing Your Changes +## πŸ§ͺ Testing Your Changes -The Komiser dashboard uses **Jest** and **React Testing Library** for unit tests. Refer the [testing section](https://github.com/tailwarden/komiser/tree/develop/dashboard#testing) of the dashboard README to understand how you can write simple unit tests, to validate your changes. +We leverage **Jest** and **React Testing Library** for unit tests. Navigate to the [testing section](https://github.com/tailwarden/komiser/tree/develop/dashboard#testing) for insights on writing tests. -### Building a Go Artifact +## πŸ“¦ Building a Go Artifact -Once you have implemented the necessary frontend changes, make sure to build a new Go artifact using the following command: +After refining the frontend, craft a new Go artifact: ``` go-bindata-assetfs -o template.go dist/ dist/assets/images/ ``` -## Contributing Best Practices -Here are some best practices to follow during the development process to make your changes more structured and making it easier for us to review: +# πŸ“š Contributing Best Practices -1. **Write Comprehensive Unit Tests:** - - - When making code changes, be sure to include well-structured unit tests. - - Utilize [Go's built-in testing framework](https://pkg.go.dev/testing) for this purpose. - - Take inspiration from existing tests in the project. - - Before submitting your pull request, run the full test suite on your development branch to ensure your changes are thoroughly validated. +> πŸ“˜ For frontend endeavors, do explore the [Getting Started Guide](https://github.com/tailwarden/komiser/tree/develop/dashboard#getting-started) in the `/dashboard` README. -2. **Keep Documentation Updated:** +Embarking on backend development? Here are golden practices to streamline your contributions and facilitate our review process: - - Ensure relevant documentation is updated or added, according to new features being added or modified. +### 1. πŸ§ͺ Write Comprehensive Unit Tests +- Every code change craves well-thought-out unit tests. +- Leverage [Go's built-in testing framework](https://pkg.go.dev/testing) for this. +- Seek inspiration from tests already enhancing the project. +- πŸš€ Before firing that pull request, run the entire test suite on your branch ensuring robust validation of your changes. -3. **Prioritize Clean Code:** +### 2. πŸ“– Keep Documentation Updated +- Newly added or tinkered features? Ensure the documentation mirrors your magic. - - Make sure to use **`go fmt`** to ensure uniform code formatting before committing your changes. - - Using IDEs and code editors like **VSCode** makes this easy, as they offer plugins that automate the formatting process. +### 3. 🎨 Prioritize Clean Code +- Seal your changes with **`go fmt`** for a consistent code style. +- Tools like **VSCode** come handy with plugins automating this formatting quest. -4. **Mindful Code Comments:** +### 4. πŸ–Š Mindful Code Comments +- Complex logic? Esoteric algorithms? Or just proud of your intricate code snippet? Comment them. +- Thoughtful comments pave way for an insightful review, letting others decipher your masterpiece with ease. - - Use comments to explain complex logic, algorithms, or any non-obvious parts of your code. - - Well-placed comments will make your code more accessible to others and will ultimately help in a smoother review process of your changes. +--- +# 🌟 Ending Note -## Ending Note +We trust this guide lights up your contribution path. Diving into Komiser's codebase should now be thrilling and engaging. -We hope this guide proves to be helpful and makes contributing to Komiser an exciting and fun process for you all. +In wrapping up, a **MASSIVE THANK YOU** echoes for your invaluable time and efforts. You're making Komiser even more radiant and welcoming for the community. -At the end, we wanna give you a **HUGE THANK YOU** for taking out your time in contributing and making Komiser better and more accessible to the community! +> πŸ“ž Need a chat? Have queries buzzing? Buzz us on our [Discord Server](https://discord.tailwarden.com). -Feel free to reach out to us on our [Discord Server](https://discord.tailwarden.com) if you need any assistance or have any questions. +--- -## License +# πŸ“œ License -Komiser is an open-source software released under the [Elastic License 2.0 (ELv2)](https://github.com/tailwarden/komiser/blob/develop/LICENSE). \ No newline at end of file +Komiser, an emblem of open-source, basks under the [Elastic License 2.0 (ELv2)](https://github.com/tailwarden/komiser/blob/develop/LICENSE). diff --git a/dashboard/README.md b/dashboard/README.md index c17f1c503..ec0690013 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -4,16 +4,18 @@ Komiser dashboard is a [Next.js](https://nextjs.org/) project bootstrapped with **Full frontend stack:** -- πŸ–₯ `Next.js` -- πŸ“œ `Typescript` -- 🎨 `Tailwind` -- πŸ“– `Storybook` -- πŸ§ͺ `Jest` -- πŸ“š `React Testing Library` +- πŸ–₯ [`Next.js`](https://nextjs.org/) +- πŸ“œ [`Typescript`](https://www.typescriptlang.org/) +- 🎨 [`Tailwind`](https://tailwindcss.com/) +- πŸ“– [`Storybook`](https://storybook.js.org/) +- πŸ§ͺ [`Jest`](https://jestjs.io/) +- πŸ“š [`React Testing Library`](https://testing-library.com/docs/react-testing-library/intro) ## πŸš€ Getting Started -First, run the development server: +Follow the [Contribution Guide](https://github.com/tailwarden/komiser/blob/develop/CONTRIBUTING.md#contributing-to-komiser-dashboard-ui) first if you haven't done so already. Then come back here and follow the next steps: + +1. Run the development server: ```shell @@ -36,6 +38,7 @@ Open [http://localhost:3002/](http://localhost:3002). If you see the dashboard, ❗ If you get an error page such as this, please refer to the logs and our [docs](https://docs.komiser.io/docs/introduction/getting-started). ![Error Image](https://user-images.githubusercontent.com/13384559/224320642-0bf6814b-d97a-4ad9-95a0-ca82e353c5d0.png) + ## 🧩 Components Komiser components are documented under `/components` @@ -52,6 +55,11 @@ Komiser components are documented under `/components` > } > ``` +You can find all the shared Components also inside [Storybook](https://storybook.komiser.io/). If you're implementing a new Story, please check for existing or new components with Storybook. +We will require a story for new shared components like icons, inputs or similar. + +Component convention: + **Component convention:** - πŸ“ Component folder: component name in `kebab-case` @@ -60,6 +68,7 @@ Komiser components are documented under `/components` - 🎭 Component story mock (if needed): component name in `UpperCamelCase.mocks.*` - πŸ§ͺ Component unit test: component name in `UpperCamelCase.test.*` - 🧐 Check `Card` example for more details: + ![Component Example](https://user-images.githubusercontent.com/13384559/224307211-2ce62245-de24-4ee7-a156-fb54d8d34b4f.png) **Additional instructions:** @@ -67,12 +76,17 @@ Komiser components are documented under `/components` - πŸ“– To view this component on Storybook, run: `npm run storybook`, then pick `Card` ![Storybook Image](https://user-images.githubusercontent.com/13384559/224320112-e21d2ed4-1e22-4a33-adb3-6c236c4d4208.png) - πŸ§ͺ To run the unit tests, run: `npm run test:watch`, hit `p`, then `card` + ![Unit Test Image](https://user-images.githubusercontent.com/13384559/224320260-19b1359e-1bfb-4db5-8379-918dacd7da44.png) ## πŸ§ͺ Testing We use Jest & React Testing Library for our unit tests. +- To run the unit tests, run: `npm run test:watch`, hit `p`, then `card` + image + + **Testing convention:** - βœ… All new Utils need to be tested. Existing ones when being changed @@ -151,6 +165,90 @@ it('opens the dropdown when clicked', () => { }); ``` +## 🎨 Adding to Storybook + +[**Storybook**](https://storybook.komiser.io/) is a tool for UI development. It makes development faster by isolating components. This allows you to work on one component at a time. If you create a new shared component or want to visualize variations of an existing one, follow these steps: + +- To view this component on Storybook locally, run: `npm run storybook`, then pick an example (`Card`) or your new component story + image + +### 1. **Create the Story**: + +In the same directory as your component, create a Storybook story: + +- Create a story file: component name in `UpperCamelCase.stories.*`. + +Here's a basic story format: + +```typescript +import React from 'react'; +import { ComponentStory, ComponentMeta } from '@storybook/react'; + +import YourComponent from './YourComponent'; + +export default { + title: 'Path/To/YourComponent', + component: YourComponent +} as ComponentMeta; + +const Template: ComponentStory = args => ( + +); + +export const Default = Template.bind({}); +Default.args = { + // default props here... +}; +``` + +### 2. **Add Variations**: + +You can create multiple variations of your component by replicating the `Default` pattern. For example, if your component has a variation for a "disabled" state: + +```typescript +export const Disabled = Template.bind({}); +Disabled.args = { + // props to set the component to its disabled state... +}; +``` + +### 3. **Mock Data**: + +If your component requires mock data, create a mock file: component name in `UpperCamelCase.mocks.*`. Import this data into your story file to use with your component variations. + +### 4. **Visual Check**: + +Run Storybook: + +```bash +npm run storybook +``` + +Your component should now appear in the Storybook UI. Navigate to it, and verify all the variations display correctly. + +### 5. **Documentation**: + +Add a brief description and any notes on your component's functionality within the Storybook UI. Use the `parameters` object in your default export: + +```typescript +export default { + title: 'Path/To/YourComponent', + component: YourComponent, + parameters: { + docs: { + description: { + component: 'Your description here...' + } + } + } +} as ComponentMeta; +``` + +--- + +> Remember: Storybook is not just a tool but also a way to document components. Ensure you provide meaningful names, descriptions, and use cases to help other developers understand the use and purpose of each component. + + ## 🀝 Contributing We welcome all contributors to join us on the mission of improving Komiser, especially when it comes to writing tests and adding documentation. diff --git a/internal/internal.go b/internal/internal.go index 1765a1171..29d8ea202 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -11,6 +11,7 @@ import ( "runtime" "strconv" "strings" + "sync" "time" "github.com/getsentry/sentry-go" @@ -54,7 +55,7 @@ var Arch = runtime.GOARCH var db *bun.DB var analytics utils.Analytics -func Exec(address string, port int, configPath string, telemetry bool, a utils.Analytics, regions []string, cmd *cobra.Command) error { +func Exec(address string, port int, configPath string, telemetry bool, a utils.Analytics, regions []string, _ *cobra.Command) error { analytics = a ctx := context.Background() @@ -79,10 +80,8 @@ func Exec(address string, port int, configPath string, telemetry bool, a utils.A _, err = cron.Every(1).Hours().Do(func() { log.Info("Fetching resources workflow has started") - err = fetchResources(ctx, clients, regions, telemetry) - if err != nil { - log.Fatal(err) - } + + fetchResources(ctx, clients, regions, telemetry) }) if err != nil { @@ -209,7 +208,7 @@ func setupDBConnection(c *models.Config) error { return nil } -func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string) { +func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string, wp *providers.WorkerPool) { localHub := sentry.CurrentHub().Clone() defer func() { @@ -233,7 +232,7 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien switch provider { case "AWS": - aws.FetchResources(ctx, client, regions, db, telemetry, analytics) + aws.FetchResources(ctx, client, regions, db, telemetry, analytics, wp) case "DigitalOcean": do.FetchResources(ctx, client, db, telemetry, analytics) case "OCI": @@ -257,33 +256,48 @@ func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClien } } -func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool) error { +func fetchResources(ctx context.Context, clients []providers.ProviderClient, regions []string, telemetry bool) { + numWorkers := 64 + wp := providers.NewWorkerPool(numWorkers) + wp.Start() + + var wwg sync.WaitGroup + workflowTrigger := func(client providers.ProviderClient, provider string) { + wwg.Add(1) + go func() { + defer wwg.Done() + triggerFetchingWorfklow(ctx, client, provider, telemetry, regions, wp) + }() + } + for _, client := range clients { if client.AWSClient != nil { - go triggerFetchingWorfklow(ctx, client, "AWS", telemetry, regions) + workflowTrigger(client, "AWS") } else if client.DigitalOceanClient != nil { - go triggerFetchingWorfklow(ctx, client, "DigitalOcean", telemetry, regions) + workflowTrigger(client, "DigitalOcean") } else if client.OciClient != nil { - go triggerFetchingWorfklow(ctx, client, "OCI", telemetry, regions) + workflowTrigger(client, "OCI") } else if client.CivoClient != nil { - go triggerFetchingWorfklow(ctx, client, "Civo", telemetry, regions) + workflowTrigger(client, "Civo") } else if client.K8sClient != nil { - go triggerFetchingWorfklow(ctx, client, "Kubernetes", telemetry, regions) + workflowTrigger(client, "Kubernetes") } else if client.LinodeClient != nil { - go triggerFetchingWorfklow(ctx, client, "Linode", telemetry, regions) + workflowTrigger(client, "Linode") } else if client.TencentClient != nil { - go triggerFetchingWorfklow(ctx, client, "Tencent", telemetry, regions) + workflowTrigger(client, "Tencent") } else if client.AzureClient != nil { - go triggerFetchingWorfklow(ctx, client, "Azure", telemetry, regions) + workflowTrigger(client, "Azure") } else if client.ScalewayClient != nil { - go triggerFetchingWorfklow(ctx, client, "Scaleway", telemetry, regions) + workflowTrigger(client, "Scaleway") } else if client.MongoDBAtlasClient != nil { - go triggerFetchingWorfklow(ctx, client, "MongoDBAtlas", telemetry, regions) + workflowTrigger(client, "MongoDBAtlas") } else if client.GCPClient != nil { - go triggerFetchingWorfklow(ctx, client, "GCP", telemetry, regions) + workflowTrigger(client, "GCP") } } - return nil + + wwg.Wait() + wp.Wait() } func checkUpgrade() { diff --git a/internal/internal_test.go b/internal/internal_test.go new file mode 100644 index 000000000..f8bfc9ee9 --- /dev/null +++ b/internal/internal_test.go @@ -0,0 +1,37 @@ +package internal + +import ( + "context" + "io" + "testing" + + log "github.com/sirupsen/logrus" + + "github.com/tailwarden/komiser/internal/config" + "github.com/tailwarden/komiser/utils" +) + +// BenchmarkFactorial benchmarks the Factorial function. +func BenchmarkFetchResources(b *testing.B) { + // Setup + ctx := context.TODO() + log.SetOutput(io.Discard) + analytics.Init() + cfg, clients, accounts, err := config.Load("/workspaces/komiser/config.toml", false, analytics, db) + if err != nil { + b.Fatalf("Error during config setup: %v", err) + } + err = setupDBConnection(cfg) + if err != nil { + b.Fatalf("Error during DB setup: %v", err) + } + err = utils.SetupSchema(db, cfg, accounts) + if err != nil { + b.Fatalf("Error during schema setup: %v", err) + } + + // The benchmark function will run b.N times + for i := 0; i < b.N; i++ { + fetchResources(ctx, clients, []string{}, false) + } +} diff --git a/providers/aws/aws.go b/providers/aws/aws.go index ecf1e862e..7b1cc8f4d 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -98,7 +98,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { } } -func FetchResources(ctx context.Context, client providers.ProviderClient, regions []string, db *bun.DB, telemetry bool, analytics utils.Analytics) { +func FetchResources(ctx context.Context, client providers.ProviderClient, regions []string, db *bun.DB, telemetry bool, analytics utils.Analytics, wp *providers.WorkerPool) { listOfSupportedRegions := getRegions() if len(regions) > 0 { log.Infof("Komiser will fetch resources from the following regions: %s", strings.Join(regions, ",")) @@ -108,23 +108,25 @@ func FetchResources(ctx context.Context, client providers.ProviderClient, region for _, region := range listOfSupportedRegions { client.AWSClient.Region = region for _, fetchResources := range listOfSupportedServices() { - resources, err := fetchResources(ctx, client) - if err != nil { - log.Warnf("[%s][AWS] %s", client.Name, err) - } else { - for _, resource := range resources { - _, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost, relations=EXCLUDED.relations").Exec(context.Background()) - if err != nil { - log.WithError(err).Errorf("db trigger failed") + wp.SubmitTask(func() { + resources, err := fetchResources(ctx, client) + if err != nil { + log.Warnf("[%s][AWS] %s", client.Name, err) + } else { + for _, resource := range resources { + _, err = db.NewInsert().Model(&resource).On("CONFLICT (resource_id) DO UPDATE").Set("cost = EXCLUDED.cost, relations=EXCLUDED.relations").Exec(context.Background()) + if err != nil { + log.WithError(err).Errorf("db trigger failed") + } + } + if telemetry { + analytics.TrackEvent("discovered_resources", map[string]interface{}{ + "provider": "AWS", + "resources": len(resources), + }) } } - if telemetry { - analytics.TrackEvent("discovered_resources", map[string]interface{}{ - "provider": "AWS", - "resources": len(resources), - }) - } - } + }) } } } diff --git a/providers/aws/ec2/instances.go b/providers/aws/ec2/instances.go index b4997579a..32eb93c08 100644 --- a/providers/aws/ec2/instances.go +++ b/providers/aws/ec2/instances.go @@ -212,12 +212,14 @@ func getEC2Relations(inst *etype.Instance, resourceArn string) (rel []models.Lin }) // Get associated Keypair - rel = append(rel, models.Link{ - ResourceID: *inst.KeyName, - Name: *inst.KeyName, - Type: "Key Pair", - Relation: "USES", - }) + if inst.KeyName != nil { + rel = append(rel, models.Link{ + ResourceID: *inst.KeyName, + Name: *inst.KeyName, + Type: "Key Pair", + Relation: "USES", + }) + } // Get associated IAM roles if inst.IamInstanceProfile != nil { diff --git a/providers/aws/lambda/functions.go b/providers/aws/lambda/functions.go index b7a345c6e..02a41e387 100644 --- a/providers/aws/lambda/functions.go +++ b/providers/aws/lambda/functions.go @@ -48,7 +48,7 @@ func Functions(ctx context.Context, client providers.ProviderClient) ([]models.R return resources, err } - priceMap, err := awsUtils.GetPriceMap(pricingOutput) + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) return resources, err diff --git a/providers/aws/s3/buckets.go b/providers/aws/s3/buckets.go index de60e6ed6..005f6104e 100644 --- a/providers/aws/s3/buckets.go +++ b/providers/aws/s3/buckets.go @@ -45,7 +45,7 @@ func Buckets(ctx context.Context, client ProviderClient) ([]Resource, error) { return resources, err } - priceMap, err := awsUtils.GetPriceMap(pricingOutput) + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") if err != nil { log.Errorf("ERROR: Failed to calculate cost per month: %v", err) return resources, err diff --git a/providers/aws/utils/utils.go b/providers/aws/utils/utils.go index bc1708972..529c00bcb 100644 --- a/providers/aws/utils/utils.go +++ b/providers/aws/utils/utils.go @@ -11,8 +11,12 @@ import ( type ProductEntry struct { Product struct { Attributes struct { - Group string `json:"group"` - Operation string `json:"operation"` + Group string `json:"group"` + Operation string `json:"operation"` + GroupDescription string `json:"groupDescription"` + RequestDescription string `json:"requestDescription"` + InstanceType string `json:"instanceType"` + InstanceTypeFamily string `json:"instanceTypeFamily"` } `json:"attributes"` } `json:"product"` Terms struct { @@ -47,7 +51,7 @@ func GetCost(pds []PriceDimensions, v float64) float64 { return total } -func GetPriceMap(pricingOutput *pricing.GetProductsOutput) (map[string][]PriceDimensions, error) { +func GetPriceMap(pricingOutput *pricing.GetProductsOutput, field string) (map[string][]PriceDimensions, error) { priceMap := make(map[string][]PriceDimensions) if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { @@ -58,7 +62,22 @@ func GetPriceMap(pricingOutput *pricing.GetProductsOutput) (map[string][]PriceDi return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) } - group := price.Product.Attributes.Group + var key string + switch field { + case "group": + key = price.Product.Attributes.Group + case "operation": + key = price.Product.Attributes.Operation + case "groupDescription": + key = price.Product.Attributes.GroupDescription + case "requestDescription": + key = price.Product.Attributes.RequestDescription + case "instanceType": + key = price.Product.Attributes.InstanceType + case "instanceTypeFamily": + key = price.Product.Attributes.InstanceTypeFamily + } + unitPrices := []PriceDimensions{} for _, pd := range price.Terms.OnDemand { for _, p := range pd.PriceDimensions { @@ -66,7 +85,7 @@ func GetPriceMap(pricingOutput *pricing.GetProductsOutput) (map[string][]PriceDi } } - priceMap[group] = unitPrices + priceMap[key] = unitPrices } } diff --git a/providers/aws/utils/utils_test.go b/providers/aws/utils/utils_test.go index 1602fb242..bf2f6fb39 100644 --- a/providers/aws/utils/utils_test.go +++ b/providers/aws/utils/utils_test.go @@ -49,6 +49,7 @@ func TestGetCost(t *testing.T) { func TestGetPriceMap(t *testing.T) { testCases := []struct { inputPriceList []string + field string expectedNumProducts int expectedNumPriceDims map[string]int }{ @@ -77,6 +78,7 @@ func TestGetPriceMap(t *testing.T) { } } }`}, + field: "group", expectedNumProducts: 1, expectedNumPriceDims: map[string]int{"TestGroup": 1}, }, @@ -153,16 +155,46 @@ func TestGetPriceMap(t *testing.T) { } }`, }, + field: "group", expectedNumProducts: 2, expectedNumPriceDims: map[string]int{"TestGroup1": 2, "TestGroup2": 3}, }, + // Minimal valid JSON input with a single product, one price dimension & "instanceType" attribute + { + inputPriceList: []string{` + { + "product": { + "attributes": { + "instanceType": "TestInstanceType" + } + }, + "terms": { + "OnDemand": { + "test_term": { + "priceDimensions": { + "test_price_dimension": { + "beginRange": "0", + "endRange": "Inf", + "pricePerUnit": { + "USD": "0.1" + } + } + } + } + } + } + }`}, + field: "instanceType", + expectedNumProducts: 1, + expectedNumPriceDims: map[string]int{"TestInstanceType": 1}, + }, } for i, testCase := range testCases { t.Run(fmt.Sprintf("Test case %d", i+1), func(t *testing.T) { output := pricing.GetProductsOutput{ PriceList: testCase.inputPriceList, } - priceMap, err := GetPriceMap(&output) + priceMap, err := GetPriceMap(&output, "group") if err != nil { t.Errorf("Expected no error, but got: %v", err) } @@ -186,7 +218,7 @@ func TestGetPriceMap_InvalidJSON(t *testing.T) { output := pricing.GetProductsOutput{ PriceList: []string{invalidJSON}, } - _, err := GetPriceMap(&output) + _, err := GetPriceMap(&output, "group") if err == nil { t.Error("Expected an error, but got nil") } @@ -194,7 +226,7 @@ func TestGetPriceMap_InvalidJSON(t *testing.T) { func TestGetPriceMap_NoPricingOutput(t *testing.T) { // PricingOutput is nil - priceMap, err := GetPriceMap(nil) + priceMap, err := GetPriceMap(nil, "group") if err != nil { t.Errorf("Expected no error, but got: %v", err) } diff --git a/providers/providers.go b/providers/providers.go index 138fa34da..ef0198714 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -2,6 +2,7 @@ package providers import ( "context" + "sync" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/aws/aws-sdk-go-v2/aws" @@ -47,3 +48,39 @@ type K8sClient struct { Client *kubernetes.Clientset OpencostBaseUrl string } + +type WorkerPool struct { + numWorkers int + tasks chan func() + wg sync.WaitGroup +} + +func NewWorkerPool(numWorkers int) *WorkerPool { + return &WorkerPool{ + numWorkers: numWorkers, + tasks: make(chan func()), + } +} + +func (wp *WorkerPool) Start() { + for i := 0; i < wp.numWorkers; i++ { + go wp.worker() + } +} + +func (wp *WorkerPool) SubmitTask(task func()) { + wp.wg.Add(1) + wp.tasks <- task +} + +func (wp *WorkerPool) Wait() { + wp.wg.Wait() + close(wp.tasks) +} + +func (wp *WorkerPool) worker() { + for task := range wp.tasks { + task() + wp.wg.Done() + } +}