Add OpenCost Integration (#133)

* add pagerduty Port ocean integration files * update poetry toml * resolve linting errors * fix type hints * format files with black and make lint * modify integration config files and refactor code * lint files * make rest api async with httpx * add changelog to feature implementation * build towncrier changelog * make lint files * improve on code review * change logger info to error where necessary * print error status and text * change http error object and lint code * Revert "change http error object and lint code" This reverts commit 513ad19. * resolve http error object * update service url and add oncall * bump integration version and add changelog * add changelog to new feature * update integration identifier * add linting * sync code with remote repo * opencost resources added * lint code * make lint to sync with remote codebase * add README.md, concurrency and changelog * build changelog * add install/prod makefile * attempt to resolve networking issue * update integration to match data schema * lint code * remove sample data * use list comprehension and lin * remove semaphore * remove async from process_cost_item --------- Co-authored-by: Yair Siman Tov <[email protected]> Co-authored-by: Daniel Sinai <[email protected]>
port-labs · Oct 11, 2023 · c0e97d1 · c0e97d1
1 parent 5d949c8
commit c0e97d1
Show file tree

Hide file tree

Showing 19 changed files with 2,250 additions and 0 deletions.
diff --git a/integrations/opencost/.dockerignore b/integrations/opencost/.dockerignore
@@ -0,0 +1,94 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+
+# CI
+.codeclimate.yml
+.travis.yml
+.taskcluster.yml
+
+# Docker
+docker-compose.yml
+Dockerfile
+.docker
+.dockerignore
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__/
+**/*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# PyCharm
+.idea
+
+# Python mode for VIM
+.ropeproject
+**/.ropeproject
+
+# Vim swap files
+**/*.swp
+
+# VS Code
+.vscode/
+
+*.md
+**/.ruff_cache
+**/cahangelog
+**/tests
diff --git a/integrations/opencost/.gitignore b/integrations/opencost/.gitignore
@@ -0,0 +1,153 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
diff --git a/integrations/opencost/.port/resources/.gitignore b/integrations/opencost/.port/resources/.gitignore
@@ -0,0 +1 @@
+!.gitignore
diff --git a/integrations/opencost/.port/resources/blueprints.json b/integrations/opencost/.port/resources/blueprints.json
@@ -0,0 +1,94 @@
+[
+    {
+        "identifier": "openCostResourceAllocation",
+        "description": "This blueprint represents an OpenCost resource allocation in our software catalog",
+        "title": "OpenCost Resource Allocation",
+        "icon": "OpenCost",
+        "schema": {
+          "properties": {
+            "cluster": {
+              "type": "string",
+              "title": "Cluster"
+            },
+            "namespace": {
+              "type": "string",
+              "title": "Namespace"
+            },
+            "startDate": {
+              "title": "Start Date",
+              "type": "string",
+              "format": "date-time"
+            },
+            "endDate": {
+              "title": "End Date",
+              "type": "string",
+              "format": "date-time"
+            },
+            "cpuCoreHours": {
+              "title": "CPU Core Hours",
+              "type": "number"
+            },
+            "cpuCost": {
+              "title": "CPU Cost",
+              "type": "number"
+            },
+            "cpuEfficiency": {
+              "title": "CPU Efficiency",
+              "type": "number"
+            },
+            "gpuHours": {
+              "title": "GPU Hours",
+              "type": "number"
+            },
+            "gpuCost": {
+              "title": "GPU Cost",
+              "type": "number"
+            },
+            "networkCost": {
+              "title": "Network Cost",
+              "type": "number"
+            },
+            "loadBalancerCost": {
+              "title": "Load Balancer Cost",
+              "type": "number"
+            },
+            "pvCost": {
+              "title": "PV Cost",
+              "type": "number"
+            },
+            "ramBytes": {
+              "title": "RAM Bytes",
+              "type": "number"
+            },
+            "ramCost": {
+              "title": "RAM Cost",
+              "type": "number"
+            },
+            "ramEfficiency": {
+              "title": "RAM Efficiency",
+              "type": "number"
+            },
+            "sharedCost": {
+              "title": "Shared Cost",
+              "type": "number"
+            },
+            "externalCost": {
+              "title": "External Cost",
+              "type": "number"
+            },
+            "totalCost": {
+              "title": "Total Cost",
+              "type": "number"
+            },
+            "totalEfficiency": {
+              "title": "Total Efficiency",
+              "type": "number"
+            }
+          },
+          "required": []
+        },
+        "mirrorProperties": {},
+        "calculationProperties": {},
+        "relations": {}
+    }
+]
diff --git a/integrations/opencost/.port/resources/port-app-config.yaml b/integrations/opencost/.port/resources/port-app-config.yaml
@@ -0,0 +1,32 @@
+createMissingRelatedEntities: true
+deleteDependentEntities: true
+resources:
+  - kind: cost
+    selector:
+      query: 'true'
+    port:
+      entity:
+        mappings:
+          blueprint: '"openCostResourceAllocation"'
+          identifier: .name
+          title: .name
+          properties:
+              cluster: .properties.cluster
+              namespace: .properties.namespace
+              startDate: .start
+              endDate: .end
+              cpuCoreHours: .cpuCoreHours
+              cpuCost: .cpuCost
+              cpuEfficiency: .cpuEfficiency
+              gpuHours: .gpuHours
+              gpuCost: .gpuCost
+              networkCost: .networkCost
+              loadBalancerCost: .loadBalancerCost
+              pvCost: .pvCost
+              ramBytes: .ramBytes
+              ramCost: .ramCost
+              ramEfficiency: .ramEfficiency
+              sharedCost: .sharedCost
+              externalCost: .externalCost
+              totalCost: .totalCost
+              totalEfficiency: .totalEfficiency
diff --git a/integrations/opencost/.port/spec.yaml b/integrations/opencost/.port/spec.yaml
@@ -0,0 +1,17 @@
+version: v0.1.0
+type: opencost
+description: OpenCost integration for Port Ocean
+icon: OpenCost
+features:
+  - type: exporter
+    section: Cloud Cost
+    resources:
+      - kind: cost
+configurations:
+  - name: window
+    type: string
+    default: today
+    description: Duration of time over which to query. Accepts words like today, week, month, yesterday, lastweek, lastmonth
+  - name: appHost
+    type: string
+    description: The URL of your OpenCost server