From 30feb1b8a1399f9a6e1a744e4ac27099a36cc178 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 14:51:39 +0000 Subject: [PATCH 1/7] Remove various legacy files --- cron/example.sh | 17 -- cron/grafana_dashboard.json | 524 ----------------------------------- doc/source/conf.py | 91 ------ doc/source/development.rst | 2 - doc/source/index.rst | 25 -- doc/source/installation.rst | 2 - doc/source/usage.rst | 2 - horizon/_footer.html.example | 56 ---- horizon/config.json.example | 17 -- horizon/generate-footer.sh | 43 --- os_capacity/prometheus.py | 6 +- setup.cfg | 19 +- setup.py | 13 +- tox.ini | 3 +- 14 files changed, 14 insertions(+), 806 deletions(-) delete mode 100755 cron/example.sh delete mode 100644 cron/grafana_dashboard.json delete mode 100644 doc/source/conf.py delete mode 100644 doc/source/development.rst delete mode 100644 doc/source/index.rst delete mode 100644 doc/source/installation.rst delete mode 100644 doc/source/usage.rst delete mode 100644 horizon/_footer.html.example delete mode 100644 horizon/config.json.example delete mode 100755 horizon/generate-footer.sh diff --git a/cron/example.sh b/cron/example.sh deleted file mode 100755 index 98acd25..0000000 --- a/cron/example.sh +++ /dev/null @@ -1,17 +0,0 @@ -# -# Example script to use with cron to regular send metrics to Monasca -# - -# Example crontab entry to run this script every 5 mins: -# -# */5 * * * * /home/stack/os-capacity/cron/example.sh > /tmp/metrics_last.log - -set -e - -source /home/stack/os-capacity/.openrc -source /home/stack/os-capacity/.venv-test/bin/activate - -export OS_CAPACITY_SEND_METRICS=1 - -os-capacity usages group -os-capacity resources group diff --git a/cron/grafana_dashboard.json b/cron/grafana_dashboard.json deleted file mode 100644 index 75fd35a..0000000 --- a/cron/grafana_dashboard.json +++ /dev/null @@ -1,524 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_MONASCA-METRICS", - "label": "Monasca-metrics", - "description": "", - "type": "datasource", - "pluginId": "monasca-datasource", - "pluginName": "Monasca" - } - ], - "__requires": [ - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "4.1.0-pre1" - }, - { - "type": "datasource", - "id": "monasca-datasource", - "name": "Monasca", - "version": "1.0.0" - } - ], - "id": null, - "title": "Resource Usage Dashboard", - "tags": [], - "style": "dark", - "timezone": "browser", - "editable": true, - "sharedCrosshair": true, - "hideControls": true, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "refresh": false, - "schemaVersion": 13, - "version": 12, - "links": [], - "gnetId": null, - "rows": [ - { - "title": "Dashboard Row", - "panels": [ - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_MONASCA-METRICS}", - "editable": true, - "error": false, - "fontSize": "100%", - "id": 2, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": null, - "desc": false - }, - "span": 4, - "styles": [ - { - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "colorMode": "cell", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 176, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "pattern": "/.*/", - "thresholds": [ - "0.1", - "1.1" - ], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "aggregator": "max", - "alias": "@flavor", - "dimensions": [ - { - "key": "flavor", - "value": "$all" - } - ], - "error": "", - "group": true, - "metric": "os_capacity.resources.free", - "period": "300", - "refId": "A" - } - ], - "title": "Free Resources", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_MONASCA-METRICS}", - "editable": true, - "error": false, - "fontSize": "100%", - "id": 3, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": null, - "desc": false - }, - "span": 4, - "styles": [ - { - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 0, - "pattern": "/.*/", - "thresholds": [ - "0" - ], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "aggregator": "max", - "alias": "@flavor", - "dimensions": [ - { - "key": "flavor", - "value": "$all" - } - ], - "error": "", - "group": true, - "metric": "os_capacity.resources.used", - "period": "300", - "refId": "A" - } - ], - "title": "Used Resources", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_MONASCA-METRICS}", - "editable": true, - "error": false, - "fontSize": "100%", - "id": 4, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": null, - "desc": false - }, - "span": 4, - "styles": [ - { - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "aggregator": "max", - "alias": "@flavor", - "dimensions": [ - { - "key": "flavor", - "value": "$all" - } - ], - "error": "", - "group": true, - "metric": "os_capacity.resources.total", - "period": "300", - "refId": "A" - } - ], - "title": "Total Resources in System", - "transform": "timeseries_aggregations", - "type": "table" - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MONASCA-METRICS}", - "editable": true, - "error": false, - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "aggregator": "none", - "alias": "@flavor", - "dimensions": [], - "error": "", - "group": true, - "hide": false, - "metric": "os_capacity.resources.used", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Servers Used per Flavor", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "${DS_MONASCA-METRICS}", - "editable": true, - "error": false, - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "aggregator": "none", - "alias": "@username", - "dimensions": [], - "error": "", - "group": true, - "hide": false, - "metric": "os_capacity.usage.user.count", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Servers Used per User", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "columns": [ - { - "text": "Current", - "value": "current" - } - ], - "datasource": "${DS_MONASCA-METRICS}", - "editable": true, - "error": false, - "fontSize": "100%", - "id": 7, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 1, - "desc": true - }, - "span": 4, - "styles": [ - { - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "aggregator": "max", - "alias": "@username", - "dimensions": [ - { - "key": "flavor", - "value": "$all" - } - ], - "error": "", - "group": true, - "metric": "os_capacity.usage.user.days.count", - "period": "300", - "refId": "A" - } - ], - "title": "Number of Server Days per User", - "transform": "timeseries_aggregations", - "type": "table" - } - ], - "showTitle": false, - "titleSize": "h6", - "height": "250px", - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - } - ] -} diff --git a/doc/source/conf.py b/doc/source/conf.py deleted file mode 100644 index e492ae3..0000000 --- a/doc/source/conf.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# -*- coding: utf-8 -*- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -sys.path.insert(0, os.path.abspath('../..')) -# -- General configuration ---------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ - 'sphinx.ext.autodoc', - #'sphinx.ext.intersphinx', - # Uncomment this to enable the OpenStack documentation style, adding - # oslosphinx to test-requirements.txt. - #'oslosphinx', -] - -# autodoc generation is a bit aggressive and a nuisance when doing heavy -# text edit cycles. -# execute "export SPHINX_DEBUG=1" in your terminal to disable - -# The suffix of source filenames. -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'os-capacity' -copyright = u'2017, StackHPC Ltd.' - -# If true, '()' will be appended to :func: etc. cross-reference text. -add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -add_module_names = True - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# -- Options for HTML output -------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. Major themes that come with -# Sphinx are currently 'default' and 'sphinxdoc'. -# html_theme_path = ["."] -# html_theme = '_theme' -# html_static_path = ['static'] - -# Output file base name for HTML help builder. -htmlhelp_basename = '%sdoc' % project - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass -# [howto/manual]). -latex_documents = [ - ('index', - '%s.tex' % project, - u'%s Documentation' % project, - u'OpenStack Foundation', 'manual'), -] - -# Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/development.rst b/doc/source/development.rst deleted file mode 100644 index 3982e78..0000000 --- a/doc/source/development.rst +++ /dev/null @@ -1,2 +0,0 @@ -TODO development -================ diff --git a/doc/source/index.rst b/doc/source/index.rst deleted file mode 100644 index ee8a183..0000000 --- a/doc/source/index.rst +++ /dev/null @@ -1,25 +0,0 @@ -Welcome to os-capacity's documentation! -======================================= - -.. include:: ../../README.rst - -Documentation -------------- - -.. note:: - - nothing to see here yet :( - -.. toctree:: - :maxdepth: 2 - - installation - usage - -Developer Documentation ------------------------ - -.. toctree:: - :maxdepth: 2 - - development diff --git a/doc/source/installation.rst b/doc/source/installation.rst deleted file mode 100644 index 30a8521..0000000 --- a/doc/source/installation.rst +++ /dev/null @@ -1,2 +0,0 @@ -TODO: Install -============= diff --git a/doc/source/usage.rst b/doc/source/usage.rst deleted file mode 100644 index 2adda45..0000000 --- a/doc/source/usage.rst +++ /dev/null @@ -1,2 +0,0 @@ -TODO: usage docs -================ diff --git a/horizon/_footer.html.example b/horizon/_footer.html.example deleted file mode 100644 index 735fdd2..0000000 --- a/horizon/_footer.html.example +++ /dev/null @@ -1,56 +0,0 @@ -

Capacity

- -

Note: this is stale data from Thu 7 Sep 13:04:05 BST 2017

- -
- - - - - - - -
FlavorsFreeUsedTotal
- - - - - diff --git a/horizon/config.json.example b/horizon/config.json.example deleted file mode 100644 index 7879b82..0000000 --- a/horizon/config.json.example +++ /dev/null @@ -1,17 +0,0 @@ -{ - "command": "/usr/sbin/httpd -DFOREGROUND", - "config_files": [ - { - "source": "/var/lib/kolla/config_files/horizon.conf", - "dest": "/etc/httpd/conf.d/horizon.conf", - "owner": "horizon", - "perm": "0600" - }, - { - "source": "/var/lib/kolla/config_files/_footer.html", - "dest": "/usr/share/openstack-dashboard/openstack_dashboard/templates/_login_footer.html", - "owner": "horizon", - "perm": "0600" - }, -... -} diff --git a/horizon/generate-footer.sh b/horizon/generate-footer.sh deleted file mode 100755 index 030aede..0000000 --- a/horizon/generate-footer.sh +++ /dev/null @@ -1,43 +0,0 @@ -set -ex - -CAPACITY=`os-capacity resources group -f json` -CURRENT_DATE=`date` - -cat >_footer.html <Capacity - -

Note: this is stale data from $CURRENT_DATE

- -
-
- - - - - - -
FlavorsFreeUsedTotal
- - - - - -EOF - -cat _footer.html diff --git a/os_capacity/prometheus.py b/os_capacity/prometheus.py index 8db0b18..309bf25 100755 --- a/os_capacity/prometheus.py +++ b/os_capacity/prometheus.py @@ -376,7 +376,7 @@ def collect(self): return guages -if __name__ == "__main__": +def main(): kwargs = { "port": int(os.environ.get("OS_CAPACITY_EXPORTER_PORT", 9000)), "addr": os.environ.get("OS_CAPACITY_EXPORTER_LISTEN_ADDRESS", "0.0.0.0"), @@ -387,3 +387,7 @@ def collect(self): # there must be a better way! while True: time.sleep(5000) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index b1ce8f4..3600d32 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,10 @@ [metadata] -name = kayobe +name = os_capacity summary = Deployment of Scientific OpenStack using OpenStack Kolla description-file = README.rst -author = Mark Goddard -author-email = mark@stackhpc.com +author = StackHPC +author-email = johng@stackhpc.com home-page = https://stackhpc.com classifier = Environment :: OpenStack @@ -12,17 +12,10 @@ classifier = Intended Audience :: System Administrators Operating System :: POSIX :: Linux Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.9 [files] packages = - kayobe + os_capacity -[build_sphinx] -all-files = 1 -source-dir = doc/source -build-dir = doc/build - -[upload_sphinx] -upload-dir = doc/build/html diff --git a/setup.py b/setup.py index 61272ae..a9061b6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ PROJECT = 'os_capacity' -VERSION = '0.2' +VERSION = '0.4' try: long_description = open('README.md', 'rt').read() @@ -48,16 +48,7 @@ entry_points={ 'console_scripts': [ - 'os-capacity = os_capacity.shell:main', - ], - 'os_capacity.commands': [ - 'flavor_list = os_capacity.commands.commands:FlavorList', - 'resources_all = os_capacity.commands.commands:ListResourcesAll', - 'resources_group = os_capacity.commands' - '.commands:ListResourcesGroups', - 'usages_all = os_capacity.commands.commands:ListUsagesAll', - 'usages_group = os_capacity.commands.commands:ListUsagesGroup', - 'prometheus = os_capacity.commands.commands:PrometheusAll', + 'os-capacity = os_capacity.prometheus:main', ], }, ) diff --git a/tox.ini b/tox.ini index 87145c5..c817894 100644 --- a/tox.ini +++ b/tox.ini @@ -5,11 +5,10 @@ skipsdist = True [testenv] usedevelop = True -install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/yoga} {opts} {packages} +install_command = pip install {opts} {packages} setenv = VIRTUAL_ENV={envdir} PYTHONWARNINGS=default::DeprecationWarning - TESTS_DIR=./kayobe/tests/unit/ deps = -r{toxinidir}/test-requirements.txt commands = stestr run {posargs} From 034ebc8fb65eb5506974c1bbc14a8e9fa59ff17b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 14:58:12 +0000 Subject: [PATCH 2/7] Remove unused code --- os_capacity/commands/__init__.py | 0 os_capacity/commands/commands.py | 91 --------- os_capacity/data/__init__.py | 0 os_capacity/data/flavors.py | 36 ---- os_capacity/data/resource_provider.py | 89 --------- os_capacity/data/server.py | 42 ----- os_capacity/data/users.py | 25 --- os_capacity/shell.py | 68 ------- os_capacity/tests/unit/__init__.py | 0 os_capacity/tests/unit/fakes.py | 189 ------------------- os_capacity/tests/unit/test_utils.py | 239 ------------------------ os_capacity/utils.py | 254 -------------------------- 12 files changed, 1033 deletions(-) delete mode 100644 os_capacity/commands/__init__.py delete mode 100644 os_capacity/commands/commands.py delete mode 100644 os_capacity/data/__init__.py delete mode 100644 os_capacity/data/flavors.py delete mode 100644 os_capacity/data/resource_provider.py delete mode 100644 os_capacity/data/server.py delete mode 100644 os_capacity/data/users.py delete mode 100644 os_capacity/shell.py delete mode 100644 os_capacity/tests/unit/__init__.py delete mode 100644 os_capacity/tests/unit/fakes.py delete mode 100644 os_capacity/tests/unit/test_utils.py delete mode 100644 os_capacity/utils.py diff --git a/os_capacity/commands/__init__.py b/os_capacity/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/os_capacity/commands/commands.py b/os_capacity/commands/commands.py deleted file mode 100644 index 2c56858..0000000 --- a/os_capacity/commands/commands.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import logging - -from cliff.lister import Lister - -from os_capacity import prometheus -from os_capacity import utils - - - -class PrometheusAll(Lister): - """To be run as node exporter textfile collector.""" - def take_action(self, parsed_args): - prometheus.print_exporter_data(self.app) - # TODO(johngarbutt) a total hack! - return (('fake'), []) - - -class FlavorList(Lister): - """List all the flavors.""" - - log = logging.getLogger(__name__) - - def take_action(self, parsed_args): - flavors = utils.get_flavors(self.app) - return (('UUID', 'Name', 'VCPUs', 'RAM MB', 'DISK GB', 'Extra Specs'), - flavors) - - -class ListResourcesAll(Lister): - """List all resource providers, with their resources and servers.""" - - def take_action(self, parsed_args): - inventories = utils.get_providers_with_resources_and_servers(self.app) - return (('Provider Name', 'Resources', 'Severs'), inventories) - - -class ListResourcesGroups(Lister): - """Lists counts of resource providers with similar inventories.""" - - def take_action(self, parsed_args): - groups = utils.group_providers_by_type_with_capacity(self.app) - return ( - ('Resource Class Groups', 'Total', 'Used', 'Free', 'Flavors'), - groups) - - -class ListUsagesAll(Lister): - """List all current resource usages.""" - - def take_action(self, parsed_args): - allocations = utils.get_allocations_with_server_info(self.app, - get_names=True) - return ( - ('Provider Name', 'Server UUID', 'Resources', - 'Flavor', 'Days', 'Project', 'User'), allocations) - - -class ListUsagesGroup(Lister): - """Group usage by specified key (by user or project). - - NOTE: The usage days is not complete as it only takes into - account any currently active servers. Any previously deleted - servers are not counted. - """ - - def get_parser(self, prog_name): - parser = super(ListUsagesGroup, self).get_parser(prog_name) - parser.add_argument('group_by', nargs='?', default='user', - help='Group by user_id or project_id or all', - choices=['user', 'project', 'all']) - return parser - - def take_action(self, parsed_args): - usages = utils.group_usage(self.app, parsed_args.group_by) - sort_key_title = parsed_args.group_by.title() - return ((sort_key_title, 'Current Usage', 'Usage Days'), usages) diff --git a/os_capacity/data/__init__.py b/os_capacity/data/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/os_capacity/data/flavors.py b/os_capacity/data/flavors.py deleted file mode 100644 index 9849725..0000000 --- a/os_capacity/data/flavors.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections - - -Flavor = collections.namedtuple( - "Flavor", ("id", "name", "vcpus", "ram_mb", "disk_gb", "extra_specs")) - - -def get_all(compute_client, include_extra_specs=True): - response = compute_client.get('/flavors/detail').json() - raw_flavors = response['flavors'] - - extra_specs = {} - if include_extra_specs: - for flavor in raw_flavors: - url = '/flavors/%s/os-extra_specs' % flavor['id'] - response = compute_client.get(url).json() - extra_specs[flavor['id']] = response['extra_specs'] - - return [Flavor(f['id'], f['name'], f['vcpus'], f['ram'], - (f['disk'] + f['OS-FLV-EXT-DATA:ephemeral']), - extra_specs.get(f['id'])) - for f in raw_flavors] diff --git a/os_capacity/data/resource_provider.py b/os_capacity/data/resource_provider.py deleted file mode 100644 index 307564a..0000000 --- a/os_capacity/data/resource_provider.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections - - -ResourceProvider = collections.namedtuple( - "ResourceProvider", ("uuid", "name")) - -Inventory = collections.namedtuple( - "Inventory", ("resource_provider_uuid", "resource_class", "total")) - -Allocation = collections.namedtuple( - "Allocation", ("resource_provider_uuid", "consumer_uuid", - "resources")) - -ResourceClassAmount = collections.namedtuple( - "ResourceClassAmount", ("resource_class", "amount")) - - -def get_all(placement_client): - response = placement_client.get("/resource_providers").json() - raw_rps = response['resource_providers'] - return [ResourceProvider(f['uuid'], f['name']) for f in raw_rps] - - -def get_inventories(placement_client, resource_provider): - uuid = resource_provider.uuid - url = "/resource_providers/%s/inventories" % uuid - response = placement_client.get(url).json() - raw_inventories = response['inventories'] - - inventories = [] - for resource_class, raw_inventory in raw_inventories.items(): - inventory = Inventory( - uuid, resource_class, raw_inventory['total']) - inventories.append(inventory) - - return inventories - - -def get_all_inventories(placement_client, resource_providers=None): - if resource_providers is None: - resource_providers = get_all(placement_client) - - inventories = [] - for resource_provider in resource_providers: - inventories += get_inventories(placement_client, resource_provider) - - return inventories - - -def get_allocations(placement_client, resource_provider): - allocations = [] - url = "/resource_providers/%s/allocations" % resource_provider.uuid - response = placement_client.get(url).json() - raw_allocations = response['allocations'] - for consumer_uuid, raw_allocation in raw_allocations.items(): - raw_resources = raw_allocation['resources'] - resources = [] - for resource_class, amount in raw_resources.items(): - resources.append(ResourceClassAmount(resource_class, amount)) - resources.sort() - - allocations.append(Allocation( - resource_provider.uuid, consumer_uuid, resources)) - return allocations - - -def get_all_allocations(placement_client, resource_providers=None): - if resource_providers is None: - resource_providers = get_all(placement_client) - - allocations = [] - for resource_provider in resource_providers: - allocations += get_allocations(placement_client, resource_provider) - - return allocations diff --git a/os_capacity/data/server.py b/os_capacity/data/server.py deleted file mode 100644 index 4a94942..0000000 --- a/os_capacity/data/server.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections -from datetime import datetime - - -Server = collections.namedtuple( - "Server", ("uuid", "name", "created", - "user_id", "project_id", "flavor_id")) - - -def _parse_created(raw_created): - """Parse a string like 2017-02-14T19:23:58Z""" - return datetime.strptime(raw_created, "%Y-%m-%dT%H:%M:%SZ") - - -def get(compute_client, uuid): - url = "/servers/%s" % uuid - response = compute_client.get(url) - if not response.ok: - return None - raw_server = response.json()['server'] - return Server( - uuid=raw_server['id'], - name=raw_server['name'], - created=_parse_created(raw_server['created']), - user_id=raw_server['user_id'], - project_id=raw_server['tenant_id'], - flavor_id=raw_server['flavor'].get('id'), - ) diff --git a/os_capacity/data/users.py b/os_capacity/data/users.py deleted file mode 100644 index f4c8a72..0000000 --- a/os_capacity/data/users.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -def get_all(identity_client): - response = identity_client.get("/users").json() - raw_users = response['users'] - return {u['id']: u['name'] for u in raw_users} - - -def get_all_projects(identity_client): - response = identity_client.get("/projects").json() - raw_projects = response['projects'] - return {u['id']: u['name'] for u in raw_projects} diff --git a/os_capacity/shell.py b/os_capacity/shell.py deleted file mode 100644 index a78ff9e..0000000 --- a/os_capacity/shell.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sys - -from cliff.app import App -from cliff.commandmanager import CommandManager -import os_client_config - -import openstack - -def get_cloud_config(): - # TODO(johngarbutt) consider passing in argument parser - return os_client_config.get_config() - - -def get_client(cloud_config, service_type): - return cloud_config.get_session_client(service_type) - - -class CapacityApp(App): - - def __init__(self): - super(CapacityApp, self).__init__( - description='OS-Capacity (StackHPC) Command Line Interface (CLI)', - version='0.1', - command_manager=CommandManager('os_capacity.commands'), - deferred_help=True, - ) - - def initialize_app(self, argv): - self.LOG.debug('initialize_app') - - conn = openstack.connect() - self.connection = conn - self.compute_client = conn.compute - self.placement_client = conn.placement - self.identity_client = conn.identity - - self.LOG.debug('setup Keystone API REST clients') - - def prepare_to_run_command(self, cmd): - self.LOG.debug('prepare_to_run_command %s', cmd.__class__.__name__) - - def clean_up(self, cmd, result, err): - self.LOG.debug('clean_up %s', cmd.__class__.__name__) - if err: - self.LOG.debug('got an error: %s', err) - - -def main(argv=sys.argv[1:]): - myapp = CapacityApp() - return myapp.run(argv) - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/os_capacity/tests/unit/__init__.py b/os_capacity/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/os_capacity/tests/unit/fakes.py b/os_capacity/tests/unit/fakes.py deleted file mode 100644 index a47dd0e..0000000 --- a/os_capacity/tests/unit/fakes.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -FLAVOR = { - 'OS-FLV-DISABLED:disabled': False, - 'OS-FLV-EXT-DATA:ephemeral': 0, - 'disk': 30, - 'id': 'd0e9df0c-34a3-4283-9547-d873e4e86a41', - 'links': [ - {'href': 'http://example.com:8774/v2.1/flavors/' - 'd0e9df0c-34a3-4283-9547-d873e4e86a41', - 'rel': 'self'}, - {'href': 'http://10.60.253.1:8774/flavors/' - 'd0e9df0c-34a3-4283-9547-d873e4e86a41', - 'rel': 'bookmark'}], - 'name': 'compute-GPU', - 'os-flavor-access:is_public': True, - 'ram': 2048, - 'rxtx_factor': 1.0, - 'swap': '', - 'vcpus': 8, -} -FLAVOR_RESPONSE = {'flavors': [FLAVOR]} - -FLAVOR_EXTRA_RESPONSE = {'extra_specs': {"example_key": "example_value"}} - -RESOURCE_PROVIDER = { - 'generation': 6, - 'name': 'name1', - 'uuid': '97585d53-67a6-4e9d-9fe7-cd75b331b17b', - 'links': [ - {'href': '/resource_providers' - '/97585d53-67a6-4e9d-9fe7-cd75b331b17c', - 'rel': 'self'}, - {'href': '/resource_providers' - '/97585d53-67a6-4e9d-9fe7-cd75b331b17c/aggregates', - 'rel': 'aggregates'}, - {'href': '/resource_providers' - '/97585d53-67a6-4e9d-9fe7-cd75b331b17c/inventories', - 'rel': 'inventories'}, - {'href': '/resource_providers' - '/97585d53-67a6-4e9d-9fe7-cd75b331b17c/usages', - 'rel': 'usages'} - ] -} -RESOURCE_PROVIDER_RESPONSE = {'resource_providers': [RESOURCE_PROVIDER]} - -INVENTORIES = { - 'DISK_GB': { - 'allocation_ratio': 1.0, - 'max_unit': 10, - 'min_unit': 1, - 'reserved': 0, - 'step_size': 1, - 'total': 10}, - 'MEMORY_MB': { - 'max_unit': 20, - 'total': 20}, - 'VCPU': { - 'max_unit': 30, - 'total': 30}, -} -INVENTORIES_RESPONSE = { - 'inventories': INVENTORIES, - 'resource_provider_generation': 7, -} - -ALLOCATIONS = { - 'consumer_uuid': { - 'resources': { - 'DISK_GB': 10, 'MEMORY_MB': 20, 'VCPU': 30} - } -} -ALLOCATIONS_RESPONSE = { - 'allocations': ALLOCATIONS, - 'resource_provider_generation': 42, -} - -SERVER = { - "OS-DCF:diskConfig": "AUTO", - "OS-EXT-AZ:availability_zone": "nova", - "OS-EXT-SRV-ATTR:host": "compute", - "OS-EXT-SRV-ATTR:hostname": "new-server-test", - "OS-EXT-SRV-ATTR:hypervisor_hostname": "fake-mini", - "OS-EXT-SRV-ATTR:instance_name": "instance-00000001", - "OS-EXT-SRV-ATTR:kernel_id": "", - "OS-EXT-SRV-ATTR:launch_index": 0, - "OS-EXT-SRV-ATTR:ramdisk_id": "", - "OS-EXT-SRV-ATTR:reservation_id": "r-ov3q80zj", - "OS-EXT-SRV-ATTR:root_device_name": "/dev/sda", - "OS-EXT-SRV-ATTR:user_data": "", - "OS-EXT-STS:power_state": 1, - "OS-EXT-STS:task_state": None, - "OS-EXT-STS:vm_state": "active", - "OS-SRV-USG:launched_at": "2017-02-14T19:23:59.895661", - "OS-SRV-USG:terminated_at": None, - "accessIPv4": "1.2.3.4", - "accessIPv6": "80fe::", - "addresses": { - "private": [ - { - "OS-EXT-IPS-MAC:mac_addr": "aa:bb:cc:dd:ee:ff", - "OS-EXT-IPS:type": "fixed", - "addr": "192.168.0.3", - "version": 4 - } - ] - }, - "config_drive": "", - "created": "2017-02-14T19:23:58Z", - "description": "fake description", - "flavor": { - "id": "7b46326c-ce48-4e43-8aca-4f5ca00d5f37", - "ephemeral": 0, - "extra_specs": { - "hw:cpu_model": "SandyBridge", - "hw:mem_page_size": "2048", - "hw:cpu_policy": "dedicated" - }, - "original_name": "m1.tiny.specs", - "ram": 512, - "swap": 0, - "vcpus": 1 - }, - "hostId": "2091634baaccdc4c5a1d57069c833e402921df696b7f970791b12ec6", - "host_status": "UP", - "id": "9168b536-cd40-4630-b43f-b259807c6e87", - "image": { - "id": "70a599e0-31e7-49b7-b260-868f441e862b", - "links": [ - { - "href": "http://openstack.example.com" - "/images/70a599e0-31e7-49b7-b260-868f441e862b", - "rel": "bookmark" - } - ] - }, - "key_name": None, - "links": [ - { - "href": "http://openstack.example.com/v2.1/servers" - "/9168b536-cd40-4630-b43f-b259807c6e87", - "rel": "self" - }, - { - "href": "http://openstack.example.com/servers" - "/9168b536-cd40-4630-b43f-b259807c6e87", - "rel": "bookmark" - } - ], - "locked": False, - "metadata": { - "My Server Name": "Apache1" - }, - "name": "new-server-test", - "os-extended-volumes:volumes_attached": [ - { - "delete_on_termination": False, - "id": "volume_id1" - }, - { - "delete_on_termination": False, - "id": "volume_id2" - } - ], - "progress": 0, - "security_groups": [ - { - "name": "default" - } - ], - "status": "ACTIVE", - "tags": [], - "tenant_id": "6f70656e737461636b20342065766572", - "updated": "2017-02-14T19:24:00Z", - "user_id": "fake" -} -SERVER_RESPONSE = {'server': SERVER} diff --git a/os_capacity/tests/unit/test_utils.py b/os_capacity/tests/unit/test_utils.py deleted file mode 100644 index 466d9af..0000000 --- a/os_capacity/tests/unit/test_utils.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime -import unittest -from unittest import mock - -from os_capacity.data import flavors -from os_capacity.data import resource_provider -from os_capacity.data import server -from os_capacity import utils - - -class TestUtils(unittest.TestCase): - - @mock.patch.object(flavors, "get_all") - def test_get_flavors(self, mock_flavors): - mock_list = [flavors.Flavor( - id="id", name="name", vcpus=1, ram_mb=2, disk_gb=3, - extra_specs={})] - mock_flavors.return_value = mock_list - app = mock.Mock() - - result = utils.get_flavors(app) - - mock_flavors.assert_called_once_with( - app.compute_client, include_extra_specs=True) - expected_flavors = [('id', 'name', 1, 2, 3, {})] - self.assertEqual(mock_list, result) - - @mock.patch.object(resource_provider, 'get_allocations') - @mock.patch.object(resource_provider, 'get_inventories') - @mock.patch.object(resource_provider, 'get_all') - def test_get_all_inventories_and_usage(self, mock_grp, mock_gi, mock_ga): - mock_grp.return_value = [ - resource_provider.ResourceProvider('uuid1', 'name1'), - resource_provider.ResourceProvider('uuid2', 'name2'), - resource_provider.ResourceProvider('uuid3', 'name3')] - fake_r = [ - resource_provider.ResourceClassAmount("CUSTOM_FOO", 3), - resource_provider.ResourceClassAmount("CUSTOM_BAR", 2), - ] - mock_ga.side_effect = [ - [resource_provider.Allocation("uuid1", "consumer_uuid1", fake_r)], - [], - [resource_provider.Allocation("uuid1", "consumer_uuid2", fake_r)], - ] - mock_gi.side_effect = [ - [resource_provider.Inventory("uuid1", "VCPU", 10), - resource_provider.Inventory("uuid1", "DISK_GB", 5)], - [], - [resource_provider.Inventory("uuid1", "VCPU", 10), - resource_provider.Inventory("uuid1", "DISK_GB", 6)], - ] - app = mock.Mock() - - result = list(utils.get_providers_with_resources_and_servers(app)) - - mock_grp.assert_called_once_with(app.placement_client) - self.assertEqual(3, mock_gi.call_count) - self.assertEqual(3, mock_ga.call_count) - - self.assertEqual([ - ('name1', 'DISK_GB:5, VCPU:10', 'consumer_uuid1'), - ('name2', '', ''), - ('name3', 'DISK_GB:6, VCPU:10', 'consumer_uuid2'), - ], result) - - @mock.patch.object(resource_provider, 'get_allocations') - @mock.patch.object(resource_provider, 'get_inventories') - @mock.patch.object(resource_provider, 'get_all') - @mock.patch.object(flavors, 'get_all') - def test_group_inventories(self, mock_flav, mock_grp, mock_gi, mock_ga): - mock_flav.return_value = [ - flavors.Flavor(id="id1", name="flavor1", - vcpus=1, ram_mb=2, disk_gb=3), - flavors.Flavor(id="id2", name="flavor2", - vcpus=1, ram_mb=2, disk_gb=30), - flavors.Flavor(id="id3", name="flavor3", - vcpus=1, ram_mb=2, disk_gb=3), - ] - mock_grp.return_value = [ - resource_provider.ResourceProvider('uuid1', 'name1'), - resource_provider.ResourceProvider('uuid2', 'name2'), - resource_provider.ResourceProvider('uuid3', 'name3')] - fake_r = [ - resource_provider.ResourceClassAmount("VCPU", 1), - resource_provider.ResourceClassAmount("MEMORY_MB", 2), - resource_provider.ResourceClassAmount("DISK_GB", 3), - ] - mock_ga.side_effect = [ - [resource_provider.Allocation("uuid1", "consumer_uuid1", fake_r)], - [], - [resource_provider.Allocation("uuid3", "consumer_uuid2", fake_r)], - ] - mock_gi.side_effect = [ - [resource_provider.Inventory("uuid1", "VCPU", 1), - resource_provider.Inventory("uuid1", "MEMORY_MB", 2), - resource_provider.Inventory("uuid1", "DISK_GB", 3)], - [resource_provider.Inventory("uuid1", "VCPU", 1), - resource_provider.Inventory("uuid1", "MEMORY_MB", 2), - resource_provider.Inventory("uuid1", "DISK_GB", 30)], - [resource_provider.Inventory("uuid1", "VCPU", 1), - resource_provider.Inventory("uuid1", "MEMORY_MB", 2), - resource_provider.Inventory("uuid1", "DISK_GB", 3)], - ] - app = mock.Mock() - - result = list(utils.group_providers_by_type_with_capacity(app)) - - self.assertEqual(2, len(result)) - expected = [ - ('VCPU:1,MEMORY_MB:2,DISK_GB:30', 1, 0, 1, 'flavor2'), - ('VCPU:1,MEMORY_MB:2,DISK_GB:3', 2, 2, 0, 'flavor1, flavor3') - ] - self.assertEqual(expected, result) - - @mock.patch.object(server, 'get') - @mock.patch.object(resource_provider, 'get_all_allocations') - @mock.patch.object(resource_provider, 'get_all') - @mock.patch.object(utils, '_get_now') - def test_get_allocation_list(self, mock_now, mock_rps, mock_allocations, - mock_server): - mock_now.return_value = datetime.datetime(2017, 3, 2) - - mock_rps.return_value = [ - resource_provider.ResourceProvider("uuid1", "name1"), - resource_provider.ResourceProvider("uuid2", "name2") - ] - mock_allocations.return_value = [ - resource_provider.Allocation( - "uuid1", "consumer_uuid2", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]), - resource_provider.Allocation( - "uuid2", "consumer_uuid1", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]), - ] - mock_server.side_effect = [ - server.Server("consumer_uuid2", "name", - datetime.datetime(2017, 3, 1), - "user_id", "project_id", "flavor"), - server.Server("consumer_uuid2", "name", - datetime.datetime(2017, 3, 1), - "user_id", "project_id", "flavor"), - ] - app = mock.Mock() - - result = utils.get_allocations_with_server_info(app) - - mock_rps.assert_called_once_with(app.placement_client) - self.assertEqual(2, len(result)) - expected1 = utils.AllocationList( - 'name1', 'consumer_uuid2', - 'DISK_GB:10, MEMORY_MB:20, VCPU:30', - 'flavor', 1, 'project_id', 'user_id') - self.assertEqual(expected1, result[0]) - expected2 = utils.AllocationList( - 'name2', 'consumer_uuid1', - 'DISK_GB:10, MEMORY_MB:20, VCPU:30', - 'flavor', 1, 'project_id', 'user_id') - self.assertEqual(expected2, result[1]) - - def _setup_fake_allocations(self, mock_get_allocations): - fake_usage = [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30), - ] - mock_get_allocations.return_value = [ - utils.AllocationList( - 'name1', 'consumer_uuid2', fake_usage, - 'flavor', 1, 'project_id', 'user_id1'), - utils.AllocationList( - 'name2', 'consumer_uuid1', fake_usage, - 'flavor', 2, 'project_id', 'user_id1'), - utils.AllocationList( - 'name2', 'consumer_uuid1', fake_usage, - 'flavor', 1, 'project_id', 'user_id2'), - ] - - @mock.patch.object(utils, "get_allocations_with_server_info") - def test_group_usage(self, mock_get_allocations): - self._setup_fake_allocations(mock_get_allocations) - app = mock.Mock() - - result = utils.group_usage(app) - - expected = [ - ('user_id1', - 'Count:2, DISK_GB:20, MEMORY_MB:40, VCPU:60', - 'Count:3, DISK_GB:30, MEMORY_MB:60, VCPU:90'), - ('user_id2', - 'Count:1, DISK_GB:10, MEMORY_MB:20, VCPU:30', - 'Count:1, DISK_GB:10, MEMORY_MB:20, VCPU:30'), - ] - self.assertEqual(expected, result) - - @mock.patch.object(utils, "get_allocations_with_server_info") - def test_group_usage_project(self, mock_get_allocations): - self._setup_fake_allocations(mock_get_allocations) - app = mock.Mock() - - result = utils.group_usage(app, "project") - - expected = [ - ('project_id', - 'Count:3, DISK_GB:30, MEMORY_MB:60, VCPU:90', - 'Count:4, DISK_GB:40, MEMORY_MB:80, VCPU:120'), - ] - self.assertEqual(expected, result) - - @mock.patch.object(utils, "get_allocations_with_server_info") - def test_group_usage_all(self, mock_get_allocations): - self._setup_fake_allocations(mock_get_allocations) - app = mock.Mock() - - result = utils.group_usage(app, "all") - - expected = [ - ('(All)', - 'Count:3, DISK_GB:30, MEMORY_MB:60, VCPU:90', - 'Count:4, DISK_GB:40, MEMORY_MB:80, VCPU:120'), - ] - self.assertEqual(expected, result) diff --git a/os_capacity/utils.py b/os_capacity/utils.py deleted file mode 100644 index 32ea515..0000000 --- a/os_capacity/utils.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections -from datetime import datetime -import os - -from os_capacity.data import flavors -from os_capacity.data import resource_provider -from os_capacity.data import server as server_data -from os_capacity.data import users - -IGNORE_CUSTOM_RC = 'OS_CAPACITY_IGNORE_CUSTOM_RC' in os.environ - - -def get_flavors(app): - app.LOG.debug("Getting flavors") - return flavors.get_all(app.compute_client, include_extra_specs=True) - - -def get_providers_with_resources_and_servers(app): - resource_providers = resource_provider.get_all(app.placement_client) - - for rp in resource_providers: - inventories = resource_provider.get_inventories( - app.placement_client, rp) - allocations = resource_provider.get_allocations( - app.placement_client, rp) - - inventory_texts = ["%s:%s" % (i.resource_class, i.total) - for i in inventories] - inventory_texts.sort() - inventory_text = ", ".join(inventory_texts) - - allocation_texts = [a.consumer_uuid for a in allocations] - allocation_texts.sort() - allocation_text = ", ".join(allocation_texts) - - yield (rp.name, inventory_text, allocation_text) - - -def group_providers_by_type_with_capacity(app): - # TODO(johngarbutt) this flavor grouping is very ironic specific - all_flavors = flavors.get_all(app.compute_client) - grouped_flavors = collections.defaultdict(list) - for flavor in all_flavors: - custom_rc = None - if not IGNORE_CUSTOM_RC: - for extra_spec in flavor.extra_specs: - if extra_spec.startswith('resources:CUSTOM'): - custom_rc = extra_spec.replace('resources:', '') - break # Assuming a good Ironic setup here - - key = (flavor.vcpus, flavor.ram_mb, flavor.disk_gb, custom_rc) - grouped_flavors[key] += [flavor.name] - - all_resource_providers = resource_provider.get_all(app.placement_client) - - inventory_counts = collections.defaultdict(int) - allocation_counts = collections.defaultdict(int) - for rp in all_resource_providers: - inventories = resource_provider.get_inventories( - app.placement_client, rp) - - # TODO(johngarbutt) much refinement needed to be general... - vcpus = 0 - ram_mb = 0 - disk_gb = 0 - custom_rc = None - for inventory in inventories: - if "VCPU" in inventory.resource_class: - vcpus += inventory.total - if "MEMORY" in inventory.resource_class: - ram_mb += inventory.total - if "DISK" in inventory.resource_class: - disk_gb += inventory.total - if inventory.resource_class.startswith('CUSTOM_'): - if not IGNORE_CUSTOM_RC: - custom_rc = inventory.resource_class # Ironic specific - key = (vcpus, ram_mb, disk_gb, custom_rc) - - inventory_counts[key] += 1 - - allocations = resource_provider.get_allocations( - app.placement_client, rp) - if allocations: - allocation_counts[key] += 1 - - for key, inventory_count in inventory_counts.items(): - resources = "VCPU:%s, MEMORY_MB:%s, DISK_GB:%s, %s" % key - matching_flavors = grouped_flavors[key] - matching_flavors.sort() - matching_flavors = ", ".join(matching_flavors) - total = inventory_count - used = allocation_counts[key] - free = total - used - - yield (resources, total, used, free, matching_flavors) - - -def _get_now(): - # To make it easy to mock in tests - return datetime.now() - - -AllocationList = collections.namedtuple( - "AllocationList", ("resource_provider_name", "consumer_uuid", - "usage", "flavor_id", "days", - "project_id", "user_id")) - - -def get_allocations_with_server_info(app, flat_usage=True, get_names=False): - """Get allocations, add in server and resource provider details.""" - resource_providers = resource_provider.get_all(app.placement_client) - rp_dict = {rp.uuid: rp.name for rp in resource_providers} - - all_allocations = resource_provider.get_all_allocations( - app.placement_client, resource_providers) - - now = _get_now() - - allocation_tuples = [] - for allocation in all_allocations: - rp_name = rp_dict[allocation.resource_provider_uuid] - - # TODO(johngarbutt) this is too presentation like for here - usage = allocation.resources - if flat_usage: - usage_amounts = ["%s:%s" % (rca.resource_class, rca.amount) - for rca in allocation.resources] - usage_amounts.sort() - usage = ", ".join(usage_amounts) - - server = server_data.get(app.compute_client, allocation.consumer_uuid) - if server: - delta = now - server.created - days_running = delta.days + 1 - - allocation_tuples.append(AllocationList( - rp_name, allocation.consumer_uuid, usage, - server.flavor_id, days_running, server.project_id, - server.user_id)) - - allocation_tuples.sort(key=lambda x: (x.project_id, x.user_id, - x.days * -1, x.flavor_id)) - - if get_names: - all_users = users.get_all(app.identity_client) - all_projects = users.get_all_projects(app.identity_client) - all_flavors_list = flavors.get_all(app.compute_client, - include_extra_specs=False) - all_flavors = {flavor.id: flavor.name for flavor in all_flavors_list} - - updated = [] - for allocation in allocation_tuples: - user_id = all_users.get(allocation.user_id) - project_id = all_projects.get(allocation.project_id) - flavor_id = all_flavors.get(allocation.flavor_id) - updated.append(AllocationList( - allocation.resource_provider_name, - allocation.consumer_uuid, - allocation.usage, - flavor_id, - allocation.days, - project_id, - user_id)) - allocation_tuples = updated - - return allocation_tuples - - -UsageSummary = collections.namedtuple( - "UsageSummary", ("resource_provider_name", "consumer_uuid", - "usage", "flavor_id", "days", - "project_id", "user_id")) - - -def group_usage(app, group_by="user"): - all_allocations = get_allocations_with_server_info(app, flat_usage=False) - - def get_key(allocation): - if group_by == "user": - return allocation.user_id - if group_by == "project": - return allocation.project_id - return "(All)" - - grouped_allocations = collections.defaultdict(list) - for allocation in all_allocations: - grouped_allocations[get_key(allocation)].append(allocation) - - all_users = users.get_all(app.identity_client) - all_projects = users.get_all_projects(app.identity_client) - - summary_tuples = [] - for key, group in grouped_allocations.items(): - grouped_usage = collections.defaultdict(int) - grouped_usage_days = collections.defaultdict(int) - for allocation in group: - for rca in allocation.usage: - grouped_usage[rca.resource_class] += rca.amount - grouped_usage_days[rca.resource_class] += ( - rca.amount * allocation.days) - grouped_usage["Count"] += 1 - grouped_usage_days["Count"] += allocation.days - - usage_amounts = ["%s:%s" % (resource_class, total) - for resource_class, total in grouped_usage.items()] - usage_amounts.sort() - usage = ", ".join(usage_amounts) - - usage_days_amounts = [ - "%s:%s" % (resource_class, total) - for resource_class, total in grouped_usage_days.items()] - usage_days_amounts.sort() - usage_days = ", ".join(usage_days_amounts) - - # Resolve id to name, if possible - key_name = None - if group_by == "user": - key_name = all_users.get(key) - elif group_by == "project": - key_name = all_projects.get(key) - - summary_tuples.append((key_name or key, usage, usage_days)) - - if group_by == "user" or group_by == "project_id": - if group_by == "user": - dimensions = {"user_id": key} - name_key = "username" - else: - dimensions = {"project_id": key} - name_key = "project_name" - - if key_name: - dimensions[name_key] = key_name - value_meta = {'usage_summary': usage} - dimensions['version'] = '2.0' - - # Sort my largest current usage first - summary_tuples.sort(key=lambda x: x[1], reverse=True) - - return summary_tuples From ed4f2cf2c46a13653164ef3cf1ae844fbf133797 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 15:41:55 +0000 Subject: [PATCH 3/7] Fix up unit tests and linter --- os_capacity/prometheus.py | 41 +++-- os_capacity/tests/unit/__init__.py | 0 os_capacity/tests/unit/test_data.py | 188 ---------------------- os_capacity/tests/unit/test_prometheus.py | 37 +++++ requirements.txt | 8 +- setup.cfg | 15 +- setup.py | 53 +----- tox.ini | 37 +++-- 8 files changed, 96 insertions(+), 283 deletions(-) create mode 100644 os_capacity/tests/unit/__init__.py delete mode 100644 os_capacity/tests/unit/test_data.py create mode 100644 os_capacity/tests/unit/test_prometheus.py diff --git a/os_capacity/prometheus.py b/os_capacity/prometheus.py index a476964..9e0ac7d 100755 --- a/os_capacity/prometheus.py +++ b/os_capacity/prometheus.py @@ -1,8 +1,18 @@ #!/usr/bin/env python3 +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. -import os import collections -import json +import os import time import uuid @@ -174,12 +184,12 @@ def get_host_details(compute_client, placement_client): counts = capacity_per_flavor.get(flavor.name, {}).values() total = 0 if not counts else sum(counts) free_by_flavor_total.add_metric([flavor.name, str(flavor.is_public)], total) - # print(f'openstack_free_capacity_by_flavor{{flavor="{flavor_name}"}} {total}') # capacity per host free_by_flavor_hypervisor = prom_core.GaugeMetricFamily( "openstack_free_capacity_hypervisor_by_flavor", - "Free capacity for each hypervisor if you fill remaining space full of each flavor", + "Free capacity for each hypervisor if you fill " + "remaining space full of each flavor", labels=["hypervisor", "flavor_name", "az_aggregate", "project_aggregate"], ) resource_providers, project_to_aggregate = get_resource_provider_info( @@ -203,7 +213,8 @@ def get_host_details(compute_client, placement_client): ) free_space_found = True if not free_space_found: - # TODO(johngarbutt) allocation candidates only returns some not all candidates! + # TODO(johngarbutt) allocation candidates only returns some, + # not all candidates! print(f"# WARNING - no free spaces found for {hostname}") project_filter_aggregates = prom_core.GaugeMetricFamily( @@ -214,9 +225,6 @@ def get_host_details(compute_client, placement_client): for project, names in project_to_aggregate.items(): for name in names: project_filter_aggregates.add_metric([project, name], 1) - # print( - # f'openstack_project_filter_aggregate{{project_id="{project}",aggregate="{name}"}} 1' - # ) return resource_providers, [ free_by_flavor_total, free_by_flavor_hypervisor, @@ -312,10 +320,6 @@ def get_host_usage(resource_providers, placement_client): return [usage_guage, capacity_guage] -def print_exporter_data(app): - print_host_free_details(app.compute_client, app.placement_client) - - class OpenStackCapacityCollector(object): def __init__(self): self.conn = openstack.connect() @@ -346,7 +350,8 @@ def collect(self): host_time = time.perf_counter() host_duration = host_time - start_time print( - f"1 of 3: host flavor capacity complete for {collect_id} it took {host_duration} seconds" + "1 of 3: host flavor capacity complete " + f"for {collect_id} it took {host_duration} seconds" ) if not skip_project_usage: @@ -354,17 +359,19 @@ def collect(self): project_time = time.perf_counter() project_duration = project_time - host_time print( - f"2 of 3: project usage complete for {collect_id} it took {project_duration} seconds" + "2 of 3: project usage complete " + f"for {collect_id} it took {project_duration} seconds" ) else: print("2 of 3: skipping project usage") - if not skip_project_usage: + if not skip_host_usage: guages += get_host_usage(resource_providers, conn.placement) host_usage_time = time.perf_counter() host_usage_duration = host_usage_time - project_time print( - f"3 of 3: host usage complete for {collect_id} it took {host_usage_duration} seconds" + "3 of 3: host usage complete for " + f"{collect_id} it took {host_usage_duration} seconds" ) else: print("3 of 3: skipping host usage") @@ -391,4 +398,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/os_capacity/tests/unit/__init__.py b/os_capacity/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/os_capacity/tests/unit/test_data.py b/os_capacity/tests/unit/test_data.py deleted file mode 100644 index 5b1b6a3..0000000 --- a/os_capacity/tests/unit/test_data.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime -import unittest -from unittest import mock - -from os_capacity.data import flavors -from os_capacity.data import resource_provider -from os_capacity.data import server -from os_capacity.tests.unit import fakes - - -class TestFlavor(unittest.TestCase): - - def test_get_all_no_extra_specs(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.FLAVOR_RESPONSE - compute_client = mock.MagicMock() - compute_client.get.return_value = fake_response - - result = flavors.get_all(compute_client, False) - - compute_client.get.assert_called_once_with("/flavors/detail") - expected_flavors = [flavors.Flavor(fakes.FLAVOR['id'], 'compute-GPU', - 8, 2048, 30, None)] - self.assertEqual(expected_flavors, result) - - def test_get_all(self): - fake_response1 = mock.MagicMock() - fake_response1.json.return_value = fakes.FLAVOR_RESPONSE - fake_response2 = mock.MagicMock() - fake_response2.json.return_value = fakes.FLAVOR_EXTRA_RESPONSE - compute_client = mock.MagicMock() - compute_client.get.side_effect = [fake_response1, fake_response2] - - result = flavors.get_all(compute_client) - - compute_client.get.assert_has_calls([ - mock.call("/flavors/detail"), - mock.call("/flavors/%s/os-extra_specs" % fakes.FLAVOR['id'])]) - expected_flavors = [flavors.Flavor(fakes.FLAVOR['id'], 'compute-GPU', - 8, 2048, 30, - {'example_key': 'example_value'})] - self.assertEqual(expected_flavors, result) - - -class TestResourceProvider(unittest.TestCase): - - def test_get_all(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.RESOURCE_PROVIDER_RESPONSE - placement_client = mock.MagicMock() - placement_client.get.return_value = fake_response - - result = resource_provider.get_all(placement_client) - - placement_client.get.assert_called_once_with("/resource_providers") - self.assertEqual([(fakes.RESOURCE_PROVIDER['uuid'], 'name1')], result) - - -class TestInventory(unittest.TestCase): - - def test_get_inventories(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.INVENTORIES_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp = resource_provider.ResourceProvider("uuid", "name") - - result = resource_provider.get_inventories(client, rp) - - client.get.assert_called_once_with( - "/resource_providers/uuid/inventories") - self.assertEqual(3, len(result)) - disk = resource_provider.Inventory("uuid", "DISK_GB", 10) - self.assertIn(disk, result) - mem = resource_provider.Inventory("uuid", "MEMORY_MB", 20) - self.assertIn(mem, result) - vcpu = resource_provider.Inventory("uuid", "VCPU", 30) - self.assertIn(vcpu, result) - - @mock.patch.object(resource_provider, 'get_all') - def test_get_all_inventories(self, mock_get_all): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.INVENTORIES_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp1 = resource_provider.ResourceProvider("uuid1", "name1") - rp2 = resource_provider.ResourceProvider("uuid2", "name2") - mock_get_all.return_value = [rp1, rp2] - - result = resource_provider.get_all_inventories(client) - - mock_get_all.assert_called_once_with(client) - client.get.assert_called_with( - "/resource_providers/uuid2/inventories") - self.assertEqual(6, len(result)) - disk1 = resource_provider.Inventory("uuid1", "DISK_GB", 10) - self.assertIn(disk1, result) - disk2 = resource_provider.Inventory("uuid2", "DISK_GB", 10) - self.assertIn(disk2, result) - - -class TestAllocations(unittest.TestCase): - - def test_get_allocations(self): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.ALLOCATIONS_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp = resource_provider.ResourceProvider("uuid", "name") - - result = resource_provider.get_allocations(client, rp) - - self.assertEqual(1, len(result)) - expected = resource_provider.Allocation( - "uuid", "consumer_uuid", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]) - self.assertEqual(expected, result[0]) - - @mock.patch.object(resource_provider, 'get_all') - def test_get_all_allocations(self, mock_get_all): - fake_response = mock.MagicMock() - fake_response.json.return_value = fakes.ALLOCATIONS_RESPONSE - client = mock.MagicMock() - client.get.return_value = fake_response - - rp1 = resource_provider.ResourceProvider("uuid1", "name1") - rp2 = resource_provider.ResourceProvider("uuid2", "name2") - mock_get_all.return_value = [rp1, rp2] - - result = resource_provider.get_all_allocations(client) - - self.assertEqual(2, len(result)) - uuid1 = resource_provider.Allocation( - "uuid1", "consumer_uuid", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]) - self.assertIn(uuid1, result) - uuid2 = resource_provider.Allocation( - "uuid1", "consumer_uuid", [ - resource_provider.ResourceClassAmount("DISK_GB", 10), - resource_provider.ResourceClassAmount("MEMORY_MB", 20), - resource_provider.ResourceClassAmount("VCPU", 30)]) - self.assertIn(uuid2, result) - - -class TestServer(unittest.TestCase): - def test_parse_created(self): - result = server._parse_created("2017-02-14T19:23:58Z") - expected = datetime.datetime(2017, 2, 14, 19, 23, 58) - self.assertEqual(expected, result) - - def test_get(self): - mock_response = mock.Mock() - mock_response.json.return_value = fakes.SERVER_RESPONSE - compute_client = mock.Mock() - compute_client.get.return_value = mock_response - - uuid = '9168b536-cd40-4630-b43f-b259807c6e87' - result = server.get(compute_client, uuid) - - expected = server.Server( - uuid=uuid, - name='new-server-test', - created=datetime.datetime(2017, 2, 14, 19, 23, 58), - user_id='fake', - project_id='6f70656e737461636b20342065766572', - flavor_id='7b46326c-ce48-4e43-8aca-4f5ca00d5f37') - self.assertEqual(expected, result) diff --git a/os_capacity/tests/unit/test_prometheus.py b/os_capacity/tests/unit/test_prometheus.py new file mode 100644 index 0000000..cc3d687 --- /dev/null +++ b/os_capacity/tests/unit/test_prometheus.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from os_capacity import prometheus + + +class FakeFlavor: + def __init__(self, id, name, vcpus, ram, disk, ephemeral, extra_specs): + self.id = id + self.name = name + self.vcpus = vcpus + self.ram = ram + self.disk = disk + self.ephemeral = ephemeral + self.extra_specs = extra_specs + + +class TestFlavor(unittest.TestCase): + def test_get_placement_request(self): + flavor = FakeFlavor( + "fake_id", "fake_name", 8, 2048, 30, 0, {"hw:cpu_policy": "dedicated"} + ) + resources, traits = prometheus.get_placement_request(flavor) + + self.assertEqual({"PCPU": 8, "MEMORY_MB": 2048, "DISK_GB": 30}, resources) + self.assertEqual([], traits) diff --git a/requirements.txt b/requirements.txt index 4e5edae..689a89b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -cliff>=2.8.0 # Apache -os-client-config>=1.28.0 # Apache-2.0 -pbr>=2.0.0,!=2.1.0 # Apache-2.0 -prometheus-client==0.16.0 -six # MIT +openstacksdk +pbr +prometheus-client diff --git a/setup.cfg b/setup.cfg index 3600d32..dd98db5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,17 +5,14 @@ description-file = README.rst author = StackHPC author-email = johng@stackhpc.com -home-page = https://stackhpc.com -classifier = - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.9 +url = https://github.com/stackhpc/os-capacity +python-requires = >=3.9 +license = Apache-2 [files] packages = os_capacity +[entry_points] +console_scripts= + os_capacity = os_capacity.prometheus:main diff --git a/setup.py b/setup.py index a9061b6..32792a1 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2017 StackHPC Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +import setuptools -from setuptools import find_packages -from setuptools import setup - - -PROJECT = 'os_capacity' -VERSION = '0.4' - -try: - long_description = open('README.md', 'rt').read() -except IOError: - long_description = '' - -setup( - name=PROJECT, - version=VERSION, - - description='OpenStack capacity tooling', - long_description=long_description, - - author='StackHPC', - author_email='john.garbutt@stackhpc.com', - - url='https://github.com/stackhpc/os-capacity', - download_url='https://github.com/stackhpc/os-capacity/tarball/master', - - provides=[], - install_requires=open('requirements.txt', 'rt').read().splitlines(), - - namespace_packages=[], - packages=find_packages(), - include_package_data=True, - - entry_points={ - 'console_scripts': [ - 'os-capacity = os_capacity.prometheus:main', - ], - }, -) +setuptools.setup(setup_requires=["pbr"], pbr=True) diff --git a/tox.ini b/tox.ini index c817894..f46a41f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 2.0 -envlist = py39,pep8 +envlist = py3,black,pep8 skipsdist = True [testenv] @@ -9,21 +9,20 @@ install_command = pip install {opts} {packages} setenv = VIRTUAL_ENV={envdir} PYTHONWARNINGS=default::DeprecationWarning -deps = -r{toxinidir}/test-requirements.txt +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt commands = stestr run {posargs} -[testenv:pep8] +[testenv:cover] +setenv = + VIRTUAL_ENV={envdir} + PYTHON=coverage run --source azimuth_caas_operator --parallel-mode commands = - flake8 {posargs} - -[testenv:venv] -commands = {posargs} - -[testenv:docs] -commands = python setup.py build_sphinx - -[testenv:debug] -commands = oslo_debug_helper {posargs} + stestr run {posargs} + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report [flake8] # E123, E125 skipped as they are invalid PEP-8. @@ -31,3 +30,15 @@ show-source = True ignore = E123,E125 builtins = _ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build +# match black +max-line-length = 88 + +[testenv:pep8] +commands = + black {tox_root} + flake8 {posargs} +allowlist_externals = black + +[testenv:black] +commands = black {tox_root} --check +allowlist_externals = black From 40de8b11e327a0356fddf6cc119e41dd73b31f34 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 15:46:39 +0000 Subject: [PATCH 4/7] Add tox workflow --- .github/workflows/tox.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/tox.yaml diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml new file mode 100644 index 0000000..aa98fa3 --- /dev/null +++ b/.github/workflows/tox.yaml @@ -0,0 +1,28 @@ +name: Tox unit tests + +on: + workflow_call: + +jobs: + build: + name: Tox unit tests and linting + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + + - name: Test with tox + run: tox From 4d70891ebab54667fc6d13865b9288253d6d79db Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 15:49:38 +0000 Subject: [PATCH 5/7] Slight update to the readme --- README.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 357461f..48a56d6 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,11 @@ os-capacity =========== -This is a prototype tool to extract capacity information. +This is a prototype prometheus exporter +to extract capacity information from OpenStack Placement. + +It includes support for both baremetal flavors +and flavors with PCPU resources implied. Install ------- From c4ae98cc7a4d5ba23f783974f28805e9246b5e57 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 15:50:47 +0000 Subject: [PATCH 6/7] Always run the tox job --- .github/workflows/tox.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index aa98fa3..82bd824 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -1,7 +1,6 @@ name: Tox unit tests -on: - workflow_call: +on: push jobs: build: From e576b358cbd4bb8f06bddad48992615172c8ec3e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 5 Nov 2024 16:04:15 +0000 Subject: [PATCH 7/7] Update container build --- .github/workflows/docker.yaml | 41 +++++++++++++---------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index fbfcc21..34fcf65 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -3,24 +3,20 @@ name: Docker image on: push jobs: build_push_api: - name: Build and push execution environment + name: Build and push image runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # needed for signing the images with GitHub OIDC Token + packages: write # required for pushing container images + security-events: write # required for pushing SARIF files + steps: - name: Check out the repository - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: actions/checkout@v4 - - name: Set up Docker layer caching - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -28,7 +24,7 @@ jobs: - name: Calculate metadata for image id: image-meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: images: ghcr.io/stackhpc/os-capacity # Produce the branch name or tag and the SHA as tags @@ -36,21 +32,14 @@ jobs: type=ref,event=branch type=ref,event=tag type=sha,prefix= + - name: Build and push image - uses: docker/build-push-action@v2 + uses: azimuth-cloud/github-actions/docker-multiarch-build-push@master with: + cache-key: os-capacity context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.image-meta.outputs.tags }} labels: ${{ steps.image-meta.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - # https://github.com/docker/buildx/pull/535 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + github-token: ${{ secrets.GITHUB_TOKEN }}