From 5817d52482fe0071f3110a98b6c681e312e14f95 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 31 Oct 2024 11:28:15 +0100 Subject: [PATCH] Implements: #1353 --- IM/InfrastructureManager.py | 19 +++++ IM/REST.py | 44 +++++++++++ IM/ServiceRequests.py | 14 ++++ IM/Stats.py | 142 +++++++++++++++++++++++++++++++++++ IM/im_service.py | 7 ++ test/unit/REST.py | 15 ++++ test/unit/ServiceRequests.py | 7 ++ test/unit/test_im_logic.py | 42 +++++++++++ 8 files changed, 290 insertions(+) create mode 100644 IM/Stats.py diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index 24a012ac..f0ed96f2 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -42,6 +42,7 @@ from IM.openid.JWT import JWT from IM.openid.OpenIDClient import OpenIDClient from IM.vault import VaultCredentials +from IM.Stats import Stats if Config.MAX_SIMULTANEOUS_LAUNCHES > 1: @@ -2036,3 +2037,21 @@ def EstimateResouces(radl_data, auth): cont += 1 return res + + @staticmethod + def GetStats(init_date, end_date, auth): + """ + Get the statistics from the IM DB. + Args: + - init_date(str): Only will be returned infrastructure created afther this date. + - end_date(str): Only will be returned infrastructure created before this date. + - auth(Authentication): parsed authentication tokens. + Return: a list of dict with the stats. + """ + # First check the auth data + auth = InfrastructureManager.check_auth_data(auth) + stats = Stats.get_stats(init_date, end_date, auth) + if not stats: + raise Exception("ERROR connecting with the database!.") + else: + return stats diff --git a/IM/REST.py b/IM/REST.py index 572f3c55..3f92a8a0 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -21,6 +21,7 @@ import flask import os import yaml +import datetime from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher from cheroot.ssl.builtin import BuiltinSSLAdapter @@ -1077,6 +1078,49 @@ def RESTChangeInfrastructureAuth(infid=None): return return_error(400, "Error modifying infrastructure owner: %s" % get_ex_error(ex)) +@app.route('/stats', methods=['GET']) +def RESTGetStats(): + try: + auth = get_auth_header() + except Exception: + return return_error(401, "No authentication data provided") + + try: + init_date = None + if "init_date" in flask.request.args.keys(): + init_date = flask.request.args.get("init_date").lower() + init_date = init_date.replace("/", "-") + parts = init_date.split("-") + try: + year = int(parts[0]) + month = int(parts[1]) + day = int(parts[2]) + datetime.date(year, month, day) + except Exception: + return return_error(400, "Incorrect format in init_date parameter: YYYY/MM/dd") + else: + init_date = "1970-01-01" + + end_date = None + if "end_date" in flask.request.args.keys(): + end_date = flask.request.args.get("end_date").lower() + end_date = end_date.replace("/", "-") + parts = end_date.split("-") + try: + year = int(parts[0]) + month = int(parts[1]) + day = int(parts[2]) + datetime.date(year, month, day) + except Exception: + return return_error(400, "Incorrect format in end_date parameter: YYYY/MM/dd") + + stats = InfrastructureManager.GetStats(init_date, end_date, auth) + return format_output(stats, default_type="application/json", field_name="stats") + except Exception as ex: + logger.exception("Error getting stats") + return return_error(400, "Error getting stats: %s" % get_ex_error(ex)) + + @app.errorhandler(403) def error_mesage_403(error): return return_error(403, error.description) diff --git a/IM/ServiceRequests.py b/IM/ServiceRequests.py index 0f8cfad1..9fba899c 100644 --- a/IM/ServiceRequests.py +++ b/IM/ServiceRequests.py @@ -60,6 +60,7 @@ class IMBaseRequest(AsyncRequest): CHANGE_INFRASTRUCTURE_AUTH = "ChangeInfrastructureAuth" GET_INFRASTRUCTURE_OWNERS = "GetInfrastructureOwners" ESTIMATE_RESOURCES = "EstimateResouces" + GET_STATS = "GetStats" @staticmethod def create_request(function, arguments=()): @@ -119,6 +120,8 @@ def create_request(function, arguments=()): return Request_GetInfrastructureOwners(arguments) elif function == IMBaseRequest.ESTIMATE_RESOURCES: return Request_EstimateResouces(arguments) + elif function == IMBaseRequest.GET_STATS: + return Request_GetStats(arguments) else: raise NotImplementedError("Function not Implemented") @@ -473,3 +476,14 @@ def _call_function(self): self._error_mesage = "Error getting the resources estimation" (radl_data, auth_data) = self.arguments return IM.InfrastructureManager.InfrastructureManager.EstimateResouces(radl_data, Authentication(auth_data)) + + +class Request_GetStats(IMBaseRequest): + """ + Request class for the GetStats function + """ + + def _call_function(self): + self._error_mesage = "Error getting stats" + (init_date, end_date, auth_data) = self.arguments + return IM.InfrastructureManager.InfrastructureManager.GetStats(init_date, end_date, Authentication(auth_data)) diff --git a/IM/Stats.py b/IM/Stats.py new file mode 100644 index 00000000..b1a0d79f --- /dev/null +++ b/IM/Stats.py @@ -0,0 +1,142 @@ +# IM - Infrastructure Manager +# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os.path +import datetime +import json +import yaml +import logging + +from IM.db import DataBase +from IM.auth import Authentication +from IM.config import Config +from IM.InfrastructureList import InfrastructureList +from radl.radl_parse import parse_radl + + +class Stats(): + + logger = logging.getLogger('InfrastructureManager') + """Logger object.""" + + @staticmethod + def _get_data(str_data, init_date=None, end_date=None): + dic = json.loads(str_data) + resp = {'creation_date': None} + if 'creation_date' in dic and dic['creation_date']: + creation_date = datetime.datetime.fromtimestamp(float(dic['creation_date'])) + resp['creation_date'] = str(creation_date) + if init_date and creation_date < init_date: + return None + if end_date and creation_date > end_date: + return None + + resp['tosca_name'] = None + if 'extra_info' in dic and dic['extra_info'] and "TOSCA" in dic['extra_info']: + try: + tosca = yaml.safe_load(dic['extra_info']['TOSCA']) + icon = tosca.get("metadata", {}).get("icon", "") + resp['tosca_name'] = os.path.basename(icon)[:-4] + except Exception: + Stats.logger.exception("Error loading TOSCA.") + + resp['vm_count'] = 0 + resp['cpu_count'] = 0 + resp['memory_size'] = 0 + resp['cloud_type'] = None + resp['cloud_host'] = None + resp['hybrid'] = False + for str_vm_data in dic['vm_list']: + vm_data = json.loads(str_vm_data) + cloud_data = json.loads(vm_data["cloud"]) + + # only get the cloud of the first VM + if not resp['cloud_type']: + resp['cloud_type'] = cloud_data["type"] + if not resp['cloud_host']: + resp['cloud_host'] = cloud_data["server"] + elif resp['cloud_host'] != cloud_data["server"]: + resp['hybrid'] = True + + vm_sys = parse_radl(vm_data['info']).systems[0] + if vm_sys.getValue('cpu.count'): + resp['cpu_count'] += vm_sys.getValue('cpu.count') + if vm_sys.getValue('memory.size'): + resp['memory_size'] += vm_sys.getFeature('memory.size').getValue('M') + resp['vm_count'] += 1 + + inf_auth = Authentication.deserialize(dic['auth']).getAuthInfo('InfrastructureManager')[0] + resp['im_user'] = inf_auth.get('username') + return resp + + @staticmethod + def get_stats(init_date="1970-01-01", end_date=None, auth=None): + """ + Get the statistics from the IM DB. + + Args: + + - init_date(str): Only will be returned infrastructure created afther this date. + - end_date(str): Only will be returned infrastructure created afther this date. + - auth(Authentication): parsed authentication tokens. + + Return: a list of dict with the stats with the following format: + {'creation_date': '2022-03-07 13:16:14', + 'tosca_name': 'kubernetes', + 'vm_count': 2, + 'cpu_count': 4, + 'memory_size': 1024, + 'cloud_type': 'OSCAR', + 'cloud_host': 'sharp-elbakyan5.im.grycap.net', + 'hybrid': False, + 'im_user': '__OPENID__mcaballer', + 'inf_id': '1', + 'last_date': '2022-03-23'} + """ + stats = [] + db = DataBase(Config.DATA_DB) + if db.connect(): + if db.db_type == DataBase.MONGO: + filt = InfrastructureList._gen_filter_from_auth(auth) + res = db.find("inf_list", filt, {"id": True, "date": True, "date": True}, [('id', -1)]) + else: + where = InfrastructureList._gen_where_from_auth(auth) + res = db.select("select data, date, id from inf_list %s order by rowid desc" % where) + + for elem in res: + if db.db_type == DataBase.MONGO: + data = elem["data"] + date = elem["date"] + inf_id = elem["id"] + else: + data = elem[0] + date = elem[1] + inf_id = elem[2] + try: + init = datetime.datetime.strptime(init_date, "%Y-%m-%d") + end = datetime.datetime.strptime(end_date, "%Y-%m-%d") if end_date else None + res = Stats._get_data(data, init, end) + if res: + res['inf_id'] = inf_id + res['last_date'] = str(date) + stats.append(res) + except Exception: + Stats.logger.exception("ERROR reading infrastructure info from Inf ID: %s" % inf_id) + db.close() + return stats + else: + Stats.logger.error("ERROR connecting with the database!.") + return None diff --git a/IM/im_service.py b/IM/im_service.py index 0edaf452..d7eff4e6 100755 --- a/IM/im_service.py +++ b/IM/im_service.py @@ -229,6 +229,12 @@ def EstimateResources(radl_data, auth_data): return WaitRequest(request) +def GetStats(init_date, end_date, auth_data): + request = IMBaseRequest.create_request( + IMBaseRequest.GET_STATS, (init_date, end_date, auth_data)) + return WaitRequest(request) + + def launch_daemon(): """ Launch the IM daemon @@ -290,6 +296,7 @@ def launch_daemon(): server.register_function(ChangeInfrastructureAuth) server.register_function(GetInfrastructureOwners) server.register_function(EstimateResources) + server.register_function(GetStats) # Launch the API XMLRPC thread server.serve_forever_in_thread() diff --git a/test/unit/REST.py b/test/unit/REST.py index 56b24ba4..3b723a9b 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -838,6 +838,21 @@ def test_GetInfrastructureOwners(self, GetInfrastructureOwners): res = self.client.get('/infrastructures/1/authorization', headers=headers) self.assertEqual(res.json, {"authorization": ["user1", "user2"]}) + @patch("IM.InfrastructureManager.InfrastructureManager.GetStats") + def test_GetStats(self, GetStats): + """Test REST GetStats.""" + headers = {"AUTHORIZATION": "type = InfrastructureManager; username = user; password = pass"} + GetStats.return_value = [{"key": 1}] + + res = self.client.get('/stats?init_date=2010-01-01&end_date=2022-01-01', headers=headers) + + self.assertEqual(res.json, {"stats": [{"key": 1}]}) + self.assertEqual(GetStats.call_args_list[0][0][0], '2010-01-01') + self.assertEqual(GetStats.call_args_list[0][0][1], '2022-01-01') + self.assertEqual(GetStats.call_args_list[0][0][2].auth_list, [{"type": "InfrastructureManager", + "username": "user", + "password": "pass"}]) + if __name__ == "__main__": unittest.main() diff --git a/test/unit/ServiceRequests.py b/test/unit/ServiceRequests.py index 4bf16b98..795baf46 100755 --- a/test/unit/ServiceRequests.py +++ b/test/unit/ServiceRequests.py @@ -214,6 +214,13 @@ def test_estimate_resources(self, inflist): IM.ServiceRequests.IMBaseRequest.ESTIMATE_RESOURCES, ("", "")) req._call_function() + @patch('IM.InfrastructureManager.InfrastructureManager') + def test_get_stats(self, inflist): + import IM.ServiceRequests + req = IM.ServiceRequests.IMBaseRequest.create_request( + IM.ServiceRequests.IMBaseRequest.GET_STATS, ("", "", "")) + req._call_function() + if __name__ == '__main__': unittest.main() diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py index ffb3383a..f2bbd33a 100644 --- a/test/unit/test_im_logic.py +++ b/test/unit/test_im_logic.py @@ -23,6 +23,7 @@ import sys import json import base64 +import yaml from mock import Mock, patch, MagicMock @@ -1560,6 +1561,47 @@ def test_estimate_resources(self): 'storage': [{'sizeInGigabytes': 100}] }}) + @patch('IM.Stats.DataBase') + @patch('IM.InfrastructureManager.InfrastructureManager.check_auth_data') + def test_get_stats(self, check_auth_data, DataBase): + radl = """ + system node ( + memory.size = 512M and + cpu.count = 2 + )""" + + auth = Authentication([{'type': 'InfrastructureManager', 'token': 'atoken', + 'username': '__OPENID__mcaballer', 'password': 'pass'}]) + check_auth_data.return_value = auth + + db = MagicMock() + inf_data = { + "id": "1", + "auth": auth.serialize(), + "creation_date": 1646655374, + "extra_info": {"TOSCA": yaml.dump({"metadata": {"icon": "kubernetes.png"}})}, + "vm_list": [ + json.dumps({"cloud": '{"type": "OSCAR", "server": "sharp-elbakyan5.im.grycap.net"}', "info": radl}), + json.dumps({"cloud": '{"type": "OSCAR", "server": "sharp-elbakyan5.im.grycap.net"}', "info": radl}) + ] + } + db.select.return_value = [(json.dumps(inf_data), '2022-03-23', '1')] + DataBase.return_value = db + + stats = IM.GetStats('2001-01-01', '2122-01-01', auth) + expected_res = [{'creation_date': '2022-03-07 13:16:14', + 'tosca_name': 'kubernetes', + 'vm_count': 2, + 'cpu_count': 4, + 'memory_size': 1024, + 'cloud_type': 'OSCAR', + 'cloud_host': 'sharp-elbakyan5.im.grycap.net', + 'hybrid': False, + 'im_user': '__OPENID__mcaballer', + 'inf_id': '1', + 'last_date': '2022-03-23'}] + self.assertEqual(stats, expected_res) + if __name__ == "__main__": unittest.main()