From 80af5a7240782229553168637528c5280fb3d86e Mon Sep 17 00:00:00 2001 From: sh0ked Date: Thu, 16 Mar 2017 17:58:42 +0700 Subject: [PATCH 1/3] Selenium server request timeout --- core/constants.py | 1 + core/exceptions.py | 10 +++++++++- core/sessions.py | 13 +++++++++---- vmmaster/webdriver/commands.py | 2 ++ vmpool/virtual_machines_pool.py | 2 +- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/core/constants.py b/core/constants.py index e0b823a5..eedac2cf 100644 --- a/core/constants.py +++ b/core/constants.py @@ -6,3 +6,4 @@ GET_ENDPOINT_WAIT_TIME_INCREMENT = 5 ENDPOINT_REBUILD_ATTEMPTS = 3 +REQUEST_TIMEOUT = 120 diff --git a/core/exceptions.py b/core/exceptions.py index 85473720..c3f88b73 100644 --- a/core/exceptions.py +++ b/core/exceptions.py @@ -23,6 +23,14 @@ class CreationException(Exception): pass +class RequestTimeoutException(Exception): + pass + + +class RequestException(Exception): + pass + + class TimeoutException(Exception): pass @@ -36,4 +44,4 @@ class NoSuchEndpoint(Exception): class QueueItemNotFound(Exception): - pass \ No newline at end of file + pass diff --git a/core/sessions.py b/core/sessions.py index 7cc1f3b5..1ba36e5c 100644 --- a/core/sessions.py +++ b/core/sessions.py @@ -9,9 +9,10 @@ from datetime import datetime from flask import current_app +from core import constants from core.db import models from core.config import config -from core.exceptions import SessionException +from core.exceptions import SessionException, RequestException, RequestTimeoutException from core.video import VNCVideoHelper log = logging.getLogger(__name__) @@ -192,7 +193,7 @@ def add_session_step(self, control_line, body=None, created=None): return step - def make_request(self, port, request): + def make_request(self, port, request, timeout=constants.REQUEST_TIMEOUT): """ Make http request to some port in session and return the response. """ @@ -206,7 +207,8 @@ def req(): return requests.request(method=request.method, url=url, headers=request.headers, - data=request.data) + data=request.data, + timeout=timeout) t = Thread(target=getresponse, args=(req, q)) t.daemon = True @@ -216,8 +218,11 @@ def req(): yield None, None, None response = q.get() + if isinstance(response, requests.Timeout): + raise RequestTimeoutException("No response for '%s' in %s sec. Original: %s" + % (url, timeout, response)) if isinstance(response, Exception): - raise response + raise RequestException("Error for '%s'. Original: %s" % (url, response)) yield response.status_code, response.headers, response.content diff --git a/vmmaster/webdriver/commands.py b/vmmaster/webdriver/commands.py index 5e2a8a53..7196ac9f 100644 --- a/vmmaster/webdriver/commands.py +++ b/vmmaster/webdriver/commands.py @@ -53,6 +53,8 @@ def wrapper(port, request, *args, **kwargs): @connection_watcher def start_session(request, session): + log.info("Start preparing selenium session for %s on %s(%s)" + % (session.id, session.endpoint_name, session.endpoint_ip)) status, headers, body = None, None, None ping_vm(session) diff --git a/vmpool/virtual_machines_pool.py b/vmpool/virtual_machines_pool.py index e8fbd673..6da33abb 100644 --- a/vmpool/virtual_machines_pool.py +++ b/vmpool/virtual_machines_pool.py @@ -88,7 +88,7 @@ def can_produce(cls, platform): return True if cls.count() >= platform_limit: - log.warning( + log.debug( 'Can\'t produce new virtual machine with platform %s: ' 'not enough Instances resources' % platform ) From c63448709ca14cb14789d8dfcde968ddc4f3013e Mon Sep 17 00:00:00 2001 From: sh0ked Date: Thu, 16 Mar 2017 18:18:49 +0700 Subject: [PATCH 2/3] Changed work with endpoints and fixes --- config_template.py | 3 +- vmmaster/api/__init__.py | 15 +++--- vmmaster/webdriver/helpers.py | 5 +- vmpool/clone.py | 91 +++++++++++++++-------------------- 4 files changed, 53 insertions(+), 61 deletions(-) diff --git a/config_template.py b/config_template.py index 61dbed50..971473ac 100644 --- a/config_template.py +++ b/config_template.py @@ -46,7 +46,8 @@ class Config(object): OPENSTACK_PING_RETRY_COUNT = 3 OPENSTACK_DEFAULT_FLAVOR = '' OPENASTACK_VM_META_DATA = { - 'admin_pass': 'testPassw0rd.' + 'admin_pass': 'testPassw0rd.', + 'adminPass': 'testPassw0rd.' } VM_CREATE_CHECK_PAUSE = 5 diff --git a/vmmaster/api/__init__.py b/vmmaster/api/__init__.py index 931065c1..a1c68059 100644 --- a/vmmaster/api/__init__.py +++ b/vmmaster/api/__init__.py @@ -169,10 +169,13 @@ def delete_vm_from_pool(endpoint_name): if endpoint: try: - endpoint.delete() + if endpoint.is_preloaded(): + endpoint.delete() + else: + endpoint.delete(try_to_rebuild=False) result = "Endpoint %s was deleted" % endpoint_name except Exception, e: - log.info("Cannot delete vm %s through api method" % endpoint.name) + log.info("Cannot delete vm %s through api method" % endpoint_name) result = "Got error during deleting vm %s. " \ "\n\n %s" % (endpoint_name, e.message) @@ -186,11 +189,11 @@ def delete_all_vm_from_pool(): for endpoint in current_app.pool.pool + current_app.pool.using: try: - endpoint.delete() - results.append(endpoint) + delete_vm_from_pool(endpoint.name) + results.append(endpoint.name) except: log.info("Cannot delete vm %s through api method" % endpoint.name) - failed.append(endpoint) + failed.append(endpoint.name) return render_json(result="This endpoints were deleted from " - "pool: %s" % results, code=200) + "pool: %s" % results, code=200) \ No newline at end of file diff --git a/vmmaster/webdriver/helpers.py b/vmmaster/webdriver/helpers.py index cb77bdb1..88448517 100644 --- a/vmmaster/webdriver/helpers.py +++ b/vmmaster/webdriver/helpers.py @@ -182,8 +182,8 @@ def check_to_exist_ip(session, tries=10, timeout=5): def get_endpoint(session_id, dc): _endpoint = None attempt = 0 - attempts = getattr(config, "GET_ENDPOINT_WAIT_TIME_INCREMENT", - constants.GET_ENDPOINT_WAIT_TIME_INCREMENT) + attempts = getattr(config, "GET_ENDPOINT_ATTEMPTS", + constants.GET_ENDPOINT_ATTEMPTS) wait_time = 0 wait_time_increment = getattr(config, "GET_ENDPOINT_WAIT_TIME_INCREMENT", constants.GET_ENDPOINT_WAIT_TIME_INCREMENT) @@ -203,6 +203,7 @@ def get_endpoint(session_id, dc): % (attempt, session_id, str(e))) if hasattr(_endpoint, "ready"): if not _endpoint.ready: + _endpoint.delete() _endpoint = None if attempt < attempts: time.sleep(wait_time) diff --git a/vmpool/clone.py b/vmpool/clone.py index c13502d1..3f4b4011 100644 --- a/vmpool/clone.py +++ b/vmpool/clone.py @@ -15,7 +15,6 @@ from core import dumpxml from core import utils -from core import constants from core.exceptions import libvirtError, CreationException from core.config import config from core.utils import network_utils @@ -275,24 +274,13 @@ def _wait_for_activated_service(self, method=None): config.VM_CREATE_CHECK_ATTEMPTS, config.VM_CREATE_CHECK_PAUSE config_ping_retry_count, config_ping_timeout = \ config.OPENSTACK_PING_RETRY_COUNT, config.PING_TIMEOUT - rebuild_attempts = getattr(config, "ENDPOINT_REBUILD_ATTEMPTS", - constants.ENDPOINT_REBUILD_ATTEMPTS) create_check_retry = 1 ping_retry = 1 - rebuild_retry = 1 while True: - try: - server = self.nova_client.servers.find(name=self.name) - except Exception as e: - log.exception( - "Can't find vm %s in openstack. Error: %s" % - (self.name, e.message) - ) - server = None - - if server is not None and server.status.lower() in \ + server = self.get_vm(self.name) + if server and server.status.lower() in \ ('build', 'rebuild'): log.info("Virtual Machine %s is spawning..." % self.name) @@ -306,6 +294,10 @@ def _wait_for_activated_service(self, method=None): time.sleep(config_create_check_pause) elif self.vm_has_created(): + if server.status.lower() in 'error': + log.error("VM %s was errored. Rebuilding..." % server.name) + self.rebuild() + break if method is not None: method() if self.ping_vm(): @@ -314,16 +306,14 @@ def _wait_for_activated_service(self, method=None): if ping_retry > config_ping_retry_count: p = config_ping_retry_count * config_ping_timeout log.info("VM %s pings more than %s seconds..." % (self.name, p)) - if rebuild_retry < rebuild_attempts: - raise CreationException("VM %s was created but does not ping" % self.name) - log.warn("Try to rebuild vm %s. Attempt %s" % (self.name, rebuild_retry)) - rebuild_retry += 1 - self.rebuild() + self.delete(try_to_rebuild=True) break ping_retry += 1 else: - raise CreationException("VM %s has not been created." % self.name) + log.error("VM %s has not been created." % self.name) + self.delete(try_to_rebuild=False) + break @property def image(self): @@ -374,15 +364,8 @@ def get_network_id(self): # create new network def vm_has_created(self): - try: - server = self.nova_client.servers.find(name=self.name) - except Exception as e: - log.exception( - "An error occurred during addition ip for vm %s: %s" % - (self.name, e.message)) - server = None - - if server is not None: + server = self.get_vm(self.name) + if server: if server.status.lower() == 'active': if getattr(server, 'addresses', None) is not None: return True @@ -390,12 +373,15 @@ def vm_has_created(self): return False def check_vm_exist(self, server_name): + return True if self.get_vm(server_name) else False + + def get_vm(self, server_name): try: server = self.nova_client.servers.find(name=server_name) - return True if server.name == server_name else False - except Exception as e: - log.exception("VM does not exist. Error: %s" % e.message) - return False + return server if server.name == server_name else None + except: + log.exception("VM %s does not exist" % server_name) + return None def delete(self, try_to_rebuild=True): if try_to_rebuild and self.is_preloaded(): @@ -404,17 +390,15 @@ def delete(self, try_to_rebuild=True): self.ready = False self.pool.remove_vm(self) - if self.check_vm_exist(self.name): + + server = self.get_vm(self.name) + if server: try: - self.nova_client.servers.find(name=self.name).delete() - except Exception as e: - log.exception("Delete vm %s was FAILED. %s" % - (self.name, e.message)) - log.info("Deleted openstack clone: {clone}".format( - clone=self.name)) - else: - log.warn("VM {clone} can not be removed because " - "it does not exist".format(clone=self.name)) + server.delete() + except: + log.exception("Delete vm %s was FAILED." % self.name) + + log.info("Deleted openstack clone: {0}".format(self.name)) VirtualMachine.delete(self) def rebuild(self): @@ -425,12 +409,15 @@ def rebuild(self): self.pool.pool.append(self) self.ready = False - try: - self.nova_client.servers.find(name=self.name).rebuild(self.image) - self._wait_for_activated_service(lambda: log.info( - "Rebuilded openstack clone: {clone}".format(clone=self.name))) - except Exception as e: - log.exception( - "Rebuild vm %s was FAILED. %s" % (self.name, e.message) - ) - self.delete(try_to_rebuild=False) + server = self.get_vm(self.name) + if server: + try: + server.rebuild(self.image) + self._wait_for_activated_service( + lambda: log.info( + "Rebuild vm %s was successful" % self.name + ) + ) + except: + log.exception("Rebuild vm %s was FAILED." % self.name) + self.delete(try_to_rebuild=False) From 1d1f70711d9d566675c935dc42b15e2abffb5e8c Mon Sep 17 00:00:00 2001 From: sh0ked Date: Thu, 16 Mar 2017 18:44:58 +0700 Subject: [PATCH 3/3] Set default admin password for endpoints --- config_template.py | 5 +- core/constants.py | 3 - core/exceptions.py | 16 -- core/sessions.py | 5 +- tests/run_unittests.py | 2 - tests/unit/data/config_openstack.py | 1 + tests/unit/data/userdata | 3 + tests/unit/test_api.py | 2 +- tests/unit/test_openstack_clone.py | 319 ++++++++++-------------- tests/unit/test_server.py | 30 ++- tests/unit/test_unit_openstack_clone.py | 32 +++ tox.ini | 2 +- vmmaster/api/__init__.py | 9 +- vmmaster/webdriver/__init__.py | 1 - vmpool/clone.py | 76 +++--- 15 files changed, 239 insertions(+), 267 deletions(-) delete mode 100644 tests/run_unittests.py create mode 100644 tests/unit/data/userdata create mode 100644 tests/unit/test_unit_openstack_clone.py diff --git a/config_template.py b/config_template.py index 971473ac..ffa87de2 100644 --- a/config_template.py +++ b/config_template.py @@ -46,9 +46,9 @@ class Config(object): OPENSTACK_PING_RETRY_COUNT = 3 OPENSTACK_DEFAULT_FLAVOR = '' OPENASTACK_VM_META_DATA = { - 'admin_pass': 'testPassw0rd.', - 'adminPass': 'testPassw0rd.' + 'admin_pass': 'testPassw0rd.' } + OPENSTACK_VM_USERDATA_FILE_PATH = "%s/userdata" % os.path.abspath(os.curdir) VM_CREATE_CHECK_PAUSE = 5 VM_CREATE_CHECK_ATTEMPTS = 1000 @@ -64,4 +64,3 @@ class Config(object): VMMASTER_AGENT_PORT = 9000 THREAD_POOL_MAX = 100 - diff --git a/core/constants.py b/core/constants.py index eedac2cf..2051d207 100644 --- a/core/constants.py +++ b/core/constants.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- - - SESSION_CLOSE_REASON_API_CALL = "Session closed via API stop_session call" GET_ENDPOINT_ATTEMPTS = 10 GET_ENDPOINT_WAIT_TIME_INCREMENT = 5 - ENDPOINT_REBUILD_ATTEMPTS = 3 REQUEST_TIMEOUT = 120 diff --git a/core/exceptions.py b/core/exceptions.py index c3f88b73..533e6a83 100644 --- a/core/exceptions.py +++ b/core/exceptions.py @@ -11,14 +11,6 @@ class SessionException(Exception): pass -class ClonesException(Exception): - pass - - -class NoMacError(Exception): - pass - - class CreationException(Exception): pass @@ -37,11 +29,3 @@ class TimeoutException(Exception): class ConnectionError(Exception): pass - - -class NoSuchEndpoint(Exception): - pass - - -class QueueItemNotFound(Exception): - pass diff --git a/core/sessions.py b/core/sessions.py index 1ba36e5c..0f25b8d5 100644 --- a/core/sessions.py +++ b/core/sessions.py @@ -118,6 +118,8 @@ def save_artifacts(self): artifacts = { "selenium_server": "/var/log/selenium_server.log" } + if not self.endpoint_ip: + return False return self.endpoint.save_artifacts(self, artifacts) def close(self, reason=None): @@ -219,8 +221,7 @@ def req(): response = q.get() if isinstance(response, requests.Timeout): - raise RequestTimeoutException("No response for '%s' in %s sec. Original: %s" - % (url, timeout, response)) + raise RequestTimeoutException("No response for '%s' in %s sec. Original: %s" % (url, timeout, response)) if isinstance(response, Exception): raise RequestException("Error for '%s'. Original: %s" % (url, response)) diff --git a/tests/run_unittests.py b/tests/run_unittests.py deleted file mode 100644 index 0229c522..00000000 --- a/tests/run_unittests.py +++ /dev/null @@ -1,2 +0,0 @@ -import nose -nose.run(defaultTest="unit") diff --git a/tests/unit/data/config_openstack.py b/tests/unit/data/config_openstack.py index 726214bc..7c64d457 100644 --- a/tests/unit/data/config_openstack.py +++ b/tests/unit/data/config_openstack.py @@ -41,6 +41,7 @@ class Config(object): OPENASTACK_VM_META_DATA = { 'admin_pass': 'testPassw0rd.' } + OPENSTACK_VM_USERDATA_FILE_PATH = "%s/tests/unit/data/userdata" % os.path.abspath(os.curdir) VM_CHECK = False VM_CHECK_FREQUENCY = 1800 diff --git a/tests/unit/data/userdata b/tests/unit/data/userdata new file mode 100644 index 00000000..80564c41 --- /dev/null +++ b/tests/unit/data/userdata @@ -0,0 +1,3 @@ +#cloud-config +password: pass +chpasswd: {expire: False} \ No newline at end of file diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 30efaa24..98033ac0 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -285,4 +285,4 @@ def delete(self): body = json.loads(response.data) self.assertEqual(200, body['metacode']) self.assertEqual("This endpoints were deleted from pool: " - "[%s]" % vm_name_1, body['result']) + "%s" % [vm_name_1], body['result']) diff --git a/tests/unit/test_openstack_clone.py b/tests/unit/test_openstack_clone.py index cca6b2b4..6346afbb 100644 --- a/tests/unit/test_openstack_clone.py +++ b/tests/unit/test_openstack_clone.py @@ -93,116 +93,98 @@ def test_creation_vm(self): self.assertTrue(self.app.pool.using[0].ready) self.assertEqual(len(self.app.pool.using), 1) + @patch.multiple( + 'vmpool.clone.OpenstackClone', + get_vm=Mock(return_value=None) + ) def test_exception_during_creation_vm(self): """ - call OpenstackClone.create() - - exception in create() + - get_vm() return None Expected: vm has been deleted """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock( - servers=Mock(create=Mock( - side_effect=Exception('Exception in create')))) - - self.app.pool.add(self.platform) - self.assertEqual(self.app.pool.count(), 0) + self.app.pool.add(self.platform) + wait_for(lambda: self.app.pool.count() == 0) + self.assertEqual(self.app.pool.count(), 0) @patch.multiple( 'vmpool.clone.OpenstackClone', - vm_has_created=Mock(return_value=False), - check_vm_exist=Mock(return_value=True), - rebuild=Mock(return_value=True) + get_vm=Mock(return_value=None) ) - def test_e_in_wait_for_activated_service_and_vm_has_not_been_created(self): + def test_exception_in_wait_for_activated_service_and_rebuild_failed(self): """ - call OpenstackClone.create() - - exception in _wait_for_activated_service - - vm_has_created is False + - get_vm() is None + - call delete() - Expected: vm has been rebuilded + Expected: vm has been deleted """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock( - find=Mock(side_effect=Exception( - 'Exception in _wait_for_activated_service')))) - - self.app.pool.add(self.platform) - self.assertEqual(self.app.pool.count(), 1) + self.app.pool.add(self.platform) + wait_for(lambda: self.app.pool.count() == 0) + self.assertEqual(self.app.pool.count(), 0) @patch.multiple( 'vmpool.clone.OpenstackClone', - vm_has_created=Mock(return_value=False), - check_vm_exist=Mock(return_value=True) + get_ip=Mock(), + get_vm=Mock(return_value=Mock(status="active")) ) - def test_exception_in_wait_for_activated_service_and_rebuild_failed(self): + @patch.multiple( + 'core.utils.network_utils', + ping=Mock(return_value=True) + ) + def test_ping_success(self): """ - call OpenstackClone.create() - - exception in _wait_for_activated_service - - vm_has_created is False - - call rebuild() and raise Exception in rebuild() + - ping success - Expected: vm has been deleted + Expected: vm has been created """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock( - find=Mock(side_effect=Exception( - 'Exception in _wait_for_activated_service')))) + self.app.pool.add(self.platform) - self.app.pool.add(self.platform) - wait_for(lambda: self.app.pool.count() == 0) - self.assertEqual(self.app.pool.count(), 0) + wait_for(lambda: self.app.pool.using[0].ready is True) + self.assertEqual(self.app.pool.count(), 1) + self.assertTrue(self.app.pool.using[0].ready) @patch.multiple( 'vmpool.clone.OpenstackClone', - vm_has_created=Mock(return_value=True), get_ip=Mock(), + get_vm=Mock(return_value=Mock(status="active")), ping_vm=Mock(return_value=True) ) def test_exception_in_wait_for_activated_service_and_ping_success(self): """ - call OpenstackClone.create() - - exception in _wait_for_activated_service - - vm_has_created is True + - get_vm() return Mock and is_active is True - ping success Expected: vm has been created """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock( - servers=Mock( - find=Mock(side_effect=Exception( - 'Exception in _wait_for_activated_service')))) - self.app.pool.add(self.platform) + self.app.pool.add(self.platform) - wait_for(lambda: self.app.pool.using[0].ready is True) - self.assertEqual(self.app.pool.count(), 1) - self.assertTrue(self.app.pool.using[0].ready) + wait_for(lambda: self.app.pool.using[0].ready is True) + self.assertEqual(self.app.pool.count(), 1) + self.assertTrue(self.app.pool.using[0].ready) @patch.multiple( 'vmpool.clone.OpenstackClone', - vm_has_created=Mock(return_value=True), get_ip=Mock(), + get_vm=Mock(return_value=Mock(status='active')), rebuild=Mock(return_value=True), - ping_vm=Mock(return_value=False) + ping_vm=Mock(side_effect=[False, True]) ) def test_exception_in_wait_for_activated_service_and_ping_failed(self): """ - call OpenstackClone.create() - - exception in _wait_for_activated_service - - vm_has_created is True + - is_active is True - ping failed Expected: vm has been rebuilded """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock( - find=Mock(side_effect=Exception( - 'Exception in _wait_for_activated_service')))) - - self.app.pool.add(self.platform) - self.assertEqual(self.app.pool.count(), 1) + self.app.pool.add(self.platform) + self.assertEqual(self.app.pool.count(), 1) + @patch('vmpool.clone.OpenstackClone._wait_for_activated_service', custom_wait) def test_exception_in_getting_image(self): """ - call OpenstackClone.create() @@ -211,7 +193,8 @@ def test_exception_in_getting_image(self): Expected: vm has been deleted """ with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(create=Mock()), + nova.return_value = Mock(servers=Mock(create=Mock(), + status="active"), images=Mock( find=Mock(side_effect=Exception( 'Exception in image')))) @@ -219,7 +202,11 @@ def test_exception_in_getting_image(self): self.app.pool.add(self.platform) self.assertEqual(self.app.pool.count(), 0) - @patch('vmpool.clone.OpenstackClone.image', Mock()) + @patch.multiple( + 'vmpool.clone.OpenstackClone', + image=Mock(), + _wait_for_activated_service=custom_wait + ) def test_exception_in_getting_flavor(self): """ - call OpenstackClone.create() @@ -231,7 +218,8 @@ def test_exception_in_getting_flavor(self): 'core.utils.openstack_utils.nova_client' ) as nova: nova.return_value = Mock( - servers=Mock(create=Mock()), + servers=Mock(create=Mock(), + status="active"), flavors=Mock(find=Mock( side_effect=Exception('Exception in flavor'))) ) @@ -241,8 +229,8 @@ def test_exception_in_getting_flavor(self): @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=True), - _wait_for_activated_service=custom_wait + _wait_for_activated_service=custom_wait, + get_vm=Mock(status="active", delete=Mock(), rebuild=Mock()) ) def test_delete_vm(self): """ @@ -251,39 +239,34 @@ def test_delete_vm(self): Expected: vm has been deleted """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(delete=Mock(), rebuild=Mock())))) - - self.app.pool.add(self.platform) - self.app.pool.using[0].delete() - self.assertEqual(self.app.pool.count(), 0) + self.app.pool.add(self.platform) + self.app.pool.using[0].delete() + self.assertEqual(self.app.pool.count(), 0) @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=False), + get_vm=Mock(side_effect=[ + Mock(status="active", delete=Mock(), rebuild=Mock()), + None + ]), _wait_for_activated_service=custom_wait ) def test_delete_vm_if_vm_does_not_exist(self): """ - call OpenstackClone.create() - - check_vm_exist is False + - get_vm return None - call OpenstackClone.delete() Expected: vm has been deleted """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(delete=Mock(), rebuild=Mock())))) - - self.app.pool.add(self.platform) - self.app.pool.using[0].delete() - self.assertEqual(self.app.pool.count(), 0) + self.app.pool.add(self.platform) + self.app.pool.using[0].delete() + self.assertEqual(self.app.pool.count(), 0) @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=True), - _wait_for_activated_service=custom_wait + _wait_for_activated_service=custom_wait, + get_vm=Mock(status="active", delete=Mock(), rebuild=Mock()) ) def test_rebuild_preload_vm(self): """ @@ -292,21 +275,17 @@ def test_rebuild_preload_vm(self): Expected: vm has been rebuilded and added in pool """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(delete=Mock(), rebuild=Mock())))) - - self.app.pool.preload(self.platform, prefix='preloaded') - wait_for(lambda: self.app.pool.pool[0].ready is True) - self.app.pool.pool[0].rebuild() - wait_for(lambda: self.app.pool.pool[0].ready is True) - self.assertEqual(self.app.pool.count(), 1) - self.assertTrue(self.app.pool.pool[0].ready) + self.app.pool.preload(self.platform, prefix='preloaded') + wait_for(lambda: self.app.pool.pool[0].ready is True) + self.app.pool.pool[0].rebuild() + wait_for(lambda: self.app.pool.pool[0].ready is True) + self.assertEqual(self.app.pool.count(), 1) + self.assertTrue(self.app.pool.pool[0].ready) @patch.multiple( 'vmpool.clone.OpenstackClone', - vm_has_created=Mock(return_value=True), get_ip=Mock(__name__='get_ip'), + get_vm=Mock(return_value=Mock(delete=Mock(), rebuild=Mock(), status='active')), ping_vm=Mock(return_value=True) ) def test_rebuild_ondemand_vm_with_wait_activate_service(self): @@ -316,21 +295,16 @@ def test_rebuild_ondemand_vm_with_wait_activate_service(self): Expected: vm has been rebuilded and added in pool """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(delete=Mock(), rebuild=Mock())))) - - self.app.pool.add(self.platform, prefix='ondemand') - wait_for(lambda: self.app.pool.using[0].ready is True) - self.app.pool.using[0].rebuild() - wait_for(lambda: self.app.pool.using[0].ready is True) - self.assertEqual(self.app.pool.count(), 1) - self.assertTrue(self.app.pool.using[0].ready) + self.app.pool.add(self.platform, prefix='ondemand') + wait_for(lambda: self.app.pool.using[0].ready is True) + self.app.pool.using[0].rebuild() + wait_for(lambda: self.app.pool.using[0].ready is True) + self.assertEqual(self.app.pool.count(), 1) + self.assertTrue(self.app.pool.using[0].ready) @patch.multiple( 'vmpool.clone.OpenstackClone', - vm_has_created=Mock(return_value=True), - get_ip=Mock(__name__='get_ip'), + get_vm=Mock(return_value=Mock(delete=Mock(), rebuild=Mock(), status='active')), ping_vm=Mock(return_value=True) ) def test_rebuild_preload_vm_with_wait_activate_service(self): @@ -340,26 +314,24 @@ def test_rebuild_preload_vm_with_wait_activate_service(self): Expected: vm has been rebuilded and added in pool """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(delete=Mock(), rebuild=Mock())))) - - self.app.pool.preload(self.platform, prefix='preloaded') - wait_for(lambda: self.app.pool.pool[0].ready is True) - self.app.pool.pool[0].rebuild() - wait_for(lambda: self.app.pool.pool[0].ready is True) - self.assertEqual(self.app.pool.count(), 1) - self.assertTrue(self.app.pool.pool[0].ready) + self.app.pool.preload(self.platform, prefix='preloaded') + wait_for(lambda: self.app.pool.pool[0].ready is True) + self.app.pool.pool[0].rebuild() + wait_for(lambda: self.app.pool.pool[0].ready is True) + self.assertEqual(self.app.pool.count(), 1) + self.assertTrue(self.app.pool.pool[0].ready) @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=False), + get_vm=Mock(side_effect=[ + Mock(delete=Mock(), rebuild=Mock(side_effect=Exception('Rebuild error')), status='active'), + None + ]), _wait_for_activated_service=custom_wait ) def test_rebuild_vm_if_vm_does_not_exist(self): """ - call OpenstackClone.create() - - check_vm_exist is False - call OpenstackClone.rebuild() Expected: vm has been deleted @@ -376,20 +348,21 @@ def test_rebuild_vm_if_vm_does_not_exist(self): @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=True), + get_vm=Mock(return_value=Mock( + delete=Mock(), rebuild=Mock(side_effect=Exception('Rebuild error')), status='active') + ), _wait_for_activated_service=custom_wait ) def test_exception_in_rebuild_vm_if_vm_exist(self): """ - call OpenstackClone.create() - - check_vm_exist is True - exception in OpenstackClone.rebuild() Expected: vm has been deleted """ with patch('core.utils.openstack_utils.nova_client') as nova: nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(delete=Mock(), rebuild=Mock( + return_value=Mock(status="active", delete=Mock(), rebuild=Mock( side_effect=Exception('Rebuild error')))))) self.app.pool.add(self.platform) @@ -399,56 +372,32 @@ def test_exception_in_rebuild_vm_if_vm_exist(self): @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=True), ping_vm=Mock(return_value=True), - rebuild=Mock(return_value=True), + get_vm=Mock(return_value=Mock( + rebuild=Mock(return_value=True), + status=Mock(lower=Mock(side_effect=['build', 'error', 'active']))) + ), get_ip=Mock(__name__='get_ip') ) - def test_exception_in_vm_has_created(self): + def test_vm_in_error_status(self): """ - call OpenstackClone.create() - - check_vm_exist is True - - ping successful - - exception in vm_has_created() - - Expected: vm has been rebuilded - """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock( - servers=Mock(find=Mock(side_effect=Exception( - 'Exception in vm_has_created')))) - - self.app.pool.add(self.platform) - self.assertEqual(self.app.pool.count(), 1) - - @patch.multiple( - 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=True), - ping_vm=Mock(return_value=True), - vm_has_created=Mock(return_value=True), - get_ip=Mock(__name__='get_ip') - ) - def test_vm_in_build_status(self): - """ - - call OpenstackClone.create() - - check_vm_exist is True - - ping successful - - vm_has_created is True - first call server.status.lower() return 'build', - second call return 'active' + second call return 'error' + third call return 'active' Expected: vm has been created """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(status=Mock(lower=Mock( - side_effect=['build', 'active'])))))) - self.app.pool.add(self.platform) - self.assertEqual(self.app.pool.count(), 1) + self.app.pool.add(self.platform) + self.assertEqual(self.app.pool.count(), 1) @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=True), - vm_has_created=Mock(return_value=True) + get_vm=Mock(return_value=Mock( + status='active', + addresses=Mock(get=Mock(side_effect=Exception('Error get addresses'))), + rebuild=Mock(side_effect=Exception('Rebuild exception')) + ), + ) ) def test_exception_in_get_ip(self, ): """ @@ -460,8 +409,7 @@ def test_exception_in_get_ip(self, ): Expected: vm has been deleted """ with patch('core.utils.openstack_utils.nova_client') as nova,\ - patch('vmpool.clone.' - 'OpenstackClone.ping_vm') as ping_mock: + patch('vmpool.clone.OpenstackClone.ping_vm') as ping_mock: nova.return_value = Mock(servers=Mock(find=Mock( return_value=Mock(addresses=Mock( get=Mock(side_effect=Exception('Error get addresses'))), @@ -474,31 +422,32 @@ def test_exception_in_get_ip(self, ): @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=True), ping_vm=Mock(return_value=True), - vm_has_created=Mock(return_value=True) + is_created=Mock(return_value=True), + get_vm=Mock( + return_value=Mock( + status="active", + addresses=Mock(get=Mock( + return_value=[{'addr': '127.0.0.1', 'OS-EXT-IPS-MAC:mac_addr': 'test_mac'}]), + status=Mock(lower=Mock(return_value='active')), + )) + ) ) def test_create_vm_with_get_ip(self): """ - call OpenstackClone.create() - check_vm_exist is True - ping successful - - vm_has_created is True + - is_created is True - get_ip return mocked ip address and mac Expected: vm has been created """ - with patch('core.utils.openstack_utils.nova_client') as nova: - nova.return_value = Mock(servers=Mock(find=Mock( - return_value=Mock(addresses=Mock(get=Mock( - return_value=[{'addr': '127.0.0.1', - 'OS-EXT-IPS-MAC:mac_addr': 'test_mac'}]))))) - ) - self.app.pool.add(self.platform) - wait_for(lambda: self.app.pool.using[0].ready is True) - self.assertEqual(self.app.pool.using[0].ip, '127.0.0.1') - self.assertEqual(self.app.pool.using[0].mac, 'test_mac') - self.assertEqual(self.app.pool.count(), 1) + self.app.pool.add(self.platform) + wait_for(lambda: self.app.pool.using[0].ready is True) + self.assertEqual(self.app.pool.using[0].ip, '127.0.0.1') + self.assertEqual(self.app.pool.using[0].mac, 'test_mac') + self.assertEqual(self.app.pool.count(), 1) @patch.multiple( @@ -509,9 +458,8 @@ def test_create_vm_with_get_ip(self): ) @patch.multiple( 'vmpool.clone.OpenstackClone', - check_vm_exist=Mock(return_value=False), ping_vm=Mock(return_value=True), - vm_has_created=Mock(return_value=True), + get_vm=Mock(return_value=Mock(rebuild=Mock(), status="active")), get_ip=Mock(__name__='get_ip') ) @patch( @@ -535,7 +483,7 @@ def setUp(self): 'core.connection.Virsh', Mock() ), patch.multiple( 'core.utils.openstack_utils', - nova_client=Mock(return_value=Mock()), + nova_client=Mock(), neutron_client=Mock(return_value=Mock()), glance_client=Mock(return_value=Mock()) ), patch( @@ -563,9 +511,8 @@ def tearDown(self): def test_create_vm_with_getting_network_id_and_name(self): """ - call OpenstackClone.create() - - check_vm_exist is True - ping successful - - vm_has_created is True + - is_created is True - call get_network_id and get_network_name Expected: vm has been created @@ -587,9 +534,8 @@ def test_create_vm_with_getting_network_id_and_name(self): def test_exception_in_get_network_id(self): """ - call OpenstackClone.create() - - check_vm_exist is True - ping successful - - vm_has_created is True + - is_created is True - exception in get_network_id Expected: vm has not been created @@ -605,9 +551,8 @@ def test_exception_in_get_network_id(self): def test_exception_in_get_network_name(self): """ - call OpenstackClone.create() - - check_vm_exist is True - ping successful - - vm_has_created is True + - is_created is True - exception in get_network_name Expected: vm has not been created @@ -625,14 +570,12 @@ def test_exception_in_get_network_name(self): self.app.pool.add(self.platform) self.assertEqual(self.app.pool.count(), 0) - @patch('vmpool.clone.OpenstackClone.get_network_id', - new=Mock(return_value=None)) + @patch('vmpool.clone.OpenstackClone.get_network_id', new=Mock(return_value=None)) def test_none_param_for_get_network_name(self): """ - call OpenstackClone.create() - - check_vm_exist is True - ping successful - - vm_has_created is True + - is_created is True - get_network_id returned None like in case with KeyError in method - call get_network_name(None) diff --git a/tests/unit/test_server.py b/tests/unit/test_server.py index 27c0ac03..6cffb4cd 100644 --- a/tests/unit/test_server.py +++ b/tests/unit/test_server.py @@ -371,6 +371,10 @@ def test_vmmaster_label(self): self.assertEqual(json.dumps({"value": "step-label"}), response.content) session.close() + @patch.multiple( + 'vmmaster.webdriver.helpers', + swap_session=Mock() + ) def test_vmmaster_no_such_platform(self): desired_caps = { 'desiredCapabilities': { @@ -418,6 +422,13 @@ def test_server_timeouted_session(self): session.close() +@patch.multiple( + "vmpool.clone.KVMClone", + clone_origin=Mock(), + define_clone=Mock(), + start_virtual_machine=Mock(), + drive_path=Mock() + ) class TestConnectionClose(BaseTestServer): def setUp(self): setup_config('data/config.py') @@ -509,13 +520,6 @@ def wait_for_platform_in_queue(): wait_for(lambda: not q, timeout=2) self.assertEqual(len(q), 0) - @patch.multiple( - "vmpool.clone.KVMClone", - clone_origin=Mock(), - define_clone=Mock(), - start_virtual_machine=Mock(), - drive_path=Mock() - ) def test_req_closed_when_vm_is_spawning(self): """ - waiting for clone spawning to begin @@ -695,8 +699,8 @@ def tearDown(self): server_is_down(self.address) self.ctx.pop() - @patch('vmpool.clone.OpenstackClone.vm_has_created', new=Mock( - __name__='vm_has_created', + @patch('vmpool.clone.OpenstackClone.is_created', new=Mock( + __name__='is_created', return_value=True )) @patch('vmpool.clone.OpenstackClone.ping_vm', new=Mock( @@ -718,8 +722,8 @@ def test_max_count_with_run_new_request_during_prevm_is_ready(self): self.assertEqual(200, response.status) self.assertEqual(1, self.pool.count()) - @patch('vmpool.clone.OpenstackClone.vm_has_created', new=Mock( - __name__='vm_has_created', + @patch('vmpool.clone.OpenstackClone.is_created', new=Mock( + __name__='is_created', return_value='False' )) @patch('vmpool.clone.OpenstackClone.ping_vm', new=Mock( @@ -738,6 +742,10 @@ def test_max_count_with_run_new_request_during_prevm_is_not_ready(self): self.assertEqual(1, self.pool.count()) +@patch.multiple( + 'vmmaster.webdriver.helpers', + swap_session=Mock() + ) class TestSessionSteps(BaseTestServer): def setUp(self): setup_config('data/config.py') diff --git a/tests/unit/test_unit_openstack_clone.py b/tests/unit/test_unit_openstack_clone.py new file mode 100644 index 00000000..a28aca6b --- /dev/null +++ b/tests/unit/test_unit_openstack_clone.py @@ -0,0 +1,32 @@ +# coding: utf-8 + +from tests.unit.helpers import BaseTestCase +from mock import Mock, patch +from core.config import setup_config + + +@patch.multiple( + 'vmpool.clone.OpenstackClone', + get_network_name=Mock(return_value='Local-Net'), + get_network_id=Mock(return_value=1) +) +@patch.multiple( + 'core.utils.openstack_utils', + neutron_client=Mock(return_value=Mock()), + nova_client=Mock(return_value=Mock()), + glance_client=Mock(return_value=Mock()) +) +class TestOpenstackCloneUnit(BaseTestCase): + def setUp(self): + setup_config('data/config_openstack.py') + self.mocked_origin = Mock(short_name="platform_1") + self.pool = Mock( + pool=dict(), + using=dict() + ) + + def test_success_openstack_set_userdata(self): + from vmpool.clone import OpenstackClone + clone = OpenstackClone(self.mocked_origin, "preloaded", self.pool) + file_object = clone.set_userdata() + self.assertTrue(hasattr(file_object, "read")) diff --git a/tox.ini b/tox.ini index 26477326..30cd2c41 100644 --- a/tox.ini +++ b/tox.ini @@ -52,4 +52,4 @@ sitepackages = False envdir = {toxworkdir} recreate = False deps = {[testenv:unit]deps} -commands = {envbindir}/coverage run --source=vmmaster,vmpool,core tests/run_unittests.py -v +commands = {envbindir}/lode_runner -vs --with-xunit --xunit-file=report_unit.xml --cover-package=vmmaster,vmpool,core --with-coverage --cover-erase --cover-html tests/unit/ diff --git a/vmmaster/api/__init__.py b/vmmaster/api/__init__.py index a1c68059..4a75f05c 100644 --- a/vmmaster/api/__init__.py +++ b/vmmaster/api/__init__.py @@ -169,10 +169,7 @@ def delete_vm_from_pool(endpoint_name): if endpoint: try: - if endpoint.is_preloaded(): - endpoint.delete() - else: - endpoint.delete(try_to_rebuild=False) + endpoint.delete() result = "Endpoint %s was deleted" % endpoint_name except Exception, e: log.info("Cannot delete vm %s through api method" % endpoint_name) @@ -189,11 +186,11 @@ def delete_all_vm_from_pool(): for endpoint in current_app.pool.pool + current_app.pool.using: try: - delete_vm_from_pool(endpoint.name) + endpoint.delete() results.append(endpoint.name) except: log.info("Cannot delete vm %s through api method" % endpoint.name) failed.append(endpoint.name) return render_json(result="This endpoints were deleted from " - "pool: %s" % results, code=200) \ No newline at end of file + "pool: %s" % results, code=200) diff --git a/vmmaster/webdriver/__init__.py b/vmmaster/webdriver/__init__.py index 84c42478..a1ef6971 100644 --- a/vmmaster/webdriver/__init__.py +++ b/vmmaster/webdriver/__init__.py @@ -5,7 +5,6 @@ from flask import Blueprint, current_app, request, jsonify, g from vmmaster.webdriver import commands, helpers -import helpers from core.exceptions import SessionException from core.auth.custom_auth import auth, anonymous diff --git a/vmpool/clone.py b/vmpool/clone.py index 3f4b4011..0365e8e9 100644 --- a/vmpool/clone.py +++ b/vmpool/clone.py @@ -1,5 +1,5 @@ # coding: utf-8 - +import os import time import netifaces import SubnetTree @@ -226,6 +226,15 @@ def __init__(self, origin, prefix, pool): self.network_id = self.get_network_id() self.network_name = self.get_network_name(self.network_id) + @staticmethod + def set_userdata(): + file_path = getattr(config, "OPENSTACK_VM_USERDATA_FILE_PATH", "userdata") + if os.path.isfile(file_path): + try: + return open(file_path) + except: + log.exception("Userdata from %s wasn't applied" % file_path) + def create(self): log.info( "Creating openstack clone of {} with image={}, " @@ -236,12 +245,11 @@ def create(self): 'image': self.image, 'flavor': self.flavor, 'nics': [{'net-id': self.network_id}], - 'meta': config.OPENASTACK_VM_META_DATA + 'meta': getattr(config, "OPENASTACK_VM_META_DATA", {}), + 'userdata': self.set_userdata() } - if bool(config.OPENSTACK_ZONE_FOR_VM_CREATE): - kwargs.update({'availability_zone': - config.OPENSTACK_ZONE_FOR_VM_CREATE}) + kwargs.update({'availability_zone': config.OPENSTACK_ZONE_FOR_VM_CREATE}) self.nova_client.servers.create(**kwargs) self._wait_for_activated_service(self.get_ip) @@ -249,8 +257,10 @@ def create(self): def get_ip(self): if self.ip is None: try: - addresses = self.nova_client.servers.find( - name=self.name).addresses.get(self.network_name, None) + server = self.get_vm(self.name) + if not server: + return + addresses = server.addresses.get(self.network_name, None) if addresses is not None: ip = addresses[0].get('addr', None) self.mac = addresses[0].get('OS-EXT-IPS-MAC:mac_addr', @@ -272,16 +282,17 @@ def get_ip(self): def _wait_for_activated_service(self, method=None): config_create_check_retry_count, config_create_check_pause = \ config.VM_CREATE_CHECK_ATTEMPTS, config.VM_CREATE_CHECK_PAUSE - config_ping_retry_count, config_ping_timeout = \ - config.OPENSTACK_PING_RETRY_COUNT, config.PING_TIMEOUT - create_check_retry = 1 ping_retry = 1 while True: server = self.get_vm(self.name) - if server and server.status.lower() in \ - ('build', 'rebuild'): + if not server: + log.error("VM %s has not been created." % self.name) + self.delete(try_to_rebuild=False) + break + + if self.is_spawning(server): log.info("Virtual Machine %s is spawning..." % self.name) if create_check_retry > config_create_check_retry_count: @@ -293,26 +304,22 @@ def _wait_for_activated_service(self, method=None): create_check_retry += 1 time.sleep(config_create_check_pause) - elif self.vm_has_created(): - if server.status.lower() in 'error': - log.error("VM %s was errored. Rebuilding..." % server.name) - self.rebuild() - break + elif self.is_created(server): if method is not None: method() if self.ping_vm(): self.ready = True break - if ping_retry > config_ping_retry_count: - p = config_ping_retry_count * config_ping_timeout + if ping_retry > config.OPENSTACK_PING_RETRY_COUNT: + p = config.OPENSTACK_PING_RETRY_COUNT * config.PING_TIMEOUT log.info("VM %s pings more than %s seconds..." % (self.name, p)) self.delete(try_to_rebuild=True) break - ping_retry += 1 - else: - log.error("VM %s has not been created." % self.name) - self.delete(try_to_rebuild=False) + + elif self.is_broken(server): + log.error("VM %s was errored. Rebuilding..." % server.name) + self.rebuild() break @property @@ -363,22 +370,26 @@ def get_network_id(self): # fixme # create new network - def vm_has_created(self): - server = self.get_vm(self.name) - if server: - if server.status.lower() == 'active': - if getattr(server, 'addresses', None) is not None: - return True + @staticmethod + def is_created(server): + if server.status.lower() == 'active': + if getattr(server, 'addresses', None) is not None: + return True return False - def check_vm_exist(self, server_name): - return True if self.get_vm(server_name) else False + @staticmethod + def is_spawning(server): + return server.status.lower() in ('build', 'rebuild') + + @staticmethod + def is_broken(server): + return server.status.lower() in 'error' def get_vm(self, server_name): try: server = self.nova_client.servers.find(name=server_name) - return server if server.name == server_name else None + return server if server else None except: log.exception("VM %s does not exist" % server_name) return None @@ -390,7 +401,6 @@ def delete(self, try_to_rebuild=True): self.ready = False self.pool.remove_vm(self) - server = self.get_vm(self.name) if server: try: